def _build_context_encoder(self):
"""
Build the context (i.e. dialogue history) encoder.
"""
if self.opt.get("share_encoder"):
self.context_encoder = self.label_encoder
else:
if (self.opt["load_context_encoder_from"] is None
and self.opt["context_encoder_embedding_type"]
== "fasttext_cc"):
embeddings = load_fasttext_embeddings(
self.dictionary, self.opt["embedding_size"],
self.opt["datapath"])
else:
embeddings = nn.Embedding(len(self.dictionary),
self.opt["embedding_size"])
self.context_encoder = TransformerEncoder(
opt=self.opt,
embedding=embeddings,
vocabulary_size=len(self.dictionary),
padding_idx=self.dictionary.tok2ind[
self.dictionary.null_token],
embeddings_scale=False,
output_scaling=1.0,
)
if self.opt.get("load_context_encoder_from") is not None:
self._load_context_encoder_state()
def _build_text_encoder(self, n_layers_text):
"""
Build the text (candidate) encoder.
:param n_layers_text:
how many layers the transformer will have
"""
self.embeddings = nn.Embedding(len(self.dictionary),
self.opt['embedding_size'])
if (self.opt.get('load_encoder_from') is None
and self.opt['embedding_type'] == 'fasttext_cc'):
self.embeddings = load_fasttext_embeddings(
self.dictionary, self.opt['embedding_size'],
self.opt['datapath'])
self.text_encoder = TransformerEncoder(
opt=self.opt,
embedding=self.embeddings,
vocabulary_size=len(self.dictionary),
padding_idx=self.dictionary.tok2ind[self.dictionary.null_token],
embeddings_scale=False,
output_scaling=1.0,
)
if self.opt.get('load_encoder_from') is not None:
self._load_text_encoder_state()
self.additional_layer = LinearWrapper(
self.opt['embedding_size'],
self.opt['hidden_dim'],
dropout=self.opt['additional_layer_dropout'],
)
def __init__(self, opt: ModelOption, tokenizer: Tokenizer,
embedding: nn.Embedding):
super(TransformerMemNet, self).__init__()
self.encoder = TransformerEncoder(
n_heads=opt.n_heads,
n_layers=opt.n_layers,
embedding_size=embedding.weight.shape[1],
vocabulary_size=embedding.weight.shape[0],
ffn_size=opt.ffn_size,
embedding=embedding,
dropout=opt.dropout,
padding_idx=tokenizer.dictionary[tokenizer.dictionary.null_token],
n_positions=opt.n_positions,
reduction_type='none')
self.decoder = TransformerDecoder(
n_heads=opt.n_heads,
n_layers=opt.n_layers,
embedding_size=embedding.weight.shape[1],
ffn_size=opt.ffn_size,
vocabulary_size=embedding.weight.shape[0],
embedding=embedding,
dropout=opt.dropout,
n_positions=opt.n_positions,
padding_idx=tokenizer.dictionary[tokenizer.dictionary.null_token])
self.encoder = ContextKnowledgeEncoder(self.encoder)
self.decoder = ContextKnowledgeDecoder(self.decoder)
self.linear = nn.Linear(300, embedding.weight.shape[0])
def __init__(self, transformer, opt, dictionary, agenttype):
super().__init__()
# The transformer takes care of most of the work, but other modules
# expect us to have an embeddings available
self.embeddings = transformer.embeddings
self.embed_dim = transformer.embeddings.embedding_dim
self.transformer = transformer
self.knowledge_transformer = TransformerEncoder(
embedding=self.embeddings,
n_heads=opt['n_heads'],
n_layers=opt['n_layers_knowledge'],
embedding_size=opt['embedding_size'],
ffn_size=opt['ffn_size'],
vocabulary_size=len(dictionary),
padding_idx=transformer.padding_idx,
learn_positional_embeddings=opt['learn_positional_embeddings'],
embeddings_scale=opt['embeddings_scale'],
reduction_type=transformer.reduction_type,
n_positions=transformer.n_positions,
activation=opt['activation'],
variant=opt['variant'],
output_scaling=opt['output_scaling'],
)
self.agenttype = agenttype
if self.agenttype == 'agent':
self.intent_head = ClassificationHead(opt['embedding_size'])
self.name_head = MultiTokenClassificationHead(opt['embedding_size'],
self.embeddings,
opt.get('name_vec_len'))
self.reservation_transformer = TransformerEncoder(
embedding=self.embeddings,
n_heads=opt['n_heads'],
n_layers=opt['n_layers_knowledge'],
embedding_size=opt['embedding_size'],
ffn_size=opt['ffn_size'],
vocabulary_size=len(dictionary),
padding_idx=transformer.padding_idx,
learn_positional_embeddings=opt['learn_positional_embeddings'],
embeddings_scale=opt['embeddings_scale'],
reduction_type=transformer.reduction_type,
n_positions=transformer.n_positions,
activation=opt['activation'],
variant=opt['variant'],
output_scaling=opt['output_scaling'],
)
self.know_use_project = nn.Linear(opt['embedding_size'],
opt['embedding_size'])
def build_encoder(
cls, opt, dictionary, embedding=None, padding_idx=None, reduction_type='mean'
):
return TransformerEncoder(
opt=opt,
embedding=embedding,
vocabulary_size=len(dictionary),
padding_idx=padding_idx,
reduction_type=reduction_type,
)
def build_encoder(self, opt, embeddings):
return TransformerEncoder(
opt=opt,
embedding=embeddings,
vocabulary_size=self.vocab_size,
padding_idx=self.pad_idx,
dropout=0.0,
n_positions=1024,
n_segments=0,
activation='relu',
variant='aiayn',
output_scaling=1.0,
)
def build_encoder(self, opt, embeddings):
return TransformerEncoder(
n_heads=opt['n_heads'],
n_layers=opt['n_layers'],
embedding_size=opt['embedding_size'],
ffn_size=opt['ffn_size'],
vocabulary_size=self.vocab_size,
embedding=embeddings,
attention_dropout=opt['attention_dropout'],
relu_dropout=opt['relu_dropout'],
padding_idx=self.pad_idx,
learn_positional_embeddings=opt.get('learn_positional_embeddings', False),
embeddings_scale=opt['embeddings_scale'],
)
def _build_text_encoder(self, n_layers_text):
"""
Build the text (candidate) encoder.
:param n_layers_text:
how many layers the transformer will have
"""
self.embeddings = nn.Embedding(len(self.dictionary),
self.opt['embedding_size'])
if (self.opt.get('load_encoder_from') is None
and self.opt['embedding_type'] == 'fasttext_cc'):
self.embeddings = load_fasttext_embeddings(
self.dictionary, self.opt['embedding_size'],
self.opt['datapath'])
self.text_encoder = TransformerEncoder(
n_heads=self.opt['n_heads'],
n_layers=self.opt['n_layers'],
embedding_size=self.opt['embedding_size'],
ffn_size=self.opt['ffn_size'],
vocabulary_size=len(self.dictionary),
embedding=self.embeddings,
dropout=self.opt['dropout'],
attention_dropout=self.opt['attention_dropout'],
relu_dropout=self.opt['relu_dropout'],
padding_idx=self.dictionary.tok2ind[self.dictionary.null_token],
learn_positional_embeddings=self.
opt['learn_positional_embeddings'],
embeddings_scale=False,
n_positions=self.opt['n_positions'],
activation=self.opt['activation'],
variant=self.opt['variant'],
n_segments=self.opt['n_segments'],
)
if self.opt.get('load_encoder_from') is not None:
self._load_text_encoder_state()
self.additional_layer = LinearWrapper(
self.opt['embedding_size'],
self.opt['hidden_dim'],
dropout=self.opt['additional_layer_dropout'],
)
def _build_context_encoder(self):
"""
Build the context (i.e. dialogue history) encoder.
"""
if self.opt.get("share_encoder"):
self.context_encoder = self.label_encoder
else:
if (self.opt["load_context_encoder_from"] is None
and self.opt["context_encoder_embedding_type"]
== "fasttext_cc"):
embeddings = load_fasttext_embeddings(
self.dictionary, self.opt["embedding_size"],
self.opt["datapath"])
else:
embeddings = nn.Embedding(len(self.dictionary),
self.opt["embedding_size"])
self.context_encoder = TransformerEncoder(
n_heads=self.opt["n_heads"],
n_layers=self.opt["n_layers"],
embedding_size=self.opt["embedding_size"],
ffn_size=self.opt["ffn_size"],
vocabulary_size=len(self.dictionary),
embedding=embeddings,
dropout=self.opt["dropout"],
attention_dropout=self.opt["attention_dropout"],
relu_dropout=self.opt["relu_dropout"],
padding_idx=self.dictionary.tok2ind[
self.dictionary.null_token],
learn_positional_embeddings=self.
opt["learn_positional_embeddings"],
embeddings_scale=False,
n_positions=self.opt["n_positions"],
activation=self.opt["activation"],
variant=self.opt["variant"],
n_segments=self.opt["n_segments"],
)
if self.opt.get("load_context_encoder_from") is not None:
self._load_context_encoder_state()