def _build_context_encoder(self): """ Build the context (i.e. dialogue history) encoder. """ if self.opt.get("share_encoder"): self.context_encoder = self.label_encoder else: if (self.opt["load_context_encoder_from"] is None and self.opt["context_encoder_embedding_type"] == "fasttext_cc"): embeddings = load_fasttext_embeddings( self.dictionary, self.opt["embedding_size"], self.opt["datapath"]) else: embeddings = nn.Embedding(len(self.dictionary), self.opt["embedding_size"]) self.context_encoder = TransformerEncoder( opt=self.opt, embedding=embeddings, vocabulary_size=len(self.dictionary), padding_idx=self.dictionary.tok2ind[ self.dictionary.null_token], embeddings_scale=False, output_scaling=1.0, ) if self.opt.get("load_context_encoder_from") is not None: self._load_context_encoder_state()
def _build_text_encoder(self, n_layers_text): """ Build the text (candidate) encoder. :param n_layers_text: how many layers the transformer will have """ self.embeddings = nn.Embedding(len(self.dictionary), self.opt['embedding_size']) if (self.opt.get('load_encoder_from') is None and self.opt['embedding_type'] == 'fasttext_cc'): self.embeddings = load_fasttext_embeddings( self.dictionary, self.opt['embedding_size'], self.opt['datapath']) self.text_encoder = TransformerEncoder( opt=self.opt, embedding=self.embeddings, vocabulary_size=len(self.dictionary), padding_idx=self.dictionary.tok2ind[self.dictionary.null_token], embeddings_scale=False, output_scaling=1.0, ) if self.opt.get('load_encoder_from') is not None: self._load_text_encoder_state() self.additional_layer = LinearWrapper( self.opt['embedding_size'], self.opt['hidden_dim'], dropout=self.opt['additional_layer_dropout'], )
def __init__(self, opt: ModelOption, tokenizer: Tokenizer, embedding: nn.Embedding): super(TransformerMemNet, self).__init__() self.encoder = TransformerEncoder( n_heads=opt.n_heads, n_layers=opt.n_layers, embedding_size=embedding.weight.shape[1], vocabulary_size=embedding.weight.shape[0], ffn_size=opt.ffn_size, embedding=embedding, dropout=opt.dropout, padding_idx=tokenizer.dictionary[tokenizer.dictionary.null_token], n_positions=opt.n_positions, reduction_type='none') self.decoder = TransformerDecoder( n_heads=opt.n_heads, n_layers=opt.n_layers, embedding_size=embedding.weight.shape[1], ffn_size=opt.ffn_size, vocabulary_size=embedding.weight.shape[0], embedding=embedding, dropout=opt.dropout, n_positions=opt.n_positions, padding_idx=tokenizer.dictionary[tokenizer.dictionary.null_token]) self.encoder = ContextKnowledgeEncoder(self.encoder) self.decoder = ContextKnowledgeDecoder(self.decoder) self.linear = nn.Linear(300, embedding.weight.shape[0])
def __init__(self, transformer, opt, dictionary, agenttype): super().__init__() # The transformer takes care of most of the work, but other modules # expect us to have an embeddings available self.embeddings = transformer.embeddings self.embed_dim = transformer.embeddings.embedding_dim self.transformer = transformer self.knowledge_transformer = TransformerEncoder( embedding=self.embeddings, n_heads=opt['n_heads'], n_layers=opt['n_layers_knowledge'], embedding_size=opt['embedding_size'], ffn_size=opt['ffn_size'], vocabulary_size=len(dictionary), padding_idx=transformer.padding_idx, learn_positional_embeddings=opt['learn_positional_embeddings'], embeddings_scale=opt['embeddings_scale'], reduction_type=transformer.reduction_type, n_positions=transformer.n_positions, activation=opt['activation'], variant=opt['variant'], output_scaling=opt['output_scaling'], ) self.agenttype = agenttype if self.agenttype == 'agent': self.intent_head = ClassificationHead(opt['embedding_size']) self.name_head = MultiTokenClassificationHead(opt['embedding_size'], self.embeddings, opt.get('name_vec_len')) self.reservation_transformer = TransformerEncoder( embedding=self.embeddings, n_heads=opt['n_heads'], n_layers=opt['n_layers_knowledge'], embedding_size=opt['embedding_size'], ffn_size=opt['ffn_size'], vocabulary_size=len(dictionary), padding_idx=transformer.padding_idx, learn_positional_embeddings=opt['learn_positional_embeddings'], embeddings_scale=opt['embeddings_scale'], reduction_type=transformer.reduction_type, n_positions=transformer.n_positions, activation=opt['activation'], variant=opt['variant'], output_scaling=opt['output_scaling'], ) self.know_use_project = nn.Linear(opt['embedding_size'], opt['embedding_size'])
def build_encoder( cls, opt, dictionary, embedding=None, padding_idx=None, reduction_type='mean' ): return TransformerEncoder( opt=opt, embedding=embedding, vocabulary_size=len(dictionary), padding_idx=padding_idx, reduction_type=reduction_type, )
def build_encoder(self, opt, embeddings): return TransformerEncoder( opt=opt, embedding=embeddings, vocabulary_size=self.vocab_size, padding_idx=self.pad_idx, dropout=0.0, n_positions=1024, n_segments=0, activation='relu', variant='aiayn', output_scaling=1.0, )
def build_encoder(self, opt, embeddings): return TransformerEncoder( n_heads=opt['n_heads'], n_layers=opt['n_layers'], embedding_size=opt['embedding_size'], ffn_size=opt['ffn_size'], vocabulary_size=self.vocab_size, embedding=embeddings, attention_dropout=opt['attention_dropout'], relu_dropout=opt['relu_dropout'], padding_idx=self.pad_idx, learn_positional_embeddings=opt.get('learn_positional_embeddings', False), embeddings_scale=opt['embeddings_scale'], )
def _build_text_encoder(self, n_layers_text): """ Build the text (candidate) encoder. :param n_layers_text: how many layers the transformer will have """ self.embeddings = nn.Embedding(len(self.dictionary), self.opt['embedding_size']) if (self.opt.get('load_encoder_from') is None and self.opt['embedding_type'] == 'fasttext_cc'): self.embeddings = load_fasttext_embeddings( self.dictionary, self.opt['embedding_size'], self.opt['datapath']) self.text_encoder = TransformerEncoder( n_heads=self.opt['n_heads'], n_layers=self.opt['n_layers'], embedding_size=self.opt['embedding_size'], ffn_size=self.opt['ffn_size'], vocabulary_size=len(self.dictionary), embedding=self.embeddings, dropout=self.opt['dropout'], attention_dropout=self.opt['attention_dropout'], relu_dropout=self.opt['relu_dropout'], padding_idx=self.dictionary.tok2ind[self.dictionary.null_token], learn_positional_embeddings=self. opt['learn_positional_embeddings'], embeddings_scale=False, n_positions=self.opt['n_positions'], activation=self.opt['activation'], variant=self.opt['variant'], n_segments=self.opt['n_segments'], ) if self.opt.get('load_encoder_from') is not None: self._load_text_encoder_state() self.additional_layer = LinearWrapper( self.opt['embedding_size'], self.opt['hidden_dim'], dropout=self.opt['additional_layer_dropout'], )
def _build_context_encoder(self): """ Build the context (i.e. dialogue history) encoder. """ if self.opt.get("share_encoder"): self.context_encoder = self.label_encoder else: if (self.opt["load_context_encoder_from"] is None and self.opt["context_encoder_embedding_type"] == "fasttext_cc"): embeddings = load_fasttext_embeddings( self.dictionary, self.opt["embedding_size"], self.opt["datapath"]) else: embeddings = nn.Embedding(len(self.dictionary), self.opt["embedding_size"]) self.context_encoder = TransformerEncoder( n_heads=self.opt["n_heads"], n_layers=self.opt["n_layers"], embedding_size=self.opt["embedding_size"], ffn_size=self.opt["ffn_size"], vocabulary_size=len(self.dictionary), embedding=embeddings, dropout=self.opt["dropout"], attention_dropout=self.opt["attention_dropout"], relu_dropout=self.opt["relu_dropout"], padding_idx=self.dictionary.tok2ind[ self.dictionary.null_token], learn_positional_embeddings=self. opt["learn_positional_embeddings"], embeddings_scale=False, n_positions=self.opt["n_positions"], activation=self.opt["activation"], variant=self.opt["variant"], n_segments=self.opt["n_segments"], ) if self.opt.get("load_context_encoder_from") is not None: self._load_context_encoder_state()