def __init__( self, n_class, cnn_kwargs=None, encoder_kwargs=None, encoder_type="Conformer", pooling="token", layer_init="pytorch", ): super(SEDModel, self).__init__() self.cnn = CNN(n_in_channel=1, **cnn_kwargs) input_dim = self.cnn.nb_filters[-1] adim = encoder_kwargs["adim"] self.pooling = pooling if encoder_type == "Transformer": self.encoder = TransformerEncoder(input_dim, **encoder_kwargs) elif encoder_type == "Conformer": self.encoder = ConformerEncoder(input_dim, **encoder_kwargs) else: raise ValueError("Choose encoder_type in ['Transformer', 'Conformer']") self.classifier = torch.nn.Linear(adim, n_class) if self.pooling == "attention": self.dense = torch.nn.Linear(adim, n_class) self.sigmoid = torch.sigmoid self.softmax = torch.nn.Softmax(dim=-1) elif self.pooling == "token": self.linear_emb = torch.nn.Linear(1, input_dim) self.reset_parameters(layer_init)
def __init__(self, d_model, num_heads, num_encoder_layers, num_decoder_layers, dropout=0.0): super().__init__() self.encoder = TransformerEncoder(num_encoder_layers, d_model, num_heads, dropout) self.decoder = TransformerDecoder(num_decoder_layers, d_model, num_heads, dropout)
def __init__(self, initialize_xavier: bool, src_vocab_size: int, tgt_vocab_size: int, src_embedding: nn.Embedding, tgt_embedding: nn.Embedding, d_word_vec: int, n_enc_blocks=constants.DEFAULT_ENCODER_BLOCKS, n_head=constants.DEFAULT_NUMBER_OF_ATTENTION_HEADS, d_model=constants.DEFAULT_LAYER_SIZE, dropout_rate=constants.DEFAULT_MODEL_DROPOUT, pointwise_layer_size=constants.DEFAULT_DIMENSION_OF_PWFC_HIDDEN_LAYER, d_k=constants.DEFAULT_DIMENSION_OF_KEYQUERY_WEIGHTS, d_v=constants.DEFAULT_DIMENSION_OF_VALUE_WEIGHTS): """Initializes a Transformer Model Arguments: initialize_xavier {boolean} -- [description] src_vocab_size {int} -- Size / Length of the source vocabulary (how many tokens) tgt_vocab_size {[type]} -- Size / Length of the target vocabulary (how many tokens) d_word_vec {[type]} -- Dimension of word vectors (default: 512) Keyword Arguments: n_enc_blocks {[type]} -- [description] (default: {constants.DEFAULT_ENCODER_BLOCKS}) n_head {[type]} -- [description] (default: {constants.DEFAULT_NUMBER_OF_ATTENTION_HEADS}) d_model {[type]} -- [description] (default: {constants.DEFAULT_LAYER_SIZE}) dropout_rate {[type]} -- [description] (default: {constants.DEFAULT_MODEL_DROPOUT}) pointwise_layer_size {[type]} -- [description] (default: {constants.DEFAULT_DIMENSION_OF_PWFC_HIDDEN_LAYER}) d_k {[type]} -- [description] (default: {constants.DEFAULT_DIMENSION_OF_KEYQUERY_WEIGHTS}) d_v {[type]} -- [description] (default: {constants.DEFAULT_DIMENSION_OF_VALUE_WEIGHTS}) """ super(GoogleTransformer, self).__init__() assert d_model == d_word_vec, \ 'To facilitate the residual connections, \ the dimensions of all module outputs shall be the same. Input (Word Embedding) = Output of Encoder Layer' self._init_loggers() self.encoder = TransformerEncoder(src_embedding) self.decoder = TransformerDecoder(tgt_embedding) # generate a last layer that projects from the last output of the decoder layer # with size d_model to the size of the target vocabulary self.decoder_to_tgt_vocabulary = nn.Linear(d_model, tgt_vocab_size, bias=False) self.generator = TransformerTargetGenerator(d_model, src_vocab_size) if initialize_xavier: for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p)
def load_model(dataset, rc, experiment_name): loss = LossCombiner(4, dataset.class_weights, NllLoss) transformer = TransformerEncoder(dataset.source_embedding, hyperparameters=rc) model = JointAspectTagger(transformer, rc, 4, 20, dataset.target_names) optimizer = get_optimizer(model, rc) trainer = Trainer(model, loss, optimizer, rc, dataset, experiment_name, enable_tensorboard=False, verbose=False) return trainer
def get_encoder(st_ds_conf: dict): emb_sz = st_ds_conf['emb_sz'] if st_ds_conf['encoder'] == 'lstm': encoder = StackedEncoder( [ PytorchSeq2SeqWrapper( torch.nn.LSTM(emb_sz, emb_sz, batch_first=True)) for _ in range(st_ds_conf['num_enc_layers']) ], emb_sz, emb_sz, input_dropout=st_ds_conf['intermediate_dropout']) elif st_ds_conf['encoder'] == 'bilstm': encoder = StackedEncoder( [ PytorchSeq2SeqWrapper( torch.nn.LSTM( emb_sz, emb_sz, batch_first=True, bidirectional=True)) ] + [ PytorchSeq2SeqWrapper( torch.nn.LSTM(emb_sz * 2, emb_sz, batch_first=True, bidirectional=True)) for _ in range(st_ds_conf['num_enc_layers'] - 1) ], emb_sz, emb_sz * 2, input_dropout=st_ds_conf['intermediate_dropout']) elif st_ds_conf['encoder'] == 'transformer': encoder = StackedEncoder([ TransformerEncoder( input_dim=emb_sz, num_layers=st_ds_conf['num_enc_layers'], num_heads=st_ds_conf['num_heads'], feedforward_hidden_dim=emb_sz, feedforward_dropout=st_ds_conf['feedforward_dropout'], residual_dropout=st_ds_conf['residual_dropout'], attention_dropout=st_ds_conf['attention_dropout'], ) for _ in range(st_ds_conf['num_enc_layers']) ], emb_sz, emb_sz, input_dropout=0.) else: assert False return encoder
def load_model(self, dataset, rc, experiment_name, iteration): loss = LossCombiner(4, dataset.class_weights, NllLoss) if self.produce_baseline: iteration = 0 if iteration == 0: self.current_transformer = TransformerEncoder( dataset.source_embedding, hyperparameters=rc) model = JointAspectTagger(self.current_transformer, rc, 4, 20, dataset.target_names, initialize_params=True) else: model = JointAspectTagger(self.current_transformer, rc, 4, 20, dataset.target_names, initialize_params=False) optimizer = get_optimizer(model, rc) trainer = Trainer(model, loss, optimizer, rc, dataset, experiment_name, enable_tensorboard=True, verbose=True) # see if we might be able to restore the source model if iteration == 0 and self.load_model_path is not None: model, optimizer, epoch = trainer.load_model( custom_path=self.load_model_path) self.skip_source_training = True return trainer
def load_model(self, dataset, rc, experiment_name): transformer = TransformerEncoder(dataset.source_embedding, hyperparameters=rc) if rc.use_random_classifier: from models.random_model import RandomModel model = RandomModel(rc, dataset.target_size, len(dataset.target_names), dataset.target_names) loss = NllLoss(dataset.target_size, dataset.class_weights[0]) else: # NER or ABSA-task? if rc.task == 'ner': from models.transformer_tagger import TransformerTagger from models.output_layers import SoftmaxOutputLayer loss = NllLoss(dataset.target_size, dataset.class_weights[0]) softmax = SoftmaxOutputLayer(rc.model_size, dataset.target_size) model = TransformerTagger(transformer, softmax) else: from models.jointAspectTagger import JointAspectTagger loss = LossCombiner(dataset.target_size, dataset.class_weights, NllLoss) model = JointAspectTagger(transformer, rc, dataset.target_size, len(dataset.target_names), dataset.target_names) optimizer = get_optimizer(model, rc) trainer = Trainer( model, loss, optimizer, rc, dataset, experiment_name, enable_tensorboard=False, verbose=False) return trainer
def main(): parser = utils.opt_parser.get_trainer_opt_parser() parser.add_argument('models', nargs='*', help='pretrained models for the same setting') parser.add_argument('--test', action="store_true", help='use testing mode') parser.add_argument('--num-layer', type=int, help="stacked layer of transformer model") args = parser.parse_args() reader = data_adapter.GeoQueryDatasetReader() training_set = reader.read(config.DATASETS[args.dataset].train_path) try: validation_set = reader.read(config.DATASETS[args.dataset].dev_path) except: validation_set = None vocab = allennlp.data.Vocabulary.from_instances(training_set) st_ds_conf = config.TRANSFORMER_CONF[args.dataset] if args.num_layer: st_ds_conf['num_layers'] = args.num_layer encoder = TransformerEncoder( input_dim=st_ds_conf['emb_sz'], num_layers=st_ds_conf['num_layers'], num_heads=st_ds_conf['num_heads'], feedforward_hidden_dim=st_ds_conf['emb_sz'], ) decoder = TransformerDecoder( input_dim=st_ds_conf['emb_sz'], num_layers=st_ds_conf['num_layers'], num_heads=st_ds_conf['num_heads'], feedforward_hidden_dim=st_ds_conf['emb_sz'], feedforward_dropout=0.1, ) source_embedding = allennlp.modules.Embedding( num_embeddings=vocab.get_vocab_size('nltokens'), embedding_dim=st_ds_conf['emb_sz']) target_embedding = allennlp.modules.Embedding( num_embeddings=vocab.get_vocab_size('lftokens'), embedding_dim=st_ds_conf['emb_sz']) model = ParallelSeq2Seq( vocab=vocab, encoder=encoder, decoder=decoder, source_embedding=source_embedding, target_embedding=target_embedding, target_namespace='lftokens', start_symbol=START_SYMBOL, eos_symbol=END_SYMBOL, max_decoding_step=st_ds_conf['max_decoding_len'], ) if args.models: model.load_state_dict(torch.load(args.models[0])) if not args.test or not args.models: iterator = BucketIterator(sorting_keys=[("source_tokens", "num_tokens") ], batch_size=st_ds_conf['batch_sz']) iterator.index_with(vocab) optim = torch.optim.Adam(model.parameters()) savepath = os.path.join( config.SNAPSHOT_PATH, args.dataset, 'transformer', datetime.datetime.now().strftime('%Y%m%d%H%M%S')) if not os.path.exists(savepath): os.makedirs(savepath, mode=0o755) trainer = allennlp.training.Trainer( model=model, optimizer=optim, iterator=iterator, train_dataset=training_set, validation_dataset=validation_set, serialization_dir=savepath, cuda_device=args.device, num_epochs=config.TRAINING_LIMIT, ) trainer.train() else: testing_set = reader.read(config.DATASETS[args.dataset].test_path) model.eval() predictor = allennlp.predictors.SimpleSeq2SeqPredictor(model, reader) for instance in testing_set: print('SRC: ', instance.fields['source_tokens'].tokens) print( 'GOLD:', ' '.join( str(x) for x in instance.fields['target_tokens'].tokens[1:-1])) del instance.fields['target_tokens'] output = predictor.predict_instance(instance) print('PRED:', ' '.join(output['predicted_tokens']))
def main(): parser = utils.opt_parser.get_trainer_opt_parser() parser.add_argument('models', nargs='*', help='pretrained models for the same setting') parser.add_argument('--test', action="store_true", help='use testing mode') parser.add_argument('--num-layer', type=int, help='maximum number of stacked layers') parser.add_argument( '--use-ut', action="store_true", help='Use universal transformer instead of transformer') args = parser.parse_args() reader = data_adapter.GeoQueryDatasetReader() training_set = reader.read(config.DATASETS[args.dataset].train_path) try: validation_set = reader.read(config.DATASETS[args.dataset].dev_path) except: validation_set = None vocab = allennlp.data.Vocabulary.from_instances(training_set) st_ds_conf = config.TRANS2SEQ_CONF[args.dataset] if args.num_layer: st_ds_conf['max_num_layers'] = args.num_layer if args.epoch: config.TRAINING_LIMIT = args.epoch if args.batch: st_ds_conf['batch_sz'] = args.batch bsz = st_ds_conf['batch_sz'] emb_sz = st_ds_conf['emb_sz'] src_embedder = BasicTextFieldEmbedder( token_embedders={ "tokens": Embedding(vocab.get_vocab_size('nltokens'), emb_sz) }) if args.use_ut: transformer_encoder = UTEncoder( input_dim=emb_sz, max_num_layers=st_ds_conf['max_num_layers'], num_heads=st_ds_conf['num_heads'], feedforward_hidden_dim=emb_sz, feedforward_dropout=st_ds_conf['feedforward_dropout'], attention_dropout=st_ds_conf['attention_dropout'], residual_dropout=st_ds_conf['residual_dropout'], use_act=st_ds_conf['act'], use_vanilla_wiring=st_ds_conf['vanilla_wiring']) else: transformer_encoder = TransformerEncoder( input_dim=emb_sz, num_layers=st_ds_conf['max_num_layers'], num_heads=st_ds_conf['num_heads'], feedforward_hidden_dim=emb_sz, feedforward_dropout=st_ds_conf['feedforward_dropout'], attention_dropout=st_ds_conf['attention_dropout'], residual_dropout=st_ds_conf['residual_dropout'], ) model = allennlp.models.SimpleSeq2Seq( vocab, source_embedder=src_embedder, encoder=transformer_encoder, max_decoding_steps=50, attention=allennlp.modules.attention.DotProductAttention(), beam_size=6, target_namespace="lftokens", use_bleu=True) if args.models: model.load_state_dict(torch.load(args.models[0])) if not args.test or not args.models: iterator = BucketIterator(sorting_keys=[("source_tokens", "num_tokens") ], batch_size=bsz) iterator.index_with(vocab) optim = torch.optim.Adam(model.parameters()) savepath = os.path.join( config.SNAPSHOT_PATH, args.dataset, 'transformer2seq', datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + "--" + args.memo) if not os.path.exists(savepath): os.makedirs(savepath, mode=0o755) trainer = allennlp.training.Trainer( model=model, optimizer=optim, iterator=iterator, train_dataset=training_set, validation_dataset=validation_set, serialization_dir=savepath, cuda_device=args.device, num_epochs=config.TRAINING_LIMIT, ) trainer.train() else: testing_set = reader.read(config.DATASETS[args.dataset].test_path) model.eval() predictor = allennlp.predictors.SimpleSeq2SeqPredictor(model, reader) for instance in tqdm.tqdm(testing_set, total=len(testing_set)): print('SRC: ', instance.fields['source_tokens'].tokens) print( 'GOLD:', ' '.join( str(x) for x in instance.fields['target_tokens'].tokens[1:-1])) del instance.fields['target_tokens'] output = predictor.predict_instance(instance) print('PRED:', ' '.join(output['predicted_tokens']))
def get_model(vocab, st_ds_conf): emb_sz = st_ds_conf['emb_sz'] source_embedding = allennlp.modules.Embedding( num_embeddings=vocab.get_vocab_size('nltokens'), embedding_dim=emb_sz) target_embedding = allennlp.modules.Embedding( num_embeddings=vocab.get_vocab_size('lftokens'), embedding_dim=emb_sz) if st_ds_conf['encoder'] == 'lstm': encoder = StackedEncoder( [ PytorchSeq2SeqWrapper( torch.nn.LSTM(emb_sz, emb_sz, batch_first=True)) for _ in range(st_ds_conf['num_enc_layers']) ], emb_sz, emb_sz, input_dropout=st_ds_conf['intermediate_dropout']) elif st_ds_conf['encoder'] == 'bilstm': encoder = StackedEncoder( [ PytorchSeq2SeqWrapper( torch.nn.LSTM( emb_sz, emb_sz, batch_first=True, bidirectional=True)) for _ in range(st_ds_conf['num_enc_layers']) ], emb_sz, emb_sz, input_dropout=st_ds_conf['intermediate_dropout']) elif st_ds_conf['encoder'] == 'transformer': encoder = StackedEncoder( [ TransformerEncoder( input_dim=emb_sz, num_layers=st_ds_conf['num_enc_layers'], num_heads=st_ds_conf['num_heads'], feedforward_hidden_dim=emb_sz, feedforward_dropout=st_ds_conf['feedforward_dropout'], residual_dropout=st_ds_conf['residual_dropout'], attention_dropout=st_ds_conf['attention_dropout'], ) for _ in range(st_ds_conf['num_enc_layers']) ], emb_sz, emb_sz, input_dropout=st_ds_conf['intermediate_dropout']) else: assert False enc_out_dim = encoder.get_output_dim() dec_out_dim = emb_sz dec_hist_attn = get_attention(st_ds_conf, st_ds_conf['dec_hist_attn']) enc_attn = get_attention(st_ds_conf, st_ds_conf['enc_attn']) if st_ds_conf['enc_attn'] == 'dot_product': assert enc_out_dim == dec_out_dim, "encoder hidden states must be able to multiply with decoder output" def sum_attn_dims(attns, dims): return sum(dim for attn, dim in zip(attns, dims) if attn is not None) if st_ds_conf['concat_attn_to_dec_input']: dec_in_dim = dec_out_dim + sum_attn_dims([enc_attn, dec_hist_attn], [enc_out_dim, dec_out_dim]) else: dec_in_dim = dec_out_dim rnn_cell = get_rnn_cell(st_ds_conf, dec_in_dim, dec_out_dim) if st_ds_conf['concat_attn_to_dec_input']: proj_in_dim = dec_out_dim + sum_attn_dims([enc_attn, dec_hist_attn], [enc_out_dim, dec_out_dim]) else: proj_in_dim = dec_out_dim word_proj = torch.nn.Linear(proj_in_dim, vocab.get_vocab_size('lftokens')) model = BaseSeq2Seq( vocab=vocab, encoder=encoder, decoder=rnn_cell, word_projection=word_proj, source_embedding=source_embedding, target_embedding=target_embedding, target_namespace='lftokens', start_symbol=START_SYMBOL, eos_symbol=END_SYMBOL, max_decoding_step=st_ds_conf['max_decoding_len'], enc_attention=enc_attn, dec_hist_attn=dec_hist_attn, intermediate_dropout=st_ds_conf['intermediate_dropout'], concat_attn_to_dec_input=st_ds_conf['concat_attn_to_dec_input'], ) return model
def get_model(vocab, st_ds_conf): emb_sz = st_ds_conf['emb_sz'] source_embedding = allennlp.modules.Embedding( num_embeddings=vocab.get_vocab_size('nltokens'), embedding_dim=emb_sz) target_embedding = allennlp.modules.Embedding( num_embeddings=vocab.get_vocab_size('lftokens'), embedding_dim=emb_sz) if st_ds_conf['encoder'] == 'lstm': encoder = allennlp.modules.seq2seq_encoders.PytorchSeq2SeqWrapper( torch.nn.LSTM(emb_sz, emb_sz, st_ds_conf['num_enc_layers'], batch_first=True)) elif st_ds_conf['encoder'] == 'bilstm': encoder = allennlp.modules.seq2seq_encoders.PytorchSeq2SeqWrapper( torch.nn.LSTM(emb_sz, emb_sz, st_ds_conf['num_enc_layers'], batch_first=True, bidirectional=True)) elif st_ds_conf['encoder'] == 'transformer': encoder = TransformerEncoder( input_dim=emb_sz, num_layers=st_ds_conf['num_enc_layers'], num_heads=st_ds_conf['num_heads'], feedforward_hidden_dim=emb_sz, feedforward_dropout=st_ds_conf['feedforward_dropout'], residual_dropout=st_ds_conf['residual_dropout'], attention_dropout=st_ds_conf['attention_dropout'], ) else: assert False enc_out_dim = encoder.get_output_dim() dec_out_dim = emb_sz dwa = get_attention(st_ds_conf, st_ds_conf['dwa']) dec_hist_attn = get_attention(st_ds_conf, st_ds_conf['dec_hist_attn']) enc_attn = get_attention(st_ds_conf, st_ds_conf['enc_attn']) if st_ds_conf['enc_attn'] == 'dot_product': assert enc_out_dim == dec_out_dim, "encoder hidden states must be able to multiply with decoder output" def sum_attn_dims(attns, dims): return sum(dim for attn, dim in zip(attns, dims) if attn is not None) dec_in_dim = dec_out_dim + 1 + sum_attn_dims([enc_attn, dec_hist_attn], [enc_out_dim, dec_out_dim]) rnn_cell = get_rnn_cell(st_ds_conf, dec_in_dim, dec_out_dim) halting_in_dim = dec_out_dim + sum_attn_dims( [enc_attn, dec_hist_attn, dwa], [enc_out_dim, dec_out_dim, dec_out_dim]) halting_fn = torch.nn.Sequential( # torch.nn.Dropout(st_ds_conf['act_dropout']), torch.nn.Linear(halting_in_dim, halting_in_dim), torch.nn.ReLU(), torch.nn.Dropout(st_ds_conf['act_dropout']), torch.nn.Linear(halting_in_dim, 1), torch.nn.Sigmoid(), ) decoder = ACTRNNCell( rnn_cell=rnn_cell, halting_fn=halting_fn, use_act=st_ds_conf['act'], act_max_layer=st_ds_conf['act_max_layer'], act_epsilon=st_ds_conf['act_epsilon'], rnn_input_dropout=st_ds_conf['decoder_dropout'], depth_wise_attention=dwa, state_mode=st_ds_conf['act_mode'], ) proj_in_dim = dec_out_dim + sum_attn_dims([enc_attn, dec_hist_attn], [enc_out_dim, dec_out_dim]) word_proj = torch.nn.Linear(proj_in_dim, vocab.get_vocab_size('lftokens')) model = AdaptiveSeq2Seq( vocab=vocab, encoder=encoder, decoder=decoder, word_projection=word_proj, source_embedding=source_embedding, target_embedding=target_embedding, target_namespace='lftokens', start_symbol=START_SYMBOL, eos_symbol=END_SYMBOL, max_decoding_step=st_ds_conf['max_decoding_len'], enc_attention=enc_attn, dec_hist_attn=dec_hist_attn, act_loss_weight=st_ds_conf['act_loss_weight'], prediction_dropout=st_ds_conf['prediction_dropout'], embedding_dropout=st_ds_conf['embedding_dropout'], ) return model