def main(): # load config parser = argparse.ArgumentParser(description='Building ptbs dictionary') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) path_m2 = config['path_m2'] generate_ptbs(path_m2)
def main(): # load config parser = argparse.ArgumentParser(description='Corrupting corpus') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) input = config['input'] output = config['output'] corrupt_corpus(input, output)
def main(): # load config parser = argparse.ArgumentParser(description='Building vocab') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) path_src = config['input'] path_src_no_punct = config['output'] print('2') building_vocab(path_src, path_src_no_punct)
def main(): # load config parser = argparse.ArgumentParser(description='Building vocab') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) path_vocab_src = config['path_vocab_src'] vocab_path_out = config['vocab_path_out'] print('2') building_vocab(path_vocab_src, vocab_path_out)
def main(): # load config parser = argparse.ArgumentParser(description='Building vocab') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) path_m2 = config['path_m2'] path_src = config['path_src'] path_tgt = config['path_tgt'] print('2') m2_to_parallel(path_m2, path_src, path_tgt, False, True)
def main(): # load config parser = argparse.ArgumentParser( description='filtering using a language model') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) src = config['src'] tgt = config['tgt'] threshold = config['threshold'] lm = config['lm'] process(src, tgt, lm, threshold)
def main(): # load config parser = argparse.ArgumentParser(description='Seq2seq Evaluation') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # load src-tgt pair test_path_src = config['test_path_src'] test_path_tgt = config['test_path_tgt'] path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] test_path_out = config['test_path_out'] load_dir = config['load'] max_seq_len = config['max_seq_len'] batch_size = config['batch_size'] beam_width = config['beam_width'] use_gpu = config['use_gpu'] seqrev = config['seqrev'] use_type = config['use_type'] if not os.path.exists(test_path_out): os.makedirs(test_path_out) config_save_dir = os.path.join(test_path_out, 'eval.cfg') save_config(config, config_save_dir) # set test mode: 1 = translate; 2 = plot MODE = config['eval_mode'] # check device: device = check_device(use_gpu) print('device: {}'.format(device)) # load test_set test_set = Dataset(test_path_src, test_path_tgt, path_vocab_src, path_vocab_tgt, seqrev=seqrev, max_seq_len=max_seq_len, batch_size=batch_size, use_gpu=use_gpu, use_type=use_type) print('Testset loaded') sys.stdout.flush() # run eval if MODE == 1: translate(test_set, load_dir, test_path_out, use_gpu, max_seq_len, beam_width, device, seqrev=seqrev)
def main(): # import pdb; pdb.set_trace() # load config parser = argparse.ArgumentParser(description='LAS + NMT Training') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # set random seed if config['random_seed'] is not None: set_global_seeds(config['random_seed']) # record config if not os.path.isabs(config['save']): config_save_dir = os.path.join(os.getcwd(), config['save']) if not os.path.exists(config['save']): os.makedirs(config['save']) # resume or not if type(config['load']) != type(None) and config['load_mode'] == 'resume': config_save_dir = os.path.join(config['save'], 'model-cont.cfg') else: config_save_dir = os.path.join(config['save'], 'model.cfg') save_config(config, config_save_dir) loss_coeff = {} loss_coeff['nll_asr'] = config['loss_nll_asr_coeff'] loss_coeff['nll_mt'] = config['loss_nll_mt_coeff'] loss_coeff['nll_st'] = config['loss_nll_st_coeff'] # contruct trainer Trainer = globals()['Trainer_{}'.format(config['mode'])] t = Trainer(expt_dir=config['save'], load_dir=config['load'], load_mode=config['load_mode'], load_freeze=config['load_freeze'], batch_size=config['batch_size'], minibatch_partition=config['minibatch_partition'], checkpoint_every=config['checkpoint_every'], print_every=config['print_every'], learning_rate=config['learning_rate'], learning_rate_init=config['learning_rate_init'], lr_warmup_steps=config['lr_warmup_steps'], eval_with_mask=config['eval_with_mask'], use_gpu=config['use_gpu'], gpu_id=config['gpu_id'], max_grad_norm=config['max_grad_norm'], max_count_no_improve=config['max_count_no_improve'], max_count_num_rollback=config['max_count_num_rollback'], keep_num=config['keep_num'], normalise_loss=config['normalise_loss'], loss_coeff=loss_coeff) # vocab path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] # ----- 3WAY ----- train_set = None dev_set = None mode = config['mode'] if 'ST' in mode: # load train set if config['st_train_path_src']: t.logger.info(' -- load ST train set -- ') train_path_src = config['st_train_path_src'] train_path_tgt = config['st_train_path_tgt'] train_acous_path = config['st_train_acous_path'] train_set = Dataset(path_src=train_path_src, path_tgt=train_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, use_type=config['use_type'], acous_path=train_acous_path, seqrev=config['seqrev'], acous_norm=config['las_acous_norm'], acous_norm_path=config['st_acous_norm_path'], acous_max_len=config['las_acous_max_len'], max_seq_len_src=config['max_seq_len_src'], max_seq_len_tgt=config['max_seq_len_tgt'], batch_size=config['batch_size'], data_ratio=config['st_data_ratio'], use_gpu=config['use_gpu'], mode='ST', logger=t.logger) vocab_size_enc = len(train_set.vocab_src) vocab_size_dec = len(train_set.vocab_tgt) src_word2id = train_set.src_word2id tgt_word2id = train_set.tgt_word2id src_id2word = train_set.src_id2word tgt_id2word = train_set.tgt_id2word # load dev set if config['st_dev_path_src']: t.logger.info(' -- load ST dev set -- ') dev_path_src = config['st_dev_path_src'] dev_path_tgt = config['st_dev_path_tgt'] dev_acous_path = config['st_dev_acous_path'] dev_set = Dataset(path_src=dev_path_src, path_tgt=dev_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, use_type=config['use_type'], acous_path=dev_acous_path, acous_norm_path=config['st_acous_norm_path'], acous_max_len=config['las_acous_max_len'], seqrev=config['seqrev'], acous_norm=config['las_acous_norm'], max_seq_len_src=config['max_seq_len_src'], max_seq_len_tgt=config['max_seq_len_tgt'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], mode='ST', logger=t.logger) else: dev_set = None # ----- ASR ----- asr_train_set = None asr_dev_set = None if 'ASR' in mode: # load train set if config['asr_train_path_src']: t.logger.info(' -- load ASR train set -- ') asr_train_path_src = config['asr_train_path_src'] asr_train_acous_path = config['asr_train_acous_path'] asr_train_set = Dataset( path_src=asr_train_path_src, path_tgt=None, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, use_type=config['use_type'], acous_path=asr_train_acous_path, acous_norm_path=config['asr_train_acous_norm_path'], seqrev=config['seqrev'], acous_norm=config['las_acous_norm'], acous_max_len=config['las_acous_max_len'], max_seq_len_src=config['max_seq_len_src'], max_seq_len_tgt=config['max_seq_len_tgt'], batch_size=config['batch_size'], data_ratio=config['asr_data_ratio'], use_gpu=config['use_gpu'], mode='ASR', logger=t.logger) vocab_size_enc = len(asr_train_set.vocab_src) vocab_size_dec = len(asr_train_set.vocab_tgt) src_word2id = asr_train_set.src_word2id tgt_word2id = asr_train_set.tgt_word2id src_id2word = asr_train_set.src_id2word tgt_id2word = asr_train_set.tgt_id2word # load dev set if config['asr_dev_path_src']: t.logger.info(' -- load ASR dev set -- ') asr_dev_path_src = config['asr_dev_path_src'] asr_dev_acous_path = config['asr_dev_acous_path'] asr_dev_set = Dataset( path_src=asr_dev_path_src, path_tgt=None, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, use_type=config['use_type'], acous_path=asr_dev_acous_path, acous_norm_path=config['asr_dev_acous_norm_path'], acous_max_len=config['las_acous_max_len'], seqrev=config['seqrev'], acous_norm=config['las_acous_norm'], max_seq_len_src=config['max_seq_len_src'], max_seq_len_tgt=config['max_seq_len_tgt'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], mode='ASR', logger=t.logger) else: asr_dev_set = None # ----- MT ----- mt_train_set = None mt_dev_set = None if 'MT' in mode: # load train set if config['mt_train_path_src']: t.logger.info(' -- load MT train set -- ') mt_train_path_src = config['mt_train_path_src'] mt_train_path_tgt = config['mt_train_path_tgt'] mt_train_set = Dataset(path_src=mt_train_path_src, path_tgt=mt_train_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, use_type=config['use_type'], acous_path=None, acous_norm_path=None, seqrev=config['seqrev'], acous_norm=config['las_acous_norm'], acous_max_len=config['las_acous_max_len'], max_seq_len_src=config['max_seq_len_src'], max_seq_len_tgt=config['max_seq_len_tgt'], batch_size=config['batch_size'], data_ratio=config['mt_data_ratio'], use_gpu=config['use_gpu'], mode='MT', logger=t.logger) vocab_size_enc = len(mt_train_set.vocab_src) vocab_size_dec = len(mt_train_set.vocab_tgt) src_word2id = mt_train_set.src_word2id tgt_word2id = mt_train_set.tgt_word2id src_id2word = mt_train_set.src_id2word tgt_id2word = mt_train_set.tgt_id2word # load dev set if config['mt_dev_path_src']: t.logger.info(' -- load MT dev set -- ') mt_dev_path_src = config['mt_dev_path_src'] mt_dev_path_tgt = config['mt_dev_path_tgt'] mt_dev_set = Dataset(path_src=mt_dev_path_src, path_tgt=mt_dev_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, use_type=config['use_type'], acous_path=None, acous_norm_path=None, acous_max_len=config['las_acous_max_len'], seqrev=config['seqrev'], acous_norm=config['las_acous_norm'], max_seq_len_src=config['max_seq_len_src'], max_seq_len_tgt=config['max_seq_len_tgt'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], mode='MT', logger=t.logger) else: mt_dev_set = None # collect all datasets train_sets = {} dev_sets = {} train_sets['st'] = train_set train_sets['asr'] = asr_train_set train_sets['mt'] = mt_train_set dev_sets['st'] = dev_set dev_sets['asr'] = asr_dev_set dev_sets['mt'] = mt_dev_set # device device = check_device(config['use_gpu']) t.logger.info('device:{}'.format(device)) # construct nmt model seq2seq = Seq2seq( vocab_size_enc, vocab_size_dec, share_embedder=config['share_embedder'], enc_embedding_size=config['embedding_size_enc'], dec_embedding_size=config['embedding_size_dec'], load_embedding_src=config['load_embedding_src'], load_embedding_tgt=config['load_embedding_tgt'], num_heads=config['num_heads'], dim_model=config['dim_model'], dim_feedforward=config['dim_feedforward'], enc_layers=config['enc_layers'], dec_layers=config['dec_layers'], embedding_dropout=config['embedding_dropout'], dropout=config['dropout'], max_seq_len_src=config['max_seq_len_src'], max_seq_len_tgt=config['max_seq_len_tgt'], act=config['act'], enc_word2id=src_word2id, dec_word2id=tgt_word2id, enc_id2word=src_id2word, dec_id2word=tgt_id2word, transformer_type=config['transformer_type'], enc_emb_proj=config['enc_emb_proj'], dec_emb_proj=config['dec_emb_proj'], # acous_dim=config['las_acous_dim'], acous_hidden_size=config['las_acous_hidden_size'], # mode=config['mode'], load_mode=config['load_mode']) seq2seq = seq2seq.to(device=device) # run training seq2seq = t.train(train_sets, seq2seq, num_epochs=config['num_epochs'], dev_sets=dev_sets, grab_memory=config['grab_memory'])
def main(): # load config parser = argparse.ArgumentParser(description='Seq2seq Evaluation') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # load src-tgt pair test_path_src = config['test_path_src'] test_path_tgt = config['test_path_tgt'] # dummy if type(test_path_tgt) == type(None): test_path_tgt = test_path_src test_path_out = config['test_path_out'] load_dir = config['load'] max_seq_len = config['max_seq_len'] batch_size = config['batch_size'] beam_width = config['beam_width'] use_gpu = config['use_gpu'] seqrev = config['seqrev'] use_type = config['use_type'] if not os.path.exists(test_path_out): os.makedirs(test_path_out) config_save_dir = os.path.join(test_path_out, 'eval.cfg') save_config(config, config_save_dir) # set test mode: 1 = translate; 3 = plot MODE = config['eval_mode'] if MODE == 3: max_seq_len = 32 batch_size = 1 beam_width = 1 use_gpu = False # check device: device = check_device(use_gpu) print('device: {}'.format(device)) # load model latest_checkpoint_path = load_dir resume_checkpoint = Checkpoint.load(latest_checkpoint_path) model = resume_checkpoint.model.to(device) vocab_src = resume_checkpoint.input_vocab vocab_tgt = resume_checkpoint.output_vocab print('Model dir: {}'.format(latest_checkpoint_path)) print('Model laoded') # load test_set test_set = Dataset(test_path_src, test_path_tgt, vocab_src_list=vocab_src, vocab_tgt_list=vocab_tgt, seqrev=seqrev, max_seq_len=max_seq_len, batch_size=batch_size, use_gpu=use_gpu, use_type=use_type) print('Test dir: {}'.format(test_path_src)) print('Testset loaded') sys.stdout.flush() # run eval if MODE == 1: # FR translate(test_set, model, test_path_out, use_gpu, max_seq_len, beam_width, device, seqrev=seqrev) if MODE == 2: translate_batch(test_set, model, test_path_out, use_gpu, max_seq_len, beam_width, device, seqrev=seqrev) elif MODE == 3: # plotting att_plot(test_set, model, test_path_out, use_gpu, max_seq_len, beam_width, device) elif MODE == 4: # TF translate_tf(test_set, model, test_path_out, use_gpu, max_seq_len, beam_width, device, seqrev=seqrev)
def main(): # load config parser = argparse.ArgumentParser(description='Seq2seq Evaluation') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # load src-tgt pair test_path_src = config['test_path_src'] test_path_tgt = test_path_src test_path_out = config['test_path_out'] load_dir = config['load'] max_seq_len = config['max_seq_len'] batch_size = config['batch_size'] beam_width = config['beam_width'] use_gpu = config['use_gpu'] seqrev = config['seqrev'] use_type = config['use_type'] # set test mode: 1 = translate; 2 = plot; 3 = save comb ckpt MODE = config['eval_mode'] if MODE != 3: if not os.path.exists(test_path_out): os.makedirs(test_path_out) config_save_dir = os.path.join(test_path_out, 'eval.cfg') save_config(config, config_save_dir) # check device: device = check_device(use_gpu) print('device: {}'.format(device)) # load model latest_checkpoint_path = load_dir resume_checkpoint = Checkpoint.load(latest_checkpoint_path) model = resume_checkpoint.model.to(device) vocab_src = resume_checkpoint.input_vocab vocab_tgt = resume_checkpoint.output_vocab print('Model dir: {}'.format(latest_checkpoint_path)) print('Model laoded') # combine model if type(config['combine_path']) != type(None): model = combine_weights(config['combine_path']) # load test_set test_set = Dataset(test_path_src, test_path_tgt, vocab_src_list=vocab_src, vocab_tgt_list=vocab_tgt, seqrev=seqrev, max_seq_len=900, batch_size=batch_size, use_gpu=use_gpu, use_type=use_type) print('Test dir: {}'.format(test_path_src)) print('Testset loaded') sys.stdout.flush() # run eval if MODE == 1: translate(test_set, model, test_path_out, use_gpu, max_seq_len, beam_width, device, seqrev=seqrev) elif MODE == 2: # output posterior translate_logp(test_set, model, test_path_out, use_gpu, max_seq_len, device, seqrev=seqrev) elif MODE == 3: # save combined model ckpt = Checkpoint(model=model, optimizer=None, epoch=0, step=0, input_vocab=test_set.vocab_src, output_vocab=test_set.vocab_tgt) saved_path = ckpt.save_customise( os.path.join(config['combine_path'].strip('/')+'-combine','combine')) log_ckpts(config['combine_path'], config['combine_path'].strip('/')+'-combine') print('saving at {} ... '.format(saved_path))
def main(): # load config parser = argparse.ArgumentParser(description='Seq2seq Training') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # set random seed if config['random_seed'] is not None: set_global_seeds(config['random_seed']) # record config if not os.path.isabs(config['save']): config_save_dir = os.path.join(os.getcwd(), config['save']) if not os.path.exists(config['save']): os.makedirs(config['save']) # loading old models if config['load']: print('loading {} ...'.format(config['load'])) config_save_dir = os.path.join(config['save'], 'model-cont.cfg') else: config_save_dir = os.path.join(config['save'], 'model.cfg') save_config(config, config_save_dir) # contruct trainer t = Trainer(expt_dir=config['save'], load_dir=config['load'], load_mode=config['load_mode'], batch_size=config['batch_size'], checkpoint_every=config['checkpoint_every'], print_every=config['print_every'], eval_mode=config['eval_mode'], eval_metric=config['eval_metric'], learning_rate=config['learning_rate'], learning_rate_init=config['learning_rate_init'], lr_warmup_steps=config['lr_warmup_steps'], eval_with_mask=config['eval_with_mask'], use_gpu=config['use_gpu'], gpu_id=config['gpu_id'], max_grad_norm=config['max_grad_norm'], max_count_no_improve=config['max_count_no_improve'], max_count_num_rollback=config['max_count_num_rollback'], keep_num=config['keep_num'], normalise_loss=config['normalise_loss'], minibatch_split=config['minibatch_split'] ) # load train set train_path_src = config['train_path_src'] train_path_tgt = config['train_path_tgt'] path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] train_set = Dataset(train_path_src, train_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, seqrev=config['seqrev'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], data_ratio=config['data_ratio'], use_gpu=config['use_gpu'], logger=t.logger, use_type=config['use_type'], use_type_src=config['use_type_src']) vocab_size_enc = len(train_set.vocab_src) vocab_size_dec = len(train_set.vocab_tgt) # load dev set if config['dev_path_src'] and config['dev_path_tgt']: dev_path_src = config['dev_path_src'] dev_path_tgt = config['dev_path_tgt'] dev_set = Dataset(dev_path_src, dev_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, seqrev=config['seqrev'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], logger=t.logger, use_type=config['use_type'], use_type_src=config['use_type_src']) else: dev_set = None # construct model seq2seq = Seq2seq(vocab_size_enc, vocab_size_dec, share_embedder=config['share_embedder'], enc_embedding_size=config['embedding_size_enc'], dec_embedding_size=config['embedding_size_dec'], load_embedding_src=config['load_embedding_src'], load_embedding_tgt=config['load_embedding_tgt'], num_heads=config['num_heads'], dim_model=config['dim_model'], dim_feedforward=config['dim_feedforward'], enc_layers=config['enc_layers'], dec_layers=config['dec_layers'], embedding_dropout=config['embedding_dropout'], dropout=config['dropout'], max_seq_len=config['max_seq_len'], act=config['act'], enc_word2id=train_set.src_word2id, dec_word2id=train_set.tgt_word2id, enc_id2word=train_set.src_id2word, dec_id2word=train_set.tgt_id2word, transformer_type=config['transformer_type']) # import pdb; pdb.set_trace() t.logger.info("total #parameters:{}".format(sum(p.numel() for p in seq2seq.parameters() if p.requires_grad))) device = check_device(config['use_gpu']) t.logger.info('device: {}'.format(device)) seq2seq = seq2seq.to(device=device) # run training seq2seq = t.train(train_set, seq2seq, num_epochs=config['num_epochs'], dev_set=dev_set, grab_memory=config['grab_memory'])
def main(): # load config parser = argparse.ArgumentParser(description='PyTorch Seq2Seq-DD Training') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # record config if not os.path.isabs(config['save']): config_save_dir = os.path.join(os.getcwd(), config['save']) if not os.path.exists(config['save']): os.makedirs(config['save']) # check device: if config['use_gpu'] and torch.cuda.is_available(): global device device = torch.device('cuda') else: device = torch.device('cpu') print('device: {}'.format(device)) # resume or not if config['load']: resume = True print('resuming {} ...'.format(config['load'])) config_save_dir = os.path.join(config['save'], 'model-cont.cfg') else: resume = False config_save_dir = os.path.join(config['save'], 'model.cfg') save_config(config, config_save_dir) # load train set train_path_src = config['train_path_src'] train_path_tgt = config['train_path_tgt'] path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] train_attkey_path = config['train_attkey_path'] train_set = Dataset(train_path_src, train_path_tgt, path_vocab_src, path_vocab_tgt, attkey_path=train_attkey_path, seqrev=config['seqrev'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu']) vocab_size_enc = len(train_set.vocab_src) vocab_size_dec = len(train_set.vocab_tgt) # load dev set if config['dev_path_src'] and config['dev_path_tgt']: dev_path_src = config['dev_path_src'] dev_path_tgt = config['dev_path_tgt'] dev_attkey_path = config['dev_attkey_path'] dev_set = Dataset(dev_path_src, dev_path_tgt, path_vocab_src, path_vocab_tgt, attkey_path=dev_attkey_path, seqrev=config['seqrev'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu']) else: dev_set = None # construct model seq2seq = Seq2Seq(vocab_size_enc, vocab_size_dec, embedding_size_enc=config['embedding_size_enc'], embedding_size_dec=config['embedding_size_dec'], embedding_dropout=config['embedding_dropout'], hidden_size_enc=config['hidden_size_enc'], num_bilstm_enc=config['num_bilstm_enc'], num_unilstm_enc=config['num_unilstm_enc'], hidden_size_dec=config['hidden_size_dec'], num_unilstm_dec=config['num_unilstm_dec'], hidden_size_att=config['hidden_size_att'], hidden_size_shared=config['hidden_size_shared'], dropout=config['dropout'], residual=config['residual'], batch_first=config['batch_first'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], load_embedding_src=config['load_embedding_src'], load_embedding_tgt=config['load_embedding_tgt'], src_word2id=train_set.src_word2id, tgt_word2id=train_set.tgt_word2id, src_id2word=train_set.src_id2word, att_mode=config['att_mode'], hard_att=config['hard_att'], use_gpu=config['use_gpu'], additional_key_size=config['additional_key_size'], ptr_net=config['ptr_net'], use_bpe=config['use_bpe']) if config['use_gpu']: seq2seq = seq2seq.cuda() # contruct trainer t = Trainer(expt_dir=config['save'], load_dir=config['load'], batch_size=config['batch_size'], random_seed=config['random_seed'], checkpoint_every=config['checkpoint_every'], print_every=config['print_every'], learning_rate=config['learning_rate'], eval_with_mask=config['eval_with_mask'], scheduled_sampling=config['scheduled_sampling'], teacher_forcing_ratio=config['teacher_forcing_ratio'], use_gpu=config['use_gpu'], max_grad_norm=config['max_grad_norm'], ddatt_loss_weight=config['ddatt_loss_weight'], ddattcls_loss_weight=config['ddattcls_loss_weight'], att_scale_up=config['att_scale_up']) # run training seq2seq = t.train(train_set, seq2seq, num_epochs=config['num_epochs'], resume=resume, dev_set=dev_set)
def main(): # import pdb; pdb.set_trace() # load config warnings.filterwarnings("ignore") parser = argparse.ArgumentParser( description='PyTorch Seq2Seq Joint Training') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # record config if not os.path.isabs(config['save']): config_save_dir = os.path.join(os.getcwd(), config['save']) if not os.path.exists(config['save']): os.makedirs(config['save']) # check device: if config['use_gpu'] and torch.cuda.is_available(): global device device = torch.device('cuda') else: device = torch.device('cpu') print('device: {}'.format(device)) # set random seed if config['random_seed'] is not None: set_global_seeds(config['random_seed']) # resume or not if config['load']: print('resuming {} ...'.format(config['load'])) config_save_dir = os.path.join(config['save'], 'model-cont.cfg') elif config['restart']: print('restarting from {} ...'.format(config['restart'])) config_save_dir = os.path.join(config['save'], 'model-restart.cfg') else: config_save_dir = os.path.join(config['save'], 'model.cfg') save_config(config, config_save_dir) # load vacabulary path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] assert config['load_embedding_src'] == config['load_embedding_tgt'], \ 'src tgt embeddings are different' if type(config['load_embedding_src']) == type(None): load_embedding = False else: load_embedding = True print('load embedding: {}'.format(load_embedding)) # load dataset time_st1 = time.time() if config['train_mode'] == 'separate': ddgec_train_set = None ddgec_dev_set = None # load train set dd_train_path_src = config['dd_train_path_src'] dd_train_path_tgt = config['dd_train_path_tgt'] dd_train_tsv_path = config['dd_train_tsv_path'] dd_train_set = Dataset(dd_train_path_src, dd_train_path_tgt, path_vocab_src, path_vocab_tgt, seqrev=config['seqrev'], tsv_path=dd_train_tsv_path, set_type='dd', max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu']) gec_train_path_src = config['gec_train_path_src'] gec_train_path_tgt = config['gec_train_path_tgt'] gec_train_tsv_path = config['gec_train_tsv_path'] gec_train_set = Dataset(gec_train_path_src, gec_train_path_tgt, path_vocab_src, path_vocab_tgt, seqrev=config['seqrev'], tsv_path=gec_train_tsv_path, set_type='gec', max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu']) # note: vocab size of dd / gec should agree vocab_size_enc = len(dd_train_set.vocab_src) vocab_size_dec = len(dd_train_set.vocab_tgt) assert vocab_size_enc == vocab_size_dec, \ 'mismatch vocab size: {} - {}'.format(vocab_size_enc, vocab_size_dec) vocab_size = vocab_size_enc src_word2id = gec_train_set.src_word2id tgt_word2id = gec_train_set.tgt_word2id src_id2word = gec_train_set.src_id2word # load dev set if config['dd_dev_path_src'] and config['dd_dev_path_tgt']: dd_dev_path_src = config['dd_dev_path_src'] dd_dev_path_tgt = config['dd_dev_path_tgt'] dd_dev_tsv_path = config['dd_dev_tsv_path'] dd_dev_set = Dataset(dd_dev_path_src, dd_dev_path_tgt, path_vocab_src, path_vocab_tgt, seqrev=config['seqrev'], tsv_path=dd_dev_tsv_path, set_type='dd', max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu']) else: dd_dev_set = None if config['gec_dev_path_src'] and config['gec_dev_path_tgt']: gec_dev_path_src = config['gec_dev_path_src'] gec_dev_path_tgt = config['gec_dev_path_tgt'] gec_dev_tsv_path = config['gec_dev_tsv_path'] gec_dev_set = Dataset(gec_dev_path_src, gec_dev_path_tgt, path_vocab_src, path_vocab_tgt, seqrev=config['seqrev'], tsv_path=gec_dev_tsv_path, set_type='gec', max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu']) else: gec_dev_set = None elif config['train_mode'] == 'end2end': dd_train_set = None gec_train_set = None dd_dev_set = None gec_dev_set = None ddgec_train_path_src = config['ddgec_train_path_src'] ddgec_train_path_tgt = config['ddgec_train_path_tgt'] ddgec_train_path_flt = config['ddgec_train_path_flt'] ddgec_train_tsv_path = config['ddgec_train_tsv_path'] ddgec_train_set = Dataset(ddgec_train_path_src, ddgec_train_path_tgt, path_vocab_src, path_vocab_tgt, seqrev=config['seqrev'], tsv_path=ddgec_train_tsv_path, set_type='ddgec', max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], path_flt=ddgec_train_path_flt) vocab_size_enc = len(ddgec_train_set.vocab_src) vocab_size_dec = len(ddgec_train_set.vocab_tgt) assert vocab_size_enc == vocab_size_dec, \ 'mismatch vocab size: {} - {}'.format(vocab_size_enc, vocab_size_dec) vocab_size = vocab_size_enc src_word2id = ddgec_train_set.src_word2id tgt_word2id = ddgec_train_set.tgt_word2id src_id2word = ddgec_train_set.src_id2word if config['ddgec_dev_path_src'] and config[ 'ddgec_dev_path_tgt'] and config['ddgec_dev_path_flt']: ddgec_dev_path_src = config['ddgec_dev_path_src'] ddgec_dev_path_tgt = config['ddgec_dev_path_tgt'] ddgec_dev_path_flt = config['ddgec_dev_path_flt'] ddgec_dev_tsv_path = config['ddgec_dev_tsv_path'] ddgec_dev_set = Dataset(ddgec_dev_path_src, ddgec_dev_path_tgt, path_vocab_src, path_vocab_tgt, seqrev=config['seqrev'], tsv_path=ddgec_dev_tsv_path, set_type='ddgec', max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], path_flt=ddgec_dev_path_flt) else: ddgec_dev_set = None else: assert False, 'Not implemented mode {}'.format(config['train_mode']) time_st2 = time.time() print('data loading time: {}'.format(time_st2 - time_st1)) # construct model seq2seq = Seq2Seq( vocab_size_enc, vocab_size_dec, embedding_size_enc=config['embedding_size_enc'], embedding_size_dec=config['embedding_size_dec'], embedding_dropout=config['embedding_dropout'], hidden_size_enc=config['hidden_size_enc'], hidden_size_dec=config['hidden_size_dec'], num_bilstm_enc=config['num_bilstm_enc'], num_unilstm_enc=config['num_unilstm_enc'], dd_num_unilstm_dec=config['dd_num_unilstm_dec'], dd_hidden_size_att=config['dd_hidden_size_att'], dd_att_mode=config['dd_att_mode'], dd_additional_key_size=config['dd_additional_key_size'], gec_num_bilstm_dec=config['gec_num_bilstm_dec'], gec_num_unilstm_dec_preatt=config['gec_num_unilstm_dec_preatt'], gec_num_unilstm_dec_pstatt=config['gec_num_unilstm_dec_pstatt'], gec_hidden_size_att=config['gec_hidden_size_att'], gec_att_mode=config['gec_att_mode'], shared_embed=config['shared_embed'], dropout=config['dropout'], residual=config['residual'], batch_first=config['batch_first'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], load_embedding_src=config['load_embedding_src'], load_embedding_tgt=config['load_embedding_tgt'], src_word2id=src_word2id, tgt_word2id=tgt_word2id, src_id2word=src_id2word, hard_att=config['hard_att'], add_discriminator=config['add_discriminator'], dloss_coeff=config['dloss_coeff'], use_gpu=config['use_gpu'], ptr_net=config['ptr_net'], connect_type=config['connect_type'], dd_classifier=config['dd_classifier']).to(device) time_st3 = time.time() print('model init time: {}'.format(time_st3 - time_st2)) # contruct trainer t = Trainer(expt_dir=config['save'], load_dir=config['load'], restart_dir=config['restart'], batch_size=config['batch_size'], random_seed=config['random_seed'], checkpoint_every=config['checkpoint_every'], print_every=config['print_every'], learning_rate=config['learning_rate'], eval_with_mask=config['eval_with_mask'], scheduled_sampling=config['scheduled_sampling'], teacher_forcing_ratio=config['teacher_forcing_ratio'], use_gpu=config['use_gpu'], ddreg=config['ddreg'], max_grad_norm=config['max_grad_norm'], loss_shift=config['loss_shift'], max_count_no_improve=config['max_count_no_improve'], max_count_num_rollback=config['max_count_num_rollback'], train_mode=config['train_mode'], gec_acc_weight=config['gec_acc_weight'], gec_loss_weight=config['gec_loss_weight'], dd_loss_weight=config['dd_loss_weight'], ddatt_loss_weight=config['ddatt_loss_weight'], ddattcls_loss_weight=config['ddattcls_loss_weight'], att_scale_up=config['att_scale_up'], save_schedule=config['save_schedule']) # run training t.train_mode = config['train_mode'] if config['train_mode'] == 'separate': seq2seq = t.train_separate(dd_train_set, gec_train_set, seq2seq, num_epochs=config['num_epochs'], dd_dev_set=dd_dev_set, gec_dev_set=gec_dev_set) elif config['train_mode'] == 'end2end': seq2seq = t.train_end2end(ddgec_train_set, seq2seq, num_epochs=config['num_epochs'], ddgec_dev_set=ddgec_dev_set)
def main(): # import pdb; pdb.set_trace() # load config parser = argparse.ArgumentParser(description='LAS Training') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # set random seed if config['random_seed'] is not None: set_global_seeds(config['random_seed']) # record config if not os.path.isabs(config['save']): config_save_dir = os.path.join(os.getcwd(), config['save']) if not os.path.exists(config['save']): os.makedirs(config['save']) # resume or not if type(config['load']) != type(None): config_save_dir = os.path.join(config['save'], 'model-cont.cfg') else: config_save_dir = os.path.join(config['save'], 'model.cfg') save_config(config, config_save_dir) # contruct trainer t = Trainer(expt_dir=config['save'], load_dir=config['load'], batch_size=config['batch_size'], minibatch_partition=config['minibatch_partition'], checkpoint_every=config['checkpoint_every'], print_every=config['print_every'], learning_rate=config['learning_rate'], eval_with_mask=config['eval_with_mask'], scheduled_sampling=config['scheduled_sampling'], teacher_forcing_ratio=config['teacher_forcing_ratio'], use_gpu=config['use_gpu'], max_grad_norm=config['max_grad_norm'], max_count_no_improve=config['max_count_no_improve'], max_count_num_rollback=config['max_count_num_rollback'], keep_num=config['keep_num'], normalise_loss=config['normalise_loss']) # vocab path_vocab_src = config['path_vocab_src'] # load train set train_path_src = config['train_path_src'] train_acous_path = config['train_acous_path'] train_set = Dataset(train_path_src, path_vocab_src=path_vocab_src, use_type=config['use_type'], acous_path=train_acous_path, seqrev=config['seqrev'], acous_norm=config['acous_norm'], acous_norm_path=config['acous_norm_path'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], acous_max_len=config['acous_max_len'], use_gpu=config['use_gpu'], logger=t.logger) vocab_size = len(train_set.vocab_src) # load dev set if config['dev_path_src']: dev_path_src = config['dev_path_src'] dev_acous_path = config['dev_acous_path'] dev_set = Dataset(dev_path_src, path_vocab_src=path_vocab_src, use_type=config['use_type'], acous_path=dev_acous_path, acous_norm_path=config['acous_norm_path'], seqrev=config['seqrev'], acous_norm=config['acous_norm'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], acous_max_len=config['acous_max_len'], use_gpu=config['use_gpu'], logger=t.logger) else: dev_set = None # construct model las_model = LAS(vocab_size, embedding_size=config['embedding_size'], acous_hidden_size=config['acous_hidden_size'], acous_att_mode=config['acous_att_mode'], hidden_size_dec=config['hidden_size_dec'], hidden_size_shared=config['hidden_size_shared'], num_unilstm_dec=config['num_unilstm_dec'], # acous_dim=config['acous_dim'], acous_norm=config['acous_norm'], spec_aug=config['spec_aug'], batch_norm=config['batch_norm'], enc_mode=config['enc_mode'], use_type=config['use_type'], # embedding_dropout=config['embedding_dropout'], dropout=config['dropout'], residual=config['residual'], batch_first=config['batch_first'], max_seq_len=config['max_seq_len'], load_embedding=config['load_embedding'], word2id=train_set.src_word2id, id2word=train_set.src_id2word, use_gpu=config['use_gpu']) device = check_device(config['use_gpu']) t.logger.info('device:{}'.format(device)) las_model = las_model.to(device=device) # run training las_model = t.train( train_set, las_model, num_epochs=config['num_epochs'], dev_set=dev_set)
def main(): # load config parser = argparse.ArgumentParser(description='Seq2seq Training') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # set random seed if config['random_seed'] is not None: set_global_seeds(config['random_seed']) # record config if not os.path.isabs(config['save']): config_save_dir = os.path.join(os.getcwd(), config['save']) if not os.path.exists(config['save']): os.makedirs(config['save']) # resume or not if config['load']: resume = True print('resuming {} ...'.format(config['load'])) config_save_dir = os.path.join(config['save'], 'model-cont.cfg') else: resume = False config_save_dir = os.path.join(config['save'], 'model.cfg') save_config(config, config_save_dir) # contruct trainer t = Trainer(expt_dir=config['save'], load_dir=config['load'], batch_size=config['batch_size'], checkpoint_every=config['checkpoint_every'], print_every=config['print_every'], learning_rate=config['learning_rate'], eval_with_mask=config['eval_with_mask'], scheduled_sampling=config['scheduled_sampling'], teacher_forcing_ratio=config['teacher_forcing_ratio'], use_gpu=config['use_gpu'], max_grad_norm=config['max_grad_norm'], max_count_no_improve=config['max_count_no_improve'], max_count_num_rollback=config['max_count_num_rollback'], keep_num=config['keep_num'], normalise_loss=config['normalise_loss'], minibatch_split=config['minibatch_split']) # load train set train_path_src = config['train_path_src'] train_path_tgt = config['train_path_tgt'] path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] train_set = Dataset(train_path_src, train_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, seqrev=config['seqrev'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], logger=t.logger, use_type=config['use_type']) vocab_size_enc = len(train_set.vocab_src) vocab_size_dec = len(train_set.vocab_tgt) # load dev set if config['dev_path_src'] and config['dev_path_tgt']: dev_path_src = config['dev_path_src'] dev_path_tgt = config['dev_path_tgt'] dev_set = Dataset(dev_path_src, dev_path_tgt, path_vocab_src=path_vocab_src, path_vocab_tgt=path_vocab_tgt, seqrev=config['seqrev'], max_seq_len=config['max_seq_len'], batch_size=config['batch_size'], use_gpu=config['use_gpu'], logger=t.logger, use_type=config['use_type']) else: dev_set = None # construct model seq2seq = Seq2seq(vocab_size_enc, vocab_size_dec, share_embedder=config['share_embedder'], embedding_size_enc=config['embedding_size_enc'], embedding_size_dec=config['embedding_size_dec'], embedding_dropout=config['embedding_dropout'], hidden_size_enc=config['hidden_size_enc'], num_bilstm_enc=config['num_bilstm_enc'], num_unilstm_enc=config['num_unilstm_enc'], hidden_size_dec=config['hidden_size_dec'], num_unilstm_dec=config['num_unilstm_dec'], hidden_size_att=config['hidden_size_att'], hidden_size_shared=config['hidden_size_shared'], dropout=config['dropout'], residual=config['residual'], batch_first=config['batch_first'], max_seq_len=config['max_seq_len'], load_embedding_src=config['load_embedding_src'], load_embedding_tgt=config['load_embedding_tgt'], src_word2id=train_set.src_word2id, tgt_word2id=train_set.tgt_word2id, src_id2word=train_set.src_id2word, tgt_id2word=train_set.tgt_id2word, att_mode=config['att_mode']) device = check_device(config['use_gpu']) t.logger.info('device:{}'.format(device)) seq2seq = seq2seq.to(device=device) # run training seq2seq = t.train(train_set, seq2seq, num_epochs=config['num_epochs'], resume=resume, dev_set=dev_set)
def main(): # load config parser = argparse.ArgumentParser(description='PyTorch Seq2Seq Evaluation') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) config_save_dir = os.path.join(config['load'], 'eval.cfg') save_config(config, config_save_dir) # check device: if config['use_gpu'] and torch.cuda.is_available(): global device device = torch.device('cuda') else: device = torch.device('cpu') print('device: {}'.format(device)) # load src-tgt pair test_path_src = config['test_path_src'] test_path_tgt = config['test_path_tgt'] path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] test_path_out = config['test_path_out'] test_attkey_path = config['test_attkey_path'] load_dir = config['load'] max_seq_len = config['max_seq_len'] batch_size = config['batch_size'] beam_width = config['beam_width'] use_gpu = config['use_gpu'] seqrev = config['seqrev'] print('attkey dir: {}'.format(test_attkey_path)) print('reverse seq: {}'.format(seqrev)) print('use gpu: {}'.format(use_gpu)) if not os.path.exists(test_path_out): os.makedirs(test_path_out) if not os.path.exists(test_path_out): os.makedirs(test_path_out) config_save_dir = os.path.join(test_path_out, 'eval.cfg') save_config(config, config_save_dir) # set test mode MODE = config['eval_mode'] if MODE == 3: max_seq_len = 32 batch_size = 1 beam_width = 1 use_gpu = False # load test_set test_set = Dataset(test_path_src, test_path_tgt, path_vocab_src, path_vocab_tgt, attkey_path=test_attkey_path,seqrev=seqrev, max_seq_len=max_seq_len, batch_size=batch_size, use_gpu=use_gpu) print('Testset loaded') sys.stdout.flush() # run eval if MODE == 1: # run evaluation # print("use gpu: {}".format(config['use_gpu'])) accuracy = evaluate(test_set, load_dir, test_path_out, use_gpu, max_seq_len, beam_width, seqrev=seqrev) print(accuracy) elif MODE == 2: translate(test_set, load_dir, test_path_out, use_gpu, max_seq_len, beam_width, seqrev=seqrev) elif MODE == 3: # plotting att_plot(test_set, load_dir, test_path_out, use_gpu, max_seq_len, beam_width)
def main(): # load config parser = argparse.ArgumentParser(description='Evaluation') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) # load src-tgt pair test_path_src = config['test_path_src'] test_path_tgt = config['test_path_tgt'] if type(test_path_tgt) == type(None): test_path_tgt = test_path_src test_path_out = config['test_path_out'] test_acous_path = config['test_acous_path'] acous_norm_path = config['acous_norm_path'] load_dir = config['load'] max_seq_len = config['max_seq_len'] batch_size = config['batch_size'] beam_width = config['beam_width'] use_gpu = config['use_gpu'] seqrev = config['seqrev'] use_type = config['use_type'] # set test mode MODE = config['eval_mode'] if MODE != 2: if not os.path.exists(test_path_out): os.makedirs(test_path_out) config_save_dir = os.path.join(test_path_out, 'eval.cfg') save_config(config, config_save_dir) # check device: device = check_device(use_gpu) print('device: {}'.format(device)) # load model latest_checkpoint_path = load_dir resume_checkpoint = Checkpoint.load(latest_checkpoint_path) model = resume_checkpoint.model.to(device) vocab_src = resume_checkpoint.input_vocab vocab_tgt = resume_checkpoint.output_vocab print('Model dir: {}'.format(latest_checkpoint_path)) print('Model laoded') # combine model if type(config['combine_path']) != type(None): model = combine_weights(config['combine_path']) # import pdb; pdb.set_trace() # load test_set test_set = Dataset( path_src=test_path_src, path_tgt=test_path_tgt, vocab_src_list=vocab_src, vocab_tgt_list=vocab_tgt, use_type=use_type, acous_path=test_acous_path, seqrev=seqrev, acous_norm=config['acous_norm'], acous_norm_path=config['acous_norm_path'], acous_max_len=6000, # max 50k for mustc trainset max_seq_len_src=900, max_seq_len_tgt=900, # max 2.5k for mustc trainset batch_size=batch_size, mode='ST', use_gpu=use_gpu) print('Test dir: {}'.format(test_path_src)) print('Testset loaded') sys.stdout.flush() # '{AE|ASR|MT|ST}-{REF|HYP}' if len(config['gen_mode'].split('-')) == 2: gen_mode = config['gen_mode'].split('-')[0] history = config['gen_mode'].split('-')[1] elif len(config['gen_mode'].split('-')) == 1: gen_mode = config['gen_mode'] history = 'HYP' # add external language model lm_mode = config['lm_mode'] # run eval: if MODE == 1: translate(test_set, model, test_path_out, use_gpu, max_seq_len, beam_width, device, seqrev=seqrev, gen_mode=gen_mode, lm_mode=lm_mode, history=history) elif MODE == 2: # save combined model ckpt = Checkpoint(model=model, optimizer=None, epoch=0, step=0, input_vocab=test_set.vocab_src, output_vocab=test_set.vocab_tgt) saved_path = ckpt.save_customise( os.path.join(config['combine_path'].strip('/') + '-combine', 'combine')) log_ckpts(config['combine_path'], config['combine_path'].strip('/') + '-combine') print('saving at {} ... '.format(saved_path)) elif MODE == 3: plot_emb(test_set, model, test_path_out, use_gpu, max_seq_len, device) elif MODE == 4: gather_emb(test_set, model, test_path_out, use_gpu, max_seq_len, device) elif MODE == 5: compute_kl(test_set, model, test_path_out, use_gpu, max_seq_len, device)
def main(): # load config parser = argparse.ArgumentParser(description='PyTorch LAS DD Evaluation') parser = load_arguments(parser) args = vars(parser.parse_args()) config = validate_config(args) config_save_dir = os.path.join(config['load'], 'eval.cfg') # check device: if config['use_gpu'] and torch.cuda.is_available(): global device device = torch.device('cuda') else: device = torch.device('cpu') print('device: {}'.format(device)) # load src-tgt pair test_path_src = config['test_path_src'] test_path_tgt = config['test_path_tgt'] path_vocab_src = config['path_vocab_src'] path_vocab_tgt = config['path_vocab_tgt'] test_path_out = config['test_path_out'] test_tsv_path = config['test_tsv_path'] test_acous_path = config['test_acous_path'] load_dir = config['load'] max_seq_len = config['max_seq_len'] batch_size = config['batch_size'] beam_width = config['beam_width'] use_gpu = config['use_gpu'] seqrev = config['seqrev'] print('reverse seq: {}'.format(seqrev)) print('use gpu: {}'.format(use_gpu)) if not os.path.exists(test_path_out): os.makedirs(test_path_out) config_save_dir = os.path.join(test_path_out, 'eval.cfg') save_config(config, config_save_dir) # set test mode: 3 = DEBUG; 4 = PLOT MODE = config['eval_mode'] if MODE == 3 or MODE == 4 or MODE == 6: max_seq_len = 32 batch_size = 1 beam_width = 1 use_gpu = False # load test_set test_set = Dataset(test_path_src, test_path_tgt, path_vocab_src, path_vocab_tgt, use_type=config['use_type'], seqrev=config['seqrev'], add_acous=config['add_acous'], acous_path=test_acous_path, acous_norm=config['acous_norm'], tsv_path=test_tsv_path, keep_filler=config['keep_filler'], tag_rev=config['tag_rev'], add_timestamp=config['add_times'], timestamp_path=config['test_times_path'], max_seq_len=max_seq_len, batch_size=batch_size, use_gpu=use_gpu) print('Testset loaded') sys.stdout.flush() # run eval if MODE == 2: if config['add_acous']: translate_acous(test_set, load_dir, test_path_out, use_gpu, max_seq_len, beam_width, seqrev=seqrev) else: translate(test_set, load_dir, test_path_out, use_gpu, max_seq_len, beam_width, seqrev=seqrev) elif MODE == 5: # debug for beam search debug_beam_search(test_set, load_dir, use_gpu, max_seq_len, beam_width) elif MODE == 6: # plotting las attn acous_att_plot(test_set, load_dir, test_path_out, use_gpu, max_seq_len, beam_width)