def __init__(self, vocab, options): random.seed(1) self.model = dy.ParameterCollection() self.trainer = helpers.get_trainer(options, self.model) self.get_violation = helpers.update_method(options) word_count = vocab.word_freq word_vocab = vocab.wordlookup_tbl pos_vocab = vocab.poslookup_tbl rel_vocab = vocab.rellookup_tbl self.rels = rel_vocab self._enc = helpers.get_encoder(self.model, options, word_count, word_vocab, pos_vocab) self._tree_enc = TreeEncoder.get_tree_encoder(self.model, options, rel_vocab) self.mlp_rel_size = options.mlp_rel_size self.hidden_dim = options.compos_outdim W = orthonormal_initializer(self.mlp_rel_size, self.hidden_dim) self.mlp_dep_W = self.model.parameters_from_numpy(W) self.mlp_head_W = self.model.parameters_from_numpy(W) self.mlp_dep_b = self.model.add_parameters((self.mlp_rel_size,), init = dy.ConstInitializer(0.)) self.mlp_head_b = self.model.add_parameters((self.mlp_rel_size,), init = dy.ConstInitializer(0.)) # self.dropout_mlp = options.dropout_mlp self.rel_W = self.model.add_parameters((len(rel_vocab)*(self.mlp_rel_size +1) , self.mlp_rel_size + 1), init = dy.ConstInitializer(0.)) self._train_flag = True self.oracle = options.oracle self.exploration_rate = options.exploration_rate
def __init__(self, vocab, options): random.seed(1) self.model = dy.ParameterCollection() self.trainer = helpers.get_trainer(options, self.model) self.get_violation = helpers.update_method(options) word_count = vocab.word_freq word_vocab = vocab.wordlookup_tbl pos_vocab = vocab.poslookup_tbl rel_vocab = vocab.rellookup_tbl self.rels = rel_vocab self.__enc = helpers.get_encoder(self.model, options, word_count, word_vocab, pos_vocab) self.__scr = helpers.get_scorer(self.model, options, rel_vocab) self.__tree_enc = TreeEncoder.get_tree_encoder(self.model, options, rel_vocab) self.__train_flag = True self.oracle = options.oracle self.exploration_rate = options.exploration_rate
def train(options): log = helpers.Logger(options.verbose) timer = helpers.Timer() # Load data ========================================================= log.info('Reading corpora') # Read vocabs vocab = helpers.get_dictionaries(options) src_dic, trg_dic = vocab['src'], vocab['trg'] # Read training train_src_data = src_dic.read_corpus(options.train_src) train_trg_data = trg_dic.read_corpus(options.train_dst) max_src_len, max_trg_len = options.max_src_len, options.max_trg_len if max_src_len > 0 or max_trg_len > 0: train_src_data, train_trg_data = corpus_filter(train_src_data, train_trg_data, max_src_len, max_trg_len) assert len(train_src_data) == len( train_trg_data ), 'Size of source corpus and the target corpus must be the same!!' # Read validation valid_src_data = src_dic.read_corpus(options.valid_src) valid_trg_data = trg_dic.read_corpus(options.valid_dst) # Validation output if not options.valid_out: options.valid_out = helpers.exp_filename(options, 'valid.out') # Get target language model lang_model = helpers.get_language_model(options, train_trg_data, trg_dic.size()) # Create model ====================================================== log.info('Creating model') s2s = helpers.build_model(options, vocab, lang_model) # Trainer ========================================================== trainer = helpers.get_trainer(options, s2s) log.info('Using ' + options.trainer + ' optimizer') # Print configuration =============================================== if options.verbose: Opts.print_config(options, src_dict_size=src_dic.size(), trg_dict_size=trg_dic.size()) # Creat batch loaders =============================================== log.info('Creating batch loaders') trainbatchloader = BatchLoader(train_src_data, train_trg_data, options.batch_size) devbatchloader = BatchLoader(valid_src_data, valid_trg_data, options.dev_batch_size) # Start training ==================================================== log.info('starting training') timer.restart() train_loss = 0. processed = 0 best_bleu = -1 bleu = -1 deadline = 0 i = 0 for epoch in xrange(options.num_epochs): for x, y in trainbatchloader: s2s.set_train_flag() processed += sum(map(len, y)) bsize = len(y) # Compute loss loss = s2s.calculate_loss(x, y) # Backward pass and parameter update train_loss += loss.scalar_value() * bsize loss.backward() trainer.update() if (i + 1) % options.check_train_error_every == 0: # Check average training error from time to time logloss = train_loss / processed ppl = np.exp(logloss) trainer.status() log.info( " Training_loss=%f, ppl=%f, time=%f s, tokens processed=%d" % (logloss, ppl, timer.tick(), processed)) train_loss = 0 processed = 0 if (i + 1) % options.check_valid_error_every == 0: # Check generalization error on the validation set from time to time s2s.set_test_flag() dev_loss = 0 dev_processed = 0 timer.restart() for x, y in devbatchloader: dev_processed += sum(map(len, y)) bsize = len(y) loss = s2s.calculate_loss(x, y) dev_loss += loss.scalar_value() * bsize dev_logloss = dev_loss / dev_processed dev_ppl = np.exp(dev_logloss) log.info( "[epoch %d] Dev loss=%f, ppl=%f, time=%f s, tokens processed=%d" % (epoch, dev_logloss, dev_ppl, timer.tick(), dev_processed)) if (i + 1) % options.valid_bleu_every == 0: # Check BLEU score on the validation set from time to time s2s.set_test_flag() log.info('Start translating validation set, buckle up!') timer.restart() with open(options.valid_out, 'w+') as fp: for x in valid_src_data: y_hat = s2s.translate(x, beam_size=options.beam_size) translation = [ trg_dic.get_word(w) for w in y_hat[1:-1] ] fp.write(' '.join(translation)) fp.write('\n') bleu, details = evaluation.bleu_score(options.valid_dst, options.valid_out) log.info('Finished translating validation set %.2f elapsed.' % timer.tick()) log.info(details) # Early stopping : save the latest best model if bleu > best_bleu: best_bleu = bleu log.info('Best BLEU score up to date, saving model to %s' % options.model) s2s.save(options.model) deadline = 0 else: deadline += 1 if options.patience > 0 and deadline > options.patience: log.info('No improvement since %d epochs, early stopping ' 'with best validation BLEU score: %.3f' % (deadline, best_bleu)) sys.exit() # i += 1 # trainer.update() #if bleu > best_bleu: # s2s.save(options.model) s2s.save(options.model)
def eval_user_adaptation(opt): log = utils.Logger(opt.verbose) timer = utils.Timer() # Read vocabs lexicon = helpers.get_lexicon(opt) # Read data filepairs = load_user_filepairs(opt.usr_file_list) # Get target language model lang_model = None # Load model s2s = helpers.build_model(opt, lexicon, lang_model, test=True) if opt.update_mode == 'mixture_weights' and not opt.user_recognizer == 'fact_voc': log.info('Updating only the mixture weights doesn\'t make sense here') exit() s2s.lm = lexicon.trg_unigrams # s2s.freeze_parameters() # Trainer trainer = helpers.get_trainer(opt, s2s) # print config if opt.verbose: options.print_config(opt, src_dict_size=len(lexicon.w2ids), trg_dict_size=len(lexicon.w2idt)) # This will store translations and gold sentences base_translations = [] adapt_translations = [] gold = [] # Run training for usr_id, (src_file, trg_file) in enumerate(filepairs): log.info('Evaluating on files %s' % os.path.basename(src_file).split()[0]) # Load file pair src_data = data.read_corpus(src_file, lexicon.w2ids, raw=True) trg_data = data.read_corpus(trg_file, lexicon.w2idt, raw=True) # split train/test train_src, test_src, train_trg, test_trg, order = split_user_data( src_data, trg_data, n_test=opt.n_test) # Convert train data to indices train_src = lexicon.sents_to_ids(train_src) train_trg = lexicon.sents_to_ids(train_trg, trg=True) # Save test data for s in test_trg: gold.append(' '.join(s)) # Reset model s2s.load() s2s.reset_usr_vec() # Translate with baseline model base_translations.extend(evaluate_model(s2s, test_src, opt.beam_size)) # Start loop n_train = opt.max_n_train adapt_translations.extend( adapt_user(s2s, trainer, train_src[:n_train], train_trg[:n_train], test_src, opt)) # Temp files temp_gold = utils.exp_temp_filename(opt, 'gold.txt') temp_base = utils.exp_temp_filename(opt, '%s_base.txt' % opt.update_mode) temp_adapt = utils.exp_temp_filename(opt, '%s_adapt.txt' % opt.update_mode) utils.savetxt(temp_gold, gold) utils.savetxt(temp_base, base_translations) utils.savetxt(temp_adapt, adapt_translations) # Evaluate base translations bleu, details = evaluation.bleu_score(temp_gold, temp_base) log.info('Base BLEU score: %.2f' % bleu) # Evaluate base translations bleu, details = evaluation.bleu_score(temp_gold, temp_adapt) log.info('Adaptation BLEU score: %.2f' % bleu) # Compare both temp_bootstrap_gold = utils.exp_temp_filename(opt, 'bootstrap_gold.txt') temp_bootstrap_base = utils.exp_temp_filename(opt, 'bootstrap_base.txt') temp_bootstrap_adapt = utils.exp_temp_filename(opt, 'bootstrap_adapt.txt') bleus = evaluation.paired_bootstrap_resampling( temp_gold, temp_base, temp_adapt, opt.bootstrap_num_samples, opt.bootstrap_sample_size, temp_bootstrap_gold, temp_bootstrap_base, temp_bootstrap_adapt) evaluation.print_paired_stats(bleus) os.remove(temp_bootstrap_gold) os.remove(temp_bootstrap_base) os.remove(temp_bootstrap_adapt)
def train(opt): log = helpers.Logger(opt.verbose) timer = helpers.Timer() # Load data ========================================================= log.info('Reading corpora') # Read vocabs widss, ids2ws, widst, ids2wt = helpers.get_dictionaries(opt) # Read training trainings_data = data.read_corpus(opt.train_src, widss) trainingt_data = data.read_corpus(opt.train_dst, widst) # Read validation valids_data = data.read_corpus(opt.valid_src, widss) validt_data = data.read_corpus(opt.valid_dst, widst) # Validation output if not opt.valid_out: opt.valid_out = helpers.exp_filename(opt, 'valid.out') # Get target language model lang_model = helpers.get_language_model(opt, trainingt_data, widst) # Create model ====================================================== log.info('Creating model') s2s = helpers.build_model(opt, widss, widst, lang_model) # Trainer ========================================================== trainer = helpers.get_trainer(opt, s2s) log.info('Using ' + opt.trainer + ' optimizer') # Print configuration =============================================== if opt.verbose: options.print_config(opt, src_dict_size=len(widss), trg_dict_size=len(widst)) # Creat batch loaders =============================================== log.info('Creating batch loaders') trainbatchloader = data.BatchLoader(trainings_data, trainingt_data, opt.batch_size) devbatchloader = data.BatchLoader(valids_data, validt_data, opt.dev_batch_size) # Start training ==================================================== log.info('starting training') timer.restart() train_loss = 0 processed = 0 best_bleu = -1 deadline = 0 i = 0 for epoch in range(opt.num_epochs): for x, y in trainbatchloader: s2s.set_train_mode() processed += sum(map(len, y)) bsize = len(y) # Compute loss loss = s2s.calculate_loss(x, y) # Backward pass and parameter update loss.backward() trainer.update() train_loss += loss.scalar_value() * bsize if (i + 1) % opt.check_train_error_every == 0: # Check average training error from time to time logloss = train_loss / processed ppl = np.exp(logloss) trainer.status() log.info( " Training_loss=%f, ppl=%f, time=%f s, tokens processed=%d" % (logloss, ppl, timer.tick(), processed)) train_loss = 0 processed = 0 if (i + 1) % opt.check_valid_error_every == 0: # Check generalization error on the validation set from time to time s2s.set_test_mode() dev_loss = 0 dev_processed = 0 timer.restart() for x, y in devbatchloader: dev_processed += sum(map(len, y)) bsize = len(y) loss = s2s.calculate_loss(x, y, test=True) dev_loss += loss.scalar_value() * bsize dev_logloss = dev_loss / dev_processed dev_ppl = np.exp(dev_logloss) log.info( "[epoch %d] Dev loss=%f, ppl=%f, time=%f s, tokens processed=%d" % (epoch, dev_logloss, dev_ppl, timer.tick(), dev_processed)) if (i + 1) % opt.valid_bleu_every == 0: # Check BLEU score on the validation set from time to time s2s.set_test_mode() log.info('Start translating validation set, buckle up!') timer.restart() with open(opt.valid_out, 'w+') as f: for x in valids_data: y_hat = s2s.translate(x, beam_size=opt.beam_size) translation = [ids2wt[w] for w in y_hat[1:-1]] print(' '.join(translation), file=f) bleu, details = evaluation.bleu_score(opt.valid_dst, opt.valid_out) log.info('Finished translating validation set %.2f elapsed.' % timer.tick()) log.info(details) # Early stopping : save the latest best model if bleu > best_bleu: best_bleu = bleu log.info('Best BLEU score up to date, saving model to %s' % s2s.model_file) s2s.save() deadline = 0 else: deadline += 1 if opt.patience > 0 and deadline > opt.patience: log.info('No improvement since %d epochs, early stopping ' 'with best validation BLEU score: %.3f' % (deadline, best_bleu)) exit() i = i + 1 trainer.update_epoch()
def eval_user_adaptation(opt): log = utils.Logger(opt.verbose) timer = utils.Timer() # Read vocabs lexicon = helpers.get_lexicon(opt) # Read data filepairs = load_user_filepairs(opt.usr_file_list) # Get target language model lang_model = None # Load model s2s = helpers.build_model(opt, lexicon, lang_model, test=True) #if not opt.full_training: # s2s.freeze_parameters() # Trainer trainer = helpers.get_trainer(opt, s2s) # print config if opt.verbose: options.print_config(opt, src_dict_size=len(lexicon.w2ids), trg_dict_size=len(lexicon.w2idt)) # This will store translations and gold sentences translations = dict([(i, []) for i in range(opt.min_n_train, opt.max_n_train)]) gold = [] # Run training for usr_id, (src_file, trg_file) in enumerate(filepairs): log.info('Evaluating on files %s' % os.path.basename(src_file).split()[0]) # Load file pair src_data = data.read_corpus(src_file, lexicon.w2ids, raw=True) trg_data = data.read_corpus(trg_file, lexicon.w2idt, raw=True) # split train/test train_src, test_src, train_trg, test_trg, order = split_user_data( src_data, trg_data, n_test=opt.n_test) # Convert train data to indices train_src = lexicon.sents_to_ids(train_src) train_trg = lexicon.sents_to_ids(train_trg, trg=True) # Save test data for s in test_trg: gold.append(' '.join(s)) # Start loop for n_train in range(opt.min_n_train, opt.max_n_train): log.info('Training on %d sentence pairs' % n_train) # Train on n_train first sentences X, Y = train_src[:n_train], train_trg[:n_train] temp_out = utils.exp_temp_filename(opt, str(n_train) + 'out.txt') if opt.full_training: s2s.load() if opt.log_unigram_bias: if opt.use_trg_unigrams: unigrams = lexicon.compute_unigrams(Y, lang='trg') else: unigrams = lexicon.estimate_unigrams(X) log_unigrams = np.log(unigrams + opt.log_unigrams_eps) s2s.reset_usr_vec(log_unigrams) elif n_train > 0: adapt(s2s, trainer, X, Y, opt.num_epochs, opt.check_train_error_every) log.info('Translating test file') s2s.set_test_mode() # Test on test split for x in test_src: y_hat = s2s.translate(x, 0, beam_size=opt.beam_size) translations[n_train].append(y_hat) # Temp files temp_gold = utils.exp_temp_filename(opt, 'gold.txt') np.savetxt(temp_gold, gold, fmt='%s') # Results test_bleus = np.zeros(opt.max_n_train - opt.min_n_train) for n_train in range(opt.min_n_train, opt.max_n_train): log.info('Evaluation for %d sentence pairs' % n_train) temp_out = utils.exp_temp_filename(opt, str(n_train) + 'out.txt') temp_bootstrap_out = utils.exp_temp_filename( opt, str(n_train) + '_bootstrap_out.txt') temp_bootstrap_ref = utils.exp_temp_filename( opt, str(n_train) + '_bootstrap_ref.txt') np.savetxt(temp_out, translations[n_train], fmt='%s') bleu, details = evaluation.bleu_score(temp_gold, temp_out) log.info('BLEU score: %.2f' % bleu) bleus = evaluation.bootstrap_resampling(temp_gold, temp_out, opt.bootstrap_num_samples, opt.bootstrap_sample_size, temp_bootstrap_ref, temp_bootstrap_out) evaluation.print_stats(bleus) test_bleus[n_train - opt.min_n_train] = bleu np.savetxt(utils.exp_filename(opt, 'bleu_scores.txt'), test_bleus, fmt='%.3f')