def run_epoch(self, sess, train, dev, tags, epoch): """ Performs one complete pass over the train set and evaluate on dev Args: sess: tensorflow session train: dataset that yields tuple of sentences, tags dev: dataset tags: {tag: index} dictionary epoch: (int) number of the epoch """ nbatches = ( len(train) + self.config.batch_size - 1) // self.config.batch_size prog = Progbar(target=nbatches) for i, (words, labels ) in enumerate(minibatches(train, self.config.batch_size)): fd, _ = self.get_feed_dict(words, labels, self.config.LR, self.config.dropout) _, train_loss, summary = sess.run( [self.train_op, self.loss, self.merged], feed_dict=fd) prog.update(i + 1, [("train loss", train_loss)]) # tensorboard if i % 10 == 0: self.file_writer.add_summary(summary, epoch * nbatches + i) acc, f1 = self.run_evaluate(sess, dev, tags) self.logger.info( "- dev acc {:04.2f} - f1 {:04.2f}".format(100 * acc, 100 * f1)) return acc, f1
def _valid(data_loader, model, criterion, optimizer, epoch, opt, is_train=False): progbar = Progbar(title='Validating', target=len(data_loader), batch_size=opt.batch_size, total_examples=len(data_loader.dataset)) if is_train: model.train() else: model.eval() losses = [] # Note that the data should be shuffled every time for i, batch in enumerate(data_loader): src = batch.src trg = batch.trg if torch.cuda.is_available(): src.cuda() trg.cuda() decoder_probs, _, _ = model.forward(src, trg, must_teacher_forcing=True) start_time = time.time() loss = criterion( decoder_probs.contiguous().view(-1, opt.vocab_size), trg[:, 1:].contiguous().view(-1) ) print("--loss calculation --- %s" % (time.time() - start_time)) start_time = time.time() if is_train: optimizer.zero_grad() loss.backward() if opt.max_grad_norm > 0: torch.nn.utils.clip_grad_norm(model.parameters(), opt.max_grad_norm) optimizer.step() print("--backward function - %s seconds ---" % (time.time() - start_time)) losses.append(loss.data[0]) start_time = time.time() progbar.update(epoch, i, [('valid_loss', loss.data[0])]) print("-progbar.update --- %s" % (time.time() - start_time)) return losses
def evaluate_greedy(model, data_loader, test_examples, opt): model.eval() logging.info('====================== Checking GPU Availability =========================') if torch.cuda.is_available(): logging.info('Running on GPU!') model.cuda() else: logging.info('Running on CPU!') logging.info('====================== Start Predicting =========================') progbar = Progbar(title='Testing', target=len(data_loader), batch_size=data_loader.batch_size, total_examples=len(data_loader.dataset)) ''' Note here each batch only contains one data example, thus decoder_probs is flattened ''' for i, (batch, example) in enumerate(zip(data_loader, test_examples)): src = batch.src logging.info('====================== %d =========================' % (i + 1)) logging.info('\nSource text: \n %s\n' % (' '.join([opt.id2word[wi] for wi in src.data.numpy()[0]]))) if torch.cuda.is_available(): src.cuda() # trg = Variable(torch.from_numpy(np.zeros((src.size(0), opt.max_sent_length), dtype='int64'))) trg = Variable(torch.LongTensor([[opt.word2id[pykp.io.BOS_WORD]] * opt.max_sent_length])) max_words_pred = model.greedy_predict(src, trg) progbar.update(None, i, []) sentence_pred = [opt.id2word[x] for x in max_words_pred] sentence_real = example['trg_str'] if '</s>' in sentence_real: index = sentence_real.index('</s>') sentence_pred = sentence_pred[:index] logging.info('\t\tPredicted : %s ' % (' '.join(sentence_pred))) logging.info('\t\tReal : %s ' % (sentence_real))
def _valid_error(data_loader, model, criterion, epoch, opt): progbar = Progbar(title='Validating', target=len(data_loader), batch_size=data_loader.batch_size, total_examples=len(data_loader.dataset)) model.eval() losses = [] # Note that the data should be shuffled every time for i, batch in enumerate(data_loader): # if i >= 100: # break one2many_batch, one2one_batch = batch src, trg, trg_target, trg_copy_target, src_ext, oov_lists = one2one_batch if torch.cuda.is_available(): src = src.cuda() trg = trg.cuda() trg_target = trg_target.cuda() trg_copy_target = trg_copy_target.cuda() src_ext = src_ext.cuda() decoder_log_probs, _, _ = model.forward(src, trg, src_ext) if not opt.copy_model: loss = criterion( decoder_log_probs.contiguous().view(-1, opt.vocab_size), trg_target.contiguous().view(-1) ) else: loss = criterion( decoder_log_probs.contiguous().view(-1, opt.vocab_size + opt.max_unk_words), trg_copy_target.contiguous().view(-1) ) losses.append(loss.data[0]) progbar.update(epoch, i, [('valid_loss', loss.data[0]), ('PPL', loss.data[0])]) return losses
def train(self, trainset, devset, testset, batch_size=64, epochs=50, shuffle=True): self.logger.info('Start training...') init_lr = self.cfg.lr # initial learning rate, used for decay learning rate best_score = 0.0 # record the best score best_score_epoch = 1 # record the epoch of the best score obtained no_imprv_epoch = 0 # no improvement patience counter for epoch in range(self.start_epoch, epochs + 1): self.logger.info('Epoch %2d/%2d:' % (epoch, epochs)) progbar = Progbar(target=(len(trainset) + batch_size - 1) // batch_size) # number of batches if shuffle: np.random.shuffle(trainset) # shuffle training dataset each epoch # training each epoch for i, (words, labels) in enumerate(batch_iter(trainset, batch_size)): feed_dict = self._get_feed_dict(words, labels, lr=self.cfg.lr, is_train=True) _, train_loss = self.sess.run([self.train_op, self.loss], feed_dict=feed_dict) progbar.update(i + 1, [("train loss", train_loss)]) if devset is not None: self.evaluate(devset, batch_size) cur_score = self.evaluate(testset, batch_size, is_devset=False) # learning rate decay if self.cfg.decay_lr: self.cfg.lr = init_lr / (1 + self.cfg.lr_decay * epoch) # performs model saving and evaluating on test dataset if cur_score > best_score: no_imprv_epoch = 0 self.save_session(epoch) best_score = cur_score best_score_epoch = epoch self.logger.info(' -- new BEST score on TEST dataset: {:05.3f}'.format(best_score)) else: no_imprv_epoch += 1 if no_imprv_epoch >= self.cfg.no_imprv_patience: self.logger.info('early stop at {}th epoch without improvement for {} epochs, BEST score: ' '{:05.3f} at epoch {}'.format(epoch, no_imprv_epoch, best_score, best_score_epoch)) break self.logger.info('Training process done...')
def run_simultrans(model, options_file=None, config=None, policy=None, id=None, remote=False): # check envoriments check_env() if id is not None: fcon = '.config/{}.conf'.format(id) if os.path.exists(fcon): print 'load config files' policy, config = pkl.load(open(fcon, 'r')) # ======================================================================= # # load model model_options # ======================================================================= # _model = model model = '.pretrained/{}'.format(model) if options_file is not None: with open(options_file, 'rb') as f: options = pkl.load(f) else: with open('%s.pkl' % model, 'rb') as f: options = pkl.load(f) options['birnn'] = True print 'load options...' for w, p in sorted(options.items(), key=lambda x:x[0]): print '{}: {}'.format(w, p) # load detail settings from option file: dictionary, dictionary_target = options['dictionaries'] def _iter(fname): with open(fname, 'r') as f: for line in f: words = line.strip().split() x = map(lambda w: word_dict[w] if w in word_dict else 1, words) x = map(lambda ii: ii if ii < options['n_words'] else 1, x) x += [0] yield x def _check_length(fname): f = open(fname, 'r') count = 0 for _ in f: count += 1 f.close() return count # load source dictionary and invert with open(dictionary, 'rb') as f: word_dict = pkl.load(f) word_idict = dict() for kk, vv in word_dict.iteritems(): word_idict[vv] = kk word_idict[0] = '<eos>' word_idict[1] = 'UNK' # load target dictionary and invert with open(dictionary_target, 'rb') as f: word_dict_trg = pkl.load(f) word_idict_trg = dict() for kk, vv in word_dict_trg.iteritems(): word_idict_trg[vv] = kk word_idict_trg[0] = '<eos>' word_idict_trg[1] = 'UNK' # ======================================================================== # # Build a Translator # ======================================================================== # # allocate model parameters params = init_params(options) params = load_params(model, params) tparams = init_tparams(params) # print 'build the model for computing cost (full source sentence).' trng, use_noise, \ _x, _x_mask, _y, _y_mask, \ opt_ret, \ cost, f_cost = build_model(tparams, options) print 'done.' # functions for sampler # f_sim_ctx, f_sim_init, f_sim_next = build_simultaneous_sampler(tparams, options, trng) f_sim_ctx, f_sim_init, f_sim_next = build_noisy_sampler(tparams, options, trng) print 'build sampler done.' # check the ID: policy['base'] = _model _policy = Policy(trng, options, policy, config, n_out=options['dim'], recurrent=True, id=id) # DATASET trainIter = TextIterator(options['datasets'][0], options['datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=config['batchsize'], maxlen=options['maxlen']) train_num = trainIter.num validIter = TextIterator(options['valid_datasets'][0], options['valid_datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=1, maxlen=options['maxlen']) valid_num = validIter.num valid_ = options['valid_datasets'][0] valid_num = _check_length(valid_) print 'training set {} lines / validation set {} lines'.format(train_num, valid_num) print 'use the reward function {}'.format(chr(config['Rtype'] + 65)) # Translator model def _translate(src, trg, train=False, samples=80): ret = noisy_decoding( f_sim_ctx, f_sim_init, f_sim_next, f_cost, src, trg, word_idict_trg, n_samples=samples, train=train, _policy=_policy) if not train: sample, score, actions, R, tracks, attentions = ret return sample, score, actions, R, tracks else: sample, score, actions, R, info = ret return sample, score, actions, R, info # ======================================================================== # # Main Loop: Run # ======================================================================== # print 'Start Simultaneous Translator...' probar = Progbar(train_num / config['batchsize'], with_history=False) # freqs save_freq = 2000 sample_freq = 10 valid_freq = 1000 valid_size = 200 display_freq = 50 history, last_it = _policy.load() time0 = timer() for it, (srcs, trgs) in enumerate(trainIter): # only one sentence each iteration if it < last_it: # go over the scanned lines. continue samples, scores, actions, rewards, info = _translate(srcs, trgs, train=True) if it % sample_freq == 0: print '\nModel has been trained for {} seconds'.format(timer() - time0) print 'source: ', _bpe2words(_seqs2words([srcs[0]], word_idict))[0] print 'target: ', _bpe2words(_seqs2words([trgs[0]], word_idict_trg))[0] # obtain the translation results samples = _bpe2words(_seqs2words(samples, word_idict_trg)) print '---' print 'sample: ', samples[40] print 'sample: ', samples[60] values = [(w, info[w]) for w in info] probar.update(it + 1, values=values) # NaN detector for w in info: if numpy.isnan(info[w]) or numpy.isinf(info[w]): raise RuntimeError, 'NaN/INF is detected!! {} : ID={}'.format(w, id)
def train_model(model, optimizer, criterion, train_data_loader, valid_data_loader, test_data_loader, opt): generator = SequenceGenerator(model, eos_id=opt.word2id[pykp.io.EOS_WORD], beam_size=opt.beam_size, max_sequence_length=opt.max_sent_length ) logging.info('====================== Checking GPU Availability =========================') if torch.cuda.is_available(): if isinstance(opt.gpuid, int): opt.gpuid = [opt.gpuid] logging.info('Running on GPU! devices=%s' % str(opt.gpuid)) # model = nn.DataParallel(model, device_ids=opt.gpuid) else: logging.info('Running on CPU!') logging.info('====================== Start Training =========================') checkpoint_names = [] train_history_losses = [] valid_history_losses = [] test_history_losses = [] # best_loss = sys.float_info.max # for normal training/testing loss (likelihood) best_loss = 0.0 # for f-score stop_increasing = 0 train_losses = [] total_batch = 0 early_stop_flag = False if opt.train_from: state_path = opt.train_from.replace('.model', '.state') logging.info('Loading training state from: %s' % state_path) if os.path.exists(state_path): (epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_history_losses, valid_history_losses, test_history_losses) = torch.load(open(state_path, 'rb')) opt.start_epoch = epoch for epoch in range(opt.start_epoch , opt.epochs): if early_stop_flag: break progbar = Progbar(title='Training', target=len(train_data_loader), batch_size=train_data_loader.batch_size, total_examples=len(train_data_loader.dataset)) for batch_i, batch in enumerate(train_data_loader): model.train() batch_i += 1 # for the aesthetics of printing total_batch += 1 one2many_batch, one2one_batch = batch src, trg, trg_target, trg_copy_target, src_ext, oov_lists = one2one_batch max_oov_number = max([len(oov) for oov in oov_lists]) print("src size - ",src.size()) print("target size - ",trg.size()) if torch.cuda.is_available(): src = src.cuda() trg = trg.cuda() trg_target = trg_target.cuda() trg_copy_target = trg_copy_target.cuda() src_ext = src_ext.cuda() optimizer.zero_grad() ''' Training with Maximum Likelihood (word-level error) ''' decoder_log_probs, _, _ = model.forward(src, trg, src_ext, oov_lists) # simply average losses of all the predicitons # IMPORTANT, must use logits instead of probs to compute the loss, otherwise it's super super slow at the beginning (grads of probs are small)! start_time = time.time() if not opt.copy_model: ml_loss = criterion( decoder_log_probs.contiguous().view(-1, opt.vocab_size), trg_target.contiguous().view(-1) ) else: ml_loss = criterion( decoder_log_probs.contiguous().view(-1, opt.vocab_size + max_oov_number), trg_copy_target.contiguous().view(-1) ) ''' Training with Reinforcement Learning (instance-level reward f-score) ''' src_list, trg_list, _, trg_copy_target_list, src_oov_map_list, oov_list, src_str_list, trg_str_list = one2many_batch if torch.cuda.is_available(): src_list = src_list.cuda() src_oov_map_list = src_oov_map_list.cuda() rl_loss = get_loss_rl() start_time = time.time() ml_loss.backward() print("--backward- %s seconds ---" % (time.time() - start_time)) if opt.max_grad_norm > 0: pre_norm = torch.nn.utils.clip_grad_norm(model.parameters(), opt.max_grad_norm) after_norm = (sum([p.grad.data.norm(2) ** 2 for p in model.parameters() if p.grad is not None])) ** (1.0 / 2) logging.info('clip grad (%f -> %f)' % (pre_norm, after_norm)) optimizer.step() train_losses.append(ml_loss.data[0]) progbar.update(epoch, batch_i, [('train_loss', ml_loss.data[0]), ('PPL', ml_loss.data[0])]) if batch_i > 1 and batch_i % opt.report_every == 0: logging.info('====================== %d =========================' % (batch_i)) logging.info('Epoch : %d Minibatch : %d, Loss=%.5f' % (epoch, batch_i, np.mean(ml_loss.data[0]))) sampled_size = 2 logging.info('Printing predictions on %d sampled examples by greedy search' % sampled_size) if torch.cuda.is_available(): src = src.data.cpu().numpy() decoder_log_probs = decoder_log_probs.data.cpu().numpy() max_words_pred = decoder_log_probs.argmax(axis=-1) trg_target = trg_target.data.cpu().numpy() trg_copy_target = trg_copy_target.data.cpu().numpy() else: src = src.data.numpy() decoder_log_probs = decoder_log_probs.data.numpy() max_words_pred = decoder_log_probs.argmax(axis=-1) trg_target = trg_target.data.numpy() trg_copy_target = trg_copy_target.data.numpy() sampled_trg_idx = np.random.random_integers(low=0, high=len(trg) - 1, size=sampled_size) src = src[sampled_trg_idx] oov_lists = [oov_lists[i] for i in sampled_trg_idx] max_words_pred = [max_words_pred[i] for i in sampled_trg_idx] decoder_log_probs = decoder_log_probs[sampled_trg_idx] if not opt.copy_model: trg_target = [trg_target[i] for i in sampled_trg_idx] # use the real target trg_loss (the starting <BOS> has been removed and contains oov ground-truth) else: trg_target = [trg_copy_target[i] for i in sampled_trg_idx] for i, (src_wi, pred_wi, trg_i, oov_i) in enumerate(zip(src, max_words_pred, trg_target, oov_lists)): nll_prob = -np.sum([decoder_log_probs[i][l][pred_wi[l]] for l in range(len(trg_i))]) find_copy = np.any([x >= opt.vocab_size for x in src_wi]) has_copy = np.any([x >= opt.vocab_size for x in trg_i]) sentence_source = [opt.id2word[x] if x < opt.vocab_size else oov_i[x-opt.vocab_size] for x in src_wi] sentence_pred = [opt.id2word[x] if x < opt.vocab_size else oov_i[x-opt.vocab_size] for x in pred_wi] sentence_real = [opt.id2word[x] if x < opt.vocab_size else oov_i[x-opt.vocab_size] for x in trg_i] sentence_source = sentence_source[:sentence_source.index('<pad>')] if '<pad>' in sentence_source else sentence_source sentence_pred = sentence_pred[:sentence_pred.index('<pad>')] if '<pad>' in sentence_pred else sentence_pred sentence_real = sentence_real[:sentence_real.index('<pad>')] if '<pad>' in sentence_real else sentence_real logging.info('==================================================') logging.info('Source: %s ' % (' '.join(sentence_source))) logging.info('\t\tPred : %s (%.4f)' % (' '.join(sentence_pred), nll_prob) + (' [FIND COPY]' if find_copy else '')) logging.info('\t\tReal : %s ' % (' '.join(sentence_real)) + (' [HAS COPY]' + str(trg_i) if has_copy else '')) if total_batch > 1 and total_batch % opt.run_valid_every == 0: logging.info('*' * 50) logging.info('Run validing and testing @Epoch=%d,#(Total batch)=%d' % (epoch, total_batch)) # valid_losses = _valid_error(valid_data_loader, model, criterion, epoch, opt) # valid_history_losses.append(valid_losses) valid_score_dict = evaluate_beam_search(generator, valid_data_loader, opt, title='valid', epoch=epoch, save_path=opt.exp_path + '/epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch)) test_score_dict = evaluate_beam_search(generator, test_data_loader, opt, title='test', epoch=epoch, save_path=opt.exp_path + '/epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch)) checkpoint_names.append('epoch=%d-batch=%d-total_batch=%d' % (epoch, batch_i, total_batch)) train_history_losses.append(copy.copy(train_losses)) valid_history_losses.append(valid_score_dict) test_history_losses.append(test_score_dict) train_losses = [] scores = [train_history_losses] curve_names = ['Training Error'] scores += [[result_dict[name] for result_dict in valid_history_losses] for name in opt.report_score_names] curve_names += ['Valid-'+name for name in opt.report_score_names] scores += [[result_dict[name] for result_dict in test_history_losses] for name in opt.report_score_names] curve_names += ['Test-'+name for name in opt.report_score_names] scores = [np.asarray(s) for s in scores] # Plot the learning curve plot_learning_curve(scores=scores, curve_names=curve_names, checkpoint_names=checkpoint_names, title='Training Validation & Test', save_path=opt.exp_path + '/[epoch=%d,batch=%d,total_batch=%d]train_valid_test_curve.png' % (epoch, batch_i, total_batch)) ''' determine if early stop training (whether f-score increased, before is if valid error decreased) ''' valid_loss = np.average(valid_history_losses[-1][opt.report_score_names[0]]) is_best_loss = valid_loss > best_loss rate_of_change = float(valid_loss - best_loss) / float(best_loss) if float(best_loss) > 0 else 0.0 # valid error doesn't increase if rate_of_change <= 0: stop_increasing += 1 else: stop_increasing = 0 if is_best_loss: logging.info('Validation: update best loss (%.4f --> %.4f), rate of change (ROC)=%.2f' % ( best_loss, valid_loss, rate_of_change * 100)) else: logging.info('Validation: best loss is not updated for %d times (%.4f --> %.4f), rate of change (ROC)=%.2f' % ( stop_increasing, best_loss, valid_loss, rate_of_change * 100)) best_loss = max(valid_loss, best_loss) # only store the checkpoints that make better validation performances if total_batch > 1 and (total_batch % opt.save_model_every == 0 or is_best_loss): #epoch >= opt.start_checkpoint_at and # Save the checkpoint logging.info('Saving checkpoint to: %s' % os.path.join(opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d.error=%f' % (opt.exp, epoch, batch_i, total_batch, valid_loss) + '.model')) torch.save( model.state_dict(), open(os.path.join(opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.model'), 'wb') ) torch.save( (epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_history_losses, valid_history_losses, test_history_losses), open(os.path.join(opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.state'), 'wb') ) if stop_increasing >= opt.early_stop_tolerance: logging.info('Have not increased for %d epoches, early stop training' % stop_increasing) early_stop_flag = True break logging.info('*' * 50)
def train_model(model, optimizer, criterion, train_data_loader, valid_data_loader, test_data_loader, opt): generator = SequenceGenerator(model, eos_id=opt.word2id[pykp.io.EOS_WORD], beam_size=opt.beam_size, max_sequence_length=opt.max_sent_length) logging.info( '====================== Checking GPU Availability =========================' ) if torch.cuda.is_available(): if isinstance(opt.gpuid, int): opt.gpuid = [opt.gpuid] logging.info('Running on GPU! devices=%s' % str(opt.gpuid)) # model = nn.DataParallel(model, device_ids=opt.gpuid) else: logging.info('Running on CPU!') logging.info( '====================== Start Training =========================') checkpoint_names = [] train_history_losses = [] valid_history_losses = [] test_history_losses = [] # best_loss = sys.float_info.max # for normal training/testing loss (likelihood) best_loss = 0.0 # for f-score stop_increasing = 0 train_losses = [] total_batch = 0 early_stop_flag = False if opt.train_from: state_path = opt.train_from.replace('.model', '.state') logging.info('Loading training state from: %s' % state_path) if os.path.exists(state_path): (epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_history_losses, valid_history_losses, test_history_losses) = torch.load(open(state_path, 'rb')) opt.start_epoch = epoch for epoch in range(opt.start_epoch, opt.epochs): if early_stop_flag: break progbar = Progbar(title='Training', target=len(train_data_loader), batch_size=train_data_loader.batch_size, total_examples=len(train_data_loader.dataset)) for batch_i, batch in enumerate(train_data_loader): model.train() batch_i += 1 # for the aesthetics of printing total_batch += 1 one2many_batch, one2one_batch = batch src, trg, trg_target, trg_copy_target, src_ext, oov_lists = one2one_batch max_oov_number = max([len(oov) for oov in oov_lists]) print("src size - ", src.size()) print("target size - ", trg.size()) if torch.cuda.is_available(): src = src.cuda() trg = trg.cuda() trg_target = trg_target.cuda() trg_copy_target = trg_copy_target.cuda() src_ext = src_ext.cuda() optimizer.zero_grad() ''' Training with Maximum Likelihood (word-level error) ''' decoder_log_probs, _, _ = model.forward(src, trg, src_ext, oov_lists) # simply average losses of all the predicitons # IMPORTANT, must use logits instead of probs to compute the loss, otherwise it's super super slow at the beginning (grads of probs are small)! start_time = time.time() if not opt.copy_model: ml_loss = criterion( decoder_log_probs.contiguous().view(-1, opt.vocab_size), trg_target.contiguous().view(-1)) else: ml_loss = criterion( decoder_log_probs.contiguous().view( -1, opt.vocab_size + max_oov_number), trg_copy_target.contiguous().view(-1)) ''' Training with Reinforcement Learning (instance-level reward f-score) ''' src_list, trg_list, _, trg_copy_target_list, src_oov_map_list, oov_list, src_str_list, trg_str_list = one2many_batch if torch.cuda.is_available(): src_list = src_list.cuda() src_oov_map_list = src_oov_map_list.cuda() rl_loss = get_loss_rl() start_time = time.time() ml_loss.backward() print("--backward- %s seconds ---" % (time.time() - start_time)) if opt.max_grad_norm > 0: pre_norm = torch.nn.utils.clip_grad_norm( model.parameters(), opt.max_grad_norm) after_norm = (sum([ p.grad.data.norm(2)**2 for p in model.parameters() if p.grad is not None ]))**(1.0 / 2) logging.info('clip grad (%f -> %f)' % (pre_norm, after_norm)) optimizer.step() train_losses.append(ml_loss.data[0]) progbar.update(epoch, batch_i, [('train_loss', ml_loss.data[0]), ('PPL', ml_loss.data[0])]) if batch_i > 1 and batch_i % opt.report_every == 0: logging.info( '====================== %d =========================' % (batch_i)) logging.info('Epoch : %d Minibatch : %d, Loss=%.5f' % (epoch, batch_i, np.mean(ml_loss.data[0]))) sampled_size = 2 logging.info( 'Printing predictions on %d sampled examples by greedy search' % sampled_size) if torch.cuda.is_available(): src = src.data.cpu().numpy() decoder_log_probs = decoder_log_probs.data.cpu().numpy() max_words_pred = decoder_log_probs.argmax(axis=-1) trg_target = trg_target.data.cpu().numpy() trg_copy_target = trg_copy_target.data.cpu().numpy() else: src = src.data.numpy() decoder_log_probs = decoder_log_probs.data.numpy() max_words_pred = decoder_log_probs.argmax(axis=-1) trg_target = trg_target.data.numpy() trg_copy_target = trg_copy_target.data.numpy() sampled_trg_idx = np.random.random_integers(low=0, high=len(trg) - 1, size=sampled_size) src = src[sampled_trg_idx] oov_lists = [oov_lists[i] for i in sampled_trg_idx] max_words_pred = [max_words_pred[i] for i in sampled_trg_idx] decoder_log_probs = decoder_log_probs[sampled_trg_idx] if not opt.copy_model: trg_target = [ trg_target[i] for i in sampled_trg_idx ] # use the real target trg_loss (the starting <BOS> has been removed and contains oov ground-truth) else: trg_target = [trg_copy_target[i] for i in sampled_trg_idx] for i, (src_wi, pred_wi, trg_i, oov_i) in enumerate( zip(src, max_words_pred, trg_target, oov_lists)): nll_prob = -np.sum([ decoder_log_probs[i][l][pred_wi[l]] for l in range(len(trg_i)) ]) find_copy = np.any([x >= opt.vocab_size for x in src_wi]) has_copy = np.any([x >= opt.vocab_size for x in trg_i]) sentence_source = [ opt.id2word[x] if x < opt.vocab_size else oov_i[x - opt.vocab_size] for x in src_wi ] sentence_pred = [ opt.id2word[x] if x < opt.vocab_size else oov_i[x - opt.vocab_size] for x in pred_wi ] sentence_real = [ opt.id2word[x] if x < opt.vocab_size else oov_i[x - opt.vocab_size] for x in trg_i ] sentence_source = sentence_source[:sentence_source.index( '<pad>' )] if '<pad>' in sentence_source else sentence_source sentence_pred = sentence_pred[:sentence_pred.index( '<pad>' )] if '<pad>' in sentence_pred else sentence_pred sentence_real = sentence_real[:sentence_real.index( '<pad>' )] if '<pad>' in sentence_real else sentence_real logging.info( '==================================================') logging.info('Source: %s ' % (' '.join(sentence_source))) logging.info('\t\tPred : %s (%.4f)' % (' '.join(sentence_pred), nll_prob) + (' [FIND COPY]' if find_copy else '')) logging.info('\t\tReal : %s ' % (' '.join(sentence_real)) + (' [HAS COPY]' + str(trg_i) if has_copy else '')) if total_batch > 1 and total_batch % opt.run_valid_every == 0: logging.info('*' * 50) logging.info( 'Run validing and testing @Epoch=%d,#(Total batch)=%d' % (epoch, total_batch)) # valid_losses = _valid_error(valid_data_loader, model, criterion, epoch, opt) # valid_history_losses.append(valid_losses) valid_score_dict = evaluate_beam_search( generator, valid_data_loader, opt, title='valid', epoch=epoch, save_path=opt.exp_path + '/epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch)) test_score_dict = evaluate_beam_search( generator, test_data_loader, opt, title='test', epoch=epoch, save_path=opt.exp_path + '/epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch)) checkpoint_names.append('epoch=%d-batch=%d-total_batch=%d' % (epoch, batch_i, total_batch)) train_history_losses.append(copy.copy(train_losses)) valid_history_losses.append(valid_score_dict) test_history_losses.append(test_score_dict) train_losses = [] scores = [train_history_losses] curve_names = ['Training Error'] scores += [[ result_dict[name] for result_dict in valid_history_losses ] for name in opt.report_score_names] curve_names += [ 'Valid-' + name for name in opt.report_score_names ] scores += [[ result_dict[name] for result_dict in test_history_losses ] for name in opt.report_score_names] curve_names += [ 'Test-' + name for name in opt.report_score_names ] scores = [np.asarray(s) for s in scores] # Plot the learning curve plot_learning_curve( scores=scores, curve_names=curve_names, checkpoint_names=checkpoint_names, title='Training Validation & Test', save_path=opt.exp_path + '/[epoch=%d,batch=%d,total_batch=%d]train_valid_test_curve.png' % (epoch, batch_i, total_batch)) ''' determine if early stop training (whether f-score increased, before is if valid error decreased) ''' valid_loss = np.average( valid_history_losses[-1][opt.report_score_names[0]]) is_best_loss = valid_loss > best_loss rate_of_change = float(valid_loss - best_loss) / float( best_loss) if float(best_loss) > 0 else 0.0 # valid error doesn't increase if rate_of_change <= 0: stop_increasing += 1 else: stop_increasing = 0 if is_best_loss: logging.info( 'Validation: update best loss (%.4f --> %.4f), rate of change (ROC)=%.2f' % (best_loss, valid_loss, rate_of_change * 100)) else: logging.info( 'Validation: best loss is not updated for %d times (%.4f --> %.4f), rate of change (ROC)=%.2f' % (stop_increasing, best_loss, valid_loss, rate_of_change * 100)) best_loss = max(valid_loss, best_loss) # only store the checkpoints that make better validation performances if total_batch > 1 and ( total_batch % opt.save_model_every == 0 or is_best_loss): #epoch >= opt.start_checkpoint_at and # Save the checkpoint logging.info('Saving checkpoint to: %s' % os.path.join( opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d.error=%f' % (opt.exp, epoch, batch_i, total_batch, valid_loss) + '.model')) torch.save( model.state_dict(), open( os.path.join( opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.model'), 'wb')) torch.save((epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_history_losses, valid_history_losses, test_history_losses), open( os.path.join( opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.state'), 'wb')) if stop_increasing >= opt.early_stop_tolerance: logging.info( 'Have not increased for %d epoches, early stop training' % stop_increasing) early_stop_flag = True break logging.info('*' * 50)
def evaluate_beam_search(generator, data_loader, opt, title='', epoch=1, predict_save_path=None): logger = config.init_logging(title, predict_save_path + '/%s.log' % title, redirect_to_stdout=False) progbar = Progbar(logger=logger, title=title, target=len(data_loader.dataset.examples), batch_size=data_loader.batch_size, total_examples=len(data_loader.dataset.examples)) topk_range = [5, 10] score_names = ['precision', 'recall', 'f_score'] example_idx = 0 score_dict = { } # {'precision@5':[],'recall@5':[],'f1score@5':[], 'precision@10':[],'recall@10':[],'f1score@10':[]} for i, batch in enumerate(data_loader): # if i > 5: # break one2many_batch, one2one_batch = batch src_list, src_len, trg_list, _, trg_copy_target_list, src_oov_map_list, oov_list, src_str_list, trg_str_list = one2many_batch if torch.cuda.is_available(): src_list = src_list.cuda() src_oov_map_list = src_oov_map_list.cuda() print("batch size - %s" % str(src_list.size(0))) print("src size - %s" % str(src_list.size())) print("target size - %s" % len(trg_copy_target_list)) pred_seq_list = generator.beam_search(src_list, src_len, src_oov_map_list, oov_list, opt.word2id) ''' process each example in current batch ''' for src, src_str, trg, trg_str_seqs, trg_copy, pred_seq, oov in zip( src_list, src_str_list, trg_list, trg_str_list, trg_copy_target_list, pred_seq_list, oov_list): logger.info( '====================== %d =========================' % (i)) print_out = '' print_out += '[Source][%d]: %s \n' % (len(src_str), ' '.join(src_str)) src = src.cpu().data.numpy() if torch.cuda.is_available( ) else src.data.numpy() print_out += '\nSource Input: \n %s\n' % (' '.join( [opt.id2word[x] for x in src[:len(src_str) + 5]])) print_out += 'Real Target String [%d] \n\t\t%s \n' % ( len(trg_str_seqs), trg_str_seqs) print_out += 'Real Target Input: \n\t\t%s \n' % str( [[opt.id2word[x] for x in t] for t in trg]) print_out += 'Real Target Copy: \n\t\t%s \n' % str([[ opt.id2word[x] if x < opt.vocab_size else oov[x - opt.vocab_size] for x in t ] for t in trg_copy]) trg_str_is_present_flags, _ = if_present_duplicate_phrases( src_str, trg_str_seqs) # ignore the cases that there's no present phrases if opt.must_appear_in_src and np.sum( trg_str_is_present_flags) == 0: logger.error('found no present targets') continue print_out += '[GROUND-TRUTH] #(present)/#(all targets)=%d/%d\n' % ( sum(trg_str_is_present_flags), len(trg_str_is_present_flags)) print_out += '\n'.join([ '\t\t[%s]' % ' '.join(phrase) if is_present else '\t\t%s' % ' '.join(phrase) for phrase, is_present in zip( trg_str_seqs, trg_str_is_present_flags) ]) print_out += '\noov_list: \n\t\t%s \n' % str(oov) # 1st filtering pred_is_valid_flags, processed_pred_seqs, processed_pred_str_seqs, processed_pred_score = process_predseqs( pred_seq, oov, opt.id2word, opt) # 2nd filtering: if filter out phrases that don't appear in text, and keep unique ones after stemming if opt.must_appear_in_src: pred_is_present_flags, _ = if_present_duplicate_phrases( src_str, processed_pred_str_seqs) filtered_trg_str_seqs = np.asarray( trg_str_seqs)[trg_str_is_present_flags] else: pred_is_present_flags = [True] * len(processed_pred_str_seqs) valid_and_present = np.asarray(pred_is_valid_flags) * np.asarray( pred_is_present_flags) match_list = get_match_result(true_seqs=filtered_trg_str_seqs, pred_seqs=processed_pred_str_seqs) print_out += '[PREDICTION] #(valid)=%d, #(present)=%d, #(retained&present)=%d, #(all)=%d\n' % ( sum(pred_is_valid_flags), sum(pred_is_present_flags), sum(valid_and_present), len(pred_seq)) print_out += '' ''' Print and export predictions ''' preds_out = '' for p_id, (seq, word, score, match, is_valid, is_present) in enumerate( zip(processed_pred_seqs, processed_pred_str_seqs, processed_pred_score, match_list, pred_is_valid_flags, pred_is_present_flags)): # if p_id > 5: # break preds_out += '%s\n' % (' '.join(word)) if is_present: print_phrase = '[%s]' % ' '.join(word) else: print_phrase = ' '.join(word) if is_valid: print_phrase = '*%s' % print_phrase if match == 1.0: correct_str = '[correct!]' else: correct_str = '' if any([t >= opt.vocab_size for t in seq.sentence]): copy_str = '[copied!]' else: copy_str = '' print_out += '\t\t[%.4f]\t%s \t %s %s%s\n' % ( -score, print_phrase, str( seq.sentence), correct_str, copy_str) ''' Evaluate predictions w.r.t different filterings and metrics ''' processed_pred_seqs = np.asarray( processed_pred_seqs)[valid_and_present] filtered_processed_pred_str_seqs = np.asarray( processed_pred_str_seqs)[valid_and_present] filtered_processed_pred_score = np.asarray( processed_pred_score)[valid_and_present] # 3rd round filtering (one-word phrases) num_oneword_seq = -1 filtered_pred_seq, filtered_pred_str_seqs, filtered_pred_score = post_process_predseqs( (processed_pred_seqs, filtered_processed_pred_str_seqs, filtered_processed_pred_score), num_oneword_seq) match_list_exact = get_match_result( true_seqs=filtered_trg_str_seqs, pred_seqs=filtered_pred_str_seqs, type='exact') match_list_soft = get_match_result( true_seqs=filtered_trg_str_seqs, pred_seqs=filtered_pred_str_seqs, type='partial') assert len(filtered_pred_seq) == len( filtered_pred_str_seqs) == len(filtered_pred_score) == len( match_list_exact) == len(match_list_soft) print_out += "\n =======================================================" print_pred_str_seqs = [ " ".join(item) for item in filtered_pred_str_seqs ] print_trg_str_seqs = [ " ".join(item) for item in filtered_trg_str_seqs ] # print_out += "\n PREDICTION: " + " / ".join(print_pred_str_seqs) # print_out += "\n GROUND TRUTH: " + " / ".join(print_trg_str_seqs) for topk in topk_range: results_exact = evaluate(match_list_exact, filtered_pred_str_seqs, filtered_trg_str_seqs, topk=topk) for k, v in zip(score_names, results_exact): if '%s@%d_exact' % (k, topk) not in score_dict: score_dict['%s@%d_exact' % (k, topk)] = [] score_dict['%s@%d_exact' % (k, topk)].append(v) print_out += "\n ------------------------------------------------- EXACT, k=%d" % ( topk) print_out += "\n --- batch precision, recall, fscore: " + str( results_exact[0]) + " , " + str( results_exact[1]) + " , " + str(results_exact[2]) print_out += "\n --- total precision, recall, fscore: " + str(np.average(score_dict['precision@%d_exact' % (topk)])) + " , " +\ str(np.average(score_dict['recall@%d_exact' % (topk)])) + " , " +\ str(np.average(score_dict['f_score@%d_exact' % (topk)])) for topk in topk_range: results_soft = evaluate(match_list_soft, filtered_pred_str_seqs, filtered_trg_str_seqs, topk=topk) for k, v in zip(score_names, results_soft): if '%s@%d_soft' % (k, topk) not in score_dict: score_dict['%s@%d_soft' % (k, topk)] = [] score_dict['%s@%d_soft' % (k, topk)].append(v) print_out += "\n ------------------------------------------------- SOFT, k=%d" % ( topk) print_out += "\n --- batch precision, recall, fscore: " + str( results_soft[0]) + " , " + str( results_soft[1]) + " , " + str(results_soft[2]) print_out += "\n --- total precision, recall, fscore: " + str(np.average(score_dict['precision@%d_soft' % (topk)])) + " , " +\ str(np.average(score_dict['recall@%d_soft' % (topk)])) + " , " +\ str(np.average(score_dict['f_score@%d_soft' % (topk)])) print_out += "\n =======================================================" logger.info(print_out) ''' write predictions to disk ''' if predict_save_path: if not os.path.exists( os.path.join(predict_save_path, title + '_detail')): os.makedirs( os.path.join(predict_save_path, title + '_detail')) with open( os.path.join(predict_save_path, title + '_detail', str(example_idx) + '_print.txt'), 'w') as f_: f_.write(print_out) with open( os.path.join(predict_save_path, title + '_detail', str(example_idx) + '_prediction.txt'), 'w') as f_: f_.write(preds_out) out_dict = {} out_dict['src_str'] = src_str out_dict['trg_str'] = trg_str_seqs out_dict['trg_present_flag'] = trg_str_is_present_flags out_dict['pred_str'] = processed_pred_str_seqs out_dict['pred_score'] = [ float(s) for s in processed_pred_score ] out_dict['present_flag'] = pred_is_present_flags out_dict['valid_flag'] = pred_is_valid_flags out_dict['match_flag'] = [float(m) for m in match_list] for k, v in out_dict.items(): out_dict[k] = list(v) # print('len(%s) = %d' % (k, len(v))) # print(out_dict) assert len(out_dict['trg_str']) == len( out_dict['trg_present_flag']) assert len(out_dict['pred_str']) == len(out_dict['present_flag']) \ == len(out_dict['valid_flag']) == len(out_dict['match_flag']) == len(out_dict['pred_score']) with open( os.path.join(predict_save_path, title + '_detail', str(example_idx) + '.json'), 'w') as f_: f_.write(json.dumps(out_dict)) progbar.update(epoch, example_idx, [ ('f_score@5_exact', np.average(score_dict['f_score@5_exact'])), ('f_score@5_soft', np.average(score_dict['f_score@5_soft'])), ('f_score@10_exact', np.average( score_dict['f_score@10_exact'])), ('f_score@10_soft', np.average(score_dict['f_score@10_soft'])), ]) example_idx += 1 # print('#(f_score@5#oneword=-1)=%d, sum=%f' % (len(score_dict['f_score@5#oneword=-1']), sum(score_dict['f_score@5#oneword=-1']))) # print('#(f_score@10#oneword=-1)=%d, sum=%f' % (len(score_dict['f_score@10#oneword=-1']), sum(score_dict['f_score@10#oneword=-1']))) # print('#(f_score@5#oneword=1)=%d, sum=%f' % (len(score_dict['f_score@5#oneword=1']), sum(score_dict['f_score@5#oneword=1']))) # print('#(f_score@10#oneword=1)=%d, sum=%f' % (len(score_dict['f_score@10#oneword=1']), sum(score_dict['f_score@10#oneword=1']))) if predict_save_path: # export scores. Each row is scores (precision, recall and f-score) of different way of filtering predictions (how many one-word predictions to keep) with open(predict_save_path + os.path.sep + title + '_result.csv', 'w') as result_csv: csv_lines = [] for mode in ["exact", "soft"]: for topk in topk_range: csv_line = "" for k in score_names: csv_line += ',%f' % np.average( score_dict['%s@%d_%s' % (k, topk, mode)]) csv_lines.append(csv_line + '\n') result_csv.writelines(csv_lines) # precision, recall, f_score = macro_averaged_score(precisionlist=score_dict['precision'], recalllist=score_dict['recall']) # logging.info("Macro@5\n\t\tprecision %.4f\n\t\tmacro recall %.4f\n\t\tmacro fscore %.4f " % (np.average(score_dict['precision@5']), np.average(score_dict['recall@5']), np.average(score_dict['f1score@5']))) # logging.info("Macro@10\n\t\tprecision %.4f\n\t\tmacro recall %.4f\n\t\tmacro fscore %.4f " % (np.average(score_dict['precision@10']), np.average(score_dict['recall@10']), np.average(score_dict['f1score@10']))) # precision, recall, f_score = evaluate(true_seqs=target_all, pred_seqs=prediction_all, topn=5) # logging.info("micro precision %.4f , micro recall %.4f, micro fscore %.4f " % (precision, recall, f_score)) for k, v in score_dict.items(): print('#(%s) = %d' % (k, len(v))) return score_dict
def run_epoch(self, sess, src_train, src_dev, tags, target_train, target_dev, n_epoch_noimprove): nbatces = (len(target_train) + self.target_batch_size - 1) // self.target_batch_size prog = Progbar(target=nbatces) total_loss = 0 src = minibatches(src_train, self.src_batch_size, circle=True) target = minibatches(target_train, self.target_batch_size, circle=True) for i in range(nbatces): src_words, src_tags, _ = next(src) target_words, target_tags, _ = next(target) labels = src_tags + target_tags feed_dict, _ = self.get_feed_dict(src_words, labels, target_words, self.args.learning_rate, self.args.dropout, self.src_batch_size, is_training=True) if self.args.penalty_ratio > 0: _, src_crf_loss, target_crf_loss, penalty_loss, loss = sess.run( [ self.train_op, self.src_crf_loss, self.target_crf_loss, self.penalty_loss, self.loss ], feed_dict=feed_dict) try: prog.update( i + 1, [("train loss", loss[0]), ("src crf", src_crf_loss), ("target crf", target_crf_loss), ("{} loss".format(self.args.penalty), penalty_loss)]) except: prog.update( i + 1, [("train loss", loss), ("src crf", src_crf_loss), ("target crf", target_crf_loss), ("{} loss".format(self.args.penalty), penalty_loss)]) else: _, src_crf_loss, target_crf_loss, loss = sess.run( [ self.train_op, self.src_crf_loss, self.target_crf_loss, self.loss ], feed_dict=feed_dict) try: prog.update(i + 1, [("train loss", loss[0]), ("src crf", src_crf_loss), ("target crf", target_crf_loss)]) except: prog.update(i + 1, [("train loss", loss), ("src crf", src_crf_loss), ("target crf", target_crf_loss)]) total_loss += loss self.info['loss'] += [total_loss / nbatces] acc, p, r, f1 = self.run_evaluate(sess, target_train, tags, target='target') self.info['dev'].append((acc, p, r, f1)) self.logger.critical( "target train acc {:04.2f} f1 {:04.2f} p {:04.2f} r {:04.2f}". format(100 * acc, 100 * f1, 100 * p, 100 * r)) acc, p, r, f1 = self.run_evaluate(sess, target_dev, tags, target='target') self.info['dev'].append((acc, p, r, f1)) self.logger.info( "dev acc {:04.2f} f1 {:04.2f} p {:04.2f} r {:04.2f}".format( 100 * acc, 100 * f1, 100 * p, 100 * r)) return acc, p, r, f1
def run( kn=1.0, sigma=lambda v, w: 1.0, G=lambda x: 0.0, xmin=0.0, xmax=1.0, nx=40, dt=0.01, nt=1000, coll="linear", scheme="Euler", BC="periodic", f_l=lambda v: 0.0, f_r=lambda v: 0.0, init_func=lambda vmesh, u, T, rho: 0.0, ): # Load config config = collision.utils.CollisionConfig.from_json( "./linear_boltz/configs/" + "linear" + ".json" ) # Collision vmesh = collision.CartesianMesh(config) if coll == "linear": coll_op = collision.LinearBotlzmannCollision(config, vmesh, sigma=sigma) elif coll == "rbm": coll_op = collision.RandomBatchLinearBoltzmannCollision( config, vmesh, sigma=sigma ) elif coll == "rbm_symm": coll_op = collision.SymmetricRBMLinearCollision(config, vmesh, sigma=sigma) else: raise NotImplementedError( "Collision method {} is not implemented.".format(coll) ) # x domian x = pykinetic.Dimension(xmin, xmax, nx, name="x") domain = pykinetic.Domain([x]) # Riemann solver rp = pykinetic.riemann.advection_1D solver = DiffusiveRegimeSolver1D( rp, [coll_op], kn=kn(x.centers), G=G(x.centers), ) solver.order = 1 # solver.lim_type = 2 # Time integrator if "RK" in scheme: solver.time_integrator = "RK" solver.a = rkcoeff[scheme]["a"] solver.b = rkcoeff[scheme]["b"] solver.c = rkcoeff[scheme]["c"] else: solver.time_integrator = scheme solver.dt = dt print("dt is {}".format(solver.dt)) # Boundary condition def dirichlet_lower_BC(state, dim, t, qbc, auxbc, num_ghost): v = state.problem_data["v"][0] for i in range(num_ghost): qbc[0, i, v > 0] = f_l(v[v > 0]) def dirichlet_upper_BC(state, dim, t, qbc, auxbc, num_ghost): v = state.problem_data["v"][0] for i in range(num_ghost): qbc[0, -i - 1, v < 0] = f_r(v[v < 0]) if BC == "periodic": solver.bc_lower[0] = pykinetic.BC.periodic solver.bc_upper[0] = pykinetic.BC.periodic elif BC == "dirichlet": solver.bc_lower[0] = pykinetic.BC.custom solver.bc_upper[0] = pykinetic.BC.custom solver.user_bc_lower = dirichlet_lower_BC solver.user_bc_upper = dirichlet_upper_BC else: raise ValueError("Given BC type is not avaliable!") state = pykinetic.State(domain, vmesh, 1) state.problem_data["v"] = vmesh.centers qinit(state, vmesh, init_func) sol = pykinetic.Solution(state, domain) output_dict = {} sol_frames, macro_frames, ts = ( [copy.deepcopy(sol)], [compute_rho(sol.state, vmesh)], [0.0], ) pbar = Progbar(nt) for t in range(nt): solver.evolve_to_time(sol) if (t + 1) % 1 == 0: sol_frames.append(copy.deepcopy(sol)) macro_frames.append(compute_rho(sol.state, vmesh)) ts.append(sol.t) pbar.update(t + 1, finalize=False) pbar.update(nt, finalize=True) output_dict["macro_frames"] = np.asarray(macro_frames) output_dict["x"] = np.asarray(x.centers) output_dict["t"] = np.asarray(ts) return output_dict
def train(self, trainset, devset, testset, batch_size=64, epochs=50, shuffle=True): ''' :param trainset: 是所有的数据 分batch是在这个函数完成的 :param devset: :param testset: :param batch_size: :param epochs: :param shuffle: :return: ''' self.logger.info('Start training...') init_lr = self.cfg.lr # initial learning rate, used for decay learning rate best_score = 0.0 # record the best score best_score_epoch = 1 # record the epoch of the best score obtained no_imprv_epoch = 0 # no improvement patience counter for epoch in range(self.start_epoch, epochs + 1): self.logger.info('Epoch %2d/%2d:' % (epoch, epochs)) progbar = Progbar(target=(len(trainset) + batch_size - 1) // batch_size) # number of batches if shuffle: np.random.shuffle( trainset) # shuffle training dataset each epoch # training each epoch for i, (words, labels) in enumerate(batch_iter(trainset, batch_size)): feed_dict = self._get_feed_dict(words, labels, lr=self.cfg.lr, is_train=True) _, train_loss = self.sess.run([self.train_op, self.loss], feed_dict=feed_dict) progbar.update(i + 1, [("train loss", train_loss)]) if devset is not None: self.evaluate(devset, batch_size) cur_score = self.evaluate(testset, batch_size, is_devset=False) # learning rate decay if self.cfg.decay_lr: self.cfg.lr = init_lr / (1 + self.cfg.lr_decay * epoch) # performs model saving and evaluating on test dataset if cur_score > best_score: no_imprv_epoch = 0 self.save_session(epoch) best_score = cur_score best_score_epoch = epoch self.logger.info( ' -- new BEST score on TEST dataset: {:05.3f}'.format( best_score)) else: no_imprv_epoch += 1 if no_imprv_epoch >= self.cfg.no_imprv_patience: self.logger.info( 'early stop at {}th epoch without improvement for {} epochs, BEST score: ' '{:05.3f} at epoch {}'.format(epoch, no_imprv_epoch, best_score, best_score_epoch)) break self.logger.info('Training process done...')
def train_model(model, optimizer_ml, optimizer_rl, criterion, train_data_loader, valid_data_loader, test_data_loader, opt): generator = SequenceGenerator(model, eos_id=opt.word2id[pykp.io.EOS_WORD], beam_size=opt.beam_size, max_sequence_length=opt.max_sent_length) logging.info( '====================== Checking GPU Availability =========================' ) if torch.cuda.is_available(): if isinstance(opt.gpuid, int): opt.gpuid = [opt.gpuid] logging.info('Running on GPU! devices=%s' % str(opt.gpuid)) # model = nn.DataParallel(model, device_ids=opt.gpuid) else: logging.info('Running on CPU!') logging.info( '====================== Start Training =========================') checkpoint_names = [] train_ml_history_losses = [] train_rl_history_losses = [] valid_history_losses = [] test_history_losses = [] # best_loss = sys.float_info.max # for normal training/testing loss (likelihood) best_loss = 0.0 # for f-score stop_increasing = 0 train_ml_losses = [] train_rl_losses = [] total_batch = -1 early_stop_flag = False if opt.train_rl: reward_cache = RewardCache(2000) if False: # opt.train_from: state_path = opt.train_from.replace('.model', '.state') logging.info('Loading training state from: %s' % state_path) if os.path.exists(state_path): (epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_ml_history_losses, train_rl_history_losses, valid_history_losses, test_history_losses) = torch.load(open(state_path, 'rb')) opt.start_epoch = epoch for epoch in range(opt.start_epoch, opt.epochs): if early_stop_flag: break progbar = Progbar(logger=logging, title='Training', target=len(train_data_loader), batch_size=train_data_loader.batch_size, total_examples=len( train_data_loader.dataset.examples)) for batch_i, batch in enumerate(train_data_loader): model.train() total_batch += 1 one2many_batch, one2one_batch = batch report_loss = [] # Training if opt.train_ml: loss_ml, decoder_log_probs = train_ml(one2one_batch, model, optimizer_ml, criterion, opt) train_ml_losses.append(loss_ml) report_loss.append(('train_ml_loss', loss_ml)) report_loss.append(('PPL', loss_ml)) # Brief report if batch_i % opt.report_every == 0: brief_report(epoch, batch_i, one2one_batch, loss_ml, decoder_log_probs, opt) # do not apply rl in 0th epoch, need to get a resonable model before that. if opt.train_rl: if epoch >= opt.rl_start_epoch: loss_rl = train_rl(one2many_batch, model, optimizer_rl, generator, opt, reward_cache) else: loss_rl = 0.0 train_rl_losses.append(loss_rl) report_loss.append(('train_rl_loss', loss_rl)) progbar.update(epoch, batch_i, report_loss) # Validate and save checkpoint if (opt.run_valid_every == -1 and batch_i == len(train_data_loader) - 1) or\ (opt.run_valid_every > -1 and total_batch > 1 and total_batch % opt.run_valid_every == 0): logging.info('*' * 50) logging.info( 'Run validing and testing @Epoch=%d,#(Total batch)=%d' % (epoch, total_batch)) # valid_losses = _valid_error(valid_data_loader, model, criterion, epoch, opt) # valid_history_losses.append(valid_losses) valid_score_dict = evaluate_beam_search( generator, valid_data_loader, opt, title='Validating, epoch=%d, batch=%d, total_batch=%d' % (epoch, batch_i, total_batch), epoch=epoch, predict_save_path=opt.pred_path + '/epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch)) test_score_dict = evaluate_beam_search( generator, test_data_loader, opt, title='Testing, epoch=%d, batch=%d, total_batch=%d' % (epoch, batch_i, total_batch), epoch=epoch, predict_save_path=opt.pred_path + '/epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch)) checkpoint_names.append('epoch=%d-batch=%d-total_batch=%d' % (epoch, batch_i, total_batch)) curve_names = [] scores = [] if opt.train_ml: train_ml_history_losses.append(copy.copy(train_ml_losses)) scores += [train_ml_history_losses] curve_names += ['Training ML Error'] train_ml_losses = [] if opt.train_rl: train_rl_history_losses.append(copy.copy(train_rl_losses)) scores += [train_rl_history_losses] curve_names += ['Training RL Reward'] train_rl_losses = [] valid_history_losses.append(valid_score_dict) test_history_losses.append(test_score_dict) scores += [[ result_dict[name] for result_dict in valid_history_losses ] for name in opt.report_score_names] curve_names += [ 'Valid-' + name for name in opt.report_score_names ] scores += [[ result_dict[name] for result_dict in test_history_losses ] for name in opt.report_score_names] curve_names += [ 'Test-' + name for name in opt.report_score_names ] scores = [np.asarray(s) for s in scores] # Plot the learning curve plot_learning_curve_and_write_csv( scores=scores, curve_names=curve_names, checkpoint_names=checkpoint_names, title='Training Validation & Test', save_path=opt.exp_path + '/[epoch=%d,batch=%d,total_batch=%d]train_valid_test_curve.png' % (epoch, batch_i, total_batch)) ''' determine if early stop training (whether f-score increased, before is if valid error decreased) ''' valid_loss = np.average( valid_history_losses[-1][opt.report_score_names[0]]) is_best_loss = valid_loss > best_loss rate_of_change = float(valid_loss - best_loss) / float( best_loss) if float(best_loss) > 0 else 0.0 # valid error doesn't increase if rate_of_change <= 0: stop_increasing += 1 else: stop_increasing = 0 if is_best_loss: logging.info( 'Validation: update best loss (%.4f --> %.4f), rate of change (ROC)=%.2f' % (best_loss, valid_loss, rate_of_change * 100)) else: logging.info( 'Validation: best loss is not updated for %d times (%.4f --> %.4f), rate of change (ROC)=%.2f' % (stop_increasing, best_loss, valid_loss, rate_of_change * 100)) best_loss = max(valid_loss, best_loss) # only store the checkpoints that make better validation performances if total_batch > 1 and ( total_batch % opt.save_model_every == 0 or is_best_loss): # epoch >= opt.start_checkpoint_at and # Save the checkpoint logging.info('Saving checkpoint to: %s' % os.path.join( opt.model_path, '%s.epoch=%d.batch=%d.total_batch=%d.error=%f' % (opt.exp, epoch, batch_i, total_batch, valid_loss) + '.model')) torch.save( model.state_dict(), open( os.path.join( opt.model_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.model'), 'wb')) torch.save((epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_ml_history_losses, train_rl_history_losses, valid_history_losses, test_history_losses), open( os.path.join( opt.model_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.state'), 'wb')) if stop_increasing >= opt.early_stop_tolerance: logging.info( 'Have not increased for %d epoches, early stop training' % stop_increasing) early_stop_flag = True break logging.info('*' * 50)
def run( kn=lambda x: 1.0, sigma_s=lambda x: 1.0, sigma_a=lambda x: 0.0, Q=lambda x: 0.0, xmin=0.0, xmax=1.0, nx=40, dt=0.01, nt=1000, coll="linear", scheme="Euler", BC="periodic", f_l=lambda v: 1.0, f_r=lambda v: 0.0, init_func=lambda vmesh, u, T, rho: 0.0, ): # Load config config = collision.utils.CollisionConfig.from_json( "./linear_transport/configs/" + "parity" + ".json") # Collision vmesh = collision.CartesianMesh(config) # print(vmesh.centers[0], vmesh.weights) if coll == "linear": coll_op = collision.LinearCollision(config, vmesh) elif coll == "rbm": coll_op = collision.RandomBatchLinearCollision(config, vmesh) elif coll == "rbm_symm": coll_op = collision.SymmetricRBMLinearCollision(config, vmesh) else: raise NotImplementedError( "Collision method {} is not implemented.".format(coll)) # x domian x = pykinetic.Dimension(xmin, xmax, nx, name="x") # print(x.centers_with_ghost(2)) domain = pykinetic.Domain([x]) # Riemann solver rp = pykinetic.riemann.parity_1D solver = APNeutronTransportSolver1D( rp, [coll_op], kn=kn(x.centers), sigma_s=sigma_s(x.centers), sigma_a=sigma_a(x.centers), Q=Q(x.centers), ) # print(solver.kn) solver.order = 1 # solver.lim_type = -1 solver.time_integrator = scheme solver.dt = dt print("dt is {}".format(solver.dt)) sigma = sigma_s(x.centers) + kn(x.centers)**2 * sigma_a(x.centers) sigma_l, sigma_r = None, None if isinstance(sigma, np.ndarray): sigma_l, sigma_r = sigma[0], sigma[-1] else: sigma_l = sigma_r = sigma kn_l, kn_r = None, None if isinstance(solver.kn, np.ndarray): kn_l, kn_r = solver.kn[0], solver.kn[-1] else: kn_l = kn_r = solver.kn # Boundary conditions def dirichlet_lower_BC(state, dim, t, qbc, auxbc, num_ghost): v = state.problem_data["v"][0] / sigma_l kn_dx = kn_l / state.grid.delta[0] for i in range(num_ghost): qbc[0, i, :] = (f_l(v) - (0.5 - kn_dx * v) * qbc[0, num_ghost, :]) / ( 0.5 + kn_dx * v) for i in range(num_ghost): qbc[1, i, :] = (2 * f_l(v) - (qbc[0, num_ghost - 1, :] + qbc[0, num_ghost, :])) / kn_l - qbc[1, num_ghost, :] def dirichlet_upper_BC(state, dim, t, qbc, auxbc, num_ghost): v = state.problem_data["v"][0] / sigma_r kn_dx = kn_r / state.grid.delta[0] for i in range(num_ghost): qbc[0, -i - 1, :] = (f_r(v) - (0.5 - kn_dx * v) * qbc[0, -num_ghost - 1, :]) / ( 0.5 + kn_dx * v) for i in range(num_ghost): qbc[1, -i - 1, :] = ((qbc[0, -num_ghost, :] + qbc[0, -num_ghost - 1, :]) - 2 * f_r(v)) / kn_r - qbc[1, -num_ghost - 1, :] if BC == "periodic": solver.bc_lower[0] = pykinetic.BC.periodic solver.bc_upper[0] = pykinetic.BC.periodic elif BC == "dirichlet": solver.bc_lower[0] = pykinetic.BC.custom solver.bc_upper[0] = pykinetic.BC.custom solver.user_bc_lower = dirichlet_lower_BC solver.user_bc_upper = dirichlet_upper_BC else: raise ValueError("Given BC type is not avaliable!") state = pykinetic.State(domain, vmesh, 2) state.problem_data["v"] = vmesh.centers state.problem_data["phi"] = phi(solver.kn) state.problem_data["sqrt_phi"] = np.sqrt(phi(solver.kn)) qinit(state, vmesh, solver.kn, init_func) sol = pykinetic.Solution(state, domain) output_dict = {} sol_frames, macro_frames, ts = ( [copy.deepcopy(sol.q)], [compute_rho(sol.state, vmesh)], [0.0], ) pbar = Progbar(nt) for t in range(nt): solver.evolve_to_time(sol) sol_frames.append(copy.deepcopy(sol.q)) macro_frames.append(compute_rho(sol.state, vmesh)) # Test # qbc = solver.qbc # print( # np.max( # np.abs( # 5.0 * np.sin(vmesh.centers[0]) # - ( # 0.5 * (qbc[0, 1] + qbc[0, 2]) # + kn * 0.5 * (qbc[1, 1] + qbc[1, 2]) # ) # ) # ) # ) ts.append(0.0 + (t + 1) * dt) pbar.update(t + 1, finalize=False) pbar.update(nt, finalize=True) output_dict["f_frames"] = np.asarray(sol_frames) output_dict["macro_frames"] = macro_frames output_dict["x"] = x.centers output_dict["t"] = ts output_dict["v"] = np.asarray(vmesh.centers[0]) return output_dict
def train_model(model, optimizer, criterion, training_data_loader, validation_data_loader, opt): logging.info( '====================== Checking GPU Availability =========================' ) if torch.cuda.is_available(): if isinstance(opt.gpuid, int): opt.gpuid = [opt.gpuid] logging.info('Running on GPU! devices=%s' % str(opt.gpuid)) model = model.cuda() model = nn.DataParallel(model, device_ids=opt.gpuid) criterion.cuda() else: logging.info('Running on CPU!') logging.info( '====================== Start Training =========================') train_history_losses = [] valid_history_losses = [] best_loss = sys.float_info.max train_losses = [] total_batch = 0 early_stop_flag = False for epoch in range(opt.start_epoch, opt.epochs): if early_stop_flag: break progbar = Progbar(title='Training', target=len(training_data_loader), batch_size=opt.batch_size, total_examples=len(training_data_loader.dataset)) model.train() for batch_i, batch in enumerate(training_data_loader): batch_i += 1 # for the aesthetics of printing total_batch += 1 src = batch.src trg = batch.trg # print("src size - ",src.size()) # print("target size - ",trg.size()) if torch.cuda.is_available(): src.cuda() trg.cuda() optimizer.zero_grad() decoder_logits, _, _ = model.forward(src, trg, must_teacher_forcing=False) start_time = time.time() # remove the 1st word in trg to let predictions and real goal match loss = criterion( decoder_logits.contiguous().view(-1, opt.vocab_size), trg[:, 1:].contiguous().view(-1)) print("--loss calculation- %s seconds ---" % (time.time() - start_time)) start_time = time.time() loss.backward() print("--backward- %s seconds ---" % (time.time() - start_time)) if opt.max_grad_norm > 0: pre_norm = torch.nn.utils.clip_grad_norm( model.parameters(), opt.max_grad_norm) after_norm = (sum([ p.grad.data.norm(2)**2 for p in model.parameters() if p.grad is not None ]))**(1.0 / 2) logging.info('clip grad (%f -> %f)' % (pre_norm, after_norm)) optimizer.step() train_losses.append(loss.data[0]) perplexity = np.math.exp(loss.data[0]) progbar.update(epoch, batch_i, [('train_loss', loss.data[0]), ('perplexity', perplexity)]) if batch_i > 1 and batch_i % opt.report_every == 0: logging.info( '====================== %d =========================' % (batch_i)) logging.info( 'Epoch : %d Minibatch : %d, Loss=%.5f, PPL=%.5f' % (epoch, batch_i, np.mean(loss.data[0]), perplexity)) sampled_size = 2 logging.info( 'Printing predictions on %d sampled examples by greedy search' % sampled_size) # softmax logits to get probabilities (batch_size, trg_len, vocab_size) # decoder_probs = torch.nn.functional.softmax(decoder_logits.view(trg.size(0) * trg.size(1), -1)).view(*trg.size(), -1) if torch.cuda.is_available(): src = src.data.cpu().numpy() decoder_logits = decoder_logits.data.cpu().numpy() max_words_pred = decoder_logits.argmax(axis=-1) trg = trg.data.cpu().numpy() else: src = src.data.numpy() decoder_logits = decoder_logits.data.numpy() max_words_pred = decoder_logits.argmax(axis=-1) trg = trg.data.numpy() sampled_trg_idx = np.random.random_integers(low=0, high=len(trg) - 1, size=sampled_size) src = src[sampled_trg_idx] max_words_pred = [max_words_pred[i] for i in sampled_trg_idx] decoder_logits = decoder_logits[sampled_trg_idx] trg = [trg[i][1:] for i in sampled_trg_idx ] # the real target has removed the starting <BOS> for i, (src_wi, pred_wi, real_wi) in enumerate(zip(src, max_words_pred, trg)): nll_prob = -np.sum( np.log2([ decoder_logits[i][l][pred_wi[l]] for l in range(len(real_wi)) ])) sentence_source = [opt.id2word[x] for x in src_wi] sentence_pred = [opt.id2word[x] for x in pred_wi] sentence_real = [opt.id2word[x] for x in real_wi] logging.info( '==================================================') logging.info('Source: %s ' % (' '.join(sentence_source))) logging.info('\t\tPred : %s (%.4f)' % (' '.join(sentence_pred), nll_prob)) logging.info('\t\tReal : %s ' % (' '.join(sentence_real))) if total_batch > 1 and total_batch % opt.run_valid_every == 0: logging.info('*' * 50) logging.info( 'Run validation test @Epoch=%d,#(Total batch)=%d' % (epoch, total_batch)) valid_losses = _valid(validation_data_loader, model, criterion, optimizer, epoch, opt, is_train=False) train_history_losses.append(copy.copy(train_losses)) valid_history_losses.append(valid_losses) train_losses = [] # Plot the learning curve plot_learning_curve( train_history_losses, valid_history_losses, 'Training and Validation', curve1_name='Training Error', curve2_name='Validation Error', save_path=opt.exp_path + '/[epoch=%d,batch=%d,total_batch=%d]train_valid_curve.png' % (epoch, batch_i, total_batch)) ''' determine if early stop training ''' valid_loss = np.average(valid_history_losses[-1]) is_best_loss = valid_loss < best_loss rate_of_change = float(valid_loss - best_loss) / float(best_loss) # only store the checkpoints that make better validation performances if total_batch > 1 and epoch >= opt.start_checkpoint_at and ( total_batch % opt.save_model_every == 0 or is_best_loss): # Save the checkpoint logging.info('Saving checkpoint to: %s' % os.path.join( opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d.error=%f' % (opt.exp, epoch, batch_i, total_batch, valid_loss) + '.model')) torch.save( model.state_dict(), open( os.path.join( opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.model'), 'wb')) # valid error doesn't decrease if rate_of_change >= 0: stop_increasing += 1 else: stop_increasing = 0 if is_best_loss: logging.info( 'Validation: update best loss (%.4f --> %.4f), rate of change (ROC)=%.2f' % (best_loss, valid_loss, rate_of_change * 100)) else: logging.info( 'Validation: best loss is not updated for %d times (%.4f --> %.4f), rate of change (ROC)=%.2f' % (stop_increasing, best_loss, valid_loss, rate_of_change * 100)) best_loss = min(valid_loss, best_loss) if stop_increasing >= opt.early_stop_tolerance: logging.info( 'Have not increased for %d epoches, early stop training' % stop_increasing) early_stop_flag = True break logging.info('*' * 50)
def main(_): random_seed = 12345 os.environ["PYTHONHASHSEED"] = str(random_seed) random.seed(random_seed) np.random.seed(random_seed) tf.set_random_seed(random_seed) start_logger(FLAGS.model_save_filename + ".train_log") atexit.register(stop_logger) print("-- Building vocabulary") #embeddings, token2id, id2token = load_glove(args.vectors_filename, args.max_vocab, args.embeddings_size) label2id = {"neutral": 0, "entailment": 1, "contradiction": 2} id2label = {v: k for k, v in label2id.items()} #num_tokens = len(token2id) num_labels = len(label2id) #print("Number of tokens: {}".format(num_tokens)) print("Number of labels: {}".format(num_labels)) # Load e_vsnli # Explanations are encoded/padded, we ignore original explanations print("-- Loading training set") train_labels, train_explanations, train_premises, train_hypotheses, train_img_names, _, _, _, train_max_length, embeddings, token2id, id2token, _ = \ load_e_vsnli_dataset_and_glove( FLAGS.train_filename, label2id, FLAGS.vectors_filename, FLAGS.max_vocab, model_config.embedding_size, buffer_size=FLAGS.buffer_size, min_threshold = FLAGS.min_threshold, ) num_tokens = len(token2id) print("Number of tokens after filtering: ", num_tokens) print("-- Loading development set") dev_labels, dev_explanations, dev_premises, dev_hypotheses, dev_img_names, dev_original_explanations, _, _, dev_max_length, _ = \ load_e_vsnli_dataset( FLAGS.dev_filename, token2id, label2id, buffer_size=FLAGS.buffer_size, padding_length=train_max_length, ) if FLAGS.imbalance == True: dev_num_examples = dev_labels.shape[0] class_freqs = np.bincount(dev_labels) / dev_num_examples class_weights = 1 / (class_freqs * num_labels) print("Class frequencies: ", class_freqs) print("Weights: ", class_weights) np.save(FLAGS.model_save_filename + '_class_freqs.npy', class_freqs) print("-- Loading images") image_reader = ImageReader(FLAGS.img_names_filename, FLAGS.img_features_filename) print("-- Saving parameters") with open(FLAGS.model_save_filename + ".params", mode="w") as out_file: json.dump(vars(FLAGS), out_file) print("Params saved to: {}".format(FLAGS.model_save_filename + ".params")) with open(FLAGS.model_save_filename + ".index", mode="wb") as out_file: pickle.dump( { "token2id": token2id, "id2token": id2token, "label2id": label2id, "id2label": id2label }, out_file) print("Index saved to: {}".format(FLAGS.model_save_filename + ".index")) model_config.set_vocab_size(num_tokens) print("Vocab size, set to %d" % model_config.vocab_size) model_config.set_alpha(FLAGS.alpha) print("alpha = %f, set!" % model_config.alpha) ilabel2itoken = {} for i in id2label: label = id2label[i] if label in token2id: j = token2id[label] else: j = token2id["#unk#"] ilabel2itoken[i] = j print("label_id --> token_id: constructed!") num_examples = train_labels.shape[0] num_batches = num_examples // FLAGS.batch_size dev_num_examples = dev_labels.shape[0] dev_batches_indexes = np.arange(dev_num_examples) num_batches_dev = dev_num_examples // FLAGS.dev_batch_size tf.reset_default_graph() # Build the TensorFlow graph and train it g = tf.Graph() with g.as_default(): model = build_model(model_config, embeddings, ilabel2itoken=ilabel2itoken, mode=mode) # Set up the learning rate. learning_rate_decay_fn = None learning_rate = tf.constant(training_config.initial_learning_rate) if training_config.learning_rate_decay_factor > 0: num_batches_per_epoch = (num_examples / FLAGS.batch_size) decay_steps = int(num_batches_per_epoch * training_config.num_epochs_per_decay) def _learning_rate_decay_fn(learning_rate, global_step): return tf.train.exponential_decay( learning_rate, global_step, decay_steps=decay_steps, decay_rate=training_config.learning_rate_decay_factor, staircase=True) learning_rate_decay_fn = _learning_rate_decay_fn # Set up the training ops. train_op = tf.contrib.layers.optimize_loss( loss=model['total_loss'], global_step=model['global_step'], learning_rate=learning_rate, optimizer=training_config.optimizer, clip_gradients=training_config.clip_gradients, learning_rate_decay_fn=learning_rate_decay_fn) dev_best_accuracy = -1 stopping_step = 0 best_epoch = None should_stop = False # initialize all variables init = tf.global_variables_initializer() with tf.Session() as session: session.run(init) #session.run(tf.initializers.tables_initializer(name='init_all_tables')) t = 0 # counting iterations time_now = datetime.now() for epoch in range(training_config.total_num_epochs): if should_stop: break print("\n==> Online epoch # {0}".format(epoch + 1)) progress = Progbar(num_batches) batches_indexes = np.arange(num_examples) np.random.shuffle(batches_indexes) np.random.shuffle(batches_indexes) batch_index = 1 loss_history = [] epoch_loss = 0 for indexes in batch(batches_indexes, FLAGS.batch_size): t += 1 batch_hypotheses = train_hypotheses[indexes] batch_labels = train_labels[indexes] # explanations have been encoded / padded when loaded batch_explanations = train_explanations[indexes] batch_explanation_lengths = [ len(expl) for expl in batch_explanations ] batch_img_names = [train_img_names[i] for i in indexes] batch_img_features = image_reader.get_features( batch_img_names) total_loss_value = _step( session, batch_hypotheses, batch_labels, batch_explanations, batch_img_features, train_op, model, model_config.lstm_dropout_keep_prob ) # run each training step progress.update(batch_index, [("Loss", total_loss_value)]) loss_history.append(total_loss_value) epoch_loss += total_loss_value batch_index += 1 if FLAGS.print_every > 0 and t % FLAGS.print_every == 0: print( '(Iteration %d) loss: %f, and time elapsed: %.2f minutes' % (t + 1, float(loss_history[-1]), (datetime.now() - time_now).seconds / 60.0)) print("Current mean training loss: {}\n".format(epoch_loss / num_batches)) print("-- Validating model") progress = Progbar(num_batches_dev) dev_num_correct = 0 dev_batch_index = 0 for indexes in batch(dev_batches_indexes, FLAGS.dev_batch_size): t += 1 dev_batch_num_correct = 0 dev_batch_index += 1 dev_batch_hypotheses = dev_hypotheses[indexes] dev_batch_labels = dev_labels[indexes] # explanations have been encoded / padded when loaded dev_batch_explanations = dev_explanations[indexes] dev_batch_img_names = [dev_img_names[i] for i in indexes] dev_batch_img_features = image_reader.get_features( dev_batch_img_names) pred_explanations, pred_labels = _run_validation( session, dev_batch_hypotheses, dev_batch_labels, dev_batch_explanations, dev_batch_img_features, len(indexes), ilabel2itoken, model, 1.0) if FLAGS.imbalance == True: dev_batch_num_correct += np.dot( pred_labels == dev_batch_labels, class_weights[dev_batch_labels]) else: dev_batch_num_correct += ( pred_labels == dev_batch_labels).sum() dev_num_correct += dev_batch_num_correct progress.update( dev_batch_index, [("Proportion of correct labels", float(dev_batch_num_correct) / len(indexes))]) if FLAGS.sample_every > 0 and ( t + 1) % FLAGS.sample_every == 0: pred_explanations = [ unpack.reshape(-1, 1) for unpack in pred_explanations ] pred_explanations = np.concatenate( pred_explanations, 1) pred_explanations_decoded = [ decode(pred_explanations[i], id2token) for i in range(len(indexes)) ] print("\nExample generated explanation: ", pred_explanations_decoded[0]) #TODO: decode it #print("Original explanation: ", dev_original_explanations[indexes][0]) dev_accuracy = float(dev_num_correct) / dev_num_examples print("Current mean validation accuracy: {}".format( dev_accuracy)) #if True: if dev_accuracy > dev_best_accuracy: stopping_step = 0 best_epoch = epoch + 1 dev_best_accuracy = dev_accuracy model['saver'].save(session, FLAGS.model_save_filename + ".ckpt") print( "Best mean validation accuracy: {} (reached at epoch {})" .format(dev_best_accuracy, best_epoch)) print("Best model saved to: {}".format( FLAGS.model_save_filename)) else: stopping_step += 1 print("Current stopping step: {}".format(stopping_step)) if stopping_step >= FLAGS.patience: print("Early stopping at epoch {}!".format(epoch + 1)) print( "Best mean validation accuracy: {} (reached at epoch {})" .format(dev_best_accuracy, best_epoch)) should_stop = True if epoch + 1 >= training_config.total_num_epochs: print("Stopping at epoch {}!".format(epoch + 1)) print( "Best mean validation accuracy: {} (reached at epoch {})" .format(dev_best_accuracy, best_epoch))
def train_model(model, optimizer_ml, optimizer_rl, criterion, train_data_loader, valid_data_loaders, test_data_loaders, opt): generator = SequenceGenerator(model, eos_id=opt.word2id[pykp.io.EOS_WORD], beam_size=opt.beam_size, max_sequence_length=opt.max_sent_length) logger = logging.getLogger('train.py') logger.info( '====================== Checking GPU Availability =========================' ) if torch.cuda.is_available(): if isinstance(opt.gpuid, int): opt.gpuid = [opt.gpuid] logger.info('Running on GPU! devices=%s' % str(opt.gpuid)) # model = nn.DataParallel(model, device_ids=opt.gpuid) model = model.cuda() else: logger.info('Running on CPU!') logger.info( '====================== Start Training =========================' ) checkpoint_names = [] train_ml_history_losses = [] train_rl_history_losses = [] valid_history_scores = {} test_history_scores = {} # best_loss = sys.float_info.max # for normal training/testing loss (likelihood) best_loss = 0.0 # for f-score stop_increasing = 0 train_ml_losses = [] train_rl_losses = [] total_batch = -1 early_stop_flag = False if opt.train_rl: reward_cache = RewardCache(2000) # if False: # opt.train_from: # state_path = opt.train_from.replace('.model', '.state') # logger.info('Loading training state from: %s' % state_path) # if os.path.exists(state_path): # (epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_ml_history_losses, train_rl_history_losses, valid_history_scores, # test_history_losses) = torch.load(open(state_path, 'rb')) # opt.start_epoch = epoch for epoch in range(opt.start_epoch, opt.epochs): if early_stop_flag: break progbar = Progbar(logger=logger, title='Training', target=len(train_data_loader), batch_size=train_data_loader.batch_size, total_examples=len(train_data_loader.dataset)) for batch_i, batch in enumerate(train_data_loader): model.train() total_batch += 1 one2many_batch, one2one_batch = batch report_loss = [] # Training if opt.train_ml: loss_ml, decoder_log_probs = train_ml(one2one_batch, model, optimizer_ml, criterion, opt) # len(decoder_log_probs) == 0 if encountered OOM if len(decoder_log_probs) == 0: continue train_ml_losses.append(loss_ml) report_loss.append(('train_ml_loss', loss_ml)) report_loss.append(('PPL', loss_ml)) # Brief report if batch_i % opt.report_every == 0: brief_report(epoch, batch_i, one2one_batch, loss_ml, decoder_log_probs, opt) # do not apply rl in 0th epoch, need to get a resonable model before that. if opt.train_rl: if epoch >= opt.rl_start_epoch: loss_rl = train_rl(one2many_batch, model, optimizer_rl, generator, opt, reward_cache) else: loss_rl = 0.0 train_rl_losses.append(loss_rl) report_loss.append(('train_rl_loss', loss_rl)) progbar.update(epoch, batch_i, report_loss) ''' Validate and save checkpoint ''' if (opt.run_valid_every == -1 and batch_i == len(train_data_loader) - 1) or\ (opt.run_valid_every > -1 and total_batch > 1 and total_batch % opt.run_valid_every == 0): logger.info('*' * 50) logger.info( 'Run validing and testing @Epoch=%d,#(Total batch)=%d' % (epoch, total_batch)) # return a dict, key is the dataset name and value is a score dict valid_score_dict = evaluate.evaluate_multiple_datasets( generator, valid_data_loaders, opt, epoch=epoch, title='valid.epoch=%d.total_batch=%d' % (epoch, total_batch), predict_save_path=os.path.join( opt.pred_path, 'epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch))) test_score_dict = evaluate.evaluate_multiple_datasets( generator, test_data_loaders, opt, epoch=epoch, title='test.epoch=%d.total_batch=%d' % (epoch, total_batch), predict_save_path=os.path.join( opt.pred_path, 'epoch%d_batch%d_total_batch%d' % (epoch, batch_i, total_batch))) ''' Merge scores of current round into history_score ''' for dataset_name, score_dict in valid_score_dict.items(): # each history_loss is a dict, specific to a dataset # key is score name and value is a list, each element is a list of scores (e.g. f1_score) of all examples valid_history_score = valid_history_scores.get( dataset_name, {}) for score_name, score_values in score_dict.items(): history_score_values = valid_history_score.get( score_name, []) history_score_values.append(score_values) valid_history_score[score_name] = history_score_values valid_history_scores[dataset_name] = valid_history_score for dataset_name, score_dict in test_score_dict.items(): test_history_score = test_history_scores.get( dataset_name, {}) for score_name, score_values in score_dict.items(): history_score_values = test_history_score.get( score_name, []) history_score_values.append(score_values) test_history_score[score_name] = history_score_values test_history_scores[dataset_name] = test_history_score if opt.train_ml: train_ml_history_losses.append(copy.copy(train_ml_losses)) train_ml_losses = [] if opt.train_rl: train_rl_history_losses.append(copy.copy(train_rl_losses)) train_rl_losses = [] ''' Iterate each dataset (including a merged 'all_datasets') and plot learning curves ''' for dataset_name in opt.test_dataset_names + ['all_datasets']: valid_history_score = valid_history_scores[dataset_name] test_history_score = test_history_scores[dataset_name] curve_names = [] scores_for_plot = [] if opt.train_ml: scores_for_plot += [train_ml_history_losses] curve_names += ['Training ML Error'] if opt.train_rl: scores_for_plot += [train_rl_history_losses] curve_names += ['Training RL Reward'] scores_for_plot += [ valid_history_score[name] for name in opt.report_score_names ] curve_names += [ 'Valid-' + name for name in opt.report_score_names ] scores_for_plot += [ test_history_score[name] for name in opt.report_score_names ] curve_names += [ 'Test-' + name for name in opt.report_score_names ] scores_for_plot = [np.asarray(s) for s in scores_for_plot] ''' Plot the learning curve ''' plot_learning_curve_and_write_csv( scores=scores_for_plot, curve_names=curve_names, checkpoint_names=checkpoint_names, title='Training Validation & Test of %s' % dataset_name, save_path=opt.plot_path + '/[epoch=%d,batch=%d,total_batch=%d].%s.learning_curve' % (epoch, batch_i, total_batch, dataset_name)) ''' determine if early stop training (whether f-score increased, previously is if valid error decreased) opt.report_score_names[0] is 'f_score@5_exact' ''' valid_loss = np.average(valid_history_scores['all_datasets'][ opt.report_score_names[0]][-1]) is_best_loss = valid_loss > best_loss rate_of_change = float(valid_loss - best_loss) / float( best_loss) if float(best_loss) > 0 else 0.0 # valid error doesn't increase if rate_of_change <= 0: stop_increasing += 1 else: stop_increasing = 0 if is_best_loss: logging.info( 'Validation: update best loss (%.4f --> %.4f), rate of change (ROC)=%.2f' % (best_loss, valid_loss, rate_of_change * 100)) else: logging.info( 'Validation: best loss is not updated for %d times (%.4f --> %.4f), rate of change (ROC)=%.2f' % (stop_increasing, best_loss, valid_loss, rate_of_change * 100)) logging.info( 'Current test loss (over %d datasets): %s\n' % (len(opt.test_dataset_names), str(opt.test_dataset_names))) for report_score_name in opt.report_score_names: test_loss = np.average(test_history_scores['all_datasets'] [report_score_name][-1]) logging.info('\t\t %s = %.4f' % (report_score_name, test_loss)) best_loss = max(valid_loss, best_loss) ''' Save checkpoints, only store the ones that make better validation performances ''' checkpoint_names.append('epoch=%d-batch=%d-total_batch=%d' % (epoch, batch_i, total_batch)) if total_batch > 1 and ( total_batch % opt.save_model_every == 0 or is_best_loss): # epoch >= opt.start_checkpoint_at and # Save the checkpoint logging.info('Saving checkpoint to: %s' % os.path.join( opt.model_path, '%s.epoch=%d.batch=%d.total_batch=%d.error=%f' % (opt.exp, epoch, batch_i, total_batch, valid_loss) + '.model')) torch.save( model.state_dict(), open( os.path.join( opt.model_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.model'), 'wb')) torch.save((epoch, total_batch, best_loss, stop_increasing, checkpoint_names, train_ml_history_losses, train_rl_history_losses, valid_history_scores, test_history_scores), open( os.path.join( opt.model_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.state'), 'wb')) if stop_increasing >= opt.early_stop_tolerance: logging.info( 'Have not increased for %d epoches, early stop training' % stop_increasing) early_stop_flag = True break logging.info('*' * 50)
def train(self, save_dir=None): criterion = Dynamic_Cross_Entropy_Loss() padded_q_word, q_word_lengths, padded_q_dep, q_dep_lengths, padded_rel_words, rel_word_lengths, \ batch_rel_ids, padded_cons_words, cons_word_lengths, padded_cons_id, cons_id_lengths, batch_prior_weights, \ batch_labels = None, None, None, None, None, None, None, None, None, None, None, None, None epoch_loss_history = [float("inf"), ] sys.stderr.write('Max Epoch: {}\n'.format(self.max_epoch)) for epoch in range(self.max_epoch): self.scheduler.step(epoch) progbar = Progbar(len(self.data_loader), file=sys.stderr) prog_idx = 0 shuffled_indices = random.sample(range(len(self.data_loader)), len(self.data_loader)) random.shuffle(shuffled_indices) for curr_index in shuffled_indices: data_dict = self.data_loader.get_one_batch(curr_index) padded_q_word, q_word_lengths, padded_q_dep, q_dep_lengths, padded_rel_words, rel_word_lengths, batch_rel_ids, \ padded_cons_words,cons_word_lengths, padded_cons_id, cons_id_lengths, batch_prior_weights, batch_labels \ = self.unpack_data_dict(data_dict) # print "curr_idx: {}".format(curr_index) # print "padded_q_word: {}".format(padded_q_word) self.model.zero_grad() self.optimizer.zero_grad() # train if epoch <= self.pooling_threshold: if self.data_loader.use_entity_type: self.model.q_encoder.q_word_emb.weight[50:].requres_grad = False else: self.model.q_encoder.q_word_emb.weight[4:].requres_grad = False '''don't need VAR(use_constraint) because model has been initialized''' out = None # placeholder if not self.data_loader.cpu_data: out = self.model(padded_q_word_seq=padded_q_word, q_word_lengths=q_word_lengths, padded_q_dep_seq=padded_q_dep, q_dep_lengths=q_dep_lengths, padded_rel_words_seq=padded_rel_words, rel_word_lengths=rel_word_lengths, rel_ids=batch_rel_ids, padded_constraint_words_seq=padded_cons_words, constraint_word_lengths=cons_word_lengths, padded_constraint_ids=padded_cons_id, constraint_id_lengths=cons_id_lengths, pooling=self.pooling_criterion(epoch)) else: out = self.model(padded_q_word_seq=cuda_wrapper(padded_q_word), q_word_lengths=q_word_lengths, padded_q_dep_seq=cuda_wrapper(padded_q_dep), q_dep_lengths=q_dep_lengths, padded_rel_words_seq=cuda_wrapper(padded_rel_words), rel_word_lengths=rel_word_lengths, rel_ids=cuda_wrapper(batch_rel_ids), padded_constraint_words_seq=cuda_wrapper(padded_cons_words), constraint_word_lengths=cons_word_lengths, padded_constraint_ids=cuda_wrapper(padded_cons_id), constraint_id_lengths=cons_id_lengths, pooling=self.pooling_criterion(epoch)) loss = None # placeholder if self.use_prior_weights: loss = criterion.forward(out, cuda_wrapper(batch_labels.long()), cuda_wrapper(batch_prior_weights)) else: loss = criterion.forward(out, cuda_wrapper(batch_labels.long()), None) loss.backward() # print "epoch: {}, iter: {}, loss: {}".format(epoch, prog_idx, loss.item()) self.optimizer.step() if epoch <= self.pooling_threshold: self.model.q_encoder.q_word_emb.weight.requires_grad = True if self.use_constraint: self.model.query_graph_encoder.constraint_word_emb.weight.requires_grad = True self.model.query_graph_encoder.constraint_id_emb.weight.requires_grad = True progbar.update(prog_idx + 1, [("loss", loss.item())]) prog_idx += 1 #epoch_loss = self.eval() #epoch_loss_history.append(epoch_loss) #sys.stderr.write("Epoch: {}, Loss: {}\n".format(epoch, epoch_loss)) #print "Epoch: {}, Loss: {}".format(epoch, epoch_loss) if epoch == self.max_epoch - 1 or epoch % 3 == 2: epoch_loss = self.eval() sys.stderr.write("Epoch: {}, Loss: {}\n".format(epoch, epoch_loss)) if save_dir is not None: check_point = { #'loss': epoch_loss, 'state_dict': self.model.state_dict() } torch.save(check_point, os.path.join(save_dir, str(epoch)))
def train(self, X_train, X_mask, Y_train, Y_mask, input, output, verbose, optimizer): train_set_x = theano.shared(np.asarray(X_train, dtype="int32"), borrow=True) train_set_y = theano.shared(np.asarray(Y_train, dtype="int32"), borrow=True) mask_set_x = theano.shared(np.asarray(X_mask, dtype="float32"), borrow=True) mask_set_y = theano.shared(np.asarray(Y_mask, dtype="float32"), borrow=True) index = T.lscalar("index") # index to a case lr = T.scalar("lr", dtype=theano.config.floatX) mom = T.scalar("mom", dtype=theano.config.floatX) # momentum n_ex = T.lscalar("n_ex") sindex = T.lscalar("sindex") # index to a case ### batch batch_start = index * self.n_batch batch_stop = T.minimum(n_ex, (index + 1) * self.n_batch) effective_batch_size = batch_stop - batch_start get_batch_size = theano.function(inputs=[index, n_ex], outputs=effective_batch_size) cost = self.loss(self.y, self.y_mask) + self.L1_reg * self.L1 updates = eval(optimizer)(self.params, cost, mom, lr) """ compute_val_error = theano.function(inputs = [index,n_ex ], outputs = self.loss(self.y,self.y_mask), givens = { self.x: train_set_x[:,batch_start:batch_stop], self.y: train_set_y[:,batch_start:batch_stop], self.x_mask: mask_set_x[:,batch_start:batch_stop], self.y_mask: mask_set_y[:,batch_start:batch_stop] }, mode = mode) """ train_model = theano.function( inputs=[index, lr, mom, n_ex], outputs=[cost, self.loss(self.y, self.y_mask)], updates=updates, givens={ self.x: train_set_x[:, batch_start:batch_stop], self.y: train_set_y[:, batch_start:batch_stop], self.x_mask: mask_set_x[:, batch_start:batch_stop], self.y_mask: mask_set_y[:, batch_start:batch_stop], }, mode=mode, on_unused_input="ignore", ) ############### # TRAIN MODEL # ############### print "Training model ..." epoch = 0 n_train = train_set_x.get_value(borrow=True).shape[1] n_train_batches = int(np.ceil(1.0 * n_train / self.n_batch)) if optimizer is not "SGD": self.learning_rate_decay = 1 while epoch < self.n_epochs: epoch = epoch + 1 if verbose == 1: progbar = Progbar(n_train_batches) train_losses = [] train_batch_sizes = [] for idx in xrange(n_train_batches): effective_momentum = ( self.final_momentum if (epoch + len(self.errors)) > self.momentum_switchover else self.initial_momentum ) cost = train_model(idx, self.lr, effective_momentum, n_train) train_losses.append(cost[1]) train_batch_sizes.append(get_batch_size(idx, n_train)) if verbose == 1: progbar.update(idx + 1) this_train_loss = np.average(train_losses, weights=train_batch_sizes) self.errors.append(this_train_loss) print ("epoch %i, train loss %f " "lr: %f" % (epoch, this_train_loss, self.lr)) ### autimatically saving snapshot .. if np.mod(epoch, self.snapshot) == 0: if epoch is not n_train_batches: self.save() ### generating sample.. if np.mod(epoch, self.sample_Freq) == 0: print "Generating a sample..." i = np.random.randint(1, n_train) test = X_train[:, i] truth = Y_train[:, i] guess = self.gen_sample(test, X_mask[:, i]) print "Input: ", " ".join(input.sequences_to_text(test)) print "Truth: ", " ".join(output.sequences_to_text(truth)) print "Sample: ", " ".join(output.sequences_to_text(guess[1])) """ # compute loss on validation set if np.mod(epoch,self.val_Freq)==0: val_losses = [compute_val_error(i, n_train) for i in xrange(n_train_batches)] val_batch_sizes = [get_batch_size(i, n_train) for i in xrange(n_train_batches)] this_val_loss = np.average(val_losses, weights=val_batch_sizes) """ self.lr *= self.learning_rate_decay
args = parser.parse_args() img_filenames = [ filename for filename in os.listdir(args.img_path) if os.path.splitext(filename)[-1].lower() == args.img_extension ] num_img_filenames = len(img_filenames) img_features = [] progress = Progbar(num_img_filenames) base_model = VGG16(weights="imagenet") model = Model(input=base_model.input, output=base_model.get_layer("fc2").output) for num_filename, filename in enumerate(img_filenames, 1): img = image.load_img(os.path.join(args.img_path, filename), target_size=(224, 224)) x = image.img_to_array(img) x = np.expand_dims(x, axis=0) x = preprocess_input(x) fc7_features = model.predict(x).squeeze() img_features.append(fc7_features) progress.update(num_filename) img_features = np.array(img_features) with open(args.img_names_filename, "w") as out_file: json.dump(img_filenames, out_file) np.save(args.img_features_filename, img_features)
def evaluate_beam_search(generator, data_loader, opt, title='', epoch=1, save_path=None): logging = config.init_logging(title, save_path + '/%s.log' % title) progbar = Progbar(logger=logging, title=title, target=len(data_loader) / opt.beam_batch, batch_size=opt.beam_batch, total_examples=len(data_loader) / opt.beam_batch) example_idx = 0 score_dict = { } # {'precision@5':[],'recall@5':[],'f1score@5':[], 'precision@10':[],'recall@10':[],'f1score@10':[]} if opt.report_file: f = open(opt.report_file, 'w') for i, batch in enumerate(data_loader): # if i > 0: # break one2many_batch = batch src_list, src_len, trg_list, _, trg_copy_target_list, src_oov_map_list, \ oov_list, src_str_list, trg_str_list = one2many_batch if torch.cuda.is_available() and opt.use_gpu: src_list = src_list.cuda() src_oov_map_list = src_oov_map_list.cuda() pred_seq_list = generator.beam_search(src_list, src_len, src_oov_map_list, oov_list, opt.word2id) # print(len(pred_seq_list[0]), type(pred_seq_list[0])) ''' process each example in current batch ''' for src, src_str, trg, trg_str_seqs, trg_copy, pred_seq, oov \ in zip(src_list, src_str_list, trg_list, trg_str_list, trg_copy_target_list, pred_seq_list, oov_list): # logging.info('====================== %d =========================' % (example_idx)) # print(trg_str_seqs) # print(src_str) # print(src, 'src') # print(trg_copy, 'trg_copy') # print(pred_seq, 'pred_seq') # print(oov, 'oov') trg_str_is_present = if_present_duplicate_phrase( src_str, trg_str_seqs) # TODO # print(trg_str_is_present) # 1st filtering pred_is_valid, processed_pred_seqs, processed_pred_str_seqs, \ processed_pred_score = process_predseqs(pred_seq, oov, opt.id2word, opt) # print(pred_is_valid, 'pred_is_valid') # print(processed_pred_seqs, 'precessed_pred_seqs') # print(processed_pred_str_seqs, len(processed_pred_str_seqs)) # print(processed_pred_score, 'processed_pred_score') # print(len(processed_pred_str_seqs)) # 2nd filtering: if filter out phrases that don't appear in text, and keep unique ones after stemming # print(opt.must_appear_in_src) if opt.must_appear_in_src is True: pred_is_present = if_present_duplicate_phrase( src_str, processed_pred_str_seqs) trg_str_seqs = np.asarray(trg_str_seqs)[trg_str_is_present] else: pred_is_present = [True] * len(processed_pred_str_seqs) # print(pred_is_present) valid_and_present = np.asarray(pred_is_valid) * np.asarray( pred_is_present) match_list = get_match_result(true_seqs=trg_str_seqs, pred_seqs=processed_pred_str_seqs) # print(match_list, 'match', len(match_list)) ''' Evaluate predictions w.r.t different filterings and metrics ''' num_oneword_range = [-1, 1] topk_range = [5, 10] score_names = ['precision', 'recall', 'f_score'] processed_pred_seqs = np.asarray( processed_pred_seqs)[valid_and_present] processed_pred_str_seqs = np.asarray( processed_pred_str_seqs)[valid_and_present] processed_pred_score = np.asarray( processed_pred_score)[valid_and_present] try: data = {} data['src_str'] = src_str data['trg_str_seqs'] = trg_str_seqs data['pred'] = list(processed_pred_str_seqs) f.write(json.dumps(data) + '\n') except: pass for num_oneword_seq in num_oneword_range: # 3rd round filtering (one-word phrases) filtered_pred_seq, filtered_pred_str_seqs, filtered_pred_score = \ post_process_predseqs((processed_pred_seqs, processed_pred_str_seqs, processed_pred_score), num_oneword_seq) match_list = get_match_result(true_seqs=trg_str_seqs, pred_seqs=filtered_pred_str_seqs) assert len(filtered_pred_seq) == len( filtered_pred_str_seqs) == len(filtered_pred_score) == len( match_list) for topk in topk_range: results = evaluate(match_list, filtered_pred_seq, trg_str_seqs, topk=topk) for k, v in zip(score_names, results): if '%s@%d#oneword=%d' % ( k, topk, num_oneword_seq) not in score_dict: score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)] = [] score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)].append(v) if example_idx % 10 == 0: print('#(precision@5#oneword=-1)=%d, avg=%f' % (len(score_dict['precision@5#oneword=-1']), np.average(score_dict['precision@5#oneword=-1']))) print('#(precision@10#oneword=-1)=%d, avg=%f' % (len(score_dict['precision@10#oneword=-1']), np.average(score_dict['precision@10#oneword=-1']))) print('#(recall@5#oneword=-1)=%d, avg=%f' % (len(score_dict['recall@5#oneword=-1']), np.average(score_dict['recall@5#oneword=-1']))) print('#(recall@10#oneword=-1)=%d, avg=%f' % (len(score_dict['recall@10#oneword=-1']), np.average(score_dict['recall@10#oneword=-1']))) x, y = np.average(score_dict['f_score@5#oneword=-1']), np.average( score_dict['f_score@10#oneword=-1']) print('#(f_score@5#oneword=-1)=%d, avg=%f' % (len(score_dict['f_score@5#oneword=-1']), x)) print('#(f_score@10#oneword=-1)=%d, avg=%f' % (len(score_dict['f_score@10#oneword=-1']), y)) progbar.update(epoch, example_idx, [('f_score@5#oneword=-1', x), ('f_score@10#oneword=-1', y)]) print('*' * 50) example_idx += 1 # exit(0) print('#(f_score@5#oneword=-1)=%d, avg=%f' % (len(score_dict['f_score@5#oneword=-1']), np.average(score_dict['f_score@5#oneword=-1']))) print('#(f_score@10#oneword=-1)=%d, avg=%f' % (len(score_dict['f_score@10#oneword=-1']), np.average(score_dict['f_score@10#oneword=-1']))) # print('#(f_score@5#oneword=1)=%d, avg=%f' % (len(score_dict['f_score@5#oneword=1']), np.average(score_dict['f_score@5#oneword=1']))) # print('#(f_score@10#oneword=1)=%d, avg=%f' % (len(score_dict['f_score@10#oneword=1']), np.average(score_dict['f_score@10#oneword=1']))) if save_path: # export scores. Each row is scores (precision, recall and f-score) of different way of filtering predictions (how many one-word predictions to keep) with open(save_path + os.path.sep + title + '_result.csv', 'w') as result_csv: csv_lines = [] for num_oneword_seq in num_oneword_range: for topk in topk_range: csv_line = '#oneword=%d,@%d' % (num_oneword_seq, topk) for k in score_names: csv_line += ',%f' % np.average( score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)]) csv_lines.append(csv_line + '\n') result_csv.writelines(csv_lines) return score_dict
batch_ix += BATCH_SIZE if batch_ix >= train_x.shape[0]: batch_ix = 0 model_probs = rbm.gibbs_sampler(batch_x, k, return_probs=True) model_sample = rbm.sample(model_probs) if SAMPLE else model_probs train_loss = cross_entropy(batch_x, model_probs) avg_train_loss += train_loss rbm.backward(batch_x, model_sample) optimizer.step() # Clean Up optimizer.zero_grad() avg_train_loss /= steps val_sample = rbm.gibbs_sampler(val_x, k, return_probs=True) val_loss = cross_entropy(val_x, val_sample) bar.update(epoch + 1, values=[("train_loss", avg_train_loss), ("val_loss", val_loss)]) if (epoch + 1) % args.plot_after == 0: # index = range(train_x.shape[0]) # random.shuffle(index) # random_train = train_x[index, :][:16,:] random_train = np.random.uniform(0, 1., (100, 784)) sample = rbm.gibbs_sampler(random_train, args.gibbs_steps, return_probs=True) plot(sample, "Plots/Image_Epoch_{}.png".format(format(epoch + 1, '03')), epoch + 1) # Save the weights save_filename = "Models/weights_hdim_%d_epochs_%d_val_%.4f_k_%d.pkl" % ( args.n_hidden, args.n_epochs, val_loss, k)
#forward, loss, backward, step output = encoder(images=images, rotation=rotation, mode=1) loss_1 = noise_contrastive_estimator(representations, output[1], index, memory, negative_nb=negative_nb) loss_2 = noise_contrastive_estimator(representations, output[0], index, memory, negative_nb=negative_nb) loss = loss_weight * loss_1 + (1 - loss_weight) * loss_2 loss.backward() optimizer.step() #update representation memory memory.update(index, output[0].detach()) # update metric and bar train_loss.update(loss.item(), images.shape[0]) bar.update(step, values=[('train_loss', train_loss.return_avg())]) #save model if improved checkpoint.save_model(encoder, optimizer, train_loss.return_avg(), epoch) logger.update(epoch, train_loss.return_avg())
def train(config, checkpoint_dir): print('Start Training the Model!') # generate data loader if config.longterm is True: config.output_window_size = 100 # from choose_dataset import DatasetChooser choose = DatasetChooser(config) train_dataset, bone_length = choose(train=True) train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True) test_dataset, _ = choose(train=False) test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=True) prediction_dataset, bone_length = choose(prediction=True) x_test, y_test, dec_in_test = prediction_dataset device = torch.device( "cuda:" + str(config.device_ids[0]) if torch.cuda.is_available() else "cpu") print('Device {} will be used to save parameters'.format(device)) torch.cuda.manual_seed_all(112858) net = choose_net(config) net.to(device) print('Total param number:' + str(sum(p.numel() for p in net.parameters()))) print('Encoder param number:' + str(sum(p.numel() for p in net.encoder_cell.parameters()))) print('Decoder param number:' + str(sum(p.numel() for p in net.decoder.parameters()))) if torch.cuda.device_count() > 1: print("{} GPUs are usable!".format(str(torch.cuda.device_count()))) net = torch.nn.DataParallel(net, device_ids=config.device_ids) if config.restore is True: dir = utils.get_file_list(checkpoint_dir) print('Load model from:' + checkpoint_dir + dir[-1]) net.load_state_dict( torch.load(checkpoint_dir + dir[-1], map_location='cuda:0')) optimizer = optim.Adam(net.parameters(), lr=config.learning_rate) # Save model if not (os.path.exists(checkpoint_dir)): os.makedirs(checkpoint_dir) best_error = float('inf') best_error_list = None # log 定义两个数组 trainloss_list = [] validloss_list = [] Error_list = [] for epoch in range(config.max_epoch): print("At epoch:{}".format(str(epoch + 1))) prog = Progbar(target=config.training_size) prog_valid = Progbar(target=config.validation_size) # Train #with torch.autograd.set_detect_anomaly(True): for it in range(config.training_size): for i, data in enumerate(train_loader, 0): encoder_inputs = data['encoder_inputs'].float().to( device) ## 前t-1帧 decoder_inputs = data['decoder_inputs'].float().to( device) ## t-1到t-1+output_window_size帧 decoder_outputs = data['decoder_outputs'].float().to( device) ## t帧到以后的 prediction = net(encoder_inputs, decoder_inputs, train=True) loss = Loss(prediction, decoder_outputs, bone_length, config) net.zero_grad() loss.backward() _ = torch.nn.utils.clip_grad_norm_(net.parameters(), 5) optimizer.step() prog.update(it + 1, [("Training Loss", loss.item())]) trainloss_list.append(loss.item) # valid with torch.no_grad(): for it in range(config.validation_size): for j in range(3): for i, data in enumerate(test_loader, 0): if j == 0 and i == 0: encoder_inputs = data['encoder_inputs'].float().to( device) decoder_inputs = data['decoder_inputs'].float().to( device) decoder_outputs = data['decoder_outputs'].float( ).to(device) else: encoder_inputs = torch.cat([ data['encoder_inputs'].float().to(device), encoder_inputs ], dim=0) decoder_inputs = torch.cat([ data['decoder_inputs'].float().to(device), decoder_inputs ], dim=0) decoder_outputs = torch.cat([ data['decoder_outputs'].float().to(device), decoder_outputs ], dim=0) prediction = net(encoder_inputs, decoder_inputs, train=True) loss = Loss(prediction, decoder_outputs, bone_length, config) prog_valid.update(it + 1, [("Testing Loss", loss.item())]) validloss_list.append(loss.item) #Test prediction actions = list(x_test.keys()) y_predict = {} with torch.no_grad(): for act in actions: x_test_ = torch.from_numpy(x_test[act]).float().to(device) dec_in_test_ = torch.from_numpy( dec_in_test[act]).float().to(device) pred = net(x_test_, dec_in_test_, train=False) pred = pred.cpu().numpy() y_predict[act] = pred error_actions = 0.0 for act in actions: if config.datatype == 'lie': mean_error, _ = utils.mean_euler_error( config, act, y_predict[act], y_test[act][:, :config.output_window_size, :]) error = mean_error[[1, 3, 7, 9]] if config.datatype == 'smpl': mean_error, _ = utils.mean_euler_error( config, act, y_predict[act], y_test[act][:, :config.output_window_size, :]) error = mean_error[[1, 3, 7, 9]] # error_actions += error.mean() error_actions += mean_error.mean() error_actions /= len(actions) Error_list.append(error_actions) if error_actions < best_error: print(error_actions) print(best_error) best_error_list = error best_error = error_actions torch.save(net.state_dict(), checkpoint_dir + 'Epoch_' + str(epoch + 1) + '.pth') print('Current best:' + str(round(best_error_list[0], 2)) + ' ' + str(round(best_error_list[1], 2)) + ' ' + str(round(best_error_list[2], 2)) + ' ' + str(round(best_error_list[3], 2))) if not (os.path.exists('./log')): os.makedirs('./log') time_now = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") np.savez('./log/' + time_now + '.npz', trainloss_list=trainloss_list, validloss_list=validloss_list, Error_list=Error_list)
parser.add_argument("--preprocessed_dataset_filename", type=str, required=True) args = parser.parse_args() nlp = en_core_web_sm.load() num_lines = len([1 for line in open(args.dataset_filename)]) images = {} with open(args.dataset_filename) as in_file: dataset = json.load(in_file) for num_image, image in enumerate(dataset["images"], 1): images[image["id"]] = image["file_name"] print("Found {} images".format(len(images))) with open(args.preprocessed_dataset_filename, mode="w") as out_file: writer = csv.writer(out_file, delimiter="\t") progress = Progbar(len(dataset["annotations"])) for num_annotation, annotation in enumerate( dataset["annotations"], 1): caption = annotation["caption"] image = images[annotation["image_id"]] label = "yes" if annotation["foil_word"] == "ORIG" else "no" # caption_tokens = [token.lower_ for token in nlp(caption)] # writer.writerow([label, " ".join(caption_tokens), image, caption]) writer.writerow([label, image, caption]) progress.update(num_annotation)
def run_simultrans(model, options_file=None, config=None, id=None, remote=False): WORK = config['workspace'] # check hidden folders paths = [ '.policy', '.pretrained', '.log', '.config', '.images', '.translate' ] for p in paths: p = WORK + p if not os.path.exists(p): os.mkdir(p) if id is not None: fcon = WORK + '.config/{}.conf'.format(id) if os.path.exists(fcon): print 'load config files' policy, config = pkl.load(open(fcon, 'r')) # ============================================================================== # # load model model_options # ============================================================================== # _model = model.split('/')[-1] if options_file is not None: with open(options_file, 'rb') as f: options = pkl.load(f) else: with open('%s.pkl' % model, 'rb') as f: options = pkl.load(f) print 'merge configuration into options' for w in config: # if (w in options) and (config[w] is not None): options[w] = config[w] print 'load options...' for w, p in sorted(options.items(), key=lambda x: x[0]): print '{}: {}'.format(w, p) # load detail settings from option file: dictionary, dictionary_target = options['dictionaries'] # load source dictionary and invert with open(dictionary, 'rb') as f: word_dict = pkl.load(f) word_idict = dict() for kk, vv in word_dict.iteritems(): word_idict[vv] = kk word_idict[0] = '<eos>' word_idict[1] = 'UNK' # load target dictionary and invert with open(dictionary_target, 'rb') as f: word_dict_trg = pkl.load(f) word_idict_trg = dict() for kk, vv in word_dict_trg.iteritems(): word_idict_trg[vv] = kk word_idict_trg[0] = '<eos>' word_idict_trg[1] = 'UNK' options['pre'] = config['pre'] # ========================================================================= # # Build a Simultaneous Translator # ========================================================================= # # allocate model parameters params = init_params(options) params = load_params(model, params) tparams = init_tparams(params) # print 'build the model for computing cost (full source sentence).' trng, use_noise, \ _x, _x_mask, _y, _y_mask, \ opt_ret, \ cost, f_cost = build_model(tparams, options) print 'done' # functions for sampler f_sim_ctx, f_sim_init, f_sim_next = build_simultaneous_sampler( tparams, options, trng) # function for finetune the underlying model if options['finetune']: ff_init, ff_cost, ff_update = build_simultaneous_model(tparams, options, rl=True) funcs = [ f_sim_ctx, f_sim_init, f_sim_next, f_cost, ff_init, ff_cost, ff_update ] else: funcs = [f_sim_ctx, f_sim_init, f_sim_next, f_cost] # build a res-predictor if options['predict']: params_act = get_actor('gru')[0](options, prefix='pdt', nin=options['dim']) pass # check the ID: options['base'] = _model agent = Policy(trng, options, n_in=options['readout_dim'] + 1 if options['coverage'] else options['readout_dim'], n_out=3 if config['forget'] else 2, recurrent=options['recurrent'], id=id) # make the dataset ready for training & validation trainIter = TextIterator(options['datasets'][0], options['datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=config['batchsize'], maxlen=options['maxlen']) train_num = trainIter.num validIter = TextIterator(options['valid_datasets'][0], options['valid_datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=20, cache=10, maxlen=1000000) valid_num = validIter.num print 'training set {} lines / validation set {} lines'.format( train_num, valid_num) print 'use the reward function {}'.format(chr(config['Rtype'] + 65)) # ========================================================================== # # Main Loop: Run # ========================================================================== # print 'Start Simultaneous Translator...' monitor = None if remote: monitor = Monitor(root='http://localhost:9000') # freqs save_freq = 200 sample_freq = 10 valid_freq = 200 valid_size = 200 display_freq = 50 finetune_freq = 5 history, last_it = agent.load() action_space = ['W', 'C', 'F'] Log_avg = {} time0 = timer() pipe = OrderedDict() for key in ['x', 'x_mask', 'y', 'y_mask', 'c_mask']: pipe[key] = [] def _translate(src, trg, samples=None, train=False, greedy=False, show=False, full=False): time0 = time.time() if full: options1 = copy.copy(options) options1['upper'] = True else: options1 = options ret = simultaneous_decoding(funcs, agent, options1, src, trg, word_idict_trg, samples, greedy, train) if show: info = ret[1] values = [(w, float(info[w])) for w in info if w != 'advantages'] print ' , '.join(['{}={:.3f}'.format(k, f) for k, f in values]), print '...{}s'.format(time.time() - time0) return ret for it, (srcs, trgs) in enumerate(trainIter): # only one sentence each iteration if it < last_it: # go over the scanned lines. continue # for validation # doing the whole validation!! reference = [] system = [] if it % valid_freq == (valid_freq - 1): print 'start validation' collections = [[], [], [], [], []] probar_v = Progbar(valid_num / 20 + 1) for ij, (srcs, trgs) in enumerate(validIter): statistics = _translate(srcs, trgs, samples=1, train=False, greedy=True) quality, delay, reward = zip(*statistics['track']) reference += statistics['Ref'] system += statistics['Sys'] # compute the average consective waiting length def _consective(action): waits = [] temp = 0 for a in action: if a == 0: temp += 1 elif temp > 0: waits += [temp] temp = 0 if temp > 0: waits += [temp] mean = numpy.mean(waits) gec = numpy.max( waits) # numpy.prod(waits) ** (1./len(waits)) return mean, gec def _max_length(action): _cur = 0 _end = 0 _max = 0 for it, a in enumerate(action): if a == 0: _cur += 1 elif a == 2: _end += 1 temp = _cur - _end if temp > _max: _max = temp return _max maxlen = [ _max_length(action) for action in statistics['action'] ] means, gecs = zip(*(_consective(action) for action in statistics['action'])) collections[0] += quality collections[1] += delay collections[2] += means collections[3] += gecs collections[4] += maxlen values = [('quality', numpy.mean(quality)), ('delay', numpy.mean(delay)), ('wait_mean', numpy.mean(means)), ('wait_max', numpy.mean(gecs)), ('max_len', numpy.mean(maxlen))] probar_v.update(ij + 1, values=values) validIter.reset() valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx = [ numpy.mean(a) for a in collections ] print 'Iter = {}: AVG BLEU = {}, DELAY = {}, WAIT(MEAN) = {}, WAIT(MAX) = {}, MaxLen={}'.format( it, valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx) print 'Compute the Corpus BLEU={} (greedy)'.format( corpus_bleu(reference, system)) with open(WORK + '.translate/test.txt', 'w') as fout: for sys in system: fout.write('{}\n'.format(' '.join(sys))) with open(WORK + '.translate/ref.txt', 'w') as fout: for ref in reference: fout.write('{}\n'.format(' '.join(ref[0]))) history += [collections] print 'done' if options['upper']: print 'done' import sys sys.exit(-1) # training set sentence tuning new_srcs, new_trgs = [], [] for src, trg in zip(srcs, trgs): if len(src) <= options['s0']: continue # ignore when the source sentence is less than sidx. else: new_srcs += [src] new_trgs += [trg] if len(new_srcs) == 0: continue srcs, trgs = new_srcs, new_trgs statistics, info = _translate(srcs, trgs, train=True, show=True) if it % sample_freq == 0: # obtain the translation results samples = _bpe2words( _seqs2words(statistics['sample'], word_idict_trg, statistics['action'], 1)) sources = _bpe2words( _seqs2words(statistics['SWord'], word_idict, statistics['action'], 0)) targets = _bpe2words( _seqs2words(statistics['TWord'], word_idict_trg)) # obtain the delay (normalized) # delays = _action2delay(srcs[0], statistics['action']) c = 0 for j in xrange(len(samples)): if statistics['seq_info'][j][0] == 0: if c < (config['sample'] / 2.): c += 1 continue print '--Iter: {}'.format(it) print 'source: ', sources[j] print 'sample: ', samples[j] print 'target: ', targets[j] print 'quality:', statistics['track'][j][0] print 'delay:', statistics['track'][j][1] print 'reward:', statistics['track'][j][2] break # NaN detector #for w in info: # if numpy.isnan(info[w]) or numpy.isinf(info[w]): # raise RuntimeError, 'NaN/INF is detected!! {} : ID={}'.format(w, id) # remote display if remote: logs = { 'R': info['R'], 'Q': info['Q'], 'D': info['D'], 'P': float(info['P']) } if 'a_cost' in info: logs['A'] = info['a_cost'] print logs for w in logs: Log_avg[w] = Log_avg.get(w, 0) + logs[w] if it % display_freq == (display_freq - 1): for w in Log_avg: Log_avg[w] /= display_freq monitor.display(it + 1, Log_avg) Log_avg = dict() # save the history & model history += [info] if it % save_freq == 0: agent.save(history, it)
type=str, required=True) args = parser.parse_args() nlp = en_core_web_sm.load() num_lines = len([1 for line in open(args.dataset_filename)]) with open(args.dataset_filename) as in_file: reader = csv.reader(in_file, delimiter="\t") with open(args.preprocessed_dataset_filename, mode="w") as out_file: writer = csv.writer(out_file, delimiter="\t") progress = Progbar(num_lines) for row_number, row in enumerate(reader, 1): progress.update(row_number) label = row[0].strip() premise = row[1].strip() hypothesis = row[2].strip() premise_tokens = [token.lower_ for token in nlp(premise)] hypothesis_tokens = [token.lower_ for token in nlp(hypothesis)] if len(row) == 3: writer.writerow([ label, " ".join(premise_tokens), " ".join(hypothesis_tokens), premise, hypothesis ]) elif len(row) == 4: image_filename = row[3].strip() writer.writerow([ label, " ".join(premise_tokens), " ".join(hypothesis_tokens), image_filename, premise,
for att in f.readlines(): attributes.append(att.split(",")[0].lower().strip()) gpu_id = 0 caffe.set_device(gpu_id) caffe.set_mode_gpu() net = None cfg_from_file(args.cfg_filename) net = caffe.Net(args.net_def_filename, caffe.TEST, weights=args.net_weights_filename) img_filenames = os.listdir(args.img_path) num_img_filenames = len(img_filenames) progress = Progbar(num_img_filenames) bottom_up_features = {} for filename_index, filename in enumerate(img_filenames): full_filename = os.path.join(args.img_path, filename) results = get_detections_from_im(net, full_filename, conf_thresh=0.2, min_num_boxes=args.num_boxes, max_num_boxes=args.num_boxes) bottom_up_features[filename] = results progress.update(filename_index) with open(args.features_filename, mode="wb") as out_file: pickle.dump(bottom_up_features, out_file)
for indexes in batch(batches_indexes, args.batch_size): batch_premises = train_premises[indexes] batch_hypotheses = train_hypotheses[indexes] batch_labels = train_labels[indexes] batch_img_names = [train_img_names[i] for i in indexes] batch_img_features = image_reader.get_features(batch_img_names) loss, _ = session.run([loss_function, train_step], feed_dict={ premise_input: batch_premises, hypothesis_input: batch_hypotheses, img_features_input: batch_img_features, label_input: batch_labels, dropout_input: args.dropout_ratio }) progress.update(batch_index, [("Loss", loss)]) epoch_loss += loss batch_index += 1 print("Current mean training loss: {}\n".format(epoch_loss / num_batches)) print("-- Validating model") dev_num_examples = dev_labels.shape[0] dev_batches_indexes = np.arange(dev_num_examples) dev_num_correct = 0 for indexes in batch(dev_batches_indexes, args.batch_size): dev_batch_premises = dev_premises[indexes] dev_batch_hypotheses = dev_hypotheses[indexes] dev_batch_labels = dev_labels[indexes] dev_batch_img_names = [dev_img_names[i] for i in indexes] dev_batch_img_features = image_reader.get_features(dev_batch_img_names)
def train_model(model, optimizer, criterion, training_data_loader, validation_data_loader, opt): logging.info('====================== Checking GPU Availability =========================') if torch.cuda.is_available(): if isinstance(opt.gpuid, int): opt.gpuid = [opt.gpuid] logging.info('Running on GPU! devices=%s' % str(opt.gpuid)) model = model.cuda() model = nn.DataParallel(model, device_ids=opt.gpuid) criterion.cuda() else: logging.info('Running on CPU!') logging.info('====================== Start Training =========================') train_history_losses = [] valid_history_losses = [] best_loss = sys.float_info.max train_losses = [] total_batch = 0 early_stop_flag = False for epoch in range(opt.start_epoch , opt.epochs): if early_stop_flag: break progbar = Progbar(title='Training', target=len(training_data_loader), batch_size=opt.batch_size, total_examples=len(training_data_loader.dataset)) model.train() for batch_i, batch in enumerate(training_data_loader): batch_i += 1 # for the aesthetics of printing total_batch += 1 src = batch.src trg = batch.trg # print("src size - ",src.size()) # print("target size - ",trg.size()) if torch.cuda.is_available(): src.cuda() trg.cuda() optimizer.zero_grad() decoder_logits, _, _ = model.forward(src, trg, must_teacher_forcing=False) start_time = time.time() # remove the 1st word in trg to let predictions and real goal match loss = criterion( decoder_logits.contiguous().view(-1, opt.vocab_size), trg[:, 1:].contiguous().view(-1) ) print("--loss calculation- %s seconds ---" % (time.time() - start_time)) start_time = time.time() loss.backward() print("--backward- %s seconds ---" % (time.time() - start_time)) if opt.max_grad_norm > 0: pre_norm = torch.nn.utils.clip_grad_norm(model.parameters(), opt.max_grad_norm) after_norm = (sum([p.grad.data.norm(2) ** 2 for p in model.parameters() if p.grad is not None])) ** (1.0 / 2) logging.info('clip grad (%f -> %f)' % (pre_norm, after_norm)) optimizer.step() train_losses.append(loss.data[0]) perplexity = np.math.exp(loss.data[0]) progbar.update(epoch, batch_i, [('train_loss', loss.data[0]), ('perplexity', perplexity)]) if batch_i > 1 and batch_i % opt.report_every == 0: logging.info('====================== %d =========================' % (batch_i)) logging.info('Epoch : %d Minibatch : %d, Loss=%.5f, PPL=%.5f' % (epoch, batch_i, np.mean(loss.data[0]), perplexity)) sampled_size = 2 logging.info('Printing predictions on %d sampled examples by greedy search' % sampled_size) # softmax logits to get probabilities (batch_size, trg_len, vocab_size) # decoder_probs = torch.nn.functional.softmax(decoder_logits.view(trg.size(0) * trg.size(1), -1)).view(*trg.size(), -1) if torch.cuda.is_available(): src = src.data.cpu().numpy() decoder_logits = decoder_logits.data.cpu().numpy() max_words_pred = decoder_logits.argmax(axis=-1) trg = trg.data.cpu().numpy() else: src = src.data.numpy() decoder_logits = decoder_logits.data.numpy() max_words_pred = decoder_logits.argmax(axis=-1) trg = trg.data.numpy() sampled_trg_idx = np.random.random_integers(low=0, high=len(trg) - 1, size=sampled_size) src = src[sampled_trg_idx] max_words_pred = [max_words_pred[i] for i in sampled_trg_idx] decoder_logits = decoder_logits[sampled_trg_idx] trg = [trg[i][1:] for i in sampled_trg_idx] # the real target has removed the starting <BOS> for i, (src_wi, pred_wi, real_wi) in enumerate(zip(src, max_words_pred, trg)): nll_prob = -np.sum(np.log2([decoder_logits[i][l][pred_wi[l]] for l in range(len(real_wi))])) sentence_source = [opt.id2word[x] for x in src_wi] sentence_pred = [opt.id2word[x] for x in pred_wi] sentence_real = [opt.id2word[x] for x in real_wi] logging.info('==================================================') logging.info('Source: %s ' % (' '.join(sentence_source))) logging.info('\t\tPred : %s (%.4f)' % (' '.join(sentence_pred), nll_prob)) logging.info('\t\tReal : %s ' % (' '.join(sentence_real))) if total_batch > 1 and total_batch % opt.run_valid_every == 0: logging.info('*' * 50) logging.info('Run validation test @Epoch=%d,#(Total batch)=%d' % (epoch, total_batch)) valid_losses = _valid(validation_data_loader, model, criterion, optimizer, epoch, opt, is_train=False) train_history_losses.append(copy.copy(train_losses)) valid_history_losses.append(valid_losses) train_losses = [] # Plot the learning curve plot_learning_curve(train_history_losses, valid_history_losses, 'Training and Validation', curve1_name='Training Error', curve2_name='Validation Error', save_path=opt.exp_path + '/[epoch=%d,batch=%d,total_batch=%d]train_valid_curve.png' % (epoch, batch_i, total_batch)) ''' determine if early stop training ''' valid_loss = np.average(valid_history_losses[-1]) is_best_loss = valid_loss < best_loss rate_of_change = float(valid_loss - best_loss) / float(best_loss) # only store the checkpoints that make better validation performances if total_batch > 1 and epoch >= opt.start_checkpoint_at and (total_batch % opt.save_model_every == 0 or is_best_loss): # Save the checkpoint logging.info('Saving checkpoint to: %s' % os.path.join(opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d.error=%f' % (opt.exp, epoch, batch_i, total_batch, valid_loss) + '.model')) torch.save( model.state_dict(), open(os.path.join(opt.save_path, '%s.epoch=%d.batch=%d.total_batch=%d' % (opt.exp, epoch, batch_i, total_batch) + '.model'), 'wb') ) # valid error doesn't decrease if rate_of_change >= 0: stop_increasing += 1 else: stop_increasing = 0 if is_best_loss: logging.info('Validation: update best loss (%.4f --> %.4f), rate of change (ROC)=%.2f' % ( best_loss, valid_loss, rate_of_change * 100)) else: logging.info('Validation: best loss is not updated for %d times (%.4f --> %.4f), rate of change (ROC)=%.2f' % ( stop_increasing, best_loss, valid_loss, rate_of_change * 100)) best_loss = min(valid_loss, best_loss) if stop_increasing >= opt.early_stop_tolerance: logging.info('Have not increased for %d epoches, early stop training' % stop_increasing) early_stop_flag = True break logging.info('*' * 50)
file_name = MODEL_DIR + options['SAVE_PREFIX'] model = MF.BiLSTM_Model(options) n_epochs = 10 _mode = args.partial_mode if hasattr(args, 'partial_mode') else 'em' model.fit(X=train_X, y=train_y, val_split=0.9, shuffle=True, n_epochs=n_epochs, save_best=True, save_prefix=file_name, X_unlabeled=unlabeled_X, y_unlabeled=unlabeled_y, mode=_mode) bar.update(ix - start_ix + 1) else: predictions = [] model = MF.BiLSTM_Model(options) for ix in xrange(len(posts)): BASE_DIR = options['BASE_DIR'] MODEL_DIR = BASE_DIR + 'MODEL_' + str(ix) + '/' file_name = MODEL_DIR + options['SAVE_PREFIX'] best_model_file = ut.get_best_model_file(file_name) model.load_model(best_model_file) test_X, test_y = ut.data_generator([posts[ix]], options) prediction = model.predict(test_X, mode=args.inf_mode) prediction = [ options['IX_2_CLASSES'][x] if x in options['IX_2_CLASSES'] else 'O' for x in prediction ]
def run_simultrans(model, options_file=None, config=None, policy=None, id=None, remote=False): # check envoriments check_env() if id is not None: fcon = WORK + '.config/{}.conf'.format(id) if os.path.exists(fcon): print 'load config files' policy, config = pkl.load(open(fcon, 'r')) # ============================================================================== # # load model model_options # ============================================================================== # _model = model model = WORK + '.pretrained/{}'.format(model) if options_file is not None: with open(options_file, 'rb') as f: options = pkl.load(f) else: with open('%s.pkl' % model, 'rb') as f: options = pkl.load(f) print 'load options...' for w, p in sorted(options.items(), key=lambda x: x[0]): print '{}: {}'.format(w, p) # load detail settings from option file: dictionary, dictionary_target = options['dictionaries'] def _iter(fname): with open(fname, 'r') as f: for line in f: words = line.strip().split() x = map(lambda w: word_dict[w] if w in word_dict else 1, words) x = map(lambda ii: ii if ii < options['n_words'] else 1, x) x += [0] yield x def _check_length(fname): f = open(fname, 'r') count = 0 for _ in f: count += 1 f.close() return count # load source dictionary and invert with open(dictionary, 'rb') as f: word_dict = pkl.load(f) word_idict = dict() for kk, vv in word_dict.iteritems(): word_idict[vv] = kk word_idict[0] = '<eos>' word_idict[1] = 'UNK' # load target dictionary and invert with open(dictionary_target, 'rb') as f: word_dict_trg = pkl.load(f) word_idict_trg = dict() for kk, vv in word_dict_trg.iteritems(): word_idict_trg[vv] = kk word_idict_trg[0] = '<eos>' word_idict_trg[1] = 'UNK' ## use additional input for the policy network options['pre'] = config['pre'] # ================================================================================= # # Build a Simultaneous Translator # ================================================================================= # # allocate model parameters params = init_params(options) params = load_params(model, params) tparams = init_tparams(params) # print 'build the model for computing cost (full source sentence).' trng, use_noise, \ _x, _x_mask, _y, _y_mask, \ opt_ret, \ cost, f_cost = build_model(tparams, options) print 'done' # functions for sampler f_sim_ctx, f_sim_init, f_sim_next = build_simultaneous_sampler( tparams, options, trng) # function for finetune if config['finetune'] != 'nope': f_fine_init, f_fine_cost, f_fine_update = build_fine( tparams, options, fullmodel=True if config['finetune'] == 'full' else False) def _translate(src, trg, train=False, samples=config['sample'], greedy=False): ret = simultaneous_decoding( f_sim_ctx, f_sim_init, f_sim_next, f_cost, _policy, src, trg, word_idict_trg, step=config['step'], peek=config['peek'], sidx=config['s0'], n_samples=samples, reward_config={ 'target': config['target'], 'gamma': config['gamma'], 'Rtype': config['Rtype'], 'maxsrc': config['maxsrc'], 'greedy': greedy, 'upper': config['upper'] }, train=train, use_forget=config['forget'], use_newinput=config['pre'], use_coverage=config['coverage'], on_groundtruth=0 if config['finetune'] == 'nope' else 10) print ret import sys sys.exit(-1) return ret # if not train: # sample, score, actions, R, tracks, attentions = ret # return sample, score, actions, R, tracks # else: # sample, score, actions, R, info, pipe_t = ret # return sample, score, actions, R, info, pipe_t # check the ID: policy['base'] = _model _policy = Policy(trng, options, policy, config, n_in=options['readout_dim'] + 1 if config['coverage'] else options['readout_dim'], n_out=3 if config['forget'] else 2, recurrent=policy['recurrent'], id=id) # make the dataset ready for training & validation # train_ = options['datasets'][0] # train_num = _check_length trainIter = TextIterator(options['datasets'][0], options['datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=config['batchsize'], maxlen=options['maxlen']) train_num = trainIter.num validIter = TextIterator(options['valid_datasets'][0], options['valid_datasets'][1], options['dictionaries'][0], options['dictionaries'][1], n_words_source=options['n_words_src'], n_words_target=options['n_words'], batch_size=1, cache=1, maxlen=1000000) valid_num = validIter.num valid_ = options['valid_datasets'][0] valid_num = _check_length(valid_) print 'training set {} lines / validation set {} lines'.format( train_num, valid_num) print 'use the reward function {}'.format(chr(config['Rtype'] + 65)) # ================================================================================= # # Main Loop: Run # ================================================================================= # print 'Start Simultaneous Translator...' probar = Progbar(train_num / config['batchsize'], with_history=False) monitor = None if remote: monitor = Monitor(root='http://localhost:9000') # freqs save_freq = 200 sample_freq = 10 valid_freq = 200 valid_size = 200 display_freq = 50 finetune_freq = 5 history, last_it = _policy.load() action_space = ['W', 'C', 'F'] Log_avg = {} time0 = timer() pipe = PIPE(['x', 'x_mask', 'y', 'y_mask', 'c_mask']) for it, (srcs, trgs) in enumerate(trainIter): # only one sentence each iteration if it < last_it: # go over the scanned lines. continue # for validation # doing the whole validation!! reference = [] system = [] reference2 = [] system2 = [] if it % valid_freq == 0: print 'start validation' collections = [[], [], [], [], []] probar_v = Progbar(valid_num / 64 + 1) for ij, (srcs, trgs) in enumerate(validIter): # new_srcs, new_trgs = [], [] # for src, trg in zip(srcs, trgs): # if len(src) < config['s0']: # continue # ignore when the source sentence is less than sidx. we don't use the policy\ # else: # new_srcs += [src] # new_trgs += [trg] # if len(new_srcs) == 0: # continue # srcs, trgs = new_srcs, new_trgs statistics = _translate(srcs, trgs, train=False, samples=1, greedy=True) quality, delay, reward = zip(*statistics['track']) reference += statistics['Ref'] system += statistics['Sys'] # print ' '.join(reference[-1][0]) # print ' '.join(system[-1]) # compute the average consective waiting length def _consective(action): waits = [] temp = 0 for a in action: if a == 0: temp += 1 elif temp > 0: waits += [temp] temp = 0 if temp > 0: waits += [temp] mean = numpy.mean(waits) gec = numpy.max( waits) # numpy.prod(waits) ** (1./len(waits)) return mean, gec def _max_length(action): _cur = 0 _end = 0 _max = 0 for it, a in enumerate(action): if a == 0: _cur += 1 elif a == 2: _end += 1 temp = _cur - _end if temp > _max: _max = temp return _max maxlen = [ _max_length(action) for action in statistics['action'] ] means, gecs = zip(*(_consective(action) for action in statistics['action'])) collections[0] += quality collections[1] += delay collections[2] += means collections[3] += gecs collections[4] += maxlen values = [('quality', numpy.mean(quality)), ('delay', numpy.mean(delay)), ('wait_mean', numpy.mean(means)), ('wait_max', numpy.mean(gecs)), ('max_len', numpy.mean(maxlen))] probar_v.update(ij + 1, values=values) validIter.reset() valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx = [ numpy.mean(a) for a in collections ] print 'Iter = {}: AVG BLEU = {}, DELAY = {}, WAIT(MEAN) = {}, WAIT(MAX) = {}, MaxLen={}'.format( it, valid_bleu, valid_delay, valid_wait, valid_wait_gec, valid_mx) print 'Compute the Corpus BLEU={} (greedy)'.format( corpus_bleu(reference, system)) with open(WORK + '.translate/test.txt', 'w') as fout: for sys in system: fout.write('{}\n'.format(' '.join(sys))) with open(WORK + '.translate/ref.txt', 'w') as fout: for ref in reference: fout.write('{}\n'.format(' '.join(ref[0]))) if config['upper']: print 'done' import sys sys.exit(-1) # training set sentence tuning new_srcs, new_trgs = [], [] for src, trg in zip(srcs, trgs): if len(src) <= config['s0']: continue # ignore when the source sentence is less than sidx. we don't use the policy\ else: new_srcs += [src] new_trgs += [trg] if len(new_srcs) == 0: continue srcs, trgs = new_srcs, new_trgs try: statistics, info, pipe_t = _translate(srcs, trgs, train=True) except Exception: print 'translate a empty sentence. bug.' continue # samples, scores, actions, rewards, info, pipe_t = _translate(srcs, trgs, train=True) # print pipe_t if config['finetune'] != 'nope': for idx, act in enumerate(pipe_t['action']): _start = 0 _end = 0 _mask = [0 for _ in srcs[0]] _cmask = [] pipe.messages['x'] += srcs pipe.messages['y'] += [pipe_t['sample'][idx]] for a in act: # print _start, _end if a == 0: _mask[_start] = 1 _start += 1 elif a == 2: _mask[_end] = 0 _end += 1 else: _cmask.append(_mask) # print numpy.asarray(_cmask).shape pipe.messages['c_mask'].append(_cmask) if it % finetune_freq == (finetune_freq - 1): num = len(pipe.messages['x']) max_x = max([len(v) for v in pipe.messages['x']]) max_y = max([len(v) for v in pipe.messages['y']]) xx, xx_mask = _padding(pipe.messages['x'], shape=(max_x, num), return_mask=True, dtype='int64') yy, yy_mask = _padding(pipe.messages['y'], shape=(max_y, num), return_mask=True, dtype='int64') cc_mask = _padding(pipe.messages['c_mask'], shape=(max_y, num, max_x)).transpose([0, 2, 1]) # fine-tune the EncDec of translation if config['finetune'] == 'full': cost = f_fine_cost(xx, xx_mask, yy, yy_mask, cc_mask) elif config['finetune'] == 'decoder': cost = f_fine_cost(xx, xx_mask, yy, yy_mask, cc_mask) else: raise NotImplementedError print '\nIter={} || cost = {}'.format(it, cost[0]) f_fine_update(0.00001) pipe.reset() if it % sample_freq == 0: print '\nModel:{} has been trained for {} hours'.format( _policy.id, (timer() - time0) / 3600.) print 'source: ', _bpe2words(_seqs2words([srcs[0]], word_idict))[0] print 'target: ', _bpe2words(_seqs2words([trgs[0]], word_idict_trg))[0] # obtain the translation results samples = _bpe2words( _seqs2words(statistics['sample'], word_idict_trg)) # obtain the delay (normalized) # delays = _action2delay(srcs[0], statistics['action']) c = 0 for j in xrange(len(samples)): if statistics['secs'][j][0] == 0: if c < 5: c += 1 print '---ID: {}'.format(_policy.id) print 'sample: ', samples[j] # print 'action: ', ','.join( # ['{}({})'.format(action_space[t], f) # for t, f in # zip(statistics['action'][j], statistics['forgotten'][j])]) print 'action: ', ','.join([ '{}'.format(action_space[t]) for t in statistics['action'][j] ]) print 'quality:', statistics['track'][j][0] print 'delay:', statistics['track'][j][1] # print 'score:', statistics['score'][j] break values = [(w, info[w]) for w in info] probar.update(it + 1, values=values) # NaN detector for w in info: if numpy.isnan(info[w]) or numpy.isinf(info[w]): raise RuntimeError, 'NaN/INF is detected!! {} : ID={}'.format( w, id) # remote display if remote: logs = { 'R': info['R'], 'Q': info['Q'], 'D': info['D'], 'P': float(info['P']) } # print logs for w in logs: Log_avg[w] = Log_avg.get(w, 0) + logs[w] if it % display_freq == (display_freq - 1): for w in Log_avg: Log_avg[w] /= display_freq monitor.display(it + 1, Log_avg) Log_avg = dict() # save the history & model history += [info] if it % save_freq == 0: _policy.save(history, it)
for epoch in xrange(N_EPOCHS): steps = (train_x.shape[0] // BATCH_SIZE) if train_x.shape[0] % BATCH_SIZE == 0 else ( train_x.shape[0] // BATCH_SIZE) + 1 entropy_train_loss = 0. mse_train_loss = 0. for ix in xrange(steps): batch_x = train_x[ix:ix + BATCH_SIZE] mask = np.random.binomial(1, NOISE, batch_x.shape) input_x = batch_x * mask reconstruction = autoencoder(input_x) loss = cross_entropy(batch_x, reconstruction) entropy_train_loss += loss mse_train_loss += mse_loss(batch_x, reconstruction) gradient = cross_entropy.grad() # gradient = mse_loss.grad() autoencoder.backward(gradient) optimizer.step() # Cleanup optimizer.zero_grad() entropy_train_loss /= steps mse_train_loss /= steps val_preds = autoencoder(val_x) entropy_val_loss = cross_entropy(val_x, val_preds) mse_val_loss = mse_loss(val_x, val_preds) bar.update(epoch + 1, values=[("train_entropy", entropy_train_loss), ("val_entropy", entropy_val_loss), ("train_mse", mse_train_loss), ("val_mse", mse_val_loss)])
def evaluate_beam_search(generator, data_loader, opt, title='', epoch=1, save_path=None): logging = config.init_logging(title, save_path + '/%s.log' % title) progbar = Progbar(logger=logging, title=title, target=len(data_loader) / opt.beam_batch, batch_size=opt.beam_batch, total_examples=len(data_loader) / opt.beam_batch) beam_batch_idx = 0 score_dict = defaultdict( list ) # {'precision@5':[],'recall@5':[],'f1score@5':[], 'precision@10':[],'recall@10':[],'f1score@10':[]} sample_idx = 0 for i, batch in enumerate(data_loader): beam_batch_idx += 1 src_list, src_len, trg_list, _, _, src_oov_map_list, oov_list, query_lists, query_len, src_str_list, trg_str_list = batch if torch.cuda.is_available() and opt.use_gpu: src_list = src_list.cuda() src_oov_map_list = src_oov_map_list.cuda() query_lists = query_lists.cuda() pred_seq_list = generator.beam_search(src_list, src_len, src_oov_map_list, oov_list, opt.word2id, query_lists, query_len) for src, src_str, trg, trg_str_seqs, pred_seq, oov in zip( src_list, src_str_list, trg_list, trg_str_list, pred_seq_list, oov_list): # logging.info('====================== %d =========================' % (beam_batch_idx)) pred_is_valid, processed_pred_seqs, processed_pred_str_seqs, processed_pred_score = process_predseqs( pred_seq, oov, opt.id2word, opt) # 2nd filtering: if filter out phrases that don't appear in text, and keep unique ones after stemming pred_is_present = [True] * len(processed_pred_str_seqs) valid_and_present = np.asarray(pred_is_valid) * np.asarray( pred_is_present) ''' Evaluate predictions w.r.t different filterings and metrics ''' num_oneword_seq = -1 # -1,1 topk_range = [5, 10] #5,10 score_names = ['precision', 'recall', 'f_score'] processed_pred_seqs = np.asarray( processed_pred_seqs)[valid_and_present] processed_pred_str_seqs = np.asarray( processed_pred_str_seqs)[valid_and_present] processed_pred_score = np.asarray( processed_pred_score)[valid_and_present] # 3rd round filtering (one-word phrases) filtered_pred_seq, filtered_pred_str_seqs, filtered_pred_score = post_process_predseqs( (processed_pred_seqs, processed_pred_str_seqs, processed_pred_score), num_oneword_seq) match_list = get_match_result(true_seqs=trg_str_seqs, pred_seqs=filtered_pred_str_seqs, type='exact') # logging_result(src_str,trg_str_seqs,filtered_pred_str_seqs,match_list) assert len(filtered_pred_seq) == len( filtered_pred_str_seqs) == len(filtered_pred_score) == len( match_list) for topk in topk_range: results = evaluate(match_list, filtered_pred_seq, trg_str_seqs, topk=topk) for k, v in zip(score_names, results): if '%s@%d#oneword=%d' % ( k, topk, num_oneword_seq) not in score_dict: score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)] = [] score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)].append(v) if beam_batch_idx % 10 == 0: # print('#(precision@5#oneword=1)=%d, avg=%f' % (len(score_dict['precision@5#oneword=1']), np.average(score_dict['precision@5#oneword=1']))) # print('#(precision@10#oneword=1)=%d, avg=%f' % (len(score_dict['precision@10#oneword=1']), np.average(score_dict['precision@10#oneword=1']))) # print('#(recall@5#oneword=1)=%d, avg=%f' % (len(score_dict['recall@5#oneword=1']), np.average(score_dict['recall@5#oneword=1']))) # print('#(recall@10#oneword=1)=%d, avg=%f' % (len(score_dict['recall@10#oneword=1']), np.average(score_dict['recall@10#oneword=1']))) metric5 = 'f_score@5#oneword=' + str(num_oneword_seq) metric10 = 'f_score@10#oneword=' + str(num_oneword_seq) x, y = np.average(score_dict[metric5]), np.average( score_dict[metric10]) print(metric5, x) print(metric10, y) # print('#(f_score@5#oneword=1)=%d, avg=%f' % (len(score_dict['f_score@5#oneword=1']), x)) # print('#(f_score@10#oneword=1)=%d, avg=%f' % (len(score_dict['f_score@10#oneword=1']), y)) progbar.update(epoch, beam_batch_idx, [(metric5, x), (metric10, y)]) print('*' * 50) ''' process each example in current batch ''' print(metric5, x) print(metric10, y) # if save_path: # # export scores. Each row is scores (precision, recall and f-score) of different way of filtering predictions (how many one-word predictions to keep) # with open(save_path + os.path.sep + title + '_result.csv', 'w') as result_csv: # csv_lines = [] # for topk in topk_range: # csv_line = '#oneword=%d,@%d' % (num_oneword_seq, topk) # for k in score_names: # csv_line += ',%f' % np.average(score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)]) # csv_lines.append(csv_line + '\n') # result_csv.writelines(csv_lines) return score_dict
def train(self, n_iters): eval_step = 10 with tf.Session() as sess: self.train_iterator_handle = sess.run( self.train_iterator.string_handle()) self.val_iterator_handle = sess.run( self.val_iterator.string_handle()) self.train_eval_iterator_handle = sess.run( self.train_eval_iterator.string_handle()) sess.run(tf.global_variables_initializer()) # writer = tf.summary.FileWriter( # 'graphs/attention1', sess.graph) initial_step = self.gstep.eval() sess.run(self.val_iterator.initializer) sess.run(self.train_eval_iterator.initializer) variables = tf.trainable_variables() num_vars = np.sum( [np.prod(v.get_shape().as_list()) for v in variables]) print("Number of variables in models: {}".format(num_vars)) for epoch in range(n_iters): print("epoch #", epoch) num_batches = int(67978.0 / self.batch_size) progress = Progbar(target=num_batches) sess.run(self.train_iterator.initializer) index = 0 total_loss = 0 progress.update(index, [("training loss", total_loss)]) while True: index += 1 try: total_loss, opt = sess.run( [self.total_loss, self.opt], feed_dict={ self.handle: self.train_iterator_handle, self.keep_prob: 0.75 }) # , options=options, run_metadata=run_metadata) progress.update(index, [("training loss", total_loss)]) except tf.errors.OutOfRangeError: break print('evaluation on 500 training elements:') preds, contexts, answers = sess.run( [self.preds, self.contexts, self.answers], feed_dict={ self.handle: self.train_eval_iterator_handle, self.keep_prob: 1.0 }) predictions = [] ground_truths = [] for i in range(len(preds)): predictions.append( convert_indices_to_text(self.vocabulary, contexts[i], preds[i, 0], preds[i, 1])) ground_truths.append( convert_indices_to_text(self.vocabulary, contexts[i], answers[i, 0], answers[i, 1])) print(evaluate(predictions, ground_truths)) print('evaluation on 500 validation elements:') preds, contexts, answers = sess.run( [self.preds, self.contexts, self.answers], feed_dict={ self.handle: self.val_iterator_handle, self.keep_prob: 1.0 }) predictions = [] ground_truths = [] for i in range(len(preds)): predictions.append( convert_indices_to_text(self.vocabulary, contexts[i], preds[i, 0], preds[i, 1])) ground_truths.append( convert_indices_to_text(self.vocabulary, contexts[i], answers[i, 0], answers[i, 1])) print(evaluate(predictions, ground_truths)) predictions = [] ground_truths = []
def evaluate_beam_search(generator, data_loader, opt, title='', epoch=1, save_path=None): logging = config.init_logging(title, save_path + '/%s.log' % title) progbar = Progbar(logger=logging, title=title, target=len(data_loader.dataset.examples), batch_size=data_loader.batch_size, total_examples=len(data_loader.dataset.examples)) example_idx = 0 score_dict = {} # {'precision@5':[],'recall@5':[],'f1score@5':[], 'precision@10':[],'recall@10':[],'f1score@10':[]} for i, batch in enumerate(data_loader): # if i > 3: # break one2many_batch, one2one_batch = batch src_list, src_len, trg_list, _, trg_copy_target_list, src_oov_map_list, oov_list, src_str_list, trg_str_list = one2many_batch if torch.cuda.is_available(): src_list = src_list.cuda() src_oov_map_list = src_oov_map_list.cuda() print("batch size - %s" % str(src_list.size(0))) # print("src size - %s" % str(src_list.size())) # print("target size - %s" % len(trg_copy_target_list)) pred_seq_list = generator.beam_search(src_list, src_len, src_oov_map_list, oov_list, opt.word2id) ''' process each example in current batch ''' for src, src_str, trg, trg_str_seqs, trg_copy, pred_seq, oov in zip(src_list, src_str_list, trg_list, trg_str_list, trg_copy_target_list, pred_seq_list, oov_list): # logging.info('====================== %d =========================' % (example_idx)) print_out = '' print_out += '[Source][%d]: %s \n' % (len(src_str), ' '.join(src_str)) # src = src.cpu().data.numpy() if torch.cuda.is_available() else src.data.numpy() # print_out += '\nSource Input: \n %s\n' % (' '.join([opt.id2word[x] for x in src[:len(src_str) + 5]])) # print_out += 'Real Target String [%d] \n\t\t%s \n' % (len(trg_str_seqs), trg_str_seqs) # print_out += 'Real Target Input: \n\t\t%s \n' % str([[opt.id2word[x] for x in t] for t in trg]) # print_out += 'Real Target Copy: \n\t\t%s \n' % str([[opt.id2word[x] if x < opt.vocab_size else oov[x - opt.vocab_size] for x in t] for t in trg_copy]) trg_str_is_present = if_present_duplicate_phrase(src_str, trg_str_seqs) print_out += '[GROUND-TRUTH] #(present)/#(all targets)=%d/%d\n' % (sum(trg_str_is_present), len(trg_str_is_present)) print_out += '\n'.join(['\t\t[%s]' % ' '.join(phrase) if is_present else '\t\t%s' % ' '.join(phrase) for phrase, is_present in zip(trg_str_seqs, trg_str_is_present)]) print_out += '\noov_list: \n\t\t%s \n' % str(oov) # 1st filtering pred_is_valid, processed_pred_seqs, processed_pred_str_seqs, processed_pred_score = process_predseqs(pred_seq, oov, opt.id2word, opt) # 2nd filtering: if filter out phrases that don't appear in text, and keep unique ones after stemming if opt.must_appear_in_src: pred_is_present = if_present_duplicate_phrase(src_str, processed_pred_str_seqs) trg_str_seqs = np.asarray(trg_str_seqs)[trg_str_is_present] else: pred_is_present = [True] * len(processed_pred_str_seqs) valid_and_present = np.asarray(pred_is_valid) * np.asarray(pred_is_present) match_list = get_match_result(true_seqs=trg_str_seqs, pred_seqs=processed_pred_str_seqs) print_out += '[PREDICTION] #(valid)=%d, #(present)=%d, #(retained&present)=%d, #(all)=%d\n' % (sum(pred_is_valid), sum(pred_is_present), sum(valid_and_present), len(pred_seq)) print_out += '' ''' Print and export predictions ''' preds_out = '' for p_id, (seq, word, score, match, is_valid, is_present) in enumerate( zip(processed_pred_seqs, processed_pred_str_seqs, processed_pred_score, match_list, pred_is_valid, pred_is_present)): # if p_id > 5: # break preds_out += '%s\n' % (' '.join(word)) if is_present: print_phrase = '[%s]' % ' '.join(word) else: print_phrase = ' '.join(word) if is_valid: print_phrase = '*%s' % print_phrase if match == 1.0: correct_str = '[correct!]' else: correct_str = '' if any([t >= opt.vocab_size for t in seq.sentence]): copy_str = '[copied!]' else: copy_str = '' # print_out += '\t\t[%.4f]\t%s \t %s %s%s\n' % (-score, print_phrase, str(seq.sentence), correct_str, copy_str) ''' Evaluate predictions w.r.t different filterings and metrics ''' num_oneword_range = [-1, 1] topk_range = [5, 10] score_names = ['precision', 'recall', 'f_score'] processed_pred_seqs = np.asarray(processed_pred_seqs)[valid_and_present] processed_pred_str_seqs = np.asarray(processed_pred_str_seqs)[valid_and_present] processed_pred_score = np.asarray(processed_pred_score)[valid_and_present] for num_oneword_seq in num_oneword_range: # 3rd round filtering (one-word phrases) filtered_pred_seq, filtered_pred_str_seqs, filtered_pred_score = post_process_predseqs((processed_pred_seqs, processed_pred_str_seqs, processed_pred_score), num_oneword_seq) match_list = get_match_result(true_seqs=trg_str_seqs, pred_seqs=filtered_pred_str_seqs) assert len(filtered_pred_seq) == len(filtered_pred_str_seqs) == len(filtered_pred_score) == len(match_list) for topk in topk_range: results = evaluate(match_list, filtered_pred_seq, trg_str_seqs, topk=topk) for k, v in zip(score_names, results): if '%s@%d#oneword=%d' % (k, topk, num_oneword_seq) not in score_dict: score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)] = [] score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)].append(v) print_out += '\t%s@%d#oneword=%d = %f\n' % (k, topk, num_oneword_seq, v) # logging.info(print_out) if save_path: if not os.path.exists(os.path.join(save_path, title + '_detail')): os.makedirs(os.path.join(save_path, title + '_detail')) with open(os.path.join(save_path, title + '_detail', str(example_idx) + '_print.txt'), 'w') as f_: f_.write(print_out) with open(os.path.join(save_path, title + '_detail', str(example_idx) + '_prediction.txt'), 'w') as f_: f_.write(preds_out) progbar.update(epoch, example_idx, [('f_score@5#oneword=-1', np.average(score_dict['f_score@5#oneword=-1'])), ('f_score@10#oneword=-1', np.average(score_dict['f_score@10#oneword=-1']))]) example_idx += 1 print('#(f_score@5#oneword=-1)=%d, sum=%f' % (len(score_dict['f_score@5#oneword=-1']), sum(score_dict['f_score@5#oneword=-1']))) print('#(f_score@10#oneword=-1)=%d, sum=%f' % (len(score_dict['f_score@10#oneword=-1']), sum(score_dict['f_score@10#oneword=-1']))) print('#(f_score@5#oneword=1)=%d, sum=%f' % (len(score_dict['f_score@5#oneword=1']), sum(score_dict['f_score@5#oneword=1']))) print('#(f_score@10#oneword=1)=%d, sum=%f' % (len(score_dict['f_score@10#oneword=1']), sum(score_dict['f_score@10#oneword=1']))) if save_path: # export scores. Each row is scores (precision, recall and f-score) of different way of filtering predictions (how many one-word predictions to keep) with open(save_path + os.path.sep + title + '_result.csv', 'w') as result_csv: csv_lines = [] for num_oneword_seq in num_oneword_range: for topk in topk_range: csv_line = '#oneword=%d,@%d' % (num_oneword_seq, topk) for k in score_names: csv_line += ',%f' % np.average(score_dict['%s@%d#oneword=%d' % (k, topk, num_oneword_seq)]) csv_lines.append(csv_line + '\n') result_csv.writelines(csv_lines) # precision, recall, f_score = macro_averaged_score(precisionlist=score_dict['precision'], recalllist=score_dict['recall']) # logging.info("Macro@5\n\t\tprecision %.4f\n\t\tmacro recall %.4f\n\t\tmacro fscore %.4f " % (np.average(score_dict['precision@5']), np.average(score_dict['recall@5']), np.average(score_dict['f1score@5']))) # logging.info("Macro@10\n\t\tprecision %.4f\n\t\tmacro recall %.4f\n\t\tmacro fscore %.4f " % (np.average(score_dict['precision@10']), np.average(score_dict['recall@10']), np.average(score_dict['f1score@10']))) # precision, recall, f_score = evaluate(true_seqs=target_all, pred_seqs=prediction_all, topn=5) # logging.info("micro precision %.4f , micro recall %.4f, micro fscore %.4f " % (precision, recall, f_score)) return score_dict