def train(self): dataset = self.train_data nb_train_sample = dataset.count index_array = np.arange(nb_train_sample) nb_epoch = config.max_epoch batch_size = config.batch_size logging.info('begin training') cum_updates = 0 patience_counter = 0 early_stop = False history_valid_perf = [] history_valid_bleu = [] history_valid_acc = [] best_model_params = best_model_by_acc = best_model_by_bleu = None # train_data_iter = DataIterator(self.train_data, batch_size) for epoch in range(nb_epoch): # train_data_iter.reset() # if shuffle: np.random.shuffle(index_array) batches = make_batches(nb_train_sample, batch_size) # epoch begin sys.stdout.write('Epoch %d' % epoch) begin_time = time.time() cum_nb_examples = 0 loss = 0.0 for batch_index, (batch_start, batch_end) in enumerate(batches): # for batch_index, (examples, batch_ids) in enumerate(train_data_iter): cum_updates += 1 batch_ids = index_array[batch_start:batch_end] examples = dataset.get_examples(batch_ids) cur_batch_size = len(examples) inputs = dataset.get_prob_func_inputs(batch_ids) if not config.enable_copy: tgt_action_seq = inputs[1] tgt_action_seq_type = inputs[2] for i in xrange(cur_batch_size): for t in xrange(tgt_action_seq[i].shape[0]): if tgt_action_seq_type[i, t, 2] == 1: # can only be copied if tgt_action_seq_type[i, t, 1] == 0: tgt_action_seq_type[i, t, 1] = 1 tgt_action_seq[i, t, 1] = 1 # index of <unk> tgt_action_seq_type[i, t, 2] = 0 train_func_outputs = self.model.train_func(*inputs) batch_loss = train_func_outputs[0] logging.debug('prob_func finished computing') cum_nb_examples += cur_batch_size loss += batch_loss * batch_size logging.debug('Batch %d, avg. loss = %f', batch_index, batch_loss) if batch_index == 4: elapsed = time.time() - begin_time eta = nb_train_sample / (cum_nb_examples / elapsed) print ', eta %ds' % (eta) sys.stdout.flush() if cum_updates % config.valid_per_batch == 0: logging.info('begin validation') if config.data_type == 'ifttt': decode_results = decoder.decode_ifttt_dataset( self.model, self.val_data, verbose=False) channel_acc, channel_func_acc, prod_f1 = evaluation.evaluate_ifttt_results( self.val_data, decode_results, verbose=False) val_perf = channel_func_acc logging.info('channel accuracy: %f', channel_acc) logging.info('channel+func accuracy: %f', channel_func_acc) logging.info('prod F1: %f', prod_f1) else: decode_results = decoder.decode_python_dataset( self.model, self.val_data, verbose=False) bleu, accuracy = evaluation.evaluate_decode_results( self.val_data, decode_results, verbose=False) val_perf = eval(config.valid_metric) logging.info('avg. example bleu: %f', bleu) logging.info('accuracy: %f', accuracy) if len(history_valid_acc) == 0 or accuracy > np.array( history_valid_acc).max(): best_model_by_acc = self.model.pull_params() # logging.info('current model has best accuracy') history_valid_acc.append(accuracy) if len(history_valid_bleu) == 0 or bleu > np.array( history_valid_bleu).max(): best_model_by_bleu = self.model.pull_params() # logging.info('current model has best accuracy') history_valid_bleu.append(bleu) if len(history_valid_perf) == 0 or val_perf > np.array( history_valid_perf).max(): best_model_params = self.model.pull_params() patience_counter = 0 logging.info('save current best model') self.model.save( os.path.join(config.output_dir, 'model.npz')) else: patience_counter += 1 logging.info('hitting patience_counter: %d', patience_counter) if patience_counter >= config.train_patience: logging.info('Early Stop!') early_stop = True break history_valid_perf.append(val_perf) if cum_updates % config.save_per_batch == 0: self.model.save( os.path.join(config.output_dir, 'model.iter%d' % cum_updates)) logging.info('[Epoch %d] cumulative loss = %f, (took %ds)', epoch, loss / cum_nb_examples, time.time() - begin_time) np.savez( os.path.join(config.output_dir, 'model-epoch{}.npz'.format(epoch)), **self.model.pull_params()) if early_stop: break logging.info('training finished, save the best model') # np.savez(os.path.join(config.output_dir, 'model.npz'), **best_model_params) if config.data_type == 'django' or config.data_type == 'hs': logging.info('save the best model by accuracy') np.savez(os.path.join(config.output_dir, 'model.best_acc.npz'), **best_model_by_acc) logging.info('save the best model by bleu') np.savez(os.path.join(config.output_dir, 'model.best_bleu.npz'), **best_model_by_bleu)
# short_examples = [e for e in test_data.examples if e.parse_tree.size <= 2] # for e in short_examples: # print e.parse_tree # print 'short examples num: ', len(short_examples) # dataset = test_data # test_data.get_dataset_by_ids([1,2,3,4,5,6,7,8,9,10], name='sample') # cProfile.run('decode_dataset(model, dataset)', sort=2) # from evaluation import decode_and_evaluate_ifttt if args.data_type == 'ifttt': decode_results = decode_and_evaluate_ifttt_by_split( model, test_data) else: dataset = eval(args.type) decode_results = decode_python_dataset(model, dataset) serialize_to_file(decode_results, args.saveto) if args.operation == 'evaluate': dataset = eval(args.type) if config.mode == 'self': decode_results_file = args.input decode_results = deserialize_from_file(decode_results_file) evaluate_decode_results(dataset, decode_results) elif config.mode == 'seq2tree': from evaluation import evaluate_seq2tree_sample_file evaluate_seq2tree_sample_file(config.seq2tree_sample_file, config.seq2tree_id_file, dataset) elif config.mode == 'seq2seq':
def train(self, optimizer): dataset = self.train_data nb_train_sample = dataset.count index_array = np.arange(nb_train_sample) nb_epoch = config.max_epoch batch_size = config.batch_size logging.info('begin training') cum_updates = 0 patience_counter = 0 early_stop = False history_valid_perf = [] history_valid_bleu = [] history_valid_acc = [] history_losses = [] best_model_params = best_model_by_acc = best_model_by_bleu = None # train_data_iter = DataIterator(self.train_data, batch_size) original_start = time.time() start_time = original_start self.model.train() for epoch in range(nb_epoch): # train_data_iter.reset() # if shuffle: np.random.shuffle(index_array) batches = make_batches(nb_train_sample, batch_size) # epoch begin sys.stdout.write('Epoch %d' % epoch) begin_time = time.time() cum_nb_examples = 0 loss = 0.0 for batch_index, (batch_start, batch_end) in enumerate(batches): optimizer.zero_grad() cum_updates += 1 batch_ids = index_array[batch_start:batch_end] examples = dataset.get_examples(batch_ids) cur_batch_size = len(examples) inputs = dataset.get_prob_func_inputs(batch_ids) if not config.enable_copy: tgt_action_seq = inputs[1] tgt_action_seq_type = inputs[2] for i in xrange(cur_batch_size): for t in xrange(tgt_action_seq[i].shape[0]): if tgt_action_seq_type[i, t, 2] == 1: # can only be copied if tgt_action_seq_type[i, t, 1] == 0: tgt_action_seq_type[i, t, 1] = 1 tgt_action_seq[i, t, 1] = 1 # index of <unk> tgt_action_seq_type[i, t, 2] = 0 encode_outputs = self.model(*inputs) batch_loss = encode_outputs.item() # print("=" * 60) # trace_back(encode_outputs.grad_fn) # print("=" * 60) encode_outputs.backward() optimizer.step() logging.debug('prob_func finished computing') cum_nb_examples += cur_batch_size loss += batch_loss * batch_size logging.debug('Batch %d, avg. loss = %f', batch_index, batch_loss) if batch_index == 4: elapsed = time.time() - begin_time eta = nb_train_sample / (cum_nb_examples / elapsed) print(', eta %ds' % eta) sys.stdout.flush() if cum_updates % config.valid_per_batch == 0: # if cum_updates % 40 == 0: logging.info('begin validation') if config.data_type == 'ifttt': decode_results = decoder.decode_ifttt_dataset( self.model, self.val_data, verbose=False) channel_acc, channel_func_acc, prod_f1 = evaluation.evaluate_ifttt_results( self.val_data, decode_results, verbose=False) val_perf = channel_func_acc logging.info('channel accuracy: %f', channel_acc) logging.info('channel+func accuracy: %f', channel_func_acc) logging.info('prod F1: %f', prod_f1) else: decode_results = decoder.decode_python_dataset( self.model, train_data=dataset, test_data=self.val_data, verbose=True) bleu, accuracy = evaluation.evaluate_decode_results( self.val_data, decode_results, verbose=True) val_perf = eval(config.valid_metric) logging.info('avg. example bleu: %f', bleu) logging.info('accuracy: %f', accuracy) if len(history_valid_acc) == 0 or accuracy > np.array( history_valid_acc).max(): best_model_by_acc = self.model.state_dict() # logging.info('current model has best accuracy') history_valid_acc.append(accuracy) if len(history_valid_bleu) == 0 or bleu > np.array( history_valid_bleu).max(): best_model_by_bleu = self.model.state_dict() # logging.info('current model has best accuracy') history_valid_bleu.append(bleu) if len(history_valid_perf) == 0 or val_perf > np.array( history_valid_perf).max(): best_model = self.model.state_dict() patience_counter = 0 logging.info('save current best model') torch.save( self.model.state_dict(), os.path.join(config.output_dir, 'model.npz')) else: patience_counter += 1 logging.info('hitting patience_counter: %d', patience_counter) if patience_counter >= config.train_patience: logging.info('Early Stop!') early_stop = True break history_valid_perf.append(val_perf) if cum_updates % config.save_per_batch == 0: torch.save( self.model.state_dict(), os.path.join(config.output_dir, 'model.iter%d' % cum_updates)) logging.info( '[Epoch %d] cumulative loss = %f, (took %ds, total %f min) ', epoch, loss / cum_nb_examples, time.time() - begin_time, second2minute(time.time(), original_start)) history_losses.append(loss / cum_nb_examples) if early_stop: break logging.info('training finished, save the best model') # torch.save(best_model.state_dict(), os.path.join(config.output_dir, 'model.npz')) try: with open('valid_perf.txt', 'w') as fp: for x in history_valid_perf: fp.write(x) fp.write('\n') fp.close() with open('valid_bleu.txt', 'w') as fp: for x in history_valid_bleu: fp.write(x) fp.write('\n') fp.close() with open('valid_acc.txt', 'w') as fp: for x in history_valid_acc: fp.write(x) fp.write('\n') fp.close() with open('train_loss.txt', 'w') as fp: for x in history_losses: fp.write(x) fp.write('\n') fp.close() except: print "Fail to save result" pass if config.data_type == 'django' or config.data_type == 'hs': logging.info('save the best model by accuracy') torch.save(best_model_by_acc, os.path.join(config.output_dir, 'model.best_acc.npz')) logging.info('save the best model by bleu') torch.save(best_model_by_bleu, os.path.join(config.output_dir, 'model.best_bleu.npz'))
def train(self): dataset = self.train_data nb_train_sample = dataset.count index_array = np.arange(nb_train_sample) nb_epoch = config.max_epoch batch_size = config.batch_size logging.info('begin training') cum_updates = 0 patience_counter = 0 early_stop = False history_valid_perf = [] history_valid_bleu = [] history_valid_acc = [] best_model_params = best_model_by_acc = best_model_by_bleu = None # train_data_iter = DataIterator(self.train_data, batch_size) for epoch in range(nb_epoch): # train_data_iter.reset() # if shuffle: np.random.shuffle(index_array) batches = make_batches(nb_train_sample, batch_size) # epoch begin sys.stdout.write('Epoch %d' % epoch) begin_time = time.time() cum_nb_examples = 0 loss = 0.0 for batch_index, (batch_start, batch_end) in enumerate(batches): # for batch_index, (examples, batch_ids) in enumerate(train_data_iter): cum_updates += 1 batch_ids = index_array[batch_start:batch_end] examples = dataset.get_examples(batch_ids) cur_batch_size = len(examples) inputs = dataset.get_prob_func_inputs(batch_ids) if not config.enable_copy: tgt_action_seq = inputs[1] tgt_action_seq_type = inputs[2] for i in xrange(cur_batch_size): for t in xrange(tgt_action_seq[i].shape[0]): if tgt_action_seq_type[i, t, 2] == 1: # can only be copied if tgt_action_seq_type[i, t, 1] == 0: tgt_action_seq_type[i, t, 1] = 1 tgt_action_seq[i, t, 1] = 1 # index of <unk> tgt_action_seq_type[i, t, 2] = 0 train_func_outputs = self.model.train_func(*inputs) batch_loss = train_func_outputs[0] logging.debug('prob_func finished computing') cum_nb_examples += cur_batch_size loss += batch_loss * batch_size logging.debug('Batch %d, avg. loss = %f', batch_index, batch_loss) if batch_index == 4: elapsed = time.time() - begin_time eta = nb_train_sample / (cum_nb_examples / elapsed) print ', eta %ds' % (eta) sys.stdout.flush() if cum_updates % config.valid_per_batch == 0: logging.info('begin validation') if config.data_type == 'ifttt': decode_results = decoder.decode_ifttt_dataset(self.model, self.val_data, verbose=False) channel_acc, channel_func_acc, prod_f1 = evaluation.evaluate_ifttt_results(self.val_data, decode_results, verbose=False) val_perf = channel_func_acc logging.info('channel accuracy: %f', channel_acc) logging.info('channel+func accuracy: %f', channel_func_acc) logging.info('prod F1: %f', prod_f1) else: decode_results = decoder.decode_python_dataset(self.model, self.val_data, verbose=False) bleu, accuracy = evaluation.evaluate_decode_results(self.val_data, decode_results, verbose=False) val_perf = eval(config.valid_metric) logging.info('avg. example bleu: %f', bleu) logging.info('accuracy: %f', accuracy) if len(history_valid_acc) == 0 or accuracy > np.array(history_valid_acc).max(): best_model_by_acc = self.model.pull_params() # logging.info('current model has best accuracy') history_valid_acc.append(accuracy) if len(history_valid_bleu) == 0 or bleu > np.array(history_valid_bleu).max(): best_model_by_bleu = self.model.pull_params() # logging.info('current model has best accuracy') history_valid_bleu.append(bleu) if len(history_valid_perf) == 0 or val_perf > np.array(history_valid_perf).max(): best_model_params = self.model.pull_params() patience_counter = 0 logging.info('save current best model') self.model.save(os.path.join(config.output_dir, 'model.npz')) else: patience_counter += 1 logging.info('hitting patience_counter: %d', patience_counter) if patience_counter >= config.train_patience: logging.info('Early Stop!') early_stop = True break history_valid_perf.append(val_perf) if cum_updates % config.save_per_batch == 0: self.model.save(os.path.join(config.output_dir, 'model.iter%d' % cum_updates)) logging.info('[Epoch %d] cumulative loss = %f, (took %ds)', epoch, loss / cum_nb_examples, time.time() - begin_time) if early_stop: break logging.info('training finished, save the best model') np.savez(os.path.join(config.output_dir, 'model.npz'), **best_model_params) if config.data_type == 'django' or config.data_type == 'hs': logging.info('save the best model by accuracy') np.savez(os.path.join(config.output_dir, 'model.best_acc.npz'), **best_model_by_acc) logging.info('save the best model by bleu') np.savez(os.path.join(config.output_dir, 'model.best_bleu.npz'), **best_model_by_bleu)
# from evaluation import decode_and_evaluate_ifttt if args.data_type == 'ifttt': decode_results = decode_and_evaluate_ifttt_by_split(model, test_data) elif args.data_type == 'sql': if not args.enable_retrieval: dataset = eval(args.type) decode_results = decode_sql_dataset(model, dataset) else: print args.type print("sql here is with retrieval") decode_results = decode_sql_dataset_with_retrieval( model, train_data, dev_data, test_data, args.type, args.enable_retrieval) else: print args.type decode_results = decode_python_dataset( model, train_data, dev_data, test_data, args.type, args.enable_retrieval) serialize_to_file(decode_results, args.saveto) if args.operation == 'evaluate': dataset = eval(args.type) if config.mode == 'self': decode_results_file = args.input decode_results = deserialize_from_file(decode_results_file) evaluate_decode_results(dataset, decode_results) elif config.mode == 'seq2tree': from evaluation import evaluate_seq2tree_sample_file evaluate_seq2tree_sample_file(config.seq2tree_sample_file, config.seq2tree_id_file, dataset) elif config.mode == 'seq2seq':
# ========================== # short_examples = [e for e in test_data.examples if e.parse_tree.size <= 2] # for e in short_examples: # print e.parse_tree # print 'short examples num: ', len(short_examples) # dataset = test_data # test_data.get_dataset_by_ids([1,2,3,4,5,6,7,8,9,10], name='sample') # cProfile.run('decode_dataset(model, dataset)', sort=2) # from evaluation import decode_and_evaluate_ifttt if args.data_type == 'ifttt': decode_results = decode_and_evaluate_ifttt_by_split(model, test_data) else: dataset = eval(args.type) decode_results = decode_python_dataset(model, dataset) serialize_to_file(decode_results, args.saveto) if args.operation == 'evaluate': dataset = eval(args.type) if config.mode == 'self': decode_results_file = args.input decode_results = deserialize_from_file(decode_results_file) evaluate_decode_results(dataset, decode_results) elif config.mode == 'seq2tree': from evaluation import evaluate_seq2tree_sample_file evaluate_seq2tree_sample_file(config.seq2tree_sample_file, config.seq2tree_id_file, dataset) elif config.mode == 'seq2seq': from evaluation import evaluate_seq2seq_decode_results