def train(self):
dataset = self.train_data
nb_train_sample = dataset.count
index_array = np.arange(nb_train_sample)
nb_epoch = config.max_epoch
batch_size = config.batch_size
logging.info('begin training')
cum_updates = 0
patience_counter = 0
early_stop = False
history_valid_perf = []
history_valid_bleu = []
history_valid_acc = []
best_model_params = best_model_by_acc = best_model_by_bleu = None
# train_data_iter = DataIterator(self.train_data, batch_size)
for epoch in range(nb_epoch):
# train_data_iter.reset()
# if shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
# epoch begin
sys.stdout.write('Epoch %d' % epoch)
begin_time = time.time()
cum_nb_examples = 0
loss = 0.0
for batch_index, (batch_start, batch_end) in enumerate(batches):
# for batch_index, (examples, batch_ids) in enumerate(train_data_iter):
cum_updates += 1
batch_ids = index_array[batch_start:batch_end]
examples = dataset.get_examples(batch_ids)
cur_batch_size = len(examples)
inputs = dataset.get_prob_func_inputs(batch_ids)
if not config.enable_copy:
tgt_action_seq = inputs[1]
tgt_action_seq_type = inputs[2]
for i in xrange(cur_batch_size):
for t in xrange(tgt_action_seq[i].shape[0]):
if tgt_action_seq_type[i, t, 2] == 1:
# can only be copied
if tgt_action_seq_type[i, t, 1] == 0:
tgt_action_seq_type[i, t, 1] = 1
tgt_action_seq[i, t,
1] = 1 # index of <unk>
tgt_action_seq_type[i, t, 2] = 0
train_func_outputs = self.model.train_func(*inputs)
batch_loss = train_func_outputs[0]
logging.debug('prob_func finished computing')
cum_nb_examples += cur_batch_size
loss += batch_loss * batch_size
logging.debug('Batch %d, avg. loss = %f', batch_index,
batch_loss)
if batch_index == 4:
elapsed = time.time() - begin_time
eta = nb_train_sample / (cum_nb_examples / elapsed)
print ', eta %ds' % (eta)
sys.stdout.flush()
if cum_updates % config.valid_per_batch == 0:
logging.info('begin validation')
if config.data_type == 'ifttt':
decode_results = decoder.decode_ifttt_dataset(
self.model, self.val_data, verbose=False)
channel_acc, channel_func_acc, prod_f1 = evaluation.evaluate_ifttt_results(
self.val_data, decode_results, verbose=False)
val_perf = channel_func_acc
logging.info('channel accuracy: %f', channel_acc)
logging.info('channel+func accuracy: %f',
channel_func_acc)
logging.info('prod F1: %f', prod_f1)
else:
decode_results = decoder.decode_python_dataset(
self.model, self.val_data, verbose=False)
bleu, accuracy = evaluation.evaluate_decode_results(
self.val_data, decode_results, verbose=False)
val_perf = eval(config.valid_metric)
logging.info('avg. example bleu: %f', bleu)
logging.info('accuracy: %f', accuracy)
if len(history_valid_acc) == 0 or accuracy > np.array(
history_valid_acc).max():
best_model_by_acc = self.model.pull_params()
# logging.info('current model has best accuracy')
history_valid_acc.append(accuracy)
if len(history_valid_bleu) == 0 or bleu > np.array(
history_valid_bleu).max():
best_model_by_bleu = self.model.pull_params()
# logging.info('current model has best accuracy')
history_valid_bleu.append(bleu)
if len(history_valid_perf) == 0 or val_perf > np.array(
history_valid_perf).max():
best_model_params = self.model.pull_params()
patience_counter = 0
logging.info('save current best model')
self.model.save(
os.path.join(config.output_dir, 'model.npz'))
else:
patience_counter += 1
logging.info('hitting patience_counter: %d',
patience_counter)
if patience_counter >= config.train_patience:
logging.info('Early Stop!')
early_stop = True
break
history_valid_perf.append(val_perf)
if cum_updates % config.save_per_batch == 0:
self.model.save(
os.path.join(config.output_dir,
'model.iter%d' % cum_updates))
logging.info('[Epoch %d] cumulative loss = %f, (took %ds)', epoch,
loss / cum_nb_examples,
time.time() - begin_time)
np.savez(
os.path.join(config.output_dir,
'model-epoch{}.npz'.format(epoch)),
**self.model.pull_params())
if early_stop:
break
logging.info('training finished, save the best model')
# np.savez(os.path.join(config.output_dir, 'model.npz'), **best_model_params)
if config.data_type == 'django' or config.data_type == 'hs':
logging.info('save the best model by accuracy')
np.savez(os.path.join(config.output_dir, 'model.best_acc.npz'),
**best_model_by_acc)
logging.info('save the best model by bleu')
np.savez(os.path.join(config.output_dir, 'model.best_bleu.npz'),
**best_model_by_bleu)
# short_examples = [e for e in test_data.examples if e.parse_tree.size <= 2]
# for e in short_examples:
# print e.parse_tree
# print 'short examples num: ', len(short_examples)
# dataset = test_data # test_data.get_dataset_by_ids([1,2,3,4,5,6,7,8,9,10], name='sample')
# cProfile.run('decode_dataset(model, dataset)', sort=2)
# from evaluation import decode_and_evaluate_ifttt
if args.data_type == 'ifttt':
decode_results = decode_and_evaluate_ifttt_by_split(
model, test_data)
else:
dataset = eval(args.type)
decode_results = decode_python_dataset(model, dataset)
serialize_to_file(decode_results, args.saveto)
if args.operation == 'evaluate':
dataset = eval(args.type)
if config.mode == 'self':
decode_results_file = args.input
decode_results = deserialize_from_file(decode_results_file)
evaluate_decode_results(dataset, decode_results)
elif config.mode == 'seq2tree':
from evaluation import evaluate_seq2tree_sample_file
evaluate_seq2tree_sample_file(config.seq2tree_sample_file,
config.seq2tree_id_file, dataset)
elif config.mode == 'seq2seq':
def train(self, optimizer):
dataset = self.train_data
nb_train_sample = dataset.count
index_array = np.arange(nb_train_sample)
nb_epoch = config.max_epoch
batch_size = config.batch_size
logging.info('begin training')
cum_updates = 0
patience_counter = 0
early_stop = False
history_valid_perf = []
history_valid_bleu = []
history_valid_acc = []
history_losses = []
best_model_params = best_model_by_acc = best_model_by_bleu = None
# train_data_iter = DataIterator(self.train_data, batch_size)
original_start = time.time()
start_time = original_start
self.model.train()
for epoch in range(nb_epoch):
# train_data_iter.reset()
# if shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
# epoch begin
sys.stdout.write('Epoch %d' % epoch)
begin_time = time.time()
cum_nb_examples = 0
loss = 0.0
for batch_index, (batch_start, batch_end) in enumerate(batches):
optimizer.zero_grad()
cum_updates += 1
batch_ids = index_array[batch_start:batch_end]
examples = dataset.get_examples(batch_ids)
cur_batch_size = len(examples)
inputs = dataset.get_prob_func_inputs(batch_ids)
if not config.enable_copy:
tgt_action_seq = inputs[1]
tgt_action_seq_type = inputs[2]
for i in xrange(cur_batch_size):
for t in xrange(tgt_action_seq[i].shape[0]):
if tgt_action_seq_type[i, t, 2] == 1:
# can only be copied
if tgt_action_seq_type[i, t, 1] == 0:
tgt_action_seq_type[i, t, 1] = 1
tgt_action_seq[i, t,
1] = 1 # index of <unk>
tgt_action_seq_type[i, t, 2] = 0
encode_outputs = self.model(*inputs)
batch_loss = encode_outputs.item()
# print("=" * 60)
# trace_back(encode_outputs.grad_fn)
# print("=" * 60)
encode_outputs.backward()
optimizer.step()
logging.debug('prob_func finished computing')
cum_nb_examples += cur_batch_size
loss += batch_loss * batch_size
logging.debug('Batch %d, avg. loss = %f', batch_index,
batch_loss)
if batch_index == 4:
elapsed = time.time() - begin_time
eta = nb_train_sample / (cum_nb_examples / elapsed)
print(', eta %ds' % eta)
sys.stdout.flush()
if cum_updates % config.valid_per_batch == 0:
# if cum_updates % 40 == 0:
logging.info('begin validation')
if config.data_type == 'ifttt':
decode_results = decoder.decode_ifttt_dataset(
self.model, self.val_data, verbose=False)
channel_acc, channel_func_acc, prod_f1 = evaluation.evaluate_ifttt_results(
self.val_data, decode_results, verbose=False)
val_perf = channel_func_acc
logging.info('channel accuracy: %f', channel_acc)
logging.info('channel+func accuracy: %f',
channel_func_acc)
logging.info('prod F1: %f', prod_f1)
else:
decode_results = decoder.decode_python_dataset(
self.model,
train_data=dataset,
test_data=self.val_data,
verbose=True)
bleu, accuracy = evaluation.evaluate_decode_results(
self.val_data, decode_results, verbose=True)
val_perf = eval(config.valid_metric)
logging.info('avg. example bleu: %f', bleu)
logging.info('accuracy: %f', accuracy)
if len(history_valid_acc) == 0 or accuracy > np.array(
history_valid_acc).max():
best_model_by_acc = self.model.state_dict()
# logging.info('current model has best accuracy')
history_valid_acc.append(accuracy)
if len(history_valid_bleu) == 0 or bleu > np.array(
history_valid_bleu).max():
best_model_by_bleu = self.model.state_dict()
# logging.info('current model has best accuracy')
history_valid_bleu.append(bleu)
if len(history_valid_perf) == 0 or val_perf > np.array(
history_valid_perf).max():
best_model = self.model.state_dict()
patience_counter = 0
logging.info('save current best model')
torch.save(
self.model.state_dict(),
os.path.join(config.output_dir, 'model.npz'))
else:
patience_counter += 1
logging.info('hitting patience_counter: %d',
patience_counter)
if patience_counter >= config.train_patience:
logging.info('Early Stop!')
early_stop = True
break
history_valid_perf.append(val_perf)
if cum_updates % config.save_per_batch == 0:
torch.save(
self.model.state_dict(),
os.path.join(config.output_dir,
'model.iter%d' % cum_updates))
logging.info(
'[Epoch %d] cumulative loss = %f, (took %ds, total %f min) ',
epoch, loss / cum_nb_examples,
time.time() - begin_time,
second2minute(time.time(), original_start))
history_losses.append(loss / cum_nb_examples)
if early_stop:
break
logging.info('training finished, save the best model')
# torch.save(best_model.state_dict(), os.path.join(config.output_dir, 'model.npz'))
try:
with open('valid_perf.txt', 'w') as fp:
for x in history_valid_perf:
fp.write(x)
fp.write('\n')
fp.close()
with open('valid_bleu.txt', 'w') as fp:
for x in history_valid_bleu:
fp.write(x)
fp.write('\n')
fp.close()
with open('valid_acc.txt', 'w') as fp:
for x in history_valid_acc:
fp.write(x)
fp.write('\n')
fp.close()
with open('train_loss.txt', 'w') as fp:
for x in history_losses:
fp.write(x)
fp.write('\n')
fp.close()
except:
print "Fail to save result"
pass
if config.data_type == 'django' or config.data_type == 'hs':
logging.info('save the best model by accuracy')
torch.save(best_model_by_acc,
os.path.join(config.output_dir, 'model.best_acc.npz'))
logging.info('save the best model by bleu')
torch.save(best_model_by_bleu,
os.path.join(config.output_dir, 'model.best_bleu.npz'))
def train(self):
dataset = self.train_data
nb_train_sample = dataset.count
index_array = np.arange(nb_train_sample)
nb_epoch = config.max_epoch
batch_size = config.batch_size
logging.info('begin training')
cum_updates = 0
patience_counter = 0
early_stop = False
history_valid_perf = []
history_valid_bleu = []
history_valid_acc = []
best_model_params = best_model_by_acc = best_model_by_bleu = None
# train_data_iter = DataIterator(self.train_data, batch_size)
for epoch in range(nb_epoch):
# train_data_iter.reset()
# if shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
# epoch begin
sys.stdout.write('Epoch %d' % epoch)
begin_time = time.time()
cum_nb_examples = 0
loss = 0.0
for batch_index, (batch_start, batch_end) in enumerate(batches):
# for batch_index, (examples, batch_ids) in enumerate(train_data_iter):
cum_updates += 1
batch_ids = index_array[batch_start:batch_end]
examples = dataset.get_examples(batch_ids)
cur_batch_size = len(examples)
inputs = dataset.get_prob_func_inputs(batch_ids)
if not config.enable_copy:
tgt_action_seq = inputs[1]
tgt_action_seq_type = inputs[2]
for i in xrange(cur_batch_size):
for t in xrange(tgt_action_seq[i].shape[0]):
if tgt_action_seq_type[i, t, 2] == 1:
# can only be copied
if tgt_action_seq_type[i, t, 1] == 0:
tgt_action_seq_type[i, t, 1] = 1
tgt_action_seq[i, t, 1] = 1 # index of <unk>
tgt_action_seq_type[i, t, 2] = 0
train_func_outputs = self.model.train_func(*inputs)
batch_loss = train_func_outputs[0]
logging.debug('prob_func finished computing')
cum_nb_examples += cur_batch_size
loss += batch_loss * batch_size
logging.debug('Batch %d, avg. loss = %f', batch_index, batch_loss)
if batch_index == 4:
elapsed = time.time() - begin_time
eta = nb_train_sample / (cum_nb_examples / elapsed)
print ', eta %ds' % (eta)
sys.stdout.flush()
if cum_updates % config.valid_per_batch == 0:
logging.info('begin validation')
if config.data_type == 'ifttt':
decode_results = decoder.decode_ifttt_dataset(self.model, self.val_data, verbose=False)
channel_acc, channel_func_acc, prod_f1 = evaluation.evaluate_ifttt_results(self.val_data, decode_results, verbose=False)
val_perf = channel_func_acc
logging.info('channel accuracy: %f', channel_acc)
logging.info('channel+func accuracy: %f', channel_func_acc)
logging.info('prod F1: %f', prod_f1)
else:
decode_results = decoder.decode_python_dataset(self.model, self.val_data, verbose=False)
bleu, accuracy = evaluation.evaluate_decode_results(self.val_data, decode_results, verbose=False)
val_perf = eval(config.valid_metric)
logging.info('avg. example bleu: %f', bleu)
logging.info('accuracy: %f', accuracy)
if len(history_valid_acc) == 0 or accuracy > np.array(history_valid_acc).max():
best_model_by_acc = self.model.pull_params()
# logging.info('current model has best accuracy')
history_valid_acc.append(accuracy)
if len(history_valid_bleu) == 0 or bleu > np.array(history_valid_bleu).max():
best_model_by_bleu = self.model.pull_params()
# logging.info('current model has best accuracy')
history_valid_bleu.append(bleu)
if len(history_valid_perf) == 0 or val_perf > np.array(history_valid_perf).max():
best_model_params = self.model.pull_params()
patience_counter = 0
logging.info('save current best model')
self.model.save(os.path.join(config.output_dir, 'model.npz'))
else:
patience_counter += 1
logging.info('hitting patience_counter: %d', patience_counter)
if patience_counter >= config.train_patience:
logging.info('Early Stop!')
early_stop = True
break
history_valid_perf.append(val_perf)
if cum_updates % config.save_per_batch == 0:
self.model.save(os.path.join(config.output_dir, 'model.iter%d' % cum_updates))
logging.info('[Epoch %d] cumulative loss = %f, (took %ds)',
epoch,
loss / cum_nb_examples,
time.time() - begin_time)
if early_stop:
break
logging.info('training finished, save the best model')
np.savez(os.path.join(config.output_dir, 'model.npz'), **best_model_params)
if config.data_type == 'django' or config.data_type == 'hs':
logging.info('save the best model by accuracy')
np.savez(os.path.join(config.output_dir, 'model.best_acc.npz'), **best_model_by_acc)
logging.info('save the best model by bleu')
np.savez(os.path.join(config.output_dir, 'model.best_bleu.npz'), **best_model_by_bleu)
# from evaluation import decode_and_evaluate_ifttt
if args.data_type == 'ifttt':
decode_results = decode_and_evaluate_ifttt_by_split(model, test_data)
elif args.data_type == 'sql':
if not args.enable_retrieval:
dataset = eval(args.type)
decode_results = decode_sql_dataset(model, dataset)
else:
print args.type
print("sql here is with retrieval")
decode_results = decode_sql_dataset_with_retrieval(
model, train_data, dev_data, test_data, args.type, args.enable_retrieval)
else:
print args.type
decode_results = decode_python_dataset(
model, train_data, dev_data, test_data, args.type, args.enable_retrieval)
serialize_to_file(decode_results, args.saveto)
if args.operation == 'evaluate':
dataset = eval(args.type)
if config.mode == 'self':
decode_results_file = args.input
decode_results = deserialize_from_file(decode_results_file)
evaluate_decode_results(dataset, decode_results)
elif config.mode == 'seq2tree':
from evaluation import evaluate_seq2tree_sample_file
evaluate_seq2tree_sample_file(config.seq2tree_sample_file,
config.seq2tree_id_file, dataset)
elif config.mode == 'seq2seq':
# ==========================
# short_examples = [e for e in test_data.examples if e.parse_tree.size <= 2]
# for e in short_examples:
# print e.parse_tree
# print 'short examples num: ', len(short_examples)
# dataset = test_data # test_data.get_dataset_by_ids([1,2,3,4,5,6,7,8,9,10], name='sample')
# cProfile.run('decode_dataset(model, dataset)', sort=2)
# from evaluation import decode_and_evaluate_ifttt
if args.data_type == 'ifttt':
decode_results = decode_and_evaluate_ifttt_by_split(model, test_data)
else:
dataset = eval(args.type)
decode_results = decode_python_dataset(model, dataset)
serialize_to_file(decode_results, args.saveto)
if args.operation == 'evaluate':
dataset = eval(args.type)
if config.mode == 'self':
decode_results_file = args.input
decode_results = deserialize_from_file(decode_results_file)
evaluate_decode_results(dataset, decode_results)
elif config.mode == 'seq2tree':
from evaluation import evaluate_seq2tree_sample_file
evaluate_seq2tree_sample_file(config.seq2tree_sample_file, config.seq2tree_id_file, dataset)
elif config.mode == 'seq2seq':
from evaluation import evaluate_seq2seq_decode_results
