def prepare_dataset():
# vocab_freq_cutoff = 1 for atis
vocab_freq_cutoff = 2 # for geo query
grammar = ASDLGrammar.from_text(open('asdl/lang/prolog/prolog_asdl.txt').read())
transition_system = PrologTransitionSystem(grammar)
train_set = load_dataset(transition_system, 'data/jobs/train.txt')
test_set = load_dataset(transition_system, 'data/jobs/test.txt')
# generate vocabulary
src_vocab = VocabEntry.from_corpus([e.src_sent for e in train_set], size=5000, freq_cutoff=vocab_freq_cutoff)
primitive_tokens = [map(lambda a: a.action.token,
filter(lambda a: isinstance(a.action, GenTokenAction), e.tgt_actions))
for e in train_set]
primitive_vocab = VocabEntry.from_corpus(primitive_tokens, size=5000, freq_cutoff=0)
# generate vocabulary for the code tokens!
code_tokens = [transition_system.tokenize_code(e.tgt_code, mode='decoder') for e in train_set]
code_vocab = VocabEntry.from_corpus(code_tokens, size=5000, freq_cutoff=0)
vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab)
print('generated vocabulary %s' % repr(vocab), file=sys.stderr)
action_len = [len(e.tgt_actions) for e in chain(train_set, test_set)]
print('Max action len: %d' % max(action_len), file=sys.stderr)
print('Avg action len: %d' % np.average(action_len), file=sys.stderr)
print('Actions larger than 100: %d' % len(list(filter(lambda x: x > 100, action_len))), file=sys.stderr)
pickle.dump(train_set, open('data/jobs/train.bin', 'wb'))
pickle.dump(test_set, open('data/jobs/test.bin', 'wb'))
pickle.dump(vocab, open('data/jobs/vocab.freq2.bin', 'wb'))
def prepare_dataset():
# vocab_freq_cutoff = 1 for atis
vocab_freq_cutoff = 2 # for geo query
grammar = ASDLGrammar.from_text(open('asdl/lang/prolog/prolog_asdl.txt').read())
transition_system = PrologTransitionSystem(grammar)
train_set = load_dataset(transition_system, 'data/jobs/train.txt')
test_set = load_dataset(transition_system, 'data/jobs/test.txt')
# generate vocabulary
src_vocab = VocabEntry.from_corpus([e.src_sent for e in train_set], size=5000, freq_cutoff=vocab_freq_cutoff)
primitive_tokens = [map(lambda a: a.action.token,
filter(lambda a: isinstance(a.action, GenTokenAction), e.tgt_actions))
for e in train_set]
primitive_vocab = VocabEntry.from_corpus(primitive_tokens, size=5000, freq_cutoff=0)
# generate vocabulary for the code tokens!
code_tokens = [transition_system.tokenize_code(e.tgt_code, mode='decoder') for e in train_set]
code_vocab = VocabEntry.from_corpus(code_tokens, size=5000, freq_cutoff=0)
vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab)
print('generated vocabulary %s' % repr(vocab), file=sys.stderr)
action_len = [len(e.tgt_actions) for e in chain(train_set, test_set)]
print('Max action len: %d' % max(action_len), file=sys.stderr)
print('Avg action len: %d' % np.average(action_len), file=sys.stderr)
print('Actions larger than 100: %d' % len(list(filter(lambda x: x > 100, action_len))), file=sys.stderr)
pickle.dump(train_set, open('data/jobs/train.bin', 'wb'))
pickle.dump(test_set, open('data/jobs/test.bin', 'wb'))
pickle.dump(vocab, open('data/jobs/vocab.freq2.bin', 'wb'))
def process_regex_dataset():
grammar = ASDLGrammar.from_text(
open('asdl/lang/streg/streg_asdl.txt').read())
transition_system = StRegTransitionSystem(grammar)
train_set = StReg.load_regex_dataset(transition_system, "train")
val_set = StReg.load_regex_dataset(transition_system, "val")
testi_set = StReg.load_regex_dataset(transition_system, "testi")
teste_set = StReg.load_regex_dataset(transition_system, "teste")
# generate vocabulary
vocab_freq_cutoff = 2
src_vocab = VocabEntry.from_corpus([e.src_sent for e in train_set],
size=5000,
freq_cutoff=vocab_freq_cutoff)
primitive_tokens = [
map(
lambda a: a.action.token,
filter(lambda a: isinstance(a.action, GenTokenAction),
e.tgt_actions)) for e in train_set
]
primitive_vocab = VocabEntry.from_corpus(primitive_tokens,
size=5000,
freq_cutoff=0)
# generate vocabulary for the code tokens!
code_tokens = [
transition_system.tokenize_code(e.tgt_code, mode='decoder')
for e in train_set
]
code_vocab = VocabEntry.from_corpus(code_tokens,
size=5000,
freq_cutoff=0)
vocab = Vocab(source=src_vocab,
primitive=primitive_vocab,
code=code_vocab)
print('generated vocabulary %s' % repr(vocab), file=sys.stderr)
action_len = [
len(e.tgt_actions) for e in chain(train_set, testi_set, teste_set)
]
print('Max action len: %d' % max(action_len), file=sys.stderr)
print('Avg action len: %d' % np.average(action_len), file=sys.stderr)
print('Actions larger than 100: %d' %
len(list(filter(lambda x: x > 100, action_len))),
file=sys.stderr)
pickle.dump(train_set, open('data/streg/train.bin', 'wb'))
pickle.dump(val_set, open('data/streg/val.bin', 'wb'))
pickle.dump(testi_set, open('data/streg/testi.bin', 'wb'))
pickle.dump(teste_set, open('data/streg/teste.bin', 'wb'))
pickle.dump(
vocab,
open('data/streg/vocab.freq%d.bin' % (vocab_freq_cutoff), 'wb'))
def prepare_dataset(data_path):
grammar = ASDLGrammar.from_text(
open('../../asdl/lang/sql/sql_asdl.txt').read())
transition_system = SqlTransitionSystem(grammar)
datasets = []
for file in ['dev', 'test', 'train']:
print('processing %s' % file, file=sys.stderr)
dataset_path = os.path.join(data_path, file + '.jsonl')
table_path = os.path.join(data_path, file + '.tables.jsonl')
dataset = load_dataset(transition_system, dataset_path, table_path)
pickle.dump(dataset, open('../../data/wikisql1/%s.bin' % file, 'wb'))
datasets.append(dataset)
train_set = datasets[2]
dev_set = datasets[0]
test_set = datasets[1]
# generate vocabulary
src_vocab = VocabEntry.from_corpus([e.src_sent for e in train_set],
size=100000,
freq_cutoff=2)
primitive_vocab = VocabEntry()
primitive_vocab.add('</primitive>')
vocab = Vocab(source=src_vocab, primitive=primitive_vocab)
print('generated vocabulary %s' % repr(vocab), file=sys.stderr)
pickle.dump(vocab, open('../../data/wikisql1/vocab.bin', 'wb'))
action_len = [
len(e.tgt_actions) for e in chain(train_set, dev_set, test_set)
]
print('Max action len: %d' % max(action_len), file=sys.stderr)
print('Avg action len: %d' % np.average(action_len), file=sys.stderr)
print('Actions larger than 100: %d' %
len(list(filter(lambda x: x > 100, action_len))),
file=sys.stderr)
def train_rerank_feature(args):
train_set = Dataset.from_bin_file(args.train_file)
dev_set = Dataset.from_bin_file(args.dev_file)
vocab = pickle.load(open(args.vocab, 'rb'))
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = TransitionSystem.get_class_by_lang(args.lang)(grammar)
train_paraphrase_model = args.mode == 'train_paraphrase_identifier'
def _get_feat_class():
if args.mode == 'train_reconstructor':
return Reconstructor
elif args.mode == 'train_paraphrase_identifier':
return ParaphraseIdentificationModel
def _filter_hyps(_decode_results):
for i in range(len(_decode_results)):
valid_hyps = []
for hyp in _decode_results[i]:
try:
transition_system.tokenize_code(hyp.code)
valid_hyps.append(hyp)
except: pass
_decode_results[i] = valid_hyps
model = _get_feat_class()(args, vocab, transition_system)
if args.glorot_init:
print('use glorot initialization', file=sys.stderr)
nn_utils.glorot_init(model.parameters())
model.train()
if args.cuda: model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
# if training the paraphrase model, also load in decoding results
if train_paraphrase_model:
print('load training decode results [%s]' % args.train_decode_file, file=sys.stderr)
train_decode_results = pickle.load(open(args.train_decode_file, 'rb'))
_filter_hyps(train_decode_results)
train_decode_results = {e.idx: hyps for e, hyps in zip(train_set, train_decode_results)}
print('load dev decode results [%s]' % args.dev_decode_file, file=sys.stderr)
dev_decode_results = pickle.load(open(args.dev_decode_file, 'rb'))
_filter_hyps(dev_decode_results)
dev_decode_results = {e.idx: hyps for e, hyps in zip(dev_set, dev_decode_results)}
def evaluate_ppl():
model.eval()
cum_loss = 0.
cum_tgt_words = 0.
for batch in dev_set.batch_iter(args.batch_size):
loss = -model.score(batch).sum()
cum_loss += loss.data.item()
cum_tgt_words += sum(len(e.src_sent) + 1 for e in batch) # add ending </s>
ppl = np.exp(cum_loss / cum_tgt_words)
model.train()
return ppl
def evaluate_paraphrase_acc():
model.eval()
labels = []
for batch in dev_set.batch_iter(args.batch_size):
probs = model.score(batch).exp().data.cpu().numpy()
for p in probs:
labels.append(p >= 0.5)
# get negative examples
batch_decoding_results = [dev_decode_results[e.idx] for e in batch]
batch_negative_examples = [get_negative_example(e, _hyps, type='best')
for e, _hyps in zip(batch, batch_decoding_results)]
batch_negative_examples = list(filter(None, batch_negative_examples))
probs = model.score(batch_negative_examples).exp().data.cpu().numpy()
for p in probs:
labels.append(p < 0.5)
acc = np.average(labels)
model.train()
return acc
def get_negative_example(_example, _hyps, type='sample'):
incorrect_hyps = [hyp for hyp in _hyps if not hyp.is_correct]
if incorrect_hyps:
incorrect_hyp_scores = [hyp.score for hyp in incorrect_hyps]
if type in ('best', 'sample'):
if type == 'best':
sample_idx = np.argmax(incorrect_hyp_scores)
sampled_hyp = incorrect_hyps[sample_idx]
else:
incorrect_hyp_probs = [np.exp(score) for score in incorrect_hyp_scores]
incorrect_hyp_probs = np.array(incorrect_hyp_probs) / sum(incorrect_hyp_probs)
sampled_hyp = np.random.choice(incorrect_hyps, size=1, p=incorrect_hyp_probs)
sampled_hyp = sampled_hyp[0]
sample = Example(idx='negative-%s' % _example.idx,
src_sent=_example.src_sent,
tgt_code=sampled_hyp.code,
tgt_actions=None,
tgt_ast=None)
return sample
elif type == 'all':
samples = []
for i, hyp in enumerate(incorrect_hyps):
sample = Example(idx='negative-%s-%d' % (_example.idx, i),
src_sent=_example.src_sent,
tgt_code=hyp.code,
tgt_actions=None,
tgt_ast=None)
samples.append(sample)
return samples
else:
return None
print('begin training decoder, %d training examples, %d dev examples' % (len(train_set), len(dev_set)), file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size, shuffle=True):
batch_examples = [e for e in batch_examples if len(e.tgt_actions) <= args.decode_max_time_step]
if train_paraphrase_model:
positive_examples_num = len(batch_examples)
labels = [0] * len(batch_examples)
negative_samples = []
batch_decoding_results = [train_decode_results[e.idx] for e in batch_examples]
# sample negative examples
for example, hyps in zip(batch_examples, batch_decoding_results):
if hyps:
negative_sample = get_negative_example(example, hyps, type=args.negative_sample_type)
if negative_sample:
if isinstance(negative_sample, Example):
negative_samples.append(negative_sample)
labels.append(1)
else:
negative_samples.extend(negative_sample)
labels.extend([1] * len(negative_sample))
batch_examples += negative_samples
train_iter += 1
optimizer.zero_grad()
nll = -model(batch_examples)
if train_paraphrase_model:
idx_tensor = Variable(torch.LongTensor(labels).unsqueeze(-1), requires_grad=False)
if args.cuda: idx_tensor = idx_tensor.cuda()
loss = torch.gather(nll, 1, idx_tensor)
else:
loss = nll
# print(loss.data)
loss_val = torch.sum(loss).data.item()
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
loss.backward()
# clip gradient
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
print('[Iter %d] encoder loss=%.5f' %
(train_iter,
report_loss / report_examples),
file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' % (epoch, time.time() - epoch_begin), file=sys.stderr)
# perform validation
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
# evaluate dev_score
dev_acc = evaluate_paraphrase_acc() if train_paraphrase_model else -evaluate_ppl()
print('[Epoch %d] dev_score=%.5f took %ds' % (epoch, dev_acc, time.time() - eval_start), file=sys.stderr)
is_better = history_dev_scores == [] or dev_acc > max(history_dev_scores)
history_dev_scores.append(dev_acc)
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save currently the best model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if patience == args.patience:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial:
print('early stop!', file=sys.stderr)
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' % lr, file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin', map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
optimizer = torch.optim.Adam(model.inference_model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
def compare_ast(self, hyp_ast, ref_ast):
return is_equal_ast(hyp_ast, ref_ast)
def surface_code_to_ast(self, code):
return pdf_to_ast(self.grammar, code)
def ast_to_surface_code(self, asdl_ast):
return ast_to_pdf(asdl_ast)
def get_primitive_field_actions(self, realized_field):
assert realized_field.cardinality == 'single'
if realized_field.value is not None:
return [GenTokenAction(realized_field.value)]
else:
return []
if __name__ == '__main__':
path = 'pdfs'
grammar = ASDLGrammar.from_text(open('pdf_asdl.txt').read())
transition_system = PdfTransitionSystem(grammar)
for item in os.listdir(path):
item_path = os.path.join(path, item)
pdf = PdfReader(item_path)
for page in pdf.pages:
ast_tree = pdf_to_ast(grammar, page)
ast_tree.sanity_check()
print(ast_to_pdf(ast_tree))
def train_decoder(args):
train_set = Dataset.from_bin_file(args.train_file)
dev_set = Dataset.from_bin_file(args.dev_file)
vocab = pickle.load(open(args.vocab))
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = TransitionSystem.get_class_by_lang(args.lang)(grammar)
model = Reconstructor(args, vocab, transition_system)
model.train()
if args.cuda: model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
def evaluate_ppl():
model.eval()
cum_loss = 0.
cum_tgt_words = 0.
for batch in dev_set.batch_iter(args.batch_size):
loss = -model.score(batch).sum()
cum_loss += loss.data[0]
cum_tgt_words += sum(len(e.src_sent) + 1 for e in batch) # add ending </s>
ppl = np.exp(cum_loss / cum_tgt_words)
model.train()
return ppl
print('begin training decoder, %d training examples, %d dev examples' % (len(train_set), len(dev_set)), file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size, shuffle=True):
batch_examples = [e for e in batch_examples if len(e.tgt_actions) <= args.decode_max_time_step]
# batch_examples = [e for e in train_set.examples if e.idx in [10192, 10894, 9706, 4659, 5609, 1442, 5849, 10644, 4592, 1875]]
train_iter += 1
optimizer.zero_grad()
loss = -model.score(batch_examples)
# print(loss.data)
loss_val = torch.sum(loss).data[0]
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
loss.backward()
# clip gradient
grad_norm = torch.nn.utils.clip_grad_norm(model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
print('[Iter %d] encoder loss=%.5f' %
(train_iter,
report_loss / report_examples),
file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' % (epoch, time.time() - epoch_begin), file=sys.stderr)
# model_file = args.save_to + '.iter%d.bin' % train_iter
# print('save model to [%s]' % model_file, file=sys.stderr)
# model.save(model_file)
# perform validation
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
# evaluate ppl
ppl = evaluate_ppl()
print('[Epoch %d] ppl=%.5f took %ds' % (epoch, ppl, time.time() - eval_start), file=sys.stderr)
dev_acc = -ppl
is_better = history_dev_scores == [] or dev_acc > max(history_dev_scores)
history_dev_scores.append(dev_acc)
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save currently the best model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if patience == args.patience:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial:
print('early stop!', file=sys.stderr)
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' % lr, file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin', map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
optimizer = torch.optim.Adam(model.inference_model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
def train(args):
"""Maximum Likelihood Estimation"""
# load in train/dev set
print("loading files")
print(f"Loading Train at {args.train_file}")
train_set = Dataset.from_bin_file(args.train_file)
train_ids = [e.idx.split('-')[-1] for e in train_set.examples]
print(f"{len(train_set.examples)} total examples")
print('Checking ids:')
for idx in ['4170655', '13704860', '4170655', '13704860', '3862010']:
print(f"\t{idx} is {'not in' if idx not in train_ids else 'in'} train")
print("")
print(f"First 5 Examples in Train:")
for i in range(10):
print(f'\tExample {i + 1}(idx:{train_set.examples[i].idx}):')
print(f"\t\tSource:{repr(' '.join(train_set.all_source[i])[:100])}")
print(f"\t\tTarget:{repr(train_set.all_targets[i][:100])}")
if args.dev_file:
print(f"Loading dev at {args.dev_file}")
dev_set = Dataset.from_bin_file(args.dev_file)
else:
dev_set = Dataset(examples=[])
print("Loading vocab")
vocab = pickle.load(open(args.vocab, 'rb'))
print(f"Loading grammar {args.asdl_file}")
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = Registrable.by_name(args.transition_system)(grammar)
parser_cls = Registrable.by_name(args.parser) # TODO: add arg
if args.pretrain:
print('Finetune with: ', args.pretrain)
model = parser_cls.load(model_path=args.pretrain, cuda=args.cuda)
else:
model = parser_cls(args, vocab, transition_system)
model.train()
evaluator = Registrable.by_name(args.evaluator)(transition_system,
args=args)
if args.cuda: model.cuda()
optimizer_cls = eval('torch.optim.%s' %
args.optimizer) # FIXME: this is evil!
optimizer = optimizer_cls(model.parameters(), lr=args.lr)
if not args.pretrain:
if args.uniform_init:
print('uniformly initialize parameters [-%f, +%f]' %
(args.uniform_init, args.uniform_init))
nn_utils.uniform_init(-args.uniform_init, args.uniform_init,
model.parameters())
elif args.glorot_init:
print('use glorot initialization')
nn_utils.glorot_init(model.parameters())
# load pre-trained word embedding (optional)
if args.glove_embed_path:
print('load glove embedding from: %s' % args.glove_embed_path)
glove_embedding = GloveHelper(args.glove_embed_path)
glove_embedding.load_to(model.src_embed, vocab.source)
print('begin training, %d training examples, %d dev examples' %
(len(train_set), len(dev_set)),
file=sys.stderr)
print('vocab: %s' % repr(vocab))
epoch = train_iter = 0
report_loss = report_examples = report_sup_att_loss = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size,
shuffle=True):
batch_examples = [
e for e in batch_examples
if len(e.tgt_actions) <= args.decode_max_time_step
]
train_iter += 1
optimizer.zero_grad()
ret_val = model.score(batch_examples)
loss = -ret_val[0]
# print(loss.data)
loss_val = torch.sum(loss).data.item()
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
if args.sup_attention:
att_probs = ret_val[1]
if att_probs:
sup_att_loss = -torch.log(torch.cat(att_probs)).mean()
sup_att_loss_val = sup_att_loss.data[0]
report_sup_att_loss += sup_att_loss_val
loss += sup_att_loss
loss.backward()
# clip gradient
if args.clip_grad > 0.:
grad_norm = torch.nn.utils.clip_grad_norm_(
model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
log_str = '[Iter %d] encoder loss=%.5f' % (
train_iter, report_loss / report_examples)
if args.sup_attention:
log_str += ' supervised attention loss=%.5f' % (
report_sup_att_loss / report_examples)
report_sup_att_loss = 0.
print(log_str)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' %
(epoch, time.time() - epoch_begin))
if args.save_all_models:
model_file = args.save_to + '.iter%d.bin' % train_iter
print('save model to [%s]' % model_file)
model.save(model_file)
# perform validation
is_better = False
if args.dev_file:
if epoch % args.valid_every_epoch == 0:
print('[Epoch %d] begin validation' % epoch)
eval_start = time.time()
eval_results = evaluation.evaluate(
dev_set.examples,
model,
evaluator,
args,
verbose=False,
eval_top_pred_only=args.eval_top_pred_only)
dev_score = eval_results[evaluator.default_metric]
print(
'[Epoch %d] evaluate details: %s, dev %s: %.5f (took %ds)'
% (epoch, eval_results, evaluator.default_metric,
dev_score, time.time() - eval_start))
is_better = history_dev_scores == [] or dev_score > max(
history_dev_scores)
history_dev_scores.append(dev_score)
else:
is_better = True
if args.decay_lr_every_epoch and epoch > args.lr_decay_after_epoch:
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('decay learning rate to %f' % lr)
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save the current model ..')
print('save model to [%s]' % model_file)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience and epoch >= args.lr_decay_after_epoch:
patience += 1
print('hit patience %d' % patience)
if epoch == args.max_epoch:
print('reached max epoch, stop!')
exit(0)
if patience >= args.patience and epoch >= args.lr_decay_after_epoch:
num_trial += 1
print('hit #%d trial' % num_trial)
if num_trial == args.max_num_trial:
print('early stop!')
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' %
lr)
# load model
params = torch.load(args.save_to + '.bin',
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer')
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers')
optimizer.load_state_dict(
torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
def train_decoder(args):
train_set = Dataset.from_bin_file(args.train_file)
dev_set = Dataset.from_bin_file(args.dev_file)
vocab = pickle.load(open(args.vocab))
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = TransitionSystem.get_class_by_lang(args.lang)(grammar)
model = Reconstructor(args, vocab, transition_system)
model.train()
if args.cuda: model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
def evaluate_ppl():
model.eval()
cum_loss = 0.
cum_tgt_words = 0.
for batch in dev_set.batch_iter(args.batch_size):
loss = -model.score(batch).sum()
cum_loss += loss.data[0]
cum_tgt_words += sum(len(e.src_sent) + 1
for e in batch) # add ending </s>
ppl = np.exp(cum_loss / cum_tgt_words)
model.train()
return ppl
print('begin training decoder, %d training examples, %d dev examples' %
(len(train_set), len(dev_set)),
file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size,
shuffle=True):
batch_examples = [
e for e in batch_examples
if len(e.tgt_actions) <= args.decode_max_time_step
]
# batch_examples = [e for e in train_set.examples if e.idx in [10192, 10894, 9706, 4659, 5609, 1442, 5849, 10644, 4592, 1875]]
train_iter += 1
optimizer.zero_grad()
loss = -model.score(batch_examples)
# print(loss.data)
loss_val = torch.sum(loss).data[0]
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
loss.backward()
# clip gradient
grad_norm = torch.nn.utils.clip_grad_norm(model.parameters(),
args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
print('[Iter %d] encoder loss=%.5f' %
(train_iter, report_loss / report_examples),
file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' %
(epoch, time.time() - epoch_begin),
file=sys.stderr)
# model_file = args.save_to + '.iter%d.bin' % train_iter
# print('save model to [%s]' % model_file, file=sys.stderr)
# model.save(model_file)
# perform validation
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
# evaluate ppl
ppl = evaluate_ppl()
print('[Epoch %d] ppl=%.5f took %ds' %
(epoch, ppl, time.time() - eval_start),
file=sys.stderr)
dev_acc = -ppl
is_better = history_dev_scores == [] or dev_acc > max(
history_dev_scores)
history_dev_scores.append(dev_acc)
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save currently the best model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if patience == args.patience:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial:
print('early stop!', file=sys.stderr)
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' %
lr,
file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin',
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
optimizer = torch.optim.Adam(
model.inference_model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(
torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
# coding=utf-8
import ast
from asdl.asdl import ASDLGrammar
from asdl.lang.py.py_asdl_helper import *
from asdl.lang.py.py_transition_system import *
from asdl.hypothesis import *
import astor
if __name__ == '__main__':
# read in the grammar specification of Python 2.7, defined in ASDL
asdl_text = open('py_asdl.txt').read()
grammar = ASDLGrammar.from_text(asdl_text)
py_code = """pandas.read('file.csv', nrows=100)"""
# get the (domain-specific) python AST of the example Python code snippet
py_ast = ast.parse(py_code)
# convert the python AST into general-purpose ASDL AST used by tranX
asdl_ast = python_ast_to_asdl_ast(py_ast.body[0], grammar)
print('String representation of the ASDL AST: \n%s' % asdl_ast.to_string())
print('Size of the AST: %d' % asdl_ast.size)
# we can also convert the ASDL AST back into Python AST
py_ast_reconstructed = asdl_ast_to_python_ast(asdl_ast, grammar)
# initialize the Python transition parser
parser = PythonTransitionSystem(grammar)
# get the sequence of gold-standard actions to construct the ASDL AST
def train(args):
"""Maximum Likelihood Estimation"""
# load in train/dev set
train_set = Dataset.from_bin_file(args.train_file)
if args.dev_file:
dev_set = Dataset.from_bin_file(args.dev_file)
else:
dev_set = Dataset(examples=[])
vocab = pickle.load(open(args.vocab, 'rb'))
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = Registrable.by_name(args.transition_system)(grammar)
parser_cls = Registrable.by_name(args.parser) # TODO: add arg
if args.pretrain:
print('Finetune with: ', args.pretrain, file=sys.stderr)
model = parser_cls.load(model_path=args.pretrain, cuda=args.cuda)
else:
model = parser_cls(args, vocab, transition_system)
model.train()
evaluator = Registrable.by_name(args.evaluator)(transition_system,
args=args)
if args.cuda: model.cuda()
trainable_parameters = [
p for n, p in model.named_parameters()
if 'automodel' not in n and p.requires_grad
]
bert_parameters = [
p for n, p in model.named_parameters()
if 'automodel' in n and p.requires_grad
]
optimizer_cls = eval('torch.optim.%s' %
args.optimizer) # FIXME: this is evil!
if args.finetune_bert:
optimizer = optimizer_cls(trainable_parameters, lr=args.lr)
else:
optimizer = optimizer_cls(trainable_parameters + bert_parameters,
lr=args.lr)
if not args.pretrain:
if args.uniform_init:
print('uniformly initialize parameters [-%f, +%f]' %
(args.uniform_init, args.uniform_init),
file=sys.stderr)
nn_utils.uniform_init(-args.uniform_init, args.uniform_init,
trainable_parameters)
elif args.glorot_init:
print('use glorot initialization', file=sys.stderr)
nn_utils.glorot_init(trainable_parameters)
# load pre-trained word embedding (optional)
if args.glove_embed_path:
print('load glove embedding from: %s' % args.glove_embed_path,
file=sys.stderr)
glove_embedding = GloveHelper(args.glove_embed_path)
glove_embedding.load_to(model.src_embed, vocab.source)
print('begin training, %d training examples, %d dev examples' %
(len(train_set), len(dev_set)),
file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = report_sup_att_loss = 0.
history_dev_scores = []
num_trial = patience = 0
if args.warmup_step > 0 and args.annealing_step > args.warmup_step:
lr_scheduler = get_linear_schedule_with_warmup(optimizer,
args.warmup_step,
args.annealing_step)
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size,
shuffle=True):
batch_examples = [
e for e in batch_examples
if len(e.tgt_actions) <= args.decode_max_time_step
]
train_iter += 1
optimizer.zero_grad()
ret_val = model.score(batch_examples)
loss = -ret_val[0]
# print(loss.data)
loss_val = torch.sum(loss).data.item()
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
if args.sup_attention:
att_probs = ret_val[1]
if att_probs:
sup_att_loss = -torch.log(torch.cat(att_probs)).mean()
sup_att_loss_val = sup_att_loss.data[0]
report_sup_att_loss += sup_att_loss_val
loss += sup_att_loss
loss.backward()
# clip gradient
if args.clip_grad > 0.:
if args.finetune_bert:
grad_norm = torch.nn.utils.clip_grad_norm_(
trainable_parameters + bert_parameters, args.clip_grad)
else:
grad_norm = torch.nn.utils.clip_grad_norm_(
trainable_parameters, args.clip_grad)
optimizer.step()
# warmup and annealing
if args.warmup_step > 0 and args.annealing_step > args.warmup_step:
lr_scheduler.step()
if train_iter % args.log_every == 0:
lr = optimizer.param_groups[0]['lr']
log_str = '[Iter %d] encoder loss=%.5f, lr=%.6f' % (
train_iter, report_loss / report_examples, lr)
if args.sup_attention:
log_str += ' supervised attention loss=%.5f' % (
report_sup_att_loss / report_examples)
report_sup_att_loss = 0.
print(log_str, file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' %
(epoch, time.time() - epoch_begin),
file=sys.stderr)
if args.save_all_models:
model_file = args.save_to + '.iter%d.bin' % train_iter
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# perform validation
is_better = False
if args.dev_file:
if epoch % args.valid_every_epoch == 0:
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
eval_results = evaluation.evaluate(
dev_set.examples,
model,
evaluator,
args,
verbose=False,
eval_top_pred_only=args.eval_top_pred_only)
dev_score = eval_results[evaluator.default_metric]
print(
'[Epoch %d] evaluate details: %s, dev %s: %.5f (took %ds)'
% (epoch, eval_results, evaluator.default_metric,
dev_score, time.time() - eval_start),
file=sys.stderr)
is_better = history_dev_scores == [] or dev_score > max(
history_dev_scores)
history_dev_scores.append(dev_score)
else:
is_better = True
if args.decay_lr_every_epoch and epoch > args.lr_decay_after_epoch:
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('decay learning rate to %f' % lr, file=sys.stderr)
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save the current model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience and epoch >= args.lr_decay_after_epoch:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if epoch == args.max_epoch:
print('reached max epoch, stop!', file=sys.stderr)
exit(0)
if patience >= args.patience and epoch >= args.lr_decay_after_epoch:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial or (
args.warmup_step > 0
and args.annealing_step > args.warmup_step):
print('early stop!', file=sys.stderr)
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' %
lr,
file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin',
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
if args.finetune_bert:
optimizer = torch.optim.Adam(trainable_parameters +
bert_parameters,
lr=lr)
else:
optimizer = torch.optim.Adam(trainable_parameters, lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(
torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
# coding=utf-8
import ast
from asdl.asdl import ASDLGrammar
from asdl.lang.py.py_asdl_helper import *
from asdl.lang.py.py_transition_system import *
from asdl.hypothesis import *
import astor
if __name__ == '__main__':
# read in the grammar specification of Python 2.7, defined in ASDL
asdl_text = open('py_asdl.txt').read()
grammar = ASDLGrammar.from_text(asdl_text)
py_code = """pandas.read('file.csv', nrows=100)"""
# get the (domain-specific) python AST of the example Python code snippet
py_ast = ast.parse(py_code)
# convert the python AST into general-purpose ASDL AST used by tranX
asdl_ast = python_ast_to_asdl_ast(py_ast.body[0], grammar)
print('String representation of the ASDL AST: \n%s' % asdl_ast.to_string())
print('Size of the AST: %d' % asdl_ast.size)
# we can also convert the ASDL AST back into Python AST
py_ast_reconstructed = asdl_ast_to_python_ast(asdl_ast, grammar)
# initialize the Python transition parser
parser = PythonTransitionSystem(grammar)
# get the sequence of gold-standard actions to construct the ASDL AST
def train(args):
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = TransitionSystem.get_class_by_lang(args.lang)(grammar)
train_set = Dataset.from_bin_file(args.train_file)
dev_set = Dataset.from_bin_file(args.dev_file)
vocab = pkl.load(open(args.vocab, 'rb'))
model = Parser(args, vocab, transition_system)
model.train()
if args.cuda: model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
print('begin training, %d training examples, %d dev examples' %
(len(train_set), len(dev_set)),
file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size,
shuffle=True):
batch_examples = [
e for e in batch_examples
if len(e.tgt_actions) <= args.decode_max_time_step
]
train_iter += 1
optimizer.zero_grad()
loss = -model.score(batch_examples)
# print(loss.data)
loss_val = torch.sum(loss).data[0]
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
loss.backward()
# clip gradient
if args.clip_grad > 0.:
grad_norm = torch.nn.utils.clip_grad_norm(
model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
print('[Iter %d] encoder loss=%.5f' %
(train_iter, report_loss / report_examples),
file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' %
(epoch, time.time() - epoch_begin),
file=sys.stderr)
# model_file = args.save_to + '.iter%d.bin' % train_iter
# print('save model to [%s]' % model_file, file=sys.stderr)
# model.save(model_file)
# perform validation
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
eval_results = evaluation.evaluate(dev_set.examples,
model,
args,
verbose=True)
dev_acc = eval_results['accuracy']
print('[Epoch %d] code generation accuracy=%.5f took %ds' %
(epoch, dev_acc, time.time() - eval_start),
file=sys.stderr)
is_better = history_dev_scores == [] or dev_acc > max(
history_dev_scores)
history_dev_scores.append(dev_acc)
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save currently the best model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif epoch == args.max_epoch:
print('reached max epoch, stop!', file=sys.stderr)
exit(0)
elif patience < args.patience:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if patience == args.patience:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial:
print('early stop!', file=sys.stderr)
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' %
lr,
file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin',
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
optimizer = torch.optim.Adam(
model.inference_model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(
torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
def train_reranker_and_test(args):
print('load dataset [test %s], [dev %s]' % (args.test_file, args.dev_file))
test_set = Dataset.from_bin_file(args.test_file)
dev_set = Dataset.from_bin_file(args.dev_file)
features = []
i = 0
while i < len(args.features):
feat_name = args.features[i]
feat_cls = Registrable.by_name(feat_name)
print('Add feature %s' % feat_name)
if issubclass(feat_cls, nn.Module):
feat_path = os.path.join('saved_models/conala/',
args.features[i] + '.bin')
feat_inst = feat_cls.load(feat_path)
print('Load feature %s from %s' % (feat_name, feat_path))
else:
feat_inst = feat_cls()
features.append(feat_inst)
i += 1
print(features)
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = Registrable.by_name(args.parser)(grammar)
evaluator = Registrable.by_name(args.evaluator)(transition_system)
print('load dev decode results [%s]' % args.dev_decode_file)
dev_decode_results = pickle.load(open(args.dev_decode_file, 'rb'))
dev_eval_results = evaluator.evaluate_dataset(dev_set,
dev_decode_results,
fast_mode=False)
print('load test decode results [%s]' % args.test_decode_file)
test_decode_results = pickle.load(open(args.test_decode_file, 'rb'))
test_eval_results = evaluator.evaluate_dataset(test_set,
test_decode_results,
fast_mode=False)
print('Dev Eval Results')
print(dev_eval_results)
print('Test Eval Results')
print(test_eval_results)
if args.load_reranker:
reranker = GridSearchReranker.load(args.load_reranker)
else:
reranker = GridSearchReranker(features,
transition_system=transition_system)
if args.num_workers == 1:
reranker.train(dev_set.examples,
dev_decode_results,
evaluator=evaluator)
else:
reranker.train_multiprocess(dev_set.examples,
dev_decode_results,
evaluator=evaluator,
num_workers=args.num_workers)
if args.save_to:
print('Save Reranker to %s' % args.save_to)
reranker.save(args.save_to)
test_score_with_rerank = reranker.compute_rerank_performance(
test_set.examples,
test_decode_results,
verbose=True,
evaluator=evaluator,
args=args)
print('Test Eval Results After Reranking')
print(test_score_with_rerank)
def train(args):
"""Maximum Likelihood Estimation"""
# load in train/dev set
train_set = Dataset.from_bin_file(args.train_file)
vocab = pickle.load(open(args.vocab, 'rb'))
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = Registrable.by_name(args.transition_system)(grammar)
parser_cls = Registrable.by_name(args.parser) # TODO: add arg
model = parser_cls(args, vocab, transition_system)
model.train()
if args.cuda: model.cuda()
optimizer_cls = eval('torch.optim.%s' %
args.optimizer) # FIXME: this is evil!
optimizer = optimizer_cls(model.parameters(), lr=args.lr)
nn_utils.glorot_init(model.parameters())
print('begin training, %d training examples' % len(train_set),
file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = 0.
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size,
shuffle=True):
batch_examples = [e for e in batch_examples if len(e.tgt_actions)]
train_iter += 1
optimizer.zero_grad()
ret_val = model.score(batch_examples)
loss = -ret_val
# print(loss.data)
loss_val = torch.sum(loss).data
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
loss.backward()
# clip gradient
if args.clip_grad > 0.:
grad_norm = torch.nn.utils.clip_grad_norm_(
model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
log_str = '[Iter %d] loss=%.5f' % (train_iter, report_loss /
report_examples)
print(log_str, file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' %
(epoch, time.time() - epoch_begin),
file=sys.stderr)
model_file = args.save_to + '.iter%d.bin' % train_iter
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
if epoch == args.max_epoch:
print('reached max epoch, stop!', file=sys.stderr)
exit(0)
def train(args):
"""Maximum Likelihood Estimation"""
args.dropout=0.1
args.hidden_size=512
args.embed_size=512
args.beam_size=1
args.action_embed_size=512
args.field_embed_size=64
args.type_embed_size=64
args.lr=3e-5
args.max_epoch=50
args.cuda=True if torch.cuda.is_available() else False
# args.cuda=False
args.decay_lr_every_epoch=True
args.sup_attention=False
# load in train/dev set
train_set = Dataset.from_bin_file(args.train_file)
# train_set = Dataset.from_bin_file_joint('data/conala/src.bin','data/conala/pretrain.bin')
print (len(train_set),'this is the length of train set')
if args.dev_file:
dev_set = Dataset.from_bin_file_test(args.test_file)
else: dev_set = Dataset(examples=[])
vocab = pickle.load(open(args.vocab, 'rb'))
vocab_src=vocab
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = Registrable.by_name(args.transition_system)(grammar)
print (args.action_embed_size,args.no_copy,args.dropout)
parser_cls = Registrable.by_name(args.parser) # TODO: add arg
if args.pretrain:
print('Finetune with: ', args.pretrain, file=sys.stderr)
model = parser_cls.load(model_path=args.pretrain, cuda=args.cuda)
else:
model = parser_cls(args, vocab, transition_system,src_vocab=vocab)
model.train()
evaluator = Registrable.by_name(args.evaluator)(transition_system, args=args)
if args.cuda: model.cuda()
optimizer_cls = eval('torch.optim.%s' % args.optimizer)
optimizer = optimizer_cls(model.parameters(), lr=args.lr)
# if args.mass_pretrain:
# nn_utils.glorot_init(model.parameters())
# if not args.pretrain:
# if args.uniform_init:
# print('uniformly initialize parameters [-%f, +%f]' % (args.uniform_init, args.uniform_init), file=sys.stderr)
# nn_utils.uniform_init(-args.uniform_init, args.uniform_init, model.parameters())
# elif args.glorot_init:
# print('use glorot initialization', file=sys.stderr)
# # nn_utils.glorot_init(model.parameters())
print('begin training, %d training examples, %d dev examples' % (len(train_set), len(dev_set)), file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
print('vocab: %s' % repr(vocab_src), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = report_sup_att_loss = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
print ('lr for this epoch is ',optimizer.param_groups[0]['lr'],' and the current epoch is ',epoch)
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size, shuffle=True):
batch_examples = [e for e in batch_examples if len(e.tgt_actions) <= args.decode_max_time_step]
train_iter += 1
optimizer.zero_grad()
# print ('it gets here')
if args.mass_pretrain:
# print ('mass pretrain is true')
ret_val=model.score(batch_examples, pretrain=True)
else:
# print ('it gets here')
ret_val = model.score(batch_examples,pretrain=False)
loss = -ret_val[0]
# print(loss.data)
loss_val = torch.sum(loss).data.item()
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
loss.backward()
# clip gradient
if args.clip_grad > 0.:
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
log_str = '[Iter %d] encoder loss=%.5f' % (train_iter, report_loss / report_examples)
if args.sup_attention:
log_str += ' supervised attention loss=%.5f' % (report_sup_att_loss / report_examples)
report_sup_att_loss = 0.
print(log_str, file=sys.stderr)
report_loss = report_examples = 0.
if args.decay_lr_every_epoch and epoch >10:
lr = optimizer.param_groups[0]['lr'] * 0.95
print('decay learning rate to %f' % lr, file=sys.stderr)
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
print('[Epoch %d] epoch elapsed %ds' % (epoch, time.time() - epoch_begin), file=sys.stderr)
# if args.save_all_models:
# model_file = args.save_to + '.iter%d.bin' % train_iter
# print('save model to [%s]' % model_file, file=sys.stderr)
# model.save(model_file)
# if epoch>5:
if not args.mass_pretrain and epoch>32:
is_better = False
if args.dev_file:
print ('dev file validation')
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
eval_results = evaluation.evaluate(dev_set.examples, model, evaluator, args,
verbose=False, eval_top_pred_only=args.eval_top_pred_only)
# dev_score = eval_results[evaluator.default_metric]
# dev_score=eval_results
# for batch_examples in dev_set.batch_iter(batch_size=args.batch_size, shuffle=True):
# batch_examples = [e for e in batch_examples if len(e.tgt_actions) <= args.decode_max_time_step]
# # train_iter += 1
# # optimizer.zero_grad()
# model.eval()
# ret_val = model.score(batch_examples)
# loss = -ret_val[0]
# print ('the valid loss is: ',loss.mean())
print ('samip dahal')
# print('[Epoch %d] evaluate details: %s, dev %s: %.5f ' % (
# epoch, eval_results,
# evaluator.default_metric,
# dev_score))
# print("Evaluation ",epoch, eval_results,dev_score)
dev_score = eval_results[evaluator.default_metric]
print('[Epoch %d] evaluate details: %s, dev %s: %.5f (took %ds)' % (
epoch, eval_results,
evaluator.default_metric,
dev_score,
time.time() - eval_start), file=sys.stderr)
is_better = history_dev_scores == [] or dev_score > max(history_dev_scores)
history_dev_scores.append(dev_score)
model.train()
else:
is_better = True
if is_better :
print("the not mentioned tags are ",model.new_tags)
patience = 0
model_file = f'saved_models/pre_small/{epoch}.bin'
print (model_file)
if history_dev_scores:
s=history_dev_scores[-1]
model_file = f'saved_models/pre_small/{epoch}${s}.bin'
print('save the current model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), model_file + '.optim.bin')
else:
print ('it is mass')
if epoch%5==0 and args.mass_pretrain:
model_file = f'saved_models/code_fine_4/{epoch}{loss}.bin'
print('save the current model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), f'saved_models/code_fine_4/{epoch}{loss}' + '.optim.bin')
if epoch == args.max_epoch:
print('reached max epoch, stop!', file=sys.stderr)
exit(0)
def train(args):
"""Maximum Likelihood Estimation"""
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = TransitionSystem.get_class_by_lang(args.lang)(grammar)
train_set = Dataset.from_bin_file(args.train_file)
if args.dev_file:
dev_set = Dataset.from_bin_file(args.dev_file)
else:
dev_set = Dataset(examples=[])
vocab = pickle.load(open(args.vocab, 'rb'))
if args.lang == 'wikisql':
# import additional packages for wikisql dataset
from model.wikisql.dataset import WikiSqlExample, WikiSqlTable, TableColumn
parser_cls = get_parser_class(args.lang)
model = parser_cls(args, vocab, transition_system)
model.train()
if args.cuda: model.cuda()
optimizer_cls = eval('torch.optim.%s' %
args.optimizer) # FIXME: this is evil!
optimizer = optimizer_cls(model.parameters(), lr=args.lr)
if args.uniform_init:
print('uniformly initialize parameters [-%f, +%f]' %
(args.uniform_init, args.uniform_init),
file=sys.stderr)
nn_utils.uniform_init(-args.uniform_init, args.uniform_init,
model.parameters())
elif args.glorot_init:
print('use glorot initialization', file=sys.stderr)
nn_utils.glorot_init(model.parameters())
# load pre-trained word embedding (optional)
if args.glove_embed_path:
print('load glove embedding from: %s' % args.glove_embed_path,
file=sys.stderr)
glove_embedding = GloveHelper(args.glove_embed_path)
glove_embedding.load_to(model.src_embed, vocab.source)
print('begin training, %d training examples, %d dev examples' %
(len(train_set), len(dev_set)),
file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = report_sup_att_loss = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size,
shuffle=True):
batch_examples = [
e for e in batch_examples
if len(e.tgt_actions) <= args.decode_max_time_step
]
train_iter += 1
optimizer.zero_grad()
ret_val = model.score(batch_examples)
loss = -ret_val[0]
# print(loss.data)
loss_val = torch.sum(loss).data[0]
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
if args.sup_attention:
att_probs = ret_val[1]
if att_probs:
sup_att_loss = -torch.log(torch.cat(att_probs)).mean()
sup_att_loss_val = sup_att_loss.data[0]
report_sup_att_loss += sup_att_loss_val
loss += sup_att_loss
loss.backward()
# clip gradient
if args.clip_grad > 0.:
grad_norm = torch.nn.utils.clip_grad_norm(
model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
log_str = '[Iter %d] encoder loss=%.5f' % (
train_iter, report_loss / report_examples)
if args.sup_attention:
log_str += ' supervised attention loss=%.5f' % (
report_sup_att_loss / report_examples)
report_sup_att_loss = 0.
print(log_str, file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' %
(epoch, time.time() - epoch_begin),
file=sys.stderr)
if args.save_all_models:
model_file = args.save_to + '.iter%d.bin' % train_iter
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# perform validation
if args.dev_file:
if epoch % args.valid_every_epoch == 0:
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
eval_results = evaluation.evaluate(
dev_set.examples,
model,
args,
verbose=True,
eval_top_pred_only=args.eval_top_pred_only)
dev_acc = eval_results['accuracy']
print('[Epoch %d] code generation accuracy=%.5f took %ds' %
(epoch, dev_acc, time.time() - eval_start),
file=sys.stderr)
is_better = history_dev_scores == [] or dev_acc > max(
history_dev_scores)
history_dev_scores.append(dev_acc)
else:
is_better = True
if epoch > args.lr_decay_after_epoch:
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('decay learning rate to %f' % lr, file=sys.stderr)
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save the current model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience and epoch >= args.lr_decay_after_epoch:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if epoch == args.max_epoch:
print('reached max epoch, stop!', file=sys.stderr)
exit(0)
if patience >= args.patience and epoch >= args.lr_decay_after_epoch:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial:
print('early stop!', file=sys.stderr)
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' %
lr,
file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin',
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(
torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
def train(args):
"""Maximum Likelihood Estimation"""
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = TransitionSystem.get_class_by_lang(args.lang)(grammar)
train_set = Dataset.from_bin_file(args.train_file)
if args.dev_file:
dev_set = Dataset.from_bin_file(args.dev_file)
else: dev_set = Dataset(examples=[])
vocab = pickle.load(open(args.vocab, 'rb'))
if args.lang == 'wikisql':
# import additional packages for wikisql dataset
from model.wikisql.dataset import WikiSqlExample, WikiSqlTable, TableColumn
parser_cls = get_parser_class(args.lang)
model = parser_cls(args, vocab, transition_system)
model.train()
if args.cuda: model.cuda()
optimizer_cls = eval('torch.optim.%s' % args.optimizer) # FIXME: this is evil!
optimizer = optimizer_cls(model.parameters(), lr=args.lr)
if args.uniform_init:
print('uniformly initialize parameters [-%f, +%f]' % (args.uniform_init, args.uniform_init), file=sys.stderr)
nn_utils.uniform_init(-args.uniform_init, args.uniform_init, model.parameters())
elif args.glorot_init:
print('use glorot initialization', file=sys.stderr)
nn_utils.glorot_init(model.parameters())
# load pre-trained word embedding (optional)
if args.glove_embed_path:
print('load glove embedding from: %s' % args.glove_embed_path, file=sys.stderr)
glove_embedding = GloveHelper(args.glove_embed_path)
glove_embedding.load_to(model.src_embed, vocab.source)
print('begin training, %d training examples, %d dev examples' % (len(train_set), len(dev_set)), file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
epoch = train_iter = 0
report_loss = report_examples = report_sup_att_loss = 0.
history_dev_scores = []
num_trial = patience = 0
while True:
epoch += 1
epoch_begin = time.time()
for batch_examples in train_set.batch_iter(batch_size=args.batch_size, shuffle=True):
batch_examples = [e for e in batch_examples if len(e.tgt_actions) <= args.decode_max_time_step]
train_iter += 1
optimizer.zero_grad()
ret_val = model.score(batch_examples)
loss = -ret_val[0]
# print(loss.data)
loss_val = torch.sum(loss).data[0]
report_loss += loss_val
report_examples += len(batch_examples)
loss = torch.mean(loss)
if args.sup_attention:
att_probs = ret_val[1]
if att_probs:
sup_att_loss = -torch.log(torch.cat(att_probs)).mean()
sup_att_loss_val = sup_att_loss.data[0]
report_sup_att_loss += sup_att_loss_val
loss += sup_att_loss
loss.backward()
# clip gradient
if args.clip_grad > 0.:
grad_norm = torch.nn.utils.clip_grad_norm(model.parameters(), args.clip_grad)
optimizer.step()
if train_iter % args.log_every == 0:
log_str = '[Iter %d] encoder loss=%.5f' % (train_iter, report_loss / report_examples)
if args.sup_attention:
log_str += ' supervised attention loss=%.5f' % (report_sup_att_loss / report_examples)
report_sup_att_loss = 0.
print(log_str, file=sys.stderr)
report_loss = report_examples = 0.
print('[Epoch %d] epoch elapsed %ds' % (epoch, time.time() - epoch_begin), file=sys.stderr)
if args.save_all_models:
model_file = args.save_to + '.iter%d.bin' % train_iter
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# perform validation
if args.dev_file:
if epoch % args.valid_every_epoch == 0:
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
eval_results = evaluation.evaluate(dev_set.examples, model, args,
verbose=True, eval_top_pred_only=args.eval_top_pred_only)
dev_acc = eval_results['accuracy']
print('[Epoch %d] code generation accuracy=%.5f took %ds' % (epoch, dev_acc, time.time() - eval_start), file=sys.stderr)
is_better = history_dev_scores == [] or dev_acc > max(history_dev_scores)
history_dev_scores.append(dev_acc)
else:
is_better = True
if epoch > args.lr_decay_after_epoch:
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('decay learning rate to %f' % lr, file=sys.stderr)
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save the current model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience and epoch >= args.lr_decay_after_epoch:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if epoch == args.max_epoch:
print('reached max epoch, stop!', file=sys.stderr)
exit(0)
if patience >= args.patience and epoch >= args.lr_decay_after_epoch:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial:
print('early stop!', file=sys.stderr)
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' % lr, file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin', map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
# torch.manual_seed(args.seed)
# if args.cuda:
# torch.cuda.manual_seed(args.seed)
# np.random.seed(int(args.seed * 13 / 7))
#
# return args
#arg_parser = argparse.ArgumentParser()
#arg_parser.add_argument('-no_parent_production_embe',default=False, action='store_true',
# help='Do not use embedding of parent ASDL production to update decoder LSTM state')
#args = arg_parser.parse_args()
##args = init_config()
##args=init_config()
#print(args.no_parent_production_embe)
tmp=[]
engine = DBEngine('./data_model/wikisql/data/train.db')
grammar = ASDLGrammar.from_text(open('./asdl/lang/sql/sql_asdl.txt').read())
transition_system = SqlTransitionSystem(grammar)
if(True):
from asdl.hypothesis import Hypothesis
for ids,line in enumerate(open(data_file,encoding='utf-8')):
example = json.loads(line)
print(example['sql'])
query = Query.from_dict(example['sql']).lower()
print(query)
asdl_ast = sql_query_to_asdl_ast(query, grammar)
asdl_ast.sanity_check()
print(asdl_ast.to_string())
# asdl_ast.sort_removedup_self()
# print(asdl_ast.to_string())
# a=input('fff')
actions = transition_system.get_actions(asdl_ast)
def train(args):
"""Maximum Likelihood Estimation"""
tokenizer=BertTokenizer.from_pretrained('./pretrained_models/bert-base-uncased-vocab.txt')
bertmodels=BertModel.from_pretrained('./pretrained_models/base-uncased/')
print(len(tokenizer.vocab))
# load in train/dev set
tmpvocab={'null':0}
tmpvocab1={'null':0,'unk':1}
tmpvocab2={'null':0,'unk':1}
tmpvocab3={'null':0,'unk':1}
train_set = Dataset.from_bin_file(args.train_file)
from dependency import sentencetoadj,sentencetoextra_message
valid=0
# for example in tqdm.tqdm(train_set.examples):
## print(example.src_sent)
## example.mainnode,example.adj,example.edge,_,isv=sentencetoadj(example.src_sent,tmpvocab)
# example.contains,example.pos,example.ner,example.types,example.tins,example.F1=sentencetoextra_message(example.src_sent,[item.tokens for item in example.table.header],[item.type for item in example.table.header],tmpvocab1,tmpvocab2,tmpvocab3,True)
## valid+=isv
## print(example.src_sent)
### print( example.contains,example.pos,example.ner,example.types)
## a=input('gh')
# print('bukey',valid)
if args.dev_file:
dev_set = Dataset.from_bin_file(args.dev_file)
else: dev_set = Dataset(examples=[])
# for example in tqdm.tqdm(dev_set.examples):
## print(example.src_sent)
## example.mainnode,example.adj,example.edge,_,_=sentencetoadj(example.src_sent,tmpvocab)
# example.contains,example.pos,example.ner,example.types,example.tins,example.F1=sentencetoextra_message(example.src_sent,[item.tokens for item in example.table.header],[item.type for item in example.table.header],tmpvocab1,tmpvocab2,tmpvocab3,False)
vocab = pickle.load(open(args.vocab, 'rb'))
print(len(vocab.source))
vocab.source.copyandmerge(tokenizer)
print(len(vocab.source))
# tokenizer.update(vocab.source.word2id)
# print(len(tokenizer.vocab))
# print(tokenizer.vocab['metodiev'])
# bertmodels.resize_token_embeddings(len(vocab.source))
grammar = ASDLGrammar.from_text(open(args.asdl_file).read())
transition_system = Registrable.by_name(args.transition_system)(grammar)
parser_cls = Registrable.by_name(args.parser) # TODO: add arg
model = parser_cls(args, vocab, transition_system,tmpvocab,tmpvocab1,tmpvocab2,tmpvocab3)
model.train()
model.tokenizer=tokenizer
evaluator = Registrable.by_name(args.evaluator)(transition_system, args=args)
if args.cuda: model.cuda()
if args.uniform_init:
print('uniformly initialize parameters [-%f, +%f]' % (args.uniform_init, args.uniform_init), file=sys.stderr)
nn_utils.uniform_init(-args.uniform_init, args.uniform_init, model.parameters())
elif args.glorot_init:
print('use glorot initialization', file=sys.stderr)
nn_utils.glorot_init(model.parameters())
# load pre-trained word embedding (optional)
if args.glove_embed_path:
print('load glove embedding from: %s' % args.glove_embed_path, file=sys.stderr)
glove_embedding = GloveHelper(args.glove_embed_path)
glove_embedding.load_to(model.src_embed, vocab.source)
print([name for name,_ in model.named_parameters()])
model.bert_model=bertmodels
# print([name for name,_ in model.named_parameters()])
model.train()
if args.cuda: model.cuda()
# return 0
# a=input('haha')
optimizer_cls = eval('torch.optim.%s' % args.optimizer) # FIXME: this is evil!
# parameters=[p for name,p in model.named_parameters() if 'bert_model' not in name or 'embeddings' in name]
parameters=[p for name,p in model.named_parameters() if 'bert_model' not in name]
parameters1=[p for name,p in model.named_parameters() if 'bert_model' in name]
optimizer = optimizer_cls(parameters, lr=args.lr)
optimizer1 = optimizer_cls(parameters1, lr=0.00001)
print('begin training, %d training examples, %d dev examples' % (len(train_set), len(dev_set)), file=sys.stderr)
print('vocab: %s' % repr(vocab), file=sys.stderr)
is_better = False
epoch = train_iter = 0
report_loss = report_examples = report_sup_att_loss = 0.
report_loss1=0
history_dev_scores = []
num_trial = patience = 0
while True:
if epoch>40:break
epoch += 1
epoch_begin = time.time()
model.train()
for batch_examples in tqdm.tqdm(train_set.batch_iter(batch_size=args.batch_size, shuffle=True)):
def process(header,src_sent,tokenizer):
length1=len(header)
flat_src=[]
for item in src_sent:
flat_src.extend(tokenizer._tokenize(item))
flat=[token for item in header for token in item.tokens]
flat_head=[]
for item in flat:
flat_head.extend(tokenizer._tokenize(item))
# length2=len(flat)+length1+len(src_sent)
length2=len(flat_head)+length1+len(flat_src)
print(src_sent)
print([item.tokens for item in header])
print(flat_src)
print(flat)
a=input('hahaha')
return length2<130
batch_examples = [e for e in batch_examples if len(e.tgt_actions) <= args.decode_max_time_step and process(e.table.header,e.src_sent,tokenizer)]
train_iter += 1
optimizer.zero_grad()
optimizer1.zero_grad()
# params1=model.named_parameters()
# print([param for param,_ in params1])
# params=model.rnns.named_parameters()
# print([param for param,_ in params])
# print([type(param.grad) for _,param in model.rnns.named_parameters()])
# a=input('ghh;')
ret_val,_ = model.score(batch_examples)
loss = -ret_val[0]
loss1=ret_val[1]
# print(loss.data)
loss_val = torch.sum(loss).data.item()
report_loss += loss_val
report_loss1 += 1.0*torch.sum(ret_val[2])
report_examples += len(batch_examples)
loss = torch.mean(loss)+0*loss1+0*torch.mean(ret_val[2])
if args.sup_attention:
att_probs = ret_val[1]
if att_probs:
sup_att_loss = -torch.log(torch.cat(att_probs)).mean()
sup_att_loss_val = sup_att_loss.data.item()
report_sup_att_loss += sup_att_loss_val
loss += sup_att_loss
loss.backward()
# print([type(param.grad) for _,param in model.rnns.named_parameters()])
#
# print([type(param.grad) for param in model.parameters()])
# a=input('ghh;')
# clip gradient
if args.clip_grad > 0.:
grad_norm = torch.nn.utils.clip_grad_norm(model.parameters(), args.clip_grad)
optimizer.step()
optimizer1.step()
loss=None
if train_iter % args.log_every == 0:
log_str = '[Iter %d] encoder loss=%.5f,coverage loss=%.5f' % (train_iter, report_loss / report_examples,report_loss1 / report_examples)
if args.sup_attention:
log_str += ' supervised attention loss=%.5f' % (report_sup_att_loss / report_examples)
report_sup_att_loss = 0.
print(log_str, file=sys.stderr)
report_loss = report_examples = 0.
report_loss1=0
print('[Epoch %d] epoch elapsed %ds' % (epoch, time.time() - epoch_begin), file=sys.stderr)
if args.save_all_models:
model_file = args.save_to + '.iter%d.bin' % train_iter
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# perform validation
if args.dev_file and epoch>=6:
# a=input('gh')
if epoch % args.valid_every_epoch == 0:
print('[Epoch %d] begin validation' % epoch, file=sys.stderr)
eval_start = time.time()
eval_results = evaluation.evaluate(dev_set.examples, model, evaluator, args,
verbose=True, eval_top_pred_only=args.eval_top_pred_only)
dev_score = eval_results[evaluator.default_metric]
print('[Epoch %d] evaluate details: %s, dev %s: %.5f (took %ds)' % (
epoch, eval_results,
evaluator.default_metric,
dev_score,
time.time() - eval_start), file=sys.stderr)
is_better = history_dev_scores == [] or dev_score > max(history_dev_scores)
history_dev_scores.append(dev_score)
print('[Epoch %d] begin validation2' % epoch, file=sys.stderr)
# eval_start = time.time()
# eval_results = evaluation.evaluate(dev_set.examples[:2000], model, evaluator, args,
# verbose=True, eval_top_pred_only=args.eval_top_pred_only)
# dev_score = eval_results[evaluator.default_metric]
#
# print('[Epoch %d] evaluate details: %s, dev %s: %.5f (took %ds)' % (
# epoch, eval_results,
# evaluator.default_metric,
# dev_score,
# time.time() - eval_start), file=sys.stderr)
# is_better = history_dev_scores == [] or dev_score > max(history_dev_scores)
# history_dev_scores.append(dev_score)
else:
is_better = True
if args.decay_lr_every_epoch and epoch > args.lr_decay_after_epoch:
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('decay learning rate to %f' % lr, file=sys.stderr)
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if is_better:
patience = 0
model_file = args.save_to + '.bin'
print('save the current model ..', file=sys.stderr)
print('save model to [%s]' % model_file, file=sys.stderr)
model.save(model_file)
# also save the optimizers' state
torch.save(optimizer.state_dict(), args.save_to + '.optim.bin')
elif patience < args.patience and epoch >= args.lr_decay_after_epoch:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if epoch == args.max_epoch:
print('reached max epoch, stop!', file=sys.stderr)
exit(0)
if patience >= args.patience and epoch >= args.lr_decay_after_epoch:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trial:
print('early stop!', file=sys.stderr)
a=input('hj')
exit(0)
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * args.lr_decay
print('load previously best model and decay learning rate to %f' % lr, file=sys.stderr)
# load model
params = torch.load(args.save_to + '.bin', map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
if args.cuda: model = model.cuda()
# load optimizers
if args.reset_optimizer:
print('reset optimizer', file=sys.stderr)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
else:
print('restore parameters of the optimizers', file=sys.stderr)
optimizer.load_state_dict(torch.load(args.save_to + '.optim.bin'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0