def generate_pseudo_samples(self, conf, test_batchSize, beam, sp_samples=None, qg_samples=None):
domain = Example.domain
pseudo_samples = []
if sp_samples:
nsentences = len(sp_samples)
data_index = np.arange(nsentences)
self.model['sp'].eval()
for j in range(0, nsentences, test_batchSize):
inputs, lens, copy_tokens, oov_list, raw_inputs = get_minibatch(sp_samples, self.vocab['sp'],
task='unlabeled_semantic_parsing', data_index=data_index, index=j, batch_size=test_batchSize,
device=self.device['sp'], copy=self.model['sp'].copy)
with torch.no_grad():
results = self.model['sp'].decode_batch(inputs, lens, self.vocab['sp'].lf2id, copy_tokens, beam_size=beam, n_best=beam)
predictions = results["predictions"]
predictions = [pred for each in predictions for pred in each]
predictions = domain.reverse(predictions, self.vocab['sp'].id2lf, oov_list=oov_list)
_, idxs = domain.pick_predictions(domain.obtain_denotations(domain.normalize(predictions)), n_best=beam)
predictions = [predictions[each] for each in idxs]
pseudo_samples.extend([Example(' '.join(words), ' '.join(lfs), conf) for words, lfs in zip(raw_inputs, predictions)])
if qg_samples:
nsentences = len(qg_samples)
data_index = np.arange(nsentences)
self.model['qg'].eval()
for j in range(0, nsentences, test_batchSize):
inputs, lens, copy_tokens, oov_list, raw_inputs = get_minibatch(qg_samples, self.vocab['qg'],
task='unlabeled_question_generation', data_index=data_index, index=j, batch_size=test_batchSize,
device=self.device['qg'], copy=self.model['qg'].copy)
with torch.no_grad():
results = self.model['qg'].decode_batch(inputs, lens, self.vocab['qg'].word2id, copy_tokens, beam_size=beam, n_best=beam)
predictions = results["predictions"]
predictions = [each[0] for each in predictions]
predictions = domain.reverse(predictions, self.vocab['qg'].id2word, oov_list=oov_list)
pseudo_samples.extend([Example(' '.join(words), ' '.join(lfs), conf) for words, lfs in zip(predictions, raw_inputs)])
return pseudo_samples
def slu_pseudo_labeling(model,
vocab,
evaluator,
data_inputs,
test_batchSize,
device='cpu',
beam=5):
use_bert = hasattr(model.word_embed, 'plm')
pseudo_labeled_dataset = []
data_index = np.arange(len(data_inputs))
model.eval()
for j in range(0, len(data_index), test_batchSize):
inputs, _, _, lens, raw_in = get_minibatch(data_inputs,
vocab,
task='slu',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=device,
use_bert=use_bert)
with torch.no_grad():
results = model.decode_batch(inputs, lens, vocab.slot2id, beam, 1)
results = evaluator.parse_outputs(results, evaluation=True)
bio_list, intents = results['bio_list'], results['intents']
pseudo_labeled_dataset.extend([
Example(w, b, i) for w, b, i in zip(raw_in, bio_list, intents)
])
return pseudo_labeled_dataset
def lm_decode(model,
vocab,
evaluator,
data_inputs,
output_path,
test_batchSize,
device='cpu',
surface_level=False):
data_index = np.arange(len(data_inputs))
ppl_list, logprob_list, lens_list, raw_inputs_list = [], [], [], []
model.eval()
for j in range(0, len(data_index), test_batchSize):
inputs, lens, raw_inputs = get_minibatch(data_inputs,
vocab,
task='lm',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=device,
surface_level=surface_level)
with torch.no_grad():
logprob = model.sent_logprob(inputs, lens)
ppl = evaluator.parse_outputs(logprob, lens)
lens_list.extend(lens.tolist())
raw_inputs_list.extend(raw_inputs)
logprob_list.extend(logprob.tolist())
ppl_list.extend(ppl)
overall_ppl = evaluator.compare_outputs(logprob_list, lens_list)
with open(output_path, 'w') as f:
for idx, (inputs, ppl) in enumerate(zip(raw_inputs_list, ppl_list)):
if ppl >= overall_ppl:
f.write('Inputs: ' + ' '.join(inputs) + '\n')
f.write('Ppl: ' + str(ppl) + '\n\n')
f.write('Overall ppl is: %.4f' % (overall_ppl))
return overall_ppl
def nlg_pseudo_labeling(model,
vocab,
evaluator,
data_inputs,
test_batchSize,
device='cpu',
beam=5):
pseudo_labeled_dataset = []
data_index = np.arange(len(data_inputs))
model.eval()
for j in range(0, len(data_index), test_batchSize):
intents, slots, slot_lens, lens, _, _, slot_states, copy_tokens, (
raw_intents, raw_slots) = get_minibatch(data_inputs,
vocab,
task='nlg',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=device)
with torch.no_grad():
results = model.decode_batch(intents, slots, slot_lens, lens,
slot_states, copy_tokens,
vocab.word2id, beam, 1)
results = evaluator.parse_outputs(results,
slots=raw_slots,
evaluation=True)
words, bio_list = results['sentences'], results['bio_list']
pseudo_labeled_dataset.extend([
Example(w, b, i)
for w, b, i in zip(words, bio_list, raw_intents)
])
return pseudo_labeled_dataset
def decode(self,
data_inputs,
output_path,
test_batchSize,
beam=5,
n_best=1):
data_index = np.arange(len(data_inputs))
nsentences, total = len(data_index), []
domain = Example.domain
self.model.eval()
with open(output_path, 'w') as of:
for j in range(0, nsentences, test_batchSize):
###################### Obtain minibatch data ######################
inputs, lens, dec_inputs, _, _, copy_tokens, oov_list, (
raw_inputs,
raw_outputs) = get_minibatch(data_inputs,
self.vocab,
task='semantic_parsing',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=self.device,
copy=self.model.copy)
############################ Forward Model ############################
with torch.no_grad():
results = self.model.decode_batch(inputs,
lens,
self.vocab.lf2id,
copy_tokens,
beam_size=beam,
n_best=n_best)
predictions = results["predictions"]
predictions = [
pred for each in predictions for pred in each
]
predictions = domain.reverse(predictions,
self.vocab.id2lf,
oov_list=oov_list)
accuracy = domain.compare_logical_form(predictions,
raw_outputs,
pick=True)
total.extend(accuracy)
############################ Write result to file ############################
for idx in range(len(raw_inputs)):
of.write("Utterance: " + ' '.join(raw_inputs[idx]) + '\n')
of.write("Target: " + ' '.join(raw_outputs[idx]) + '\n')
for i in range(n_best):
of.write("Pred" + str(i) + ": " +
' '.join(predictions[n_best * idx + i]) +
'\n')
of.write("Correct: " +
("True" if accuracy[idx] == 1 else "False") +
'\n\n')
acc = sum(total) / float(len(total))
of.write('Overall accuracy is %.4f' % (acc))
return acc
def slu_decode(model,
vocab,
evaluator,
data_inputs,
output_path,
test_batchSize,
device='cpu',
beam=5,
n_best=1):
use_bert = hasattr(model.word_embed, 'plm')
data_index = np.arange(len(data_inputs))
pred_slots, pred_intents = [], []
ref_slots, ref_intents = [ex.label for ex in data_inputs
], [ex.intents for ex in data_inputs]
model.eval()
for j in range(0, len(data_index), test_batchSize):
inputs, _, _, lens, raw_in = get_minibatch(data_inputs,
vocab,
task='slu',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=device,
use_bert=use_bert)
with torch.no_grad():
results = model.decode_batch(inputs, lens, vocab.slot2id, beam,
n_best)
outputs = evaluator.parse_outputs(results,
words=raw_in,
evaluation=True)
pred_slots.extend(outputs['slots'])
pred_intents.extend(outputs['intents'])
(slot_fscore,
overall_fscore), (intent_acc, overall_acc) = evaluator.compare_outputs(
pred_slots, ref_slots, pred_intents, ref_intents)
with open(output_path, 'w') as f:
for idx, (sf, ia) in enumerate(zip(slot_fscore, intent_acc)):
if sf < 1.0 or not ia:
f.write('Input: ' + ' '.join(data_inputs[idx].words) + '\n')
f.write('Ref slots: ' +
' , '.join([' '.join(s)
for s in ref_slots[idx]]) + '\n')
f.write('Pred slots: ' +
' , '.join([' '.join(s)
for s in pred_slots[idx]]) + '\n')
f.write('Ref intents: ' + ' , '.join(ref_intents[idx]) + '\n')
f.write('Pred intents: ' + pred_intents[idx] + '\n\n')
f.write('Overall slot fscore: %.4f ; intent accuracy: %.4f' %
(overall_fscore, overall_acc))
return overall_fscore, overall_acc
def nlg_decode(model,
vocab,
evaluator,
data_inputs,
output_path,
test_batchSize,
device='cpu',
beam=5,
n_best=1):
data_index = np.arange(len(data_inputs))
pred_surfaces, ref_surfaces = [], [ex.surface for ex in data_inputs]
model.eval()
for j in range(0, len(data_index), test_batchSize):
intents, slots, slot_lens, lens, _, _, slot_states, copy_tokens, _ = get_minibatch(
data_inputs,
vocab,
task='nlg',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=device)
with torch.no_grad():
results = model.decode_batch(intents, slots, slot_lens, lens,
slot_states, copy_tokens,
vocab.word2id, beam, n_best)
surfaces = evaluator.parse_outputs(results,
evaluation=True)['surfaces']
pred_surfaces.extend(surfaces)
(bleu_score, overall_score), (slot_acc,
overall_acc) = evaluator.compare_outputs(
pred_surfaces, ref_surfaces)
with open(output_path, 'w') as f:
for idx, (b, s) in enumerate(zip(bleu_score, slot_acc)):
if b < overall_score or s < 1.0:
f.write('Intent: ' + ' , '.join(data_inputs[idx].intents) +
'\n')
f.write('Slots: ' + ' , '.join(
[' '.join(s_v) for s_v in data_inputs[idx].slots]) + '\n')
f.write('Ref: ' + ' '.join(ref_surfaces[idx]) + '\n')
f.write('Pred: ' + ' '.join(pred_surfaces[idx]) + '\n\n')
f.write('Overall bleu score and slot acc is: %.4f/%.4f' %
(overall_score, overall_acc))
return overall_score, overall_acc
def decode(self, data_inputs, output_path, test_batchSize):
data_index = np.arange(len(data_inputs))
count, eval_loss, length_list = 0, [], []
########################### Evaluation Phase ############################
self.model.eval()
with open(output_path, 'w') as f:
for j in range(0, len(data_index), test_batchSize):
###################### Obtain minibatch data ######################
inputs, lens, raw_inputs = get_minibatch(
data_inputs,
self.vocab,
task='language_model',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=self.device,
side=self.side)
length_list.extend((lens - 1).tolist())
########################## Calculate Sentence PPL #######################
with torch.no_grad():
scores = self.model(inputs,
lens) # bsize, seq_len, voc_size
batch_loss = self.loss_function(scores, inputs[:,
1:]).item()
eval_loss.append(batch_loss)
norm_log_prob = self.model.sent_logprobability(
inputs, lens).cpu().tolist()
############################# Writing Result to File ###########################
for idx in range(len(inputs)):
f.write('Utterance: ' + ' '.join(raw_inputs[idx]) + '\n')
f.write('NormLogProb: ' + str(norm_log_prob[idx]) + '\n')
current_ppl = np.exp(-norm_log_prob[idx])
f.write('PPL: ' + str(current_ppl) + '\n\n')
########################### Calculate Corpus PPL ###########################
word_count = np.sum(length_list, axis=0)
eval_loss = np.sum(eval_loss, axis=0)
final_ppl = np.exp(eval_loss / word_count)
f.write('Overall ppl: %.4f' % (final_ppl))
return final_ppl
def decode(self,
data_inputs,
output_path,
test_batchSize,
beam=5,
n_best=1):
data_index = np.arange(len(data_inputs))
nsentences = len(data_index)
domain = Example.domain
total, candidate_list, references_list = [], [], []
########################### Evaluation Phase ############################
with open(output_path, 'w') as of:
self.model.eval()
for j in range(0, nsentences, test_batchSize):
###################### Obtain minibatch data ######################
inputs, lens, _, _, _, copy_tokens, oov_list, (
raw_inputs,
raw_outputs) = get_minibatch(data_inputs,
self.vocab['sp'],
task='semantic_parsing',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=self.device['sp'],
copy=self.model.sp_model.copy)
############################ Forward Model ############################
with torch.no_grad():
results = self.model.decode_batch(inputs,
lens,
self.vocab['sp'].lf2id,
copy_tokens,
task='semantic_parsing',
beam_size=beam,
n_best=n_best)
predictions = results["predictions"]
predictions = [
pred for each in predictions for pred in each
]
predictions = domain.reverse(predictions,
self.vocab['sp'].id2lf,
oov_list=oov_list)
accuracy = domain.compare_logical_form(predictions,
raw_outputs,
pick=True)
total.extend(accuracy)
############################ Write result to file ############################
for idx in range(len(raw_inputs)):
of.write("Utterance: " + ' '.join(raw_inputs[idx]) + '\n')
of.write("Target: " + ' '.join(raw_outputs[idx]) + '\n')
for i in range(n_best):
of.write("Pred" + str(i) + ": " +
' '.join(predictions[n_best * idx + i]) +
'\n')
of.write("Correct: " +
("True" if accuracy[idx] == 1 else "False") +
'\n\n')
of.write('=' * 50 + '\n' + '=' * 50 + '\n\n')
for j in range(0, nsentences, test_batchSize):
###################### Obtain minibatch data ######################
inputs, lens, _, _, _, copy_tokens, oov_list, (
raw_inputs,
raw_outputs) = get_minibatch(data_inputs,
self.vocab['qg'],
task='question_generation',
data_index=data_index,
index=j,
batch_size=test_batchSize,
device=self.device['qg'],
copy=self.model.qg_model.copy)
########################## Beam Search/Greed Decode #######################
with torch.no_grad():
results = self.model.decode_batch(
inputs,
lens,
self.vocab['qg'].word2id,
copy_tokens,
task='question_generation',
beam_size=beam,
n_best=n_best)
predictions = results["predictions"]
predictions = [each[0] for each in predictions]
predictions = domain.reverse(predictions,
self.vocab['qg'].id2word,
oov_list=oov_list)
bleu_scores = domain.compare_question(predictions, raw_outputs)
candidate_list.extend(predictions)
references_list.extend([[ref] for ref in raw_outputs])
############################# Writing Result to File ###########################
for idx in range(len(raw_inputs)):
of.write("LogicalForm: " + ' '.join(raw_inputs[idx]) +
'\n')
of.write("Target: " + ' '.join(raw_outputs[idx]) + '\n')
of.write("Pred0: " + ' '.join(predictions[idx]) + '\n')
of.write("Bleu: " + str(bleu_scores[idx]) + '\n\n')
########################### Calculate accuracy ###########################
acc = sum(total) / float(len(total))
avg_bleu = get_bleu_score(candidate_list, references_list)
of.write('Overall accuracy: %.4f | Overall bleu score: %.4f' %
(acc, avg_bleu))
return acc, avg_bleu
def train_and_decode(self,
labeled_train_dataset,
q_unlabeled_train_dataset,
lf_unlabeled_train_dataset,
dev_dataset,
test_dataset,
batchSize,
test_batchSize,
cycle='sp+qg',
max_epoch=100,
beam=5,
n_best=1):
sp_unlabeled_train_index = np.arange(len(q_unlabeled_train_dataset))
qg_unlabeled_train_index = np.arange(len(lf_unlabeled_train_dataset))
labeled_train_index = np.arange(len(labeled_train_dataset))
nsentences = max([
len(q_unlabeled_train_dataset),
len(lf_unlabeled_train_dataset),
len(labeled_train_dataset)
])
for i in range(max_epoch):
########################### Training Phase ############################
start_time = time.time()
np.random.shuffle(sp_unlabeled_train_index)
np.random.shuffle(qg_unlabeled_train_index)
np.random.shuffle(labeled_train_index)
losses = {'sp': [], 'qg': []}
self.model.train()
for j in range(0, nsentences, batchSize):
self.model.zero_grad()
''' Cycle start from Semantic Parsing '''
if 'sp' in cycle:
###################### Obtain minibatch data ######################
inputs, lens, copy_tokens, oov_list, raw_in = get_minibatch(
q_unlabeled_train_dataset,
self.vocab['sp'],
task='unlabeled_semantic_parsing',
data_index=sp_unlabeled_train_index,
index=j,
batch_size=batchSize,
device=self.device['sp'],
copy=self.model.sp_model.copy)
######################## Forward Model ##########################
sp_loss, qg_loss = self.model(
inputs,
lens,
copy_tokens,
oov_list,
raw_in,
start_from='semantic_parsing')
losses['sp'].append(sp_loss.item())
losses['qg'].append(qg_loss.item())
sp_loss.backward()
qg_loss.backward()
''' Cycle start from Question Generation '''
if 'qg' in cycle:
###################### Obtain minibatch data ######################
inputs, lens, copy_tokens, oov_list, raw_in = get_minibatch(
lf_unlabeled_train_dataset,
self.vocab['qg'],
task='unlabeled_question_generation',
data_index=qg_unlabeled_train_index,
index=j,
batch_size=batchSize,
device=self.device['qg'],
copy=self.model.qg_model.copy)
########################### Forward Model ########################
sp_loss, qg_loss = self.model(
inputs,
lens,
copy_tokens,
oov_list,
raw_in,
start_from='question_generation')
losses['sp'].append(sp_loss.item())
losses['qg'].append(qg_loss.item())
sp_loss.backward()
qg_loss.backward()
''' Supervised Training '''
if True:
###################### Obtain minibatch data ######################
inputs, lens, dec_inputs, dec_outputs, out_lens, copy_tokens, _, _ = get_minibatch(
labeled_train_dataset,
self.vocab['sp'],
task='semantic_parsing',
data_index=labeled_train_index,
index=j,
batch_size=batchSize,
device=self.device['sp'],
copy=self.model.sp_model.copy)
############################ Forward Model ############################
batch_scores = self.model.sp_model(inputs, lens,
dec_inputs[:, :-1],
copy_tokens)
batch_loss = self.loss_function['sp'](batch_scores,
dec_outputs[:, 1:],
out_lens - 1)
losses['sp'].append(batch_loss.item())
batch_loss.backward()
###################### Obtain minibatch data ######################
inputs, lens, dec_inputs, dec_outputs, out_lens, copy_tokens, _, _ = get_minibatch(
labeled_train_dataset,
self.vocab['qg'],
task='question_generation',
data_index=labeled_train_index,
index=j,
batch_size=batchSize,
device=self.device['qg'],
copy=self.model.qg_model.copy)
############################ Forward Model ############################
batch_scores = self.model.qg_model(inputs, lens,
dec_inputs[:, :-1],
copy_tokens)
batch_loss = self.loss_function['qg'](batch_scores,
dec_outputs[:, 1:],
out_lens - 1)
losses['qg'].append(batch_loss.item())
batch_loss.backward()
self.model.pad_embedding_grad_zero()
self.optimizer.step()
gc.collect()
torch.cuda.empty_cache()
print('[learning] epoch %i >> %3.2f%%' % (i, 100),
'completed in %.2f (sec) <<' % (time.time() - start_time))
sp_loss, qg_loss = np.sum(losses['sp'],
axis=0), np.sum(losses['qg'], axis=0)
self.logger.info('Training:\tEpoch : %d\tTime : %.4fs\tLoss(sp loss : %.4f ; qg loss : %.4f)' \
% (i, time.time() - start_time, sp_loss, qg_loss))
########################### Evaluation Phase ############################
start_time = time.time()
dev_acc, dev_bleu = self.decode(dev_dataset,
os.path.join(
self.exp_path,
'valid.iter' + str(i)),
test_batchSize,
beam=beam,
n_best=n_best)
self.logger.info('Evaluation:\tEpoch : %d\tTime : %.4fs\tSemantic Parsing (acc : %.4f)\tQuestion Generation (bleu : %.4f)' \
% (i, time.time() - start_time, dev_acc, dev_bleu))
start_time = time.time()
test_acc, test_bleu = self.decode(test_dataset,
os.path.join(
self.exp_path,
'test.iter' + str(i)),
test_batchSize,
beam=beam,
n_best=n_best)
self.logger.info('Evaluation:\tEpoch : %d\tTime : %.4fs\tSemantic Parsing (acc : %.4f)\tQuestion Generation (bleu : %.4f)' \
% (i, time.time() - start_time, test_acc, test_bleu))
######################## Pick best result and save #####################
if dev_acc > self.best_result['dev_acc']:
self.model.save_model(
sp_save_dir=os.path.join(self.exp_path, 'sp_model.pkl'))
self.best_result['iter_sp'] = i
self.best_result['dev_acc'], self.best_result[
'test_acc'] = dev_acc, test_acc
self.logger.info('NEW BEST Semantic Parsing:\tEpoch : %d\tBest Valid (acc : %.4f)\tBest Test (acc : %.4f)' \
% (i, dev_acc, test_acc))
if dev_bleu >= self.best_result['dev_bleu']:
self.model.save_model(
qg_save_dir=os.path.join(self.exp_path, 'qg_model.pkl'))
self.best_result['iter_qg'] = i
self.best_result['dev_bleu'], self.best_result[
'test_bleu'] = dev_bleu, test_bleu
self.logger.info('NEW BEST Question Generation:\tEpoch : %d\tBest Valid (bleu : %.4f)\tBest Test (bleu : %.4f)' \
% (i, dev_bleu, test_bleu))
gc.collect()
torch.cuda.empty_cache()
######################## Reload best model for later usage #####################
self.logger.info(
'FINAL BEST Semantic Parsing RESULT: \tEpoch : %d\tBest Valid (acc : %.4f)\tBest Test (acc : %.4f)'
% (self.best_result['iter_sp'], self.best_result['dev_acc'],
self.best_result['test_acc']))
self.logger.info(
'FINAL BEST Question Generation RESULT: \tEpoch : %d\tBest Valid (bleu : %.4f)\tBest Test (bleu : %.4f)'
% (self.best_result['iter_qg'], self.best_result['dev_bleu'],
self.best_result['test_bleu']))
self.model.load_model(os.path.join(self.exp_path, 'sp_model.pkl'),
os.path.join(self.exp_path, 'qg_model.pkl'))
if not opt.testing:
word2id = vocab.sfm2id if params['surface_level'] else vocab.word2id
loss_function = set_celoss_function(ignore_index=word2id[PAD])
optimizer, scheduler = set_optimizer(train_model, lr=opt.lr, l2=opt.l2, max_norm=opt.max_norm, lr_schedule='constant')
logger.info("Training starts at %s" % (time.asctime(time.localtime(time.time()))))
train_data_index = np.arange(len(train_dataset))
nsentences = len(train_data_index)
best_result = {"losses": [], "iter": 0, "dev_ppl": float('inf'), "test_ppl": float('inf'),}
for i in range(opt.max_epoch):
start_time = time.time()
np.random.shuffle(train_data_index)
losses = []
train_model.train()
for j in range(0, nsentences, opt.batchSize):
optimizer.zero_grad()
inputs, lens, _ = get_minibatch(train_dataset, vocab, task=task, data_index=train_data_index, index=j,
batch_size=opt.batchSize, device=device, surface_level=params['surface_level'])
batch_scores = train_model(inputs, lens)
batch_loss = loss_function(batch_scores, inputs[:, 1:])
losses.append(batch_loss.item())
batch_loss.backward()
train_model.pad_embedding_grad_zero()
optimizer.step()
scheduler.step()
epoch_loss = np.sum(losses, axis=0)
best_result['losses'].append(epoch_loss)
logger.info('Training epoch : %d\tTime : %.4fs\tLoss : %.5f' % (i, time.time() - start_time, epoch_loss))
gc.collect()
torch.cuda.empty_cache()
##### Evaluate on dev and test dataset #####
def train_and_decode(self, labeled_train_dataset, q_unlabeled_train_dataset, lf_unlabeled_train_dataset, dev_dataset, test_dataset,
batchSize, test_batchSize, max_epoch=100, beam=5, n_best=1):
for i in range(max_epoch):
########################### Training Phase ############################
start_time = time.time()
if self.method == 'constant':
conf = self.discount
elif self.method == 'linear':
conf = self.discount * (i + 1) / float(max_epoch)
else:
raise ValueError("[Error]: not recognized method !")
pseudo_samples = self.generate_pseudo_samples(conf, test_batchSize, beam, q_unlabeled_train_dataset, lf_unlabeled_train_dataset)
self.logger.info('Generate %d new pseudo samples with confidence %.4f in epoch %d' % (len(pseudo_samples), conf, i))
cur_train_dataset = labeled_train_dataset + pseudo_samples
nsentences = len(cur_train_dataset)
train_index = np.arange(nsentences)
np.random.shuffle(train_index)
losses = { 'sp': [], 'qg': [] }
self.model['sp'].train()
self.model['qg'].train()
for j in range(0, nsentences, batchSize):
self.model['sp'].zero_grad()
self.model['qg'].zero_grad()
###################### Obtain minibatch data ######################
inputs, lens, dec_inputs, dec_outputs, out_lens, copy_tokens, conf = get_minibatch(
cur_train_dataset, self.vocab['sp'], task='pseudo_semantic_parsing',
data_index=train_index, index=j, batch_size=batchSize, device=self.device['sp'], copy=self.model['sp'].copy)
############################ Forward Model ############################
batch_scores = self.model['sp'](inputs, lens, dec_inputs[:, :-1], copy_tokens)
batch_loss = self.loss_function['sp'](batch_scores, dec_outputs[:, 1:], out_lens - 1, conf)
losses['sp'].append(batch_loss.item())
batch_loss.backward()
###################### Obtain minibatch data ######################
inputs, lens, dec_inputs, dec_outputs, out_lens, copy_tokens, conf = get_minibatch(
cur_train_dataset, self.vocab['qg'], task='pseudo_question_generation',
data_index=train_index, index=j, batch_size=batchSize, device=self.device['qg'], copy=self.model['qg'].copy)
############################ Forward Model ############################
batch_scores = self.model['qg'](inputs, lens, dec_inputs[:, :-1], copy_tokens)
batch_loss = self.loss_function['qg'](batch_scores, dec_outputs[:, 1:], out_lens - 1, conf)
losses['qg'].append(batch_loss.item())
batch_loss.backward()
self.model['sp'].pad_embedding_grad_zero()
self.model['qg'].pad_embedding_grad_zero()
self.optimizer.step()
print('[learning] epoch %i >> %3.2f%%' % (i, 100), 'completed in %.2f (sec) <<' % (time.time() - start_time))
sp_loss, qg_loss = np.sum(losses['sp']), np.sum(losses['qg'])
self.logger.info('Training:\tEpoch : %d\tTime : %.4fs\tSemantic Parsing Loss (loss : %.4f) ; Question Generation Loss (loss : %.4f)' \
% (i, time.time() - start_time, sp_loss, qg_loss))
gc.collect()
torch.cuda.empty_cache()
########################### Evaluation Phase ############################
start_time = time.time()
dev_acc, dev_bleu = self.decode(dev_dataset, os.path.join(self.exp_path, 'valid.iter' + str(i)), test_batchSize, beam=beam, n_best=n_best)
self.logger.info('Dev Evaluation:\tEpoch : %d\tTime : %.4fs\tSemantic Parsing (acc : %.4f)\tQuestion Generation (bleu : %.4f)' \
% (i, time.time() - start_time, dev_acc, dev_bleu))
start_time = time.time()
test_acc, test_bleu = self.decode(test_dataset, os.path.join(self.exp_path, 'test.iter' + str(i)), test_batchSize, beam=beam, n_best=n_best)
self.logger.info('Test Evaluation:\tEpoch : %d\tTime : %.4fs\tSemantic Parsing (acc : %.4f)\tQuestion Generation (bleu : %.4f)' \
% (i, time.time() - start_time, test_acc, test_bleu))
######################## Pick best result and save #####################
if dev_acc > self.best_result['dev_acc']:
self.model['sp'].save_model(os.path.join(self.exp_path, 'sp_model.pkl'))
self.best_result['iter_sp'] = i
self.best_result['dev_acc'], self.best_result['test_acc'] = dev_acc, test_acc
self.logger.info('NEW BEST Semantic Parsing:\tEpoch : %d\tBest Valid (acc : %.4f)\tBest Test (acc : %.4f)' \
% (i, dev_acc, test_acc))
if dev_bleu >= self.best_result['dev_bleu']:
self.model['qg'].save_model(os.path.join(self.exp_path, 'qg_model.pkl'))
self.best_result['iter_qg'] = i
self.best_result['dev_bleu'], self.best_result['test_bleu'] = dev_bleu, test_bleu
self.logger.info('NEW BEST Question Generation:\tEpoch : %d\tBest Valid (bleu : %.4f)\tBest Test (bleu : %.4f)' \
% (i, dev_bleu, test_bleu))
gc.collect()
torch.cuda.empty_cache()
######################## Reload best model for later usage #####################
self.logger.info('FINAL BEST Semantic Parsing RESULT: \tEpoch : %d\tBest Valid (acc : %.4f)\tBest Test (acc : %.4f)'
% (self.best_result['iter_sp'], self.best_result['dev_acc'], self.best_result['test_acc']))
self.logger.info('FINAL BEST Question Generation RESULT: \tEpoch : %d\tBest Valid (bleu : %.4f)\tBest Test (bleu : %.4f)'
% (self.best_result['iter_qg'], self.best_result['dev_bleu'], self.best_result['test_bleu']))
self.model['sp'].load_model(os.path.join(self.exp_path, 'sp_model.pkl'))
self.model['qg'].load_model(os.path.join(self.exp_path, 'qg_model.pkl'))
slot_control_function = set_scloss_function(slot_weight=opt.slot_weight)
optimizer, scheduler = set_optimizer(train_model, lr=opt.lr, l2=opt.l2, max_norm=opt.max_norm, lr_schedule='constant')
logger.info("Training starts at %s" % (time.asctime(time.localtime(time.time()))))
train_data_index = np.arange(len(train_dataset))
nsentences, coefficient = len(train_data_index), 0.5
best_result = {"losses": [], "iter": 0, "dev_bleu": 0., "dev_slot": 0.,
"test_bleu": 0., "test_slot": 0.,}
for i in range(opt.max_epoch):
start_time = time.time()
np.random.shuffle(train_data_index)
losses = []
train_model.train()
for j in range(0, nsentences, opt.batchSize):
optimizer.zero_grad()
intents, slots, slot_lens, lens, outputs, out_lens, slot_states, copy_tokens, _ = \
get_minibatch(train_dataset, vocab, task=task, data_index=train_data_index, index=j, batch_size=opt.batchSize, device=device)
surface_scores, slot_hist = train_model(intents, slots, slot_lens, lens, outputs[:, :-1], slot_states, copy_tokens)
surface_loss = surface_loss_function(surface_scores, outputs[:, 1:])
control_loss = slot_control_function(slot_hist, out_lens)
batch_loss = coefficient * surface_loss + (1 - coefficient) * control_loss
losses.append(batch_loss.item())
batch_loss.backward()
train_model.pad_embedding_grad_zero()
optimizer.step()
scheduler.step()
epoch_loss = np.sum(losses, axis=0)
best_result['losses'].append(epoch_loss)
logger.info('Training epoch : %d\tTime : %.4fs\tLoss : %.5f' % (i, time.time() - start_time, epoch_loss))
gc.collect()
torch.cuda.empty_cache()
def train_and_decode(self,
train_dataset,
dev_dataset,
test_dataset,
batchSize=16,
test_batchSize=128,
max_epoch=100,
beam=5,
n_best=1):
train_data_index = np.arange(len(train_dataset))
nsentences = len(train_data_index)
for i in range(max_epoch):
########################### Training Phase ############################
start_time = time.time()
np.random.shuffle(train_data_index)
losses = []
self.model.train()
for j in range(0, nsentences, batchSize):
###################### Obtain minibatch data ######################
inputs, lens, dec_inputs, dec_outputs, out_lens, copy_tokens, _, _ = get_minibatch(
train_dataset,
self.vocab,
task='semantic_parsing',
data_index=train_data_index,
index=j,
batch_size=batchSize,
device=self.device,
copy=self.model.copy)
############################ Forward Model ############################
self.model.zero_grad()
batch_scores = self.model(inputs, lens, dec_inputs[:, :-1],
copy_tokens)
############################ Loss Calculation #########################
batch_loss = self.loss_function(batch_scores, dec_outputs[:,
1:],
out_lens - 1)
losses.append(batch_loss.item())
########################### Backward and Optimize ######################
batch_loss.backward()
self.model.pad_embedding_grad_zero()
self.optimizer.step()
print('[learning] epoch %i >> %3.2f%%' % (i, 100),
'completed in %.2f (sec) <<' % (time.time() - start_time))
epoch_loss = np.sum(losses, axis=0)
self.best_result['losses'].append(epoch_loss)
self.logger.info('Training:\tEpoch : %d\tTime : %.4fs\t Loss: %.5f' \
% (i, time.time() - start_time, epoch_loss))
gc.collect()
torch.cuda.empty_cache()
if i < 10:
continue
########################### Evaluation Phase ############################
start_time = time.time()
dev_acc = self.decode(dev_dataset,
os.path.join(self.exp_path,
'valid.iter' + str(i)),
test_batchSize,
beam=beam,
n_best=n_best)
self.logger.info('Dev Evaluation:\tEpoch : %d\tTime : %.4fs\tAcc : %.4f' \
% (i, time.time() - start_time, dev_acc))
start_time = time.time()
test_acc = self.decode(test_dataset,
os.path.join(self.exp_path,
'test.iter' + str(i)),
test_batchSize,
beam=beam,
n_best=n_best)
self.logger.info('Test Evaluation:\tEpoch : %d\tTime : %.4fs\tAcc : %.4f' \
% (i, time.time() - start_time, test_acc))
######################## Pick best result on dev and save #####################
if dev_acc >= self.best_result['dev_acc']:
self.model.save_model(os.path.join(self.exp_path, 'model.pkl'))
self.best_result['iter'] = i
self.best_result['dev_acc'], self.best_result[
'test_acc'] = dev_acc, test_acc
self.logger.info(
'NEW BEST:\tEpoch : %d\tBest Valid Acc : %.4f;\tBest Test Acc : %.4f'
% (i, dev_acc, test_acc))
######################## Reload best model for later usage #####################
self.logger.info(
'FINAL BEST RESULT: \tEpoch : %d\tBest Valid (Acc : %.4f)\tBest Test (Acc : %.4f)'
% (self.best_result['iter'], self.best_result['dev_acc'],
self.best_result['test_acc']))
self.model.load_model(os.path.join(self.exp_path, 'model.pkl'))
"dev_intent": 0.,
"test_slot": 0.,
"test_intent": 0.,
}
for i in range(opt.max_epoch):
start_time = time.time()
np.random.shuffle(train_data_index)
losses = []
train_model.train()
for j in range(0, nsentences, opt.batchSize):
optimizer.zero_grad()
inputs, outputs, intents, lens, _ = get_minibatch(
train_dataset,
vocab,
task=task,
data_index=train_data_index,
index=j,
batch_size=opt.batchSize,
device=device,
use_bert=params['use_bert'])
slot_scores, intent_scores = train_model(inputs, lens, outputs)
slot_loss = slot_loss_function(
slot_scores, outputs[:, 1:]
) if 'crf' not in params['model_type'] else slot_scores
intent_loss = intent_loss_function(intent_scores, intents)
batch_loss = coefficient * slot_loss + (1 -
coefficient) * intent_loss
losses.append(batch_loss.item())
batch_loss.backward()
train_model.pad_embedding_grad_zero()
optimizer.step()
for i in range(opt.max_epoch):
np.random.shuffle(slu_start_train_index)
np.random.shuffle(nlg_start_train_index)
np.random.shuffle(labeled_train_index)
train_model.train()
start_time, losses = time.time(), []
for j in range(0, nsentences, opt.batchSize):
slu_optimizer.zero_grad()
nlg_optimizer.zero_grad()
# dual learning start from slu model
if 'slu' in opt.cycle_choice and len(unlabeled_dataset) != 0:
inputs, _, _, lens, raw_in = get_minibatch(
unlabeled_dataset,
slu_vocab,
task='slu',
data_index=slu_start_train_index,
index=j,
batch_size=opt.batchSize,
device=slu_device,
use_bert=slu_params['use_bert'])
slu_loss, nlg_loss = train_model(inputs,
lens,
raw_in,
start_from='slu')
slu_loss.backward()
nlg_loss.backward()
losses.append([slu_loss.item(), nlg_loss.item()])
# dual learning start from nlg model
if 'nlg' in opt.cycle_choice and len(unlabeled_dataset) != 0:
intents, slots, slot_lens, lens, _, _, slot_states, copy_tokens, (raw_intents, raw_slots) = \
get_minibatch(unlabeled_dataset, nlg_vocab, task='nlg', data_index=nlg_start_train_index, index=j, batch_size=opt.batchSize, device=nlg_device)
def train_and_decode(self,
train_inputs,
dev_inputs,
test_inputs,
batchSize=16,
test_batchSize=128,
max_epoch=100):
train_data_index = np.arange(len(train_inputs))
nsentences = len(train_data_index)
for i in range(max_epoch):
########################### Training Phase ############################
start_time = time.time()
np.random.shuffle(train_data_index)
losses = []
self.model.train()
for j in range(0, nsentences, batchSize):
###################### Obtain minibatch data ######################
inputs, lens, _ = get_minibatch(train_inputs,
self.vocab,
task='language_model',
data_index=train_data_index,
index=j,
batch_size=batchSize,
device=self.device,
side=self.side)
############################ Forward Model ############################
self.optimizer.zero_grad()
batch_scores = self.model(inputs, lens)
############################ Loss Calculation #########################
batch_loss = self.loss_function(batch_scores, inputs[:, 1:],
lens - 1)
losses.append(batch_loss.item())
########################### Backward and Optimize ######################
batch_loss.backward()
self.model.pad_embedding_grad_zero()
self.optimizer.step()
print('[learning] epoch %i >> %3.2f%%' % (i, 100),
'completed in %.2f (sec) <<' % (time.time() - start_time))
epoch_loss = np.sum(losses, axis=0)
self.best_result['losses'].append(epoch_loss)
self.logger.info('Training:\tEpoch : %d\tTime : %.4fs\t Loss of tgt : %.5f' \
% (i, time.time() - start_time, epoch_loss))
gc.collect()
torch.cuda.empty_cache()
# whether evaluate later after training for some epochs
if i < 10:
continue
########################### Evaluation Phase ############################
start_time = time.time()
dev_ppl = self.decode(
dev_inputs, os.path.join(self.exp_path, 'valid.iter' + str(i)),
test_batchSize)
self.logger.info(
'Evaluation:\tEpoch : %d\tTime : %.4fs\tppl : %.4f' %
(i, time.time() - start_time, dev_ppl))
start_time = time.time()
test_ppl = self.decode(
test_inputs, os.path.join(self.exp_path, 'test.iter' + str(i)),
test_batchSize)
self.logger.info(
'Evaluation:\tEpoch : %d\tTime : %.4fs\tppl : %.4f' %
(i, time.time() - start_time, test_ppl))
######################## Pick best result and save #####################
if dev_ppl < self.best_result['dev_ppl']:
self.model.save_model(os.path.join(self.exp_path, 'model.pkl'))
self.best_result['iter'] = i
self.best_result['dev_ppl'], self.best_result[
'test_ppl'] = dev_ppl, test_ppl
self.logger.info(
'NEW BEST:\tEpoch : %d\tBest Valid ppl : %.4f;\tBest Test ppl : %.4f'
% (i, dev_ppl, test_ppl))
######################## Reload best model for later usage #####################
self.logger.info(
'FINAL BEST RESULT: \tEpoch : %d\tBest Valid (ppl : %.4f)\tBest Test (ppl : %.4f) '
% (self.best_result['iter'], self.best_result['dev_ppl'],
self.best_result['test_ppl']))
self.model.load_model(os.path.join(self.exp_path, 'model.pkl'))