for top_k in top_ks:
for temperature in temperatures:
step='_ '.join([str(i) for i in (decoder_type,
beam_size,
length_penalty,
temperature,
top_p,
top_k)])
(task_score,
bert_score,
draft_attention_weights,
refine_attention_weights) = evaluate_validation_set(test_dataset,
step,
decoder_type,
beam_size,
length_penalty,
temperature,
top_p,
top_k)
combined_metric = (0.8*bert_score) + (0.2*task_score)
if max_combined_metric < combined_metric:
max_combined_metric = combined_metric
best_beam_size = beam_size
best_length_penalty = length_penalty
best_temperature = temperature
best_top_p = top_p
best_top_k = top_k
log.info(infer_template.format(task_score, bert_score, combined_metric, beam_size, step))
log.info(best_combo.format(
best_beam_size,
best_length_penalty,
training_loop(unit_test_dataset, True)
if config.check_training_pipeline:
training_loop(unit_test_dataset, False)
if config.check_evaluation_pipeline:
ck_pt_mgr = check_ckpt(config.checkpoint_path)
(rouge_score,
bert_score,
draft_attention_weights,
refine_attention_weights) = evaluate_validation_set(
unit_test_dataset,
config.beam_size,
config.length_penalty,
1
)
monitor_early_stop = monitor_run(
'not saving',
bert_score,
rouge_score,
0.0,
1,
copy_best_ckpt=False
)
if not monitor_early_stop:
log.info(f'Validation run completed with ROUGE-avg {rouge_score} and BERT-f1 {bert_score}\
Check the written summary file in {config.output_sequence_write_path}')
# if a checkpoint exists, restore the latest checkpoint.
ck_pt_mgr = check_ckpt(config.checkpoint_path)
total_steps = int(config.epochs * (config.gradient_accumulation_steps))
train_dataset = train_dataset.repeat(total_steps)
for (step, (input_ids, target_ids_)) in tqdm(enumerate(train_dataset),
initial=1):
start = time.time()
draft_mask, refine_mask, target_ids = mask_and_one_hot_labels(target_ids_)
grad_accum_flag = True if (
(step + 1) % config.gradient_accumulation_steps) == 0 else False
refine_predictions = train_step(input_ids, target_ids_, target_ids,
draft_mask, refine_mask, grad_accum_flag)
if grad_accum_flag:
train_loss = batch_run_check(step + 1, start)
evaluate = ((step + 1) * config.train_batch_size) % config.eval_after
if evaluate == 0:
predicted = train_sanity_check(target_tokenizer, refine_predictions,
target_ids_)
ckpt_save_path = ck_pt_mgr.save()
if predicted:
(rouge_score,
bert_score) = evaluate_validation_set(val_dataset, step + 1)
else:
rouge_score, bert_score = 0
training_results(step + 1, rouge_score, bert_score,
(time.time() - start), ckpt_save_path)
monitor_early_stop = monitor_run(ckpt_save_path, bert_score,
rouge_score, train_loss, step + 1)
if not monitor_early_stop:
break
log.info(f'Training completed at step {step+1}')
from utilities import log
from model_training_helper import (check_ckpt, evaluate_validation_set,
training_results)
val_dataset = create_dataset(split='validation',
source_tokenizer=source_tokenizer,
target_tokenizer=target_tokenizer,
from_=0,
to=100,
batch_size=8,
shuffle=True,
drop_remainder=True)
count = 0
step = 1
for (i, o) in val_dataset:
#print(f'input {tf.shape(i)}')
#print(f'output {tf.shape(o)}')
count += 1
print(f'Total records count is {count}')
#sys.exit()
#restore checkpoint
ck_pt_mgr = check_ckpt(config.checkpoint_path)
start_time = time.time()
(task_score, bert_score, draft_attention_weights,
refine_attention_weights) = evaluate_validation_set(
val_dataset.take(1), step, return_with_attention_weights=False)
training_results(step, 0, 0, task_score, bert_score,
(time.time() - start_time), 'none', log, config)
sys.exit()