def calc_validation_loss(validation_dataset, step, val_step,
valid_summary_writer, validation_accuracy):
total_val_acc_avg = tf.keras.metrics.Mean()
for (batch, (input_ids, target_ids_)) in enumerate(
validation_dataset.take(config.valid_samples_to_eval //
h_parms.validation_batch_size)):
# calculate rouge for only the first batch
if batch == 0:
draft_mask = tf.math.logical_not(
tf.math.equal(target_ids_[:, 1:], 0))
refine_mask = tf.math.logical_not(
tf.math.equal(target_ids_[:, :-1], 0))
target_ids = label_smoothing(
tf.one_hot(target_ids_, depth=config.input_vocab_size))
rouge_score, bert_score = val_step(input_ids, target_ids_, step,
config.write_summary_op)
else:
_ = val_step(input_ids, target_ids_, step, False)
if config.run_tensorboard:
with valid_summary_writer.as_default():
tf.summary.scalar('validation_accuracy',
validation_accuracy.result(),
step=batch)
return (total_val_acc_avg(validation_accuracy.result()), rouge_score,
bert_score)
def calc_validation_loss(validation_dataset, epoch, val_step,
valid_summary_writer, validation_loss,
validation_accuracy):
total_val_acc_avg = tf.keras.metrics.Mean()
total_val_loss_avg = tf.keras.metrics.Mean()
for (batch,
(input_ids, input_mask, input_segment_ids, target_ids_, target_mask,
target_segment_ids)) in enumerate(validation_dataset):
# calculate rouge for only the first batch
if batch == 0:
mask = tf.math.logical_not(tf.math.equal(target_ids_[:, 1:], 0))
target_ids = label_smoothing(
tf.one_hot(target_ids_, depth=config.input_vocab_size))
rouge_score, bert_score = val_step(input_ids, input_mask,
input_segment_ids, target_ids_,
target_mask, target_segment_ids,
target_ids, mask, epoch,
config.write_summary_op)
else:
_ = val_step(input_ids, input_mask, input_segment_ids, target_ids_,
target_mask, target_segment_ids, target_ids, mask,
epoch, False)
if config.run_tensorboard:
with valid_summary_writer.as_default():
tf.summary.scalar('validation_loss',
validation_loss.result(),
step=batch)
tf.summary.scalar('validation_accuracy',
validation_accuracy.result(),
step=batch)
return (total_val_acc_avg(validation_accuracy.result()),
total_val_loss_avg(validation_loss.result()), rouge_score,
bert_score)
# if len(target_ids_) >= 512 or len(target_mask)>512 or len(target_segment_ids)>512:
# print('maggoty bread')
# continue
print(np.where(input_mask.numpy()[0])[0].max())
print('input length',len(input_ids[0]))
print('input mask length',len(input_mask[0]))
print('input segment length', len(input_segment_ids[0]))
print('target length', len(target_ids_[0]))
print('target mask length', len(target_mask[0]))
print('target segment length', len(target_segment_ids[0]))
count+=1
start=time.time()
draft_mask = tf.math.logical_not(tf.math.equal(target_ids_[:, 1:], 0))
refine_mask = tf.math.logical_not(tf.math.equal(target_ids_[:, :-1], 0))
target_ids = label_smoothing(tf.one_hot(target_ids_, depth=config.input_vocab_size))
grad_accum_flag = True if (step+1)%h_parms.accumulation_steps == 0 else False
#target_x, refine_predictions=train_step(input_ids,input_mask,input_segment_ids, target_ids_,target_mask,target_segment_ids,target_ids,draft_mask,refine_mask, grad_accum_flag)
inputs=[input_ids,input_mask,input_segment_ids, target_ids_,target_mask,target_segment_ids,target_ids,draft_mask,refine_mask, grad_accum_flag]
#target_x, refine_predictions=distributed_train_step(inputs)
target_x, refine_predictions = train_step(inputs)
if grad_accum_flag:
batch_run_check(
step+1,
start,
train_summary_writer,
train_loss.result(),
train_accuracy.result(),
model
)
eval_frequency = ((step+1) * h_parms.batch_size) % config.eval_after