def write_summary(tar_real, predictions, step, write=config.write_summary_op):
ref_sents=[]
hyp_sents=[]
for tar, ref_hyp in zip(tar_real, predictions):
sum_ref = tokenizer.convert_ids_to_tokens([i for i in tf.squeeze(tar) if i not in [0, 101, 102]])
sum_hyp = tokenizer.convert_ids_to_tokens([i for i in tf.squeeze(ref_hyp) if i not in [0, 101, 102]])
sum_ref = convert_wordpiece_to_words(sum_ref)
sum_hyp = convert_wordpiece_to_words(sum_hyp)
ref_sents.append(sum_ref)
hyp_sents.append(sum_hyp)
try:
rouges = rouge_all.get_scores(ref_sents , hyp_sents)
avg_rouge_f1 = np.mean([np.mean([rouge_scores['rouge-1']["f"],
rouge_scores['rouge-2']["f"],
rouge_scores['rouge-l']["f"]]) for rouge_scores in rouges])
_, _, bert_f1 = b_score(ref_sents, hyp_sents, lang='en', model_type=config.pretrained_bert_model)
avg_bert_f1 = np.mean(bert_f1.numpy())
except:
avg_rouge_f1 = 0
avg_bert_f1 = 0
if write and (step)%config.write_per_step == 0:
with tf.io.gfile.GFile(file_path.summary_write_path+str(step.numpy()), 'w') as f:
for ref, hyp in zip(ref_sents, hyp_sents):
f.write(ref+'\t'+hyp+'\n')
return (avg_rouge_f1, avg_bert_f1)
def run_inference(dataset, print_output=False):
for draft_type, refine_type in draft_and_refine_decoder_combinations:
ref_sents = []
hyp_sents = []
for (doc_id, (input_ids, _, _, target_ids, _,
_)) in enumerate(dataset, 1):
start_time = time.time()
if draft_type != 'beam_search':
_, _, refined_summary, _ = predict_using_sampling(input_ids,
draft_type,
refine_type,
k=10)
else:
_, refined_summary, _ = predict_using_beam_search(
input_ids, refine_decoder_sampling_type=refine_type)
sum_ref = tokenizer.convert_ids_to_tokens(
[i for i in tf.squeeze(target_ids) if i not in [0, 101, 102]])
sum_hyp = tokenizer.convert_ids_to_tokens([
i for i in tf.squeeze(refined_summary)
if i not in [0, 101, 102]
])
sum_ref = convert_wordpiece_to_words(sum_ref)
sum_hyp = convert_wordpiece_to_words(sum_hyp)
if print_output:
print('Original summary: {}'.format(sum_ref))
print('Predicted summary: {}'.format(sum_hyp))
ref_sents.append(sum_ref)
hyp_sents.append(sum_hyp)
print(
f'Calculating scores for {len(ref_sents)} golden summaries and {len(hyp_sents)} predicted summaries'
)
try:
rouges = rouge_all.get_scores(ref_sents, hyp_sents)
avg_rouge_f1 = np.mean([
np.mean([
rouge_scores['rouge-1']["f"], rouge_scores['rouge-2']["f"],
rouge_scores['rouge-l']["f"]
]) for rouge_scores in rouges
])
_, _, bert_f1 = b_score(ref_sents,
hyp_sents,
lang='en',
model_type=config.pretrained_bert_model)
avg_bert_f1 = np.mean(bert_f1.numpy())
except:
avg_rouge_f1 = 0
avg_bert_f1 = 0
print(
infer_template.format(draft_type, refine_type, avg_rouge_f1,
avg_bert_f1))
print(f'time to process document {doc_id} : {time.time()-start_time}')
def run_inference(dataset, beam_sizes_to_try=h_parms.beam_sizes):
for beam_size in beam_sizes_to_try:
ref_sents = []
hyp_sents = []
for (doc_id, (input_ids, _, _, target_ids, _,
_)) in enumerate(dataset, 1):
start_time = time.time()
# translated_output_temp[0] (batch, beam_size, summ_length+1)
translated_output_temp, enc_output = draft_decoded_summary(
model, input_ids, target_ids[:, :-1], beam_size)
draft_predictions = translated_output_temp[0][:, 0, :]
_, _, dec_padding_mask = create_masks(input_ids,
target_ids[:, :-1])
refined_summary, attention_dists = refined_summary_greedy(
model,
input_ids,
enc_output,
draft_predictions,
dec_padding_mask,
training=False)
sum_ref = tokenizer.convert_ids_to_tokens(
[i for i in tf.squeeze(target_ids) if i not in [0, 101, 102]])
sum_hyp = tokenizer.convert_ids_to_tokens([
i for i in tf.squeeze(refined_summary)
if i not in [0, 101, 102]
])
sum_ref = convert_wordpiece_to_words(sum_ref)
sum_hyp = convert_wordpiece_to_words(sum_hyp)
print('Original summary: {}'.format(sum_ref))
print('Predicted summary: {}'.format(sum_hyp))
if sum_ref and sum_hyp:
ref_sents.append(sum_ref)
hyp_sents.append(sum_hyp)
try:
rouges = rouge_all.get_scores(ref_sents, hyp_sents)
avg_rouge_f1 = np.mean([
np.mean([
rouge_scores['rouge-1']["f"], rouge_scores['rouge-2']["f"],
rouge_scores['rouge-l']["f"]
]) for rouge_scores in rouges
])
_, _, bert_f1 = b_score(ref_sents,
hyp_sents,
lang='en',
model_type=config.pretrained_bert_model)
avg_bert_f1 = np.mean(bert_f1.numpy())
except:
avg_rouge_f1 = 0
avg_bert_f1 = 0
print(infer_template.format(beam_size, avg_rouge_f1, avg_bert_f1))
print(f'time to process document {doc_id} : {time.time()-start_time}')
def write_summary(tar_real, predictions, step, write=config.write_summary_op):
r_avg_final = []
total_summary = []
for i, sub_tar_real in enumerate(tar_real):
predicted_id = tf.cast(tf.argmax(predictions[i], axis=-1), tf.int32)
#decoded_logits = argmax(predictions[i])
#predicted_id = tf.cast(decoded_logits, tf.int32)
sum_ref = tokenizer.convert_ids_to_tokens([i for i in sub_tar_real.numpy() if i not in [0, 101, 102]])
sum_hyp = tokenizer.convert_ids_to_tokens([i for i in predicted_id.numpy() if i not in [0, 101, 102]])
sum_ref = convert_wordpiece_to_words(sum_ref)
sum_hyp = convert_wordpiece_to_words(sum_hyp)
# don't consider empty values for ROUGE and BERT score calculation
if sum_hyp and sum_ref:
total_summary.append((sum_ref, sum_hyp))
ref_sents = [ref for ref, _ in total_summary]
hyp_sents = [hyp for _, hyp in total_summary]
# returns :- dict of dicts
if ref_sents and hyp_sents:
try:
rouges = rouge_all.get_scores(ref_sents , hyp_sents)
avg_rouge_f1 = np.mean([np.mean([rouge_scores['rouge-1']["f"], rouge_scores['rouge-2']["f"], rouge_scores['rouge-l']["f"]]) for rouge_scores in rouges])
_, _, bert_f1 = b_score(ref_sents, hyp_sents, lang='en', model_type=config.pretrained_bert_model)
rouge_score = avg_rouge_f1.astype('float64')
bert_f1_score = np.mean(bert_f1.tolist(), dtype=np.float64)
except ValueError:
log.warning('Problem in calculating the ROUGE scores')
rouge_score = 0
bert_f1_score = 0
else:
log.warning('The sentences predicted by the model are empty so setting the scores to 0')
rouge_score = 0
bert_f1_score = 0
if write and (step)%config.write_per_step == 0:
with tf.io.gfile.GFile(file_path.summary_write_path+str(step.numpy()), 'w') as f:
for ref, hyp in total_summary:
f.write(ref+'\t'+hyp+'\n')
return (rouge_score, bert_f1_score)
train_loss, train_accuracy = get_loss_and_accuracy()
validation_loss, validation_accuracy = get_loss_and_accuracy()
accumulators = []
# if a checkpoint exists, restore the latest checkpoint.
ck_pt_mgr, latest_ckpt = check_ckpt(file_path.checkpoint_path)
total_steps = int(h_parms.epochs * (h_parms.accumulation_steps))
#train_dataset = train_dataset.repeat(total_steps)
count=0
ds_train_size = 287113
length = 1
start = randint(ds_train_size - length, size=1)[0]
train_buffer_size = 287113
for (step, (input_ids, input_mask, input_segment_ids, target_ids_, target_mask, target_segment_ids)) in enumerate(train_dataset):
#if step >= config.start_from_batch:
try:
sum_hyp = tokenizer.convert_ids_to_tokens([i for i in tf.squeeze(input_ids) if i not in [CLS_ID, SEP_ID, 0]])
ip_ids = tokenizer.encode(' '.join(sum_hyp))
if len(ip_ids) >= 512 or len(target_ids_)>512 :
# start = randint(ds_train_size - length, size=1)[0]
# examples, metadata = tfds.load('cnn_dailymail', with_info=True, as_supervised=True,
# data_dir='/content/drive/My Drive/Text_summarization/cnn_dataset',
# builder_kwargs={"version": "3.0.0"},
# split=tfds.core.ReadInstruction('train', from_=start, to=start + length,
# unit='abs'))
# train_examples = examples
# train_dataset = map_batch_shuffle(train_examples, train_buffer_size, split='train', shuffle=True,
# batch_size=1, filter_off=False)
# repeat = True
print(len(ip_ids))
continue
# if len(input_ids) >= 512 or len(input_mask)>=512 or len(input_segment_ids)>=512 or len(ip_ids)>=512:
refine_decoder_type=refine_dec_type,
k=k,
p=p,
temperature=temperature
)
else:
_, _, refined_summary, _ = predict_using_sampling(
input_ids,
draft_decoder_type=draft_dec_type,
refine_decoder_type=refine_dec_type,
k=k,
p=p,
temperature=temperature
)
for tar, ref_hyp in zip(target_ids, refined_summary):
sum_ref = tokenizer.convert_ids_to_tokens([i for i in tf.squeeze(tar) if i not in [0, 101, 102]])
sum_hyp = tokenizer.convert_ids_to_tokens([i for i in tf.squeeze(ref_hyp) if i not in [0, 101, 102]])
sum_ref = convert_wordpiece_to_words(sum_ref)
sum_hyp = convert_wordpiece_to_words(sum_hyp)
#print('Original summary: {}'.format(sum_ref))
#print('Predicted summary: {}'.format(sum_hyp))
ref_sents.append(sum_ref)
hyp_sents.append(sum_hyp)
try:
rouges = rouge_all.get_scores(ref_sents , hyp_sents)
avg_rouge_f1 = np.mean([np.mean([rouge_scores['rouge-1']["f"],
rouge_scores['rouge-2']["f"],
rouge_scores['rouge-l']["f"]]) for rouge_scores in rouges])
_, _, bert_f1 = b_score(ref_sents, hyp_sents, lang='en', model_type=config.pretrained_bert_model)
avg_bert_f1 = np.mean(bert_f1.numpy())
except:
def run_eval(
ckpt_path='/content/drive/My Drive/Text_summarization/BERT_text_summarisation/cnn_checkpoints/ckpt-69'
):
restore_chkpt(ckpt_path)
if config.use_tfds:
examples, metadata = tfds.load(
config.tfds_name,
with_info=True,
as_supervised=True,
data_dir='/content/drive/My Drive/Text_summarization/cnn_dataset',
builder_kwargs={"version": "2.0.0"})
test_examples = examples['test']
test_buffer_size = metadata.splits['test'].num_examples
test_dataset = map_batch_shuffle(test_examples,
test_buffer_size,
split='test',
batch_size=h_parms.batch_size)
log.info('Test TF_dataset created')
test_dataset = test_dataset.take(1)
else:
test_dataset = infer_data_from_df()
ref_sents = []
hyp_sents = []
for (doc_id, (input_ids, _, _, target_ids, _,
_)) in tqdm(enumerate(test_dataset, 1)):
start_time = time.time()
draft, refined_summary, att = predict_using_beam_search(
input_ids, beam_size=3, refine_decoder_sampling_type='greedy')
for tar, ref_hyp in zip(target_ids, refined_summary):
sum_ref = tokenizer.convert_ids_to_tokens(
[i for i in tf.squeeze(tar) if i not in [0, 101, 102]])
sum_hyp = tokenizer.convert_ids_to_tokens(
[i for i in tf.squeeze(ref_hyp) if i not in [0, 101, 102]])
sum_ref = convert_wordpiece_to_words(sum_ref)
sum_hyp = convert_wordpiece_to_words(sum_hyp)
#print('Original summary: {}'.format(sum_ref))
#print('Predicted summary: {}'.format(sum_hyp))
ref_sents.append(sum_ref)
hyp_sents.append(sum_hyp)
try:
rouges = rouge_all.get_scores(ref_sents, hyp_sents)
avg_rouge_f1 = np.mean([
np.mean([
rouge_scores['rouge-1']["f"], rouge_scores['rouge-2']["f"],
rouge_scores['rouge-l']["f"]
]) for rouge_scores in rouges
])
_, _, bert_f1 = b_score(ref_sents,
hyp_sents,
lang='en',
model_type=config.pretrained_bert_model)
avg_bert_f1 = np.mean(bert_f1.numpy())
except:
avg_rouge_f1 = 0
avg_bert_f1 = 0
print(
infer_template.format('beam_search', 'greedy', avg_rouge_f1,
avg_bert_f1, 3))
print(f'time to process document {doc_id} : {time.time()-start_time}')
print(
f'Calculating scores for {len(ref_sents)} golden summaries and {len(hyp_sents)} predicted summaries'
)