def compute_generation(model, tokenizer):
global tf_valid_ds
global tf_test_ds
global validation
global test
global exp_dir
global base_dir
global param_sets
print("Starting Generate Variants:")
param_count = 1
for param_set in param_sets:
print(f"Generate {param_count}/{len(param_sets)}")
print(str(param_set))
#Returns a dictionary for model output for test
model_ouput = get_model_output(model, tokenizer, param_set, None, None,
tf_test_ds)
#Test Out
test_out = model_ouput['test']['output']
ts_val = time.strftime("%Y%m%d_%H%M")
print(ts_val)
write_model_output(test, "test", ts_val, test_out, write_path=exp_dir)
# Let's Use E2E Evaluation Metrics
scores = compute_metrics(exp_dir, base_dir, ts_val, 'test', param_set)
print(scores)
print()
save_metrics(exp_dir, ts_val, scores)
param_count += 1
def compute_generation(model, tokenizer):
global tf_valid_ds
global validation
global exp_dir
global base_dir
global param_sets
print("Starting Generate Variants:")
param_count = 1
for param_set in param_sets:
# ### IF SOMETHING BREAKS PICKUP WHERE WE LEFT OFF
# if param_count < PICK_THE_NUM:
# print(f'Skipping: Model#: {model_count} Param#: {param_count}')
# param_count +=1
# continue
# ###
print(f"Generate {param_count}/{len(param_sets)}")
print(str(param_set))
#Returns a list of all the model generated outputs
model_ouput = get_model_output(model, tokenizer, param_set, None,
tf_valid_ds, None)
v_out = model_ouput['validation']['output']
ts_val = time.strftime("%Y%m%d_%H%M")
print(ts_val)
write_model_output(validation,
"validation",
ts_val,
v_out,
write_path=exp_dir)
# Let's Use E2E Evaluation Metrics
scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation',
param_set)
print(scores)
print()
save_metrics(exp_dir, ts_val, scores)
param_count += 1
#Returns a list of all the model generated outputs
model_ouput = get_model_output(model, tokenizer, {}, None, tf_valid_ds, None)
# -
#Write model outputs
v_out = model_ouput['validation']['output']
ts_val = time.strftime("%Y%m%d_%H%M")
print(ts_val)
write_model_output(valid_ds, "validation", ts_val, v_out, write_path=exp_dir)
# Let's Use E2E Evaluation Metrics
scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation', gen_params)
print(scores)
print(scores)
save_metrics(exp_dir, ts_val, scores)
# #### If we like the scores and want to save the scores to our model track
# (We should probably club this with when we save to S3)
# ### Save Model (only if its worth it)
# Keep for AWS path
model.save_pretrained(f'{model_path}')
# save_model_to_s3(model,base_dir, ts_val)
# ### Load Model
# +
#Below is an optional step to load a pre-trained and saved model to directly run predictions.
