def compute_generation(model, tokenizer): global tf_valid_ds global tf_test_ds global validation global test global exp_dir global base_dir global param_sets print("Starting Generate Variants:") param_count = 1 for param_set in param_sets: print(f"Generate {param_count}/{len(param_sets)}") print(str(param_set)) #Returns a dictionary for model output for test model_ouput = get_model_output(model, tokenizer, param_set, None, None, tf_test_ds) #Test Out test_out = model_ouput['test']['output'] ts_val = time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(test, "test", ts_val, test_out, write_path=exp_dir) # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'test', param_set) print(scores) print() save_metrics(exp_dir, ts_val, scores) param_count += 1
def compute_generation(model, tokenizer): global tf_valid_ds global validation global exp_dir global base_dir global param_sets print("Starting Generate Variants:") param_count = 1 for param_set in param_sets: # ### IF SOMETHING BREAKS PICKUP WHERE WE LEFT OFF # if param_count < PICK_THE_NUM: # print(f'Skipping: Model#: {model_count} Param#: {param_count}') # param_count +=1 # continue # ### print(f"Generate {param_count}/{len(param_sets)}") print(str(param_set)) #Returns a list of all the model generated outputs model_ouput = get_model_output(model, tokenizer, param_set, None, tf_valid_ds, None) v_out = model_ouput['validation']['output'] ts_val = time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(validation, "validation", ts_val, v_out, write_path=exp_dir) # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation', param_set) print(scores) print() save_metrics(exp_dir, ts_val, scores) param_count += 1
'num_beams': 1, 'max_length': 60, 'min_length': 20, 'early_stopping': True, 'do_sample': False, 'no_repeat_ngram_size': 2 } #Returns a list of all the model generated outputs model_ouput = get_model_output(model, tokenizer, {}, None, tf_valid_ds, None) # - #Write model outputs v_out = model_ouput['validation']['output'] ts_val = time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(valid_ds, "validation", ts_val, v_out, write_path=exp_dir) # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation', gen_params) print(scores) print(scores) save_metrics(exp_dir, ts_val, scores) # #### If we like the scores and want to save the scores to our model track # (We should probably club this with when we save to S3) # ### Save Model (only if its worth it) # Keep for AWS path model.save_pretrained(f'{model_path}')