def compute_generation(model, tokenizer): global tf_valid_ds global tf_test_ds global validation global test global exp_dir global base_dir global param_sets print("Starting Generate Variants:") param_count = 1 for param_set in param_sets: print(f"Generate {param_count}/{len(param_sets)}") print(str(param_set)) #Returns a dictionary for model output for test model_ouput = get_model_output(model, tokenizer, param_set, None, None, tf_test_ds) #Test Out test_out = model_ouput['test']['output'] ts_val = time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(test, "test", ts_val, test_out, write_path=exp_dir) # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'test', param_set) print(scores) print() save_metrics(exp_dir, ts_val, scores) param_count += 1
def compute_generation(model, tokenizer): global tf_valid_ds global validation global exp_dir global base_dir global param_sets print("Starting Generate Variants:") param_count = 1 for param_set in param_sets: # ### IF SOMETHING BREAKS PICKUP WHERE WE LEFT OFF # if param_count < PICK_THE_NUM: # print(f'Skipping: Model#: {model_count} Param#: {param_count}') # param_count +=1 # continue # ### print(f"Generate {param_count}/{len(param_sets)}") print(str(param_set)) #Returns a list of all the model generated outputs model_ouput = get_model_output(model, tokenizer, param_set, None, tf_valid_ds, None) v_out = model_ouput['validation']['output'] ts_val = time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(validation, "validation", ts_val, v_out, write_path=exp_dir) # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation', param_set) print(scores) print() save_metrics(exp_dir, ts_val, scores) param_count += 1
# <hr> # ### Generate Results + Metrics # + gen_params = { 'num_beams': 1, 'max_length': 60, 'min_length': 20, 'early_stopping': True, 'do_sample': False, 'no_repeat_ngram_size': 2 } #Returns a list of all the model generated outputs model_ouput = get_model_output(model, tokenizer, {}, None, tf_valid_ds, None) # - #Write model outputs v_out = model_ouput['validation']['output'] ts_val = time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(valid_ds, "validation", ts_val, v_out, write_path=exp_dir) # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation', gen_params) print(scores) print(scores) save_metrics(exp_dir, ts_val, scores) # #### If we like the scores and want to save the scores to our model track
# length_penalty (float, optional, defaults to 1.0) – Exponential penalty to the length. 1.0 means no penalty. # Set to values < 1.0 in order to encourage the model to generate shorter sequences, to a value > 1.0 in order to encourage the model to produce longer sequences. # no_repeat_ngram_size (int, optional, defaults to 0) – If set to int > 0, all ngrams of that size can only occur once. # num_return_sequences (int, optional, defaults to 1) – The number of independently computed returned sequences for each element in the batch. # use_cache – (bool, optional, defaults to True): Whether or not the model should use the past last key/values attentions (if applicable to the model) to speed up decoding. # + #Write model outputs for param_set in params_array: #Returns a list of all the model generated outputs model_ouput = get_model_output(model, tokenizer, param_set, None, tf_valid_ds, None) v_out = model_ouput['validation']['output'] ts_val=time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(valid_ds, "validation", ts_val, v_out, write_path=exp_dir) # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation', param_set) print(scores) save_metrics(exp_dir, ts_val, scores) # -
# top_p (float, optional, defaults to 1.0) – If set to float < 1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation. # repetition_penalty (float, optional, defaults to 1.0) – The parameter for repetition penalty. 1.0 means no penalty. See this paper for more details. # length_penalty (float, optional, defaults to 1.0) – Exponential penalty to the length. 1.0 means no penalty. # Set to values < 1.0 in order to encourage the model to generate shorter sequences, to a value > 1.0 in order to encourage the model to produce longer sequences. # no_repeat_ngram_size (int, optional, defaults to 0) – If set to int > 0, all ngrams of that size can only occur once. # num_return_sequences (int, optional, defaults to 1) – The number of independently computed returned sequences for each element in the batch. # use_cache – (bool, optional, defaults to True): Whether or not the model should use the past last key/values attentions (if applicable to the model) to speed up decoding. #Returns a list of all the model generated outputs model_ouput = get_model_output(model, tokenizer, gen_params, None, tf_valid_ds, None) # - #Write model outputs v_out = model_ouput['validation']['output'] ts_val = time.strftime("%Y%m%d_%H%M") print(ts_val) write_model_output(valid_ds, "validation", ts_val, v_out, write_path=exp_dir) # + # Let's Use E2E Evaluation Metrics scores = compute_metrics(exp_dir, base_dir, ts_val, 'validation', v_out['gen_params']) print(scores) # -