def main(config: configure_finetuning.FinetuningConfig, split, bioasq=False): expected_version = '1.1' # parser = argparse.ArgumentParser( # description='Evaluation for SQuAD ' + expected_version) # parser.add_argument('dataset_file', help='Dataset file') # parser.add_argument('prediction_file', help='Prediction File') # args = parser.parse_args() Args = collections.namedtuple("Args", ["dataset_file", "prediction_file"]) args = Args(dataset_file=os.path.join( config.raw_data_dir("squadv1"), split + ("-debug" if config.debug else "") + ".json"), prediction_file=config.qa_preds_file("squadv1")) if bioasq: args = Args(dataset_file=os.path.join( config.raw_data_dir("bioasq"), split + ("0debug" if config.debug else "") + ".json"), prediction_file=config.qa_preds_file("bioasq")) with tf.io.gfile.GFile(args.dataset_file) as dataset_file: dataset_json = json.load(dataset_file) if dataset_json['version'] != expected_version: print('Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'], file=sys.stderr) dataset = dataset_json['data'] with tf.io.gfile.GFile(args.prediction_file) as prediction_file: predictions = json.load(prediction_file) return evaluate(dataset, predictions)
def run_finetuning(config: configure_finetuning.FinetuningConfig): """Run finetuning.""" # Setup for training results = [] trial = 1 heading_info = "model={:}, trial {:}/{:}".format( config.model_name, trial, config.num_trials) def heading(msg): return utils.heading(msg + ": " + heading_info) heading("Config") utils.log_config(config) generic_model_dir = config.model_dir tasks = task_builder.get_tasks(config) # Train and evaluate num_trials models with different random seeds while config.num_trials < 0 or trial <= config.num_trials: config.model_dir = generic_model_dir + "_" + str(trial) if config.do_train: utils.rmkdir(config.model_dir) model_runner = ModelRunner(config, tasks) if config.do_train: heading("Start training") model_runner.train() utils.log() if config.do_eval: heading("Run dev set evaluation") results.append(model_runner.evaluate()) write_results(config, results) if config.write_test_outputs and trial <= config.n_writes_test: heading("Running on the test set and writing the predictions") for task in tasks: # Currently only writing preds for GLUE and SQuAD 2.0 is supported if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp", "sts"]: for split in task.get_test_splits(): model_runner.write_classification_outputs( [task], trial, split) elif task.name == "squad": print('start model runner') scorer = model_runner.evaluate_task( task, "test", False) print('end evaluate') scorer.write_predictions() print('end write prediction') preds = utils.load_json(config.qa_preds_file("squad")) null_odds = utils.load_json(config.qa_na_file("squad")) for q, _ in preds.items(): if null_odds[q] > config.qa_na_threshold: preds[q] = "" utils.write_json(preds, config.test_predictions( task.name, "test", trial)) else: utils.log("Skipping task", task.name, "- writing predictions is not supported for this task") if trial != config.num_trials and (not config.keep_all_models): utils.rmrf(config.model_dir) trial += 1
def set_opts(config: configure_finetuning.FinetuningConfig, split): global OPTS Options = collections.namedtuple("Options", [ "data_file", "pred_file", "out_file", "na_prob_file", "na_prob_thresh", "out_image_dir", "verbose" ]) OPTS = Options(data_file=os.path.join( config.raw_data_dir("squad"), split + ("-debug" if config.debug else "") + ".json"), pred_file=config.qa_preds_file("squad"), out_file=config.qa_eval_file("squad"), na_prob_file=config.qa_na_file("squad"), na_prob_thresh=config.qa_na_threshold, out_image_dir=None, verbose=False)
def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer): categories = read_tsv( os.path.join(config.raw_data_dir("scopefold"), "categories" + ".tsv")) # with open("./ft_data/scope/folds.tsv") as f: # categories = [line.rstrip('\n') for line in f] super(SCOPeFold, self).__init__(config, "scope", tokenizer, categories)
def run_finetuning(config: configure_finetuning.FinetuningConfig): """Run finetuning.""" os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = str(config.gpu) # Setup for training results = [] trial = 1 heading_info = "model={:}, trial {:}/{:}".format(config.model_name, trial, config.num_trials) heading = lambda msg: utils.heading(msg + ": " + heading_info) heading("Config") utils.log_config(config) generic_model_dir = config.model_dir tasks = task_builder.get_tasks(config) # Train and evaluate num_trials models with different random seeds while config.num_trials < 0 or trial <= config.num_trials: config.model_dir = generic_model_dir + "_" + str(trial) if config.do_train: utils.rmkdir(config.model_dir) model_runner = ModelRunner(config, tasks) if config.do_train: heading("Start training") model_runner.train() utils.log() if config.do_eval: heading("Run dev set evaluation") model_runner.evaluate() # results.append(model_runner.evaluate()) # write_results(config, results) # if config.write_test_outputs and trial <= config.n_writes_test: # heading("Running on the test set and writing the predictions") # for task in tasks: # # Currently only writing preds for GLUE and SQuAD 2.0 is supported # if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp","sts","conv"]: # for split in task.get_test_splits(): # model_runner.write_classification_outputs([task], trial, split) # elif task.name == "squad": # scorer = model_runner.evaluate_task(task, "test", False) # scorer.write_predictions() # preds = utils.load_json(config.qa_preds_file("squad")) # null_odds = utils.load_json(config.qa_na_file("squad")) # for q, _ in preds.items(): # if null_odds[q] > config.qa_na_threshold: # preds[q] = "" # utils.write_json(preds, config.test_predictions( # task.name, "test", trial)) # else: # utils.log("Skipping task", task.name, # "- writing predictions is not supported for this task") if trial != config.num_trials and (not config.keep_all_models): utils.rmrf(config.model_dir) trial += 1
def predict(config: configure_finetuning.FinetuningConfig): """using a trained model for task name, do predictions for test data set and save the results """ no_trials = config.num_trials if config.num_trials > 0 else 1 print(no_trials) generic_model_dir = config.model_dir for trial in range(1, no_trials + 1): utils.log_config(config) tasks = task_builder.get_tasks(config) config.model_dir = generic_model_dir + "_" + str(trial) print("config.model_dir:{}".format(config.model_dir)) model_runner = ModelRunner(config, tasks) utils.heading("Running on the test set and writing the predictions") for task in tasks: if task.name in [ "cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp", "sts", "yesno", "reranker", "weighted-reranker", "gad", "chemprot" ]: for split in task.get_test_splits(): model_runner.write_classification_outputs([task], trial, split)
def main(config: configure_finetuning.FinetuningConfig, split, task_name): answers, samples = read_answers(os.path.join(config.raw_data_dir(task_name), split + ".json")) predictions = read_predictions(config.qa_preds_file(task_name + "_" + split)) return evaluate(answers, predictions, samples, config.pred_bad_file(task_name + "_" + split))
def main(config: configure_finetuning.FinetuningConfig, split, task_name): answers = read_answers( os.path.join(config.raw_data_dir(task_name), split + ".jsonl")) predictions = read_predictions(config.qa_preds_file(task_name)) return evaluate(answers, predictions, True)
def run_finetuning(config: configure_finetuning.FinetuningConfig): """Run finetuning.""" tf.get_variable_scope().reuse_variables() #import pdb; pdb.set_trace() # Setup for training results = [] trial = 1 heading_info = "model={:}, trial {:}/{:}".format( config.model_name, trial, config.num_trials) heading = lambda msg: utils.heading(msg + ": " + heading_info) heading("Config") utils.log_config(config) generic_model_dir = config.model_dir tasks = task_builder.get_tasks(config) # Train and evaluate num_trials models with different random seeds while config.num_trials < 0 or trial <= config.num_trials: config.model_dir = generic_model_dir + "_" + str(trial) if config.do_train: utils.rmkdir(config.model_dir) model_runner = ModelRunner(config, tasks) if config.do_train: heading("Start training") model_runner.train() utils.log() if config.do_eval: heading("Run dev set evaluation") results.append(model_runner.evaluate()) write_results(config, results) if config.write_test_outputs and trial <= config.n_writes_test: heading("Running on the test set and writing the predictions") for task in tasks: # Currently only writing preds for GLUE and SQuAD 2.0 is supported if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp", "sts"]: for split in task.get_test_splits(): model_runner.write_classification_outputs([task], trial, split) elif task.name == "squad": scorer = model_runner.evaluate_task(task, "test", False) scorer.write_predictions() preds = utils.load_json(config.qa_preds_file("squad")) null_odds = utils.load_json(config.qa_na_file("squad")) for q, _ in preds.items(): if null_odds[q] > config.qa_na_threshold: preds[q] = "" utils.write_json(preds, config.test_predictions( task.name, "test", trial)) else: utils.log("Skipping task", task.name, "- writing predictions is not supported for this task") if trial != config.num_trials and (not config.keep_all_models): utils.rmrf(config.model_dir) trial += 1 # exporting the model if config.export_dir: # with tf.variable_scope(tf.get_variable_scope(), reuse=True): # model_runner = ModelRunner(config, tasks) # tf.gfile.MakeDirs(config.export_dir) # checkpoint_path = os.path.join(config.init_checkpoint, "model.ckpt-6315") # squad_serving_input_fn = ( # build_squad_serving_input_fn(config.max_seq_length)) # utils.log("Starting to export model.") # subfolder = model_runner._estimator.export_saved_model( # export_dir_base=os.path.join(config.export_dir, "saved_model"), # serving_input_receiver_fn=squad_serving_input_fn) tf.get_variable_scope().reuse_variables() model_runner = ModelRunner(config, tasks) tf.gfile.MakeDirs(config.export_dir) checkpoint_path = os.path.join(config.init_checkpoint, "model.ckpt-6315") squad_serving_input_fn = ( build_squad_serving_input_fn(config.max_seq_length)) utils.log("Starting to export model.") subfolder = model_runner._estimator.export_saved_model( export_dir_base=os.path.join(config.export_dir, "saved_model"), serving_input_receiver_fn=squad_serving_input_fn)