def main(config: configure_finetuning.FinetuningConfig, split, bioasq=False): expected_version = '1.1' # parser = argparse.ArgumentParser( # description='Evaluation for SQuAD ' + expected_version) # parser.add_argument('dataset_file', help='Dataset file') # parser.add_argument('prediction_file', help='Prediction File') # args = parser.parse_args() Args = collections.namedtuple("Args", ["dataset_file", "prediction_file"]) args = Args(dataset_file=os.path.join( config.raw_data_dir("squadv1"), split + ("-debug" if config.debug else "") + ".json"), prediction_file=config.qa_preds_file("squadv1")) if bioasq: args = Args(dataset_file=os.path.join( config.raw_data_dir("bioasq"), split + ("0debug" if config.debug else "") + ".json"), prediction_file=config.qa_preds_file("bioasq")) with tf.io.gfile.GFile(args.dataset_file) as dataset_file: dataset_json = json.load(dataset_file) if dataset_json['version'] != expected_version: print('Evaluation expects v-' + expected_version + ', but got dataset with v-' + dataset_json['version'], file=sys.stderr) dataset = dataset_json['data'] with tf.io.gfile.GFile(args.prediction_file) as prediction_file: predictions = json.load(prediction_file) return evaluate(dataset, predictions)
def __init__(self, config: configure_finetuning.FinetuningConfig, tokenizer): categories = read_tsv( os.path.join(config.raw_data_dir("scopefold"), "categories" + ".tsv")) # with open("./ft_data/scope/folds.tsv") as f: # categories = [line.rstrip('\n') for line in f] super(SCOPeFold, self).__init__(config, "scope", tokenizer, categories)
def set_opts(config: configure_finetuning.FinetuningConfig, split): global OPTS Options = collections.namedtuple("Options", [ "data_file", "pred_file", "out_file", "na_prob_file", "na_prob_thresh", "out_image_dir", "verbose" ]) OPTS = Options(data_file=os.path.join( config.raw_data_dir("squad"), split + ("-debug" if config.debug else "") + ".json"), pred_file=config.qa_preds_file("squad"), out_file=config.qa_eval_file("squad"), na_prob_file=config.qa_na_file("squad"), na_prob_thresh=config.qa_na_threshold, out_image_dir=None, verbose=False)
def main(config: configure_finetuning.FinetuningConfig, split, task_name): answers, samples = read_answers(os.path.join(config.raw_data_dir(task_name), split + ".json")) predictions = read_predictions(config.qa_preds_file(task_name + "_" + split)) return evaluate(answers, predictions, samples, config.pred_bad_file(task_name + "_" + split))
def main(config: configure_finetuning.FinetuningConfig, split, task_name): answers = read_answers( os.path.join(config.raw_data_dir(task_name), split + ".jsonl")) predictions = read_predictions(config.qa_preds_file(task_name)) return evaluate(answers, predictions, True)