def main(config: configure_finetuning.FinetuningConfig, split, bioasq=False):
expected_version = '1.1'
# parser = argparse.ArgumentParser(
# description='Evaluation for SQuAD ' + expected_version)
# parser.add_argument('dataset_file', help='Dataset file')
# parser.add_argument('prediction_file', help='Prediction File')
# args = parser.parse_args()
Args = collections.namedtuple("Args", ["dataset_file", "prediction_file"])
args = Args(dataset_file=os.path.join(
config.raw_data_dir("squadv1"),
split + ("-debug" if config.debug else "") + ".json"),
prediction_file=config.qa_preds_file("squadv1"))
if bioasq:
args = Args(dataset_file=os.path.join(
config.raw_data_dir("bioasq"),
split + ("0debug" if config.debug else "") + ".json"),
prediction_file=config.qa_preds_file("bioasq"))
with tf.io.gfile.GFile(args.dataset_file) as dataset_file:
dataset_json = json.load(dataset_file)
if dataset_json['version'] != expected_version:
print('Evaluation expects v-' + expected_version +
', but got dataset with v-' + dataset_json['version'],
file=sys.stderr)
dataset = dataset_json['data']
with tf.io.gfile.GFile(args.prediction_file) as prediction_file:
predictions = json.load(prediction_file)
return evaluate(dataset, predictions)
def run_finetuning(config: configure_finetuning.FinetuningConfig):
"""Run finetuning."""
# Setup for training
results = []
trial = 1
heading_info = "model={:}, trial {:}/{:}".format(
config.model_name, trial, config.num_trials)
def heading(msg): return utils.heading(msg + ": " + heading_info)
heading("Config")
utils.log_config(config)
generic_model_dir = config.model_dir
tasks = task_builder.get_tasks(config)
# Train and evaluate num_trials models with different random seeds
while config.num_trials < 0 or trial <= config.num_trials:
config.model_dir = generic_model_dir + "_" + str(trial)
if config.do_train:
utils.rmkdir(config.model_dir)
model_runner = ModelRunner(config, tasks)
if config.do_train:
heading("Start training")
model_runner.train()
utils.log()
if config.do_eval:
heading("Run dev set evaluation")
results.append(model_runner.evaluate())
write_results(config, results)
if config.write_test_outputs and trial <= config.n_writes_test:
heading("Running on the test set and writing the predictions")
for task in tasks:
# Currently only writing preds for GLUE and SQuAD 2.0 is supported
if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp",
"sts"]:
for split in task.get_test_splits():
model_runner.write_classification_outputs(
[task], trial, split)
elif task.name == "squad":
print('start model runner')
scorer = model_runner.evaluate_task(
task, "test", False)
print('end evaluate')
scorer.write_predictions()
print('end write prediction')
preds = utils.load_json(config.qa_preds_file("squad"))
null_odds = utils.load_json(config.qa_na_file("squad"))
for q, _ in preds.items():
if null_odds[q] > config.qa_na_threshold:
preds[q] = ""
utils.write_json(preds, config.test_predictions(
task.name, "test", trial))
else:
utils.log("Skipping task", task.name,
"- writing predictions is not supported for this task")
if trial != config.num_trials and (not config.keep_all_models):
utils.rmrf(config.model_dir)
trial += 1
def set_opts(config: configure_finetuning.FinetuningConfig, split):
global OPTS
Options = collections.namedtuple("Options", [
"data_file", "pred_file", "out_file", "na_prob_file", "na_prob_thresh",
"out_image_dir", "verbose"
])
OPTS = Options(data_file=os.path.join(
config.raw_data_dir("squad"),
split + ("-debug" if config.debug else "") + ".json"),
pred_file=config.qa_preds_file("squad"),
out_file=config.qa_eval_file("squad"),
na_prob_file=config.qa_na_file("squad"),
na_prob_thresh=config.qa_na_threshold,
out_image_dir=None,
verbose=False)
def __init__(self, config: configure_finetuning.FinetuningConfig,
tokenizer):
categories = read_tsv(
os.path.join(config.raw_data_dir("scopefold"),
"categories" + ".tsv"))
# with open("./ft_data/scope/folds.tsv") as f:
# categories = [line.rstrip('\n') for line in f]
super(SCOPeFold, self).__init__(config, "scope", tokenizer, categories)
def run_finetuning(config: configure_finetuning.FinetuningConfig):
"""Run finetuning."""
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(config.gpu)
# Setup for training
results = []
trial = 1
heading_info = "model={:}, trial {:}/{:}".format(config.model_name, trial, config.num_trials)
heading = lambda msg: utils.heading(msg + ": " + heading_info)
heading("Config")
utils.log_config(config)
generic_model_dir = config.model_dir
tasks = task_builder.get_tasks(config)
# Train and evaluate num_trials models with different random seeds
while config.num_trials < 0 or trial <= config.num_trials:
config.model_dir = generic_model_dir + "_" + str(trial)
if config.do_train:
utils.rmkdir(config.model_dir)
model_runner = ModelRunner(config, tasks)
if config.do_train:
heading("Start training")
model_runner.train()
utils.log()
if config.do_eval:
heading("Run dev set evaluation")
model_runner.evaluate()
# results.append(model_runner.evaluate())
# write_results(config, results)
# if config.write_test_outputs and trial <= config.n_writes_test:
# heading("Running on the test set and writing the predictions")
# for task in tasks:
# # Currently only writing preds for GLUE and SQuAD 2.0 is supported
# if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp","sts","conv"]:
# for split in task.get_test_splits():
# model_runner.write_classification_outputs([task], trial, split)
# elif task.name == "squad":
# scorer = model_runner.evaluate_task(task, "test", False)
# scorer.write_predictions()
# preds = utils.load_json(config.qa_preds_file("squad"))
# null_odds = utils.load_json(config.qa_na_file("squad"))
# for q, _ in preds.items():
# if null_odds[q] > config.qa_na_threshold:
# preds[q] = ""
# utils.write_json(preds, config.test_predictions(
# task.name, "test", trial))
# else:
# utils.log("Skipping task", task.name,
# "- writing predictions is not supported for this task")
if trial != config.num_trials and (not config.keep_all_models):
utils.rmrf(config.model_dir)
trial += 1
def predict(config: configure_finetuning.FinetuningConfig):
"""using a trained model for task name, do predictions for test data set and save the results """
no_trials = config.num_trials if config.num_trials > 0 else 1
print(no_trials)
generic_model_dir = config.model_dir
for trial in range(1, no_trials + 1):
utils.log_config(config)
tasks = task_builder.get_tasks(config)
config.model_dir = generic_model_dir + "_" + str(trial)
print("config.model_dir:{}".format(config.model_dir))
model_runner = ModelRunner(config, tasks)
utils.heading("Running on the test set and writing the predictions")
for task in tasks:
if task.name in [
"cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp", "sts",
"yesno", "reranker", "weighted-reranker", "gad", "chemprot"
]:
for split in task.get_test_splits():
model_runner.write_classification_outputs([task], trial,
split)
def main(config: configure_finetuning.FinetuningConfig, split, task_name):
answers, samples = read_answers(os.path.join(config.raw_data_dir(task_name), split + ".json"))
predictions = read_predictions(config.qa_preds_file(task_name + "_" + split))
return evaluate(answers, predictions, samples, config.pred_bad_file(task_name + "_" + split))
def main(config: configure_finetuning.FinetuningConfig, split, task_name):
answers = read_answers(
os.path.join(config.raw_data_dir(task_name), split + ".jsonl"))
predictions = read_predictions(config.qa_preds_file(task_name))
return evaluate(answers, predictions, True)
def run_finetuning(config: configure_finetuning.FinetuningConfig):
"""Run finetuning."""
tf.get_variable_scope().reuse_variables() #import pdb; pdb.set_trace()
# Setup for training
results = []
trial = 1
heading_info = "model={:}, trial {:}/{:}".format(
config.model_name, trial, config.num_trials)
heading = lambda msg: utils.heading(msg + ": " + heading_info)
heading("Config")
utils.log_config(config)
generic_model_dir = config.model_dir
tasks = task_builder.get_tasks(config)
# Train and evaluate num_trials models with different random seeds
while config.num_trials < 0 or trial <= config.num_trials:
config.model_dir = generic_model_dir + "_" + str(trial)
if config.do_train:
utils.rmkdir(config.model_dir)
model_runner = ModelRunner(config, tasks)
if config.do_train:
heading("Start training")
model_runner.train()
utils.log()
if config.do_eval:
heading("Run dev set evaluation")
results.append(model_runner.evaluate())
write_results(config, results)
if config.write_test_outputs and trial <= config.n_writes_test:
heading("Running on the test set and writing the predictions")
for task in tasks:
# Currently only writing preds for GLUE and SQuAD 2.0 is supported
if task.name in ["cola", "mrpc", "mnli", "sst", "rte", "qnli", "qqp",
"sts"]:
for split in task.get_test_splits():
model_runner.write_classification_outputs([task], trial, split)
elif task.name == "squad":
scorer = model_runner.evaluate_task(task, "test", False)
scorer.write_predictions()
preds = utils.load_json(config.qa_preds_file("squad"))
null_odds = utils.load_json(config.qa_na_file("squad"))
for q, _ in preds.items():
if null_odds[q] > config.qa_na_threshold:
preds[q] = ""
utils.write_json(preds, config.test_predictions(
task.name, "test", trial))
else:
utils.log("Skipping task", task.name,
"- writing predictions is not supported for this task")
if trial != config.num_trials and (not config.keep_all_models):
utils.rmrf(config.model_dir)
trial += 1
# exporting the model
if config.export_dir:
# with tf.variable_scope(tf.get_variable_scope(), reuse=True):
# model_runner = ModelRunner(config, tasks)
# tf.gfile.MakeDirs(config.export_dir)
# checkpoint_path = os.path.join(config.init_checkpoint, "model.ckpt-6315")
# squad_serving_input_fn = (
# build_squad_serving_input_fn(config.max_seq_length))
# utils.log("Starting to export model.")
# subfolder = model_runner._estimator.export_saved_model(
# export_dir_base=os.path.join(config.export_dir, "saved_model"),
# serving_input_receiver_fn=squad_serving_input_fn)
tf.get_variable_scope().reuse_variables()
model_runner = ModelRunner(config, tasks)
tf.gfile.MakeDirs(config.export_dir)
checkpoint_path = os.path.join(config.init_checkpoint, "model.ckpt-6315")
squad_serving_input_fn = (
build_squad_serving_input_fn(config.max_seq_length))
utils.log("Starting to export model.")
subfolder = model_runner._estimator.export_saved_model(
export_dir_base=os.path.join(config.export_dir, "saved_model"),
serving_input_receiver_fn=squad_serving_input_fn)