def test_evaluation():
##########################
########## Settings
##########################
lang_model = "deepset/roberta-base-squad2"
do_lower_case = False
test_assertions = True
data_dir = Path("testsave/data/squad20")
evaluation_filename = "dev-v2.0.json"
device, n_gpu = initialize_device_settings(use_cuda=True)
# loading models and evals
model = AdaptiveModel.convert_from_transformers(
lang_model, device=device, task_type="question_answering")
model.prediction_heads[0].no_ans_boost = 0
model.prediction_heads[0].n_best = 1
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=256,
label_list=["start_token", "end_token"],
metric="squad",
train_filename=None,
dev_filename=None,
dev_split=0,
test_filename=evaluation_filename,
data_dir=data_dir,
doc_stride=128,
)
starttime = time()
data_silo = DataSilo(processor=processor, batch_size=50)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
evaluator = Evaluator(data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# 1. Test FARM internal evaluation
results = evaluator.eval(model)
f1_score = results[0]["f1"] * 100
em_score = results[0]["EM"] * 100
tnrecall = results[0]["top_n_recall"] * 100
elapsed = time() - starttime
print(results)
print(elapsed)
gold_EM = 77.7478
gold_f1 = 82.1557
gold_tnrecall = 84.0646 # top 1 recall
gold_elapsed = 70 # 4x V100
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=f"FARM Eval changed for EM by: {em_score-gold_EM}")
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=f"FARM Eval changed for f1 score by: {f1_score-gold_f1}")
np.testing.assert_allclose(
tnrecall,
gold_tnrecall,
rtol=0.001,
err_msg=f"FARM Eval changed for top 1 recall by: {em_score-gold_EM}"
)
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"FARM Eval speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
# 2. Test FARM predictions with outside eval script
starttime = time()
model = Inferencer(model=model,
processor=processor,
task_type="question_answering",
batch_size=50,
gpu=device.type == "cuda")
filename = data_dir / evaluation_filename
result = model.inference_from_file(file=filename)
elapsed = time() - starttime
os.makedirs("../testsave", exist_ok=True)
write_squad_predictions(predictions=result,
predictions_filename=filename,
out_filename="testsave/predictions.json")
script_params = {
"data_file": filename,
"pred_file": "testsave/predictions.json",
"na_prob_thresh": 1,
"na_prob_file": False,
"out_file": False
}
results_official = squad_evaluation.main(OPTS=DotMap(script_params))
f1_score = results_official["f1"]
em_score = results_official["exact"]
gold_EM = 78.4890
gold_f1 = 81.7104
gold_elapsed = 66 # 4x V100
print(elapsed)
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=
f"Eval with official script changed for EM by: {em_score - gold_EM}"
)
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=
f"Eval with official script changed for f1 score by: {f1_score - gold_f1}"
)
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"Inference speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
def test_evaluation():
##########################
########## Settings
##########################
lang_model = "deepset/roberta-base-squad2"
do_lower_case = False
test_assertions = False
data_dir = Path("testsave/data/squad20")
evaluation_filename = "dev-v2.0.json"
device, n_gpu = initialize_device_settings(use_cuda=True)
# loading models and evals
model = AdaptiveModel.convert_from_transformers(
lang_model, device=device, task_type="question_answering")
model.prediction_heads[0].no_ans_boost = 0
model.prediction_heads[0].n_best = 1
model.prediction_heads[0].n_best_per_sample = 1
tokenizer = Tokenizer.load(pretrained_model_name_or_path=lang_model,
do_lower_case=do_lower_case)
processor = SquadProcessor(
tokenizer=tokenizer,
max_seq_len=256,
label_list=["start_token", "end_token"],
metric="squad",
train_filename=None,
dev_filename=None,
dev_split=0,
test_filename=evaluation_filename,
data_dir=data_dir,
doc_stride=128,
)
starttime = time()
data_silo = DataSilo(processor=processor, batch_size=40 * n_gpu_factor)
model.connect_heads_with_processor(data_silo.processor.tasks,
require_labels=True)
model, _ = optimize_model(model=model,
device=device,
local_rank=-1,
optimizer=None,
distributed=False,
use_amp=None)
evaluator = Evaluator(data_loader=data_silo.get_data_loader("test"),
tasks=data_silo.processor.tasks,
device=device)
# 1. Test FARM internal evaluation
results = evaluator.eval(model)
f1_score = results[0]["f1"] * 100
em_score = results[0]["EM"] * 100
tnacc = results[0]["top_n_accuracy"] * 100
elapsed = time() - starttime
print(results)
print(elapsed)
gold_EM = 78.4721
gold_f1 = 82.6671
gold_tnacc = 84.3594 # top 1 recall
gold_elapsed = 40 # 4x V100
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=f"FARM Eval changed for EM by: {em_score-gold_EM}")
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=f"FARM Eval changed for f1 score by: {f1_score-gold_f1}")
np.testing.assert_allclose(
tnacc,
gold_tnacc,
rtol=0.001,
err_msg=
f"FARM Eval changed for top 1 accuracy by: {tnacc-gold_tnacc}")
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"FARM Eval speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
if not np.allclose(f1_score, gold_f1, rtol=0.001):
error_messages.append(
f"FARM Eval changed for f1 score by: {round(f1_score - gold_f1, 4)}"
)
if not np.allclose(em_score, gold_EM, rtol=0.001):
error_messages.append(
f"FARM Eval changed for EM by: {round(em_score - gold_EM, 4)}")
if not np.allclose(tnacc, gold_tnacc, rtol=0.001):
error_messages.append(
f"FARM Eval changed for top 1 accuracy by: {round(tnacc-gold_tnacc, 4)}"
)
if not np.allclose(elapsed, gold_elapsed, rtol=0.1):
error_messages.append(
f"FARM Eval speed changed significantly by: {round(elapsed - gold_elapsed, 4)} seconds"
)
benchmark_result = [{
"run": "FARM internal evaluation",
"f1_change": round(f1_score - gold_f1, 4),
"em_change": round(em_score - gold_EM, 4),
"tnacc_change": round(tnacc - gold_tnacc, 4),
"elapsed_change": round(elapsed - gold_elapsed, 4),
"f1": f1_score,
"em": em_score,
"tnacc": round(tnacc, 4),
"elapsed": elapsed,
"f1_gold": gold_f1,
"em_gold": gold_EM,
"tnacc_gold": gold_tnacc,
"elapsed_gold": gold_elapsed
}]
logger.info("\n\n" + pformat(benchmark_result[0]) + "\n")
# # 2. Test FARM predictions with outside eval script
starttime = time()
model = Inferencer(model=model,
processor=processor,
task_type="question_answering",
batch_size=40 * n_gpu_factor,
gpu=device.type == "cuda")
filename = data_dir / evaluation_filename
result = model.inference_from_file(file=filename,
return_json=False,
multiprocessing_chunksize=80)
results_squad = [x.to_squad_eval() for x in result]
model.close_multiprocessing_pool()
elapsed = time() - starttime
os.makedirs("../testsave", exist_ok=True)
write_squad_predictions(predictions=results_squad,
predictions_filename=filename,
out_filename="testsave/predictions.json")
script_params = {
"data_file": filename,
"pred_file": "testsave/predictions.json",
"na_prob_thresh": 1,
"na_prob_file": False,
"out_file": False
}
results_official = squad_evaluation.main(OPTS=DotMap(script_params))
f1_score = results_official["f1"]
em_score = results_official["exact"]
gold_EM = 79.878
gold_f1 = 82.917
gold_elapsed = 27 # 4x V100
print(elapsed)
if test_assertions:
np.testing.assert_allclose(
em_score,
gold_EM,
rtol=0.001,
err_msg=
f"Eval with official script changed for EM by: {em_score - gold_EM}"
)
np.testing.assert_allclose(
f1_score,
gold_f1,
rtol=0.001,
err_msg=
f"Eval with official script changed for f1 score by: {f1_score - gold_f1}"
)
np.testing.assert_allclose(
elapsed,
gold_elapsed,
rtol=0.1,
err_msg=
f"Inference speed changed significantly by: {elapsed - gold_elapsed} seconds"
)
if not np.allclose(f1_score, gold_f1, rtol=0.001):
error_messages.append(
f"Eval with official script changed for f1 score by: {round(f1_score - gold_f1, 4)}"
)
if not np.allclose(em_score, gold_EM, rtol=0.001):
error_messages.append(
f"Eval with official script changed for EM by: {round(em_score - gold_EM, 4)}"
)
if not np.allclose(elapsed, gold_elapsed, rtol=0.1):
error_messages.append(
f"Inference speed changed significantly by: {round(elapsed - gold_elapsed,4)} seconds"
)
benchmark_result.append({
"run": "outside eval script",
"f1_change": round(f1_score - gold_f1, 4),
"em_change": round(em_score - gold_EM, 4),
"tnacc_change": "-",
"elapsed_change": round(elapsed - gold_elapsed, 4),
"f1": f1_score,
"em": em_score,
"tnacc": "-",
"elapsed": elapsed,
"f1_gold": gold_f1,
"em_gold": gold_EM,
"tnacc_gold": "-",
"elapsed_gold": gold_elapsed
})
logger.info("\n\n" + pformat(benchmark_result[1]) + "\n")
return benchmark_result
