def main():
parser = argparse.ArgumentParser()
parser.add_argument("predictions")
parser.add_argument("input_df_pkl")
args = parser.parse_args()
test_df = pd.read_pickle(args.input_df_pkl)
y_pred = []
with open(args.predictions) as fp:
for line in fp:
y_pred += line.strip().split()
y_true = reduce(lambda acc, l: acc + l, test_df.entity_type, [])
y_true = pd.Series(y_true)
y_pred = pd.Series(y_pred)
print(len(y_true))
print(len(y_pred))
print("Token level")
eval_d = ner_report(y_true, y_pred, mode="token", return_dict=True)
eval_d = dict(eval_d["PATHWAY"])
with open("pathway_metrics_token.json", "w") as fp:
json.dump(eval_d, fp)
fp.write("\n")
pprint(eval_d)
print("Entity level")
y_pred_corr = pd.Series(correct_iob(y_pred))
eval_d = ner_report(y_true, y_pred_corr, mode="entity", return_dict=True)
eval_d = dict(eval_d["PATHWAY"])
with open("pathway_metrics_entity.json", "w") as fp:
json.dump(eval_d, fp)
fp.write("\n")
pprint(eval_d)
print("Seqeval")
y_true = list(test_df.entity_type)
y_pred = []
with open(args.predictions) as fp:
for line in fp:
y_pred.append(line.strip().split())
from collections import Counter
c = Counter()
for x in y_true:
c.update(x)
print(c)
total = 0
for s1, s2 in zip(y_true, y_pred):
total += sum(t1 == t2 for t1, t2 in zip(s1, s2))
acc = total / sum(len(s) for s in y_true)
print("acc:", acc)
print("acc_score:", accuracy_score(y_true, y_pred))
print(classification_report(y_true, y_pred, scheme=IOB2, mode="strict"))
print(performance_measure(y_true, y_pred))
def test_ner_report(ner_annotations, dataset, mode, etypes_map, dict_tp_fn_fp):
report_str = ner_report(
ner_annotations[dataset]["annotator_1"],
ner_annotations[dataset]["annotator_2"],
mode=mode,
etypes_map=etypes_map,
return_dict=False,
)
report_dict = ner_report(
ner_annotations[dataset]["annotator_1"],
ner_annotations[dataset]["annotator_2"],
mode=mode,
etypes_map=etypes_map,
return_dict=True,
)
assert isinstance(report_str, str)
assert isinstance(report_dict, OrderedDict)
etypes = sorted(dict_tp_fn_fp.keys())
assert list(report_dict.keys()) == etypes
for etype in etypes:
assert set(report_dict[etype].keys()) == {
"precision",
"recall",
"f1-score",
"support",
}
tp, fn, fp = dict_tp_fn_fp[etype]
tot_true_pos = tp + fn
tot_pred_pos = tp + fp
prec_ = (tp / tot_pred_pos) if tot_pred_pos > 0 else 0
recall_ = tp / tot_true_pos
f1_ = (2 * prec_ * recall_ /
(prec_ + recall_)) if tot_pred_pos > 0 else 0
np.testing.assert_almost_equal(prec_, report_dict[etype]["precision"])
np.testing.assert_almost_equal(recall_, report_dict[etype]["recall"])
np.testing.assert_almost_equal(f1_, report_dict[etype]["f1-score"])
np.testing.assert_almost_equal(tot_true_pos,
report_dict[etype]["support"])
def main():
print("Read params.yaml...")
params = yaml.safe_load(open("params.yaml"))["eval"][args.etype]
# Load and preprocess the annotations
df = annotations2df(args.annotation_files.split(","))
ner_model = spacy.load(args.model)
df_pred = []
for source, df_ in df.groupby("source"):
df_ = df_.sort_values(by="id", ignore_index=True)
df_sentence = spacy2df(spacy_model=ner_model,
ground_truth_tokenization=df_["text"].to_list())
df_sentence["id"] = df_["id"].values
df_sentence["source"] = source
df_pred.append(df_sentence)
df_pred = pd.concat(
df_pred, ignore_index=True).rename(columns={"class": "class_pred"})
df = df.merge(df_pred, on=["source", "id", "text"], how="inner")
df = remove_punctuation(df)
iob_true = df["class"]
iob_pred = df["class_pred"]
output_file = pathlib.Path(args.output_file)
with output_file.open("w") as f:
all_metrics_dict = OrderedDict()
for mode in ["entity", "token"]:
metrics_dict = ner_report(
iob_true,
iob_pred,
mode=mode,
return_dict=True,
etypes_map={args.etype: params["etype_name"]},
)[args.etype]
metrics_dict = OrderedDict([(f"{mode}_{k}", v)
for k, v in metrics_dict.items()])
all_metrics_dict.update(metrics_dict)
json.dump(all_metrics_dict, f)
def main():
# Load and preprocess the annotations
print("Loading data and model")
df = annotations2df(args.annotation_files.split(","))
ner_model = spacy.load(args.model)
print("Computing predictions")
df_pred = []
for source, df_ in df.groupby("source"):
df_ = df_.sort_values(by="id", ignore_index=True)
df_sentence = spacy2df(spacy_model=ner_model,
ground_truth_tokenization=df_["text"].to_list())
df_sentence["id"] = df_["id"].values
df_sentence["source"] = source
df_pred.append(df_sentence)
print("Formatting predctions")
df_pred = pd.concat(
df_pred, ignore_index=True).rename(columns={"class": "class_pred"})
df = df.merge(df_pred, on=["source", "id", "text"], how="inner")
#df = remove_punctuation(df)
iob_true = df["class"]
iob_pred = df["class_pred"]
print("Saving predictions")
df.to_pickle("df_test_pred.pkl")
print("Computing and saving metrics")
output_file = pathlib.Path(args.output_file)
metrics_dict = ner_report(
iob_true,
iob_pred,
mode="token",
return_dict=True,
)
metrics_dict = dict(metrics_dict[args.etype])
with output_file.open("w") as f:
json.dump(metrics_dict, f)
f.write("\n")
pprint(metrics_dict)
def main():
# Load and preprocess the annotations
df_a1 = annotations2df(args.annotations1.split(","))
df_a2 = annotations2df(args.annotations2.split(","))
# Merge the common sentences between annotators
df = df_a2.merge(
df_a1,
on=["source", "id", "text", "start_char", "end_char"],
suffixes=("_annotator_2", "_annotator_1"),
how="inner",
)
df = remove_punctuation(df)
iob_true = df["class_annotator_1"]
iob_pred = df["class_annotator_2"]
metrics_dict = {}
for mode in ["entity", "token"]:
metrics_dict[mode] = ner_report(
iob_true,
iob_pred,
mode=mode,
return_dict=True,
)
for etype in metrics_dict["entity"]:
all_metrics_dict = OrderedDict()
for mode in ["entity", "token"]:
all_metrics_dict.update([
(f"{mode}_{k}", v)
for k, v in metrics_dict[mode][etype].items()
])
filename = etype.lower() + ".json"
output_file = pathlib.Path(args.output_dir) / filename
with output_file.open("w") as f:
json.dump(all_metrics_dict, f)