def plot_normal_decomposition(decomp: str):
normalizer = NormalizedGraphMatchScorer()
decomposition = Decomposition.from_str(decomp)
print("decomposition:", decomposition.to_string())
norm_decomposition = normalizer.normalized_decomposition(decomposition,
verbose=True)
print("normal form:", norm_decomposition.to_string())
print("=========================================================")
draw_decomposition_graph(decomposition.to_graph(), title="decomposition")
draw_decomposition_graph(norm_decomposition.to_graph(),
title="normal form")
def format_qdmr(input:str):
# replace multiple whitespaces with a single whitespace.
input = ' '.join(input.split())
# replace semi-colons with @@SEP@@ token, remove 'return' statements.
parts = input.split(';')
parts = [re.sub(r'return', '', part.strip().strip('\r')) for part in parts]
# replacing references with special tokens, for example replacing #2 with @@2@@.
parts = [re.sub(r'#(\d+)', '@@\g<1>@@', part) for part in parts]
return Decomposition(parts)
def predictions_to_norm(df, qdmr_col, norm_col):
df[norm_col] = df[qdmr_col]
for index, row in df.iterrows():
dec = row[norm_col]
try:
decomposition = norm_g.normalized_decomposition(
Decomposition.from_str(dec))
df.loc[index, norm_col] = decomposition.to_string()
except Exception as ex:
print(f"error in index {index}:{str(ex)}\n{dec}", flush=True)
traceback.print_exc()
df.loc[index, norm_col] = "ERROR"
def normalized_decomposition(self,
decomposition: Decomposition,
verbose: bool = False) -> Decomposition:
norm_g = self.normalize_graph(graph=decomposition.to_graph(),
verbose=verbose)
return Decomposition.from_graph(graph=norm_g)
def str_test_create_graphs(inp: str, output:[str]):
in_g = Decomposition([inp]).to_graph()
out_g = Decomposition(output).to_graph()
return in_g, out_g
def update_sucessors(graph, n_id, doc:[Token]):
refs = Decomposition._get_references_ids(" ".join([t.text for t in doc]))
graph.remove_edges_from([(n_id, s_id) for s_id in graph.successors(n_id)])
graph.add_edges_from([(n_id, r) for r in refs])
def evaluate(ids, questions, decompositions, golds, metadata,
output_path_base,
metrics=None):
decompositions_str = [d.to_string() for d in decompositions]
golds_str = [g.to_string() for g in golds]
# calculating exact match scores
exact_match = get_exact_match(decompositions_str, golds_str) \
if (metrics is None) or 'exact_match' in metrics else None
# evaluate using SARI
sari = get_sari_score(decompositions_str, golds_str, questions) \
if (metrics is None) or 'sari' in metrics else None
# evaluate using sequence matcher
match_ratio = get_match_ratio(decompositions_str, golds_str) \
if (metrics is None) or 'match' in metrics else None
structural_match_ratio = get_structural_match_ratio(decompositions_str, golds_str) \
if (metrics is None) or 'structural_match' in metrics else None
# evaluate using graph distances
graph_scorer = GraphMatchScorer()
decomposition_graphs = [d.to_graph() for d in decompositions]
gold_graphs = [g.to_graph() for g in golds]
ged_scores = graph_scorer.get_edit_distance_match_scores(decomposition_graphs, gold_graphs)
# structural_ged_scores = graph_scorer.get_edit_distance_match_scores(decomposition_graphs, gold_graphs,
# structure_only=True)
# ged_plus_scores = get_ged_plus_scores(decomposition_graphs, gold_graphs,
# exclude_thr=5, num_processes=num_processes)
# calculate normalized match scores
normalize_scorer = NormalizedGraphMatchScorer()
def try_invoke(func, graph, default=None):
try:
return func(graph)
except Exception as ex:
return default
decomposition_norm_graphs = [try_invoke(normalize_scorer.normalize_graph, g, default=g) for g in
decomposition_graphs]
decomposition_norm_str = [try_invoke(lambda x: Decomposition.from_graph(x).to_string(), g) for g in
decomposition_norm_graphs]
gold_norm_graphs = [try_invoke(normalize_scorer.normalize_graph, g, default=g) for g in gold_graphs]
gold_norm_str = [try_invoke(lambda x: Decomposition.from_graph(x).to_string(), g) for g in gold_norm_graphs]
normalized_exact_match = skip_none(get_exact_match, decomposition_norm_str, gold_norm_str) \
if (metrics is None) or 'normalized_exact_match' in metrics else None
normalized_sari = skip_none(get_sari_score, decomposition_norm_str, gold_norm_str, questions) \
if (metrics is None) or 'normalized_sari' in metrics else None
normalized_match_ratio = skip_none(get_match_ratio, decomposition_norm_str, gold_norm_str) \
if (metrics is None) or 'normalized_match' in metrics else None
normalized_structural_match_ratio = skip_none(get_structural_match_ratio, decomposition_norm_str, gold_norm_str) \
if (metrics is None) or 'normalized_structural_match' in metrics else None
evaluation_dict = {
"id": ids,
"question": questions,
"gold": golds_str,
"prediction": decompositions_str,
"exact_match": exact_match,
"match": match_ratio,
"structural_match": structural_match_ratio,
"sari": sari,
"ged": ged_scores,
# "structural_ged": structural_ged_scores,
# "ged_plus": ged_plus_scores,
"normalized_exact_match": normalized_exact_match,
"normalized_match": normalized_match_ratio,
"normalized_structural_match": normalized_structural_match_ratio,
"normalized_sari": normalized_sari,
}
evaluation_dict = {k: v for k, v in evaluation_dict.items() if v is not None}
num_examples = len(questions)
print_first_example_scores(evaluation_dict, min(5, num_examples))
mean_scores = print_score_stats(evaluation_dict)
if output_path_base:
write_evaluation_output(output_path_base, num_examples, **evaluation_dict)
### Addition write the mean scores json
write_evaluation_results(mean_scores, output_path_base)
if metadata is not None:
#metadata = metadata[metadata["question_text"].isin(evaluation_dict["question"])]
metadata = metadata[metadata['question_id'].isin(evaluation_dict['id'])]
metadata["dataset"] = metadata["question_id"].apply(lambda x: x.split("_")[0])
metadata["num_steps"] = metadata["decomposition"].apply(lambda x: len(x.split(";")))
score_keys = [key for key in evaluation_dict if key not in ["id", "question", "gold", "prediction"]]
for key in score_keys:
metadata[key] = evaluation_dict[key]
for agg_field in ["dataset", "num_steps"]:
df = metadata[[agg_field] + score_keys].groupby(agg_field).agg("mean")
print(df.round(decimals=3))
return mean_scores