def _unpack_model_batch_prediction(self,
batch,
coerce_tree=False) -> np.ndarray:
"""
Interpret prediction result per batch
coerce_tree = True if you want to ensure that the output forms a tree
"""
out_dict = self.model(**batch)
pred_matrix = out_dict["pred_matrix"]
batch_interpretation = []
for es in range(len(pred_matrix)):
essay_pred = tonp(pred_matrix[es])
# decoding using simple argmax
essay_pred = np.argmax(essay_pred, axis=-1)
dist_interpretation = []
for i in range(len(essay_pred)):
dist_interpretation.append(essay_pred[i] - i)
# check if the output is a tree
rep = TreeBuilder(dist_interpretation)
if (not rep.is_tree()) and (coerce_tree == True):
# run MINIMUM spanning tree
attn_matrix = tonp(pred_matrix[es])
attn_matrix = np.array(attn_matrix)
rank_order = get_rank_order(attn_matrix)
dist_interpretation = run_MST(
rank_order, rank_order, verdict="min"
) # --> use rank as the weight, "minimum" spanning tree, lower_rank number in rank is better
# add the decoding result to the batch result
batch_interpretation.append(dist_interpretation)
return batch_interpretation
def structured_output_quality(links) -> (List, float, float, float):
"""
Infer component labels automatically from the structure
"""
component_labels = []
tree_ratio = 0
avg_depth = 0
avg_leaf_prop = 0
all_depths = []
n_essays = len(links)
for i in range(len(links)):
rep = TreeBuilder(links[i])
component_labels.append(rep.auto_component_labels(AC_breakdown=True))
if rep.is_tree():
tree_ratio += 1
# evaluate this only when the output forms a tree
depth, leaf_prop = rep.tree_depth_and_leaf_proportion()
avg_depth += depth
all_depths.append(depth)
avg_leaf_prop += leaf_prop
return component_labels, float(tree_ratio) / float(n_essays), float(
avg_depth) / float(tree_ratio), float(avg_leaf_prop) / float(
tree_ratio), all_depths
def f1_per_depth(dist_gold: List, dist_prediction: List, max_depth: int):
"""
Find at which depth prediction mismatches happen (when the output forms a tree)
Args:
dist_gold (List): gold answer per essay
dist_prediction (List): predicted answer per essay
max_depth (int): max structure depth in the dataset
Returns:
tuple, i.e., (list, list, list)
"""
gold_all_depth = []
pred_all_depth = []
for i in range(len(dist_gold)):
rep_gold = TreeBuilder(dist_gold[i])
rep_pred = TreeBuilder(dist_prediction[i])
if rep_pred.is_tree():
g_depths = rep_gold.node_depths()
p_depths = rep_pred.node_depths()
gold_all_depth.append(g_depths)
pred_all_depth.append(p_depths)
gold_all_depth_flat = flatten_list(gold_all_depth)
pred_all_depth_flat = flatten_list(pred_all_depth)
print("=== Depth prediction performance when output forms a tree ===")
print(
classification_report(y_true=gold_all_depth_flat,
y_pred=pred_all_depth_flat,
digits=3))
report = classification_report(y_true=gold_all_depth_flat,
y_pred=pred_all_depth_flat,
output_dict=True)
f1s = []
for i in range(max_depth):
try:
f1s.append(report[str(i)]['f1-score'])
except:
f1s.append(0.0)
return f1s
def _unpack_model_batch_prediction(self,
batch,
coerce_tree=False) -> np.ndarray:
"""
Interpret prediction result per batch
coerce_tree = True if we want to make sure that the predictions form a tree (using MST (min or max) algorithm)
"""
out_dict = self.model(**batch)
pred_linking_softmax = tonp(out_dict["pred_linking_softmax"])
pred_node_labelling_softmax = tonp(
out_dict["pred_node_labelling_softmax"])
linking_preds = []
node_labelling_preds = []
for es in range(len(pred_linking_softmax)):
essay_linking = []
essay_labelling = []
max_seq_len = batch["seq_len"][es]
# simple decoding using argmax
for s in range(
max_seq_len
): # iterate each sentence in the essay, s is the index of the current sentence
# perform constrained argmax for linking
curr_link_softmax = pred_linking_softmax[es][s]
ranked_pred = [
i for i in reversed(
sorted(enumerate(curr_link_softmax),
key=lambda x: x[1]))
]
for i in range(len(ranked_pred)):
tmp_dist = self.dist_idx_to_dist(ranked_pred[i][0])
if 0 <= tmp_dist + s <= max_seq_len - 1:
pred_dist = tmp_dist
break
# argmax for labelling
curr_label_softmax = pred_node_labelling_softmax[es][s]
pred_idx = np.argmax(curr_label_softmax)
pred_label = self.component_idx_to_label(pred_idx)
# essay-level result
essay_linking.append(pred_dist)
essay_labelling.append(pred_label)
# check if the output is tree
rep = TreeBuilder(essay_linking)
if (not rep.is_tree()) and (coerce_tree == True):
attn_matrix = [
] # element [i,j] denotes the probability of sentence i connects to sentence j (j as the target)
for s in range(
max_seq_len
): # iterate each sentence in the essay, s is the index of the current sentence
curr_pred = pred_linking_softmax[es][s]
# get the prediction to each possible target sentence in the text
row_pred = [0] * max_seq_len
for i in range(len(curr_pred)):
temp_dist = self.dist_idx_to_dist(i)
value = curr_pred[i]
if 0 <= temp_dist + s <= max_seq_len - 1:
row_pred[temp_dist + s] = value
attn_matrix.append(row_pred)
# run MAXIMUM spanning tree
attn_matrix = np.array(attn_matrix)
rank_order = get_rank_order(attn_matrix)
essay_linking = run_MST(
rank_order, attn_matrix, verdict="max"
) # --> use the softmax probability as the weight, we run the maximum spanning tree here because higher probability means better
# batch-level result
linking_preds.append(essay_linking)
node_labelling_preds.append(essay_labelling)
return linking_preds, node_labelling_preds
def _unpack_model_batch_prediction(self,
batch,
coerce_tree=False) -> np.ndarray:
"""
Interpret prediction result per batch
"""
out_dict = self.model(**batch)
pred_softmax = tonp(out_dict["pred_softmax"])
# print("seq len", batch["seq_len"])
# print(pred_softmax.shape)
batch_interpretation = []
for es in range(len(pred_softmax)):
essay_interpretation = []
max_seq_len = batch["seq_len"][es]
# simple decoding using argmax
for s in range(
max_seq_len
): # iterate each sentence in the essay, s is the index of the current sentence
curr_pred = pred_softmax[es][s]
# perform constrained argmax
ranked_pred = [
i for i in reversed(
sorted(enumerate(curr_pred), key=lambda x: x[1]))
]
# print(ranked_pred)
for i in range(len(ranked_pred)):
tmp_dist = self.dist_idx_to_dist(ranked_pred[i][0])
# print(tmp_dist, tmp_dist+s)
# input()
if 0 <= tmp_dist + s <= max_seq_len - 1:
pred_dist = tmp_dist
break
essay_interpretation.append(pred_dist)
# check if the output is tree
rep = TreeBuilder(essay_interpretation)
if (not rep.is_tree()) and (coerce_tree == True):
attn_matrix = [
] # element [i,j] denotes the probability of sentence i connects to sentence j (j as the target)
for s in range(
max_seq_len
): # iterate each sentence in the essay, s is the index of the current sentence
curr_pred = pred_softmax[es][s]
# get the prediction to each possible target sentence in the text
row_pred = [0] * max_seq_len
for i in range(len(curr_pred)):
temp_dist = self.dist_idx_to_dist(i)
value = curr_pred[i]
if 0 <= temp_dist + s <= max_seq_len - 1:
row_pred[temp_dist + s] = value
attn_matrix.append(row_pred)
# run MAXIMUM spanning tree
attn_matrix = np.array(attn_matrix)
rank_order = get_rank_order(attn_matrix)
essay_interpretation = run_MST(
rank_order, attn_matrix, verdict="max"
) # --> use the softmax probability as the weight, we run the maximum spanning tree here because higher probability means better
batch_interpretation.append(essay_interpretation)
return batch_interpretation
