def test_merge_scores(self):
s1 = scores.Stats_datastruct()
s2 = scores.Stats_datastruct()
truth_set = {2, 3, 4}
candidate_set_1 = {1, 2, 3}
candidate_set_2 = {2, 3, 4, 5, 6}
s1.compute_all(truth_set, candidate_set_1, 6)
s2.compute_all(truth_set, candidate_set_2, 6)
total = scores.merge_scores([s1, s2])
pprint.pprint(total)
self.assertEqual(total.ACC, (s1.ACC + s2.ACC) / 2)
self.assertEqual(total.F1, (s1.F1 + s2.F1) / 2)
self.assertEqual(total.TPR, (s1.TPR + s2.TPR) / 2)
def get_decreasing_graph(self):
perf_list = []
MAX_VAL = 20
for i in range(0, MAX_VAL):
t = i / MAX_VAL
tmp_score = stats_datastruct.Stats_datastruct()
tmp_score.TPR = 1 - (i / MAX_VAL)
perf_list.append(perf_datastruct.Perf(score=tmp_score,
threshold=t))
return perf_list
def get_real_graph(self):
perf_list = []
MAX_VAL = 20
last_F1 = 0
for i in range(0, MAX_VAL):
t = i / MAX_VAL
tmp_score = stats_datastruct.Stats_datastruct()
delta = 0.05
tmp_score.TPR = (i / MAX_VAL) # Increasing
tmp_score.FPR = (i / MAX_VAL) + delta # Increasing
tmp_score.TNR = 1 - (i / MAX_VAL) # Decreasing
tmp_score.FNR = 1 - (i / MAX_VAL) - delta # Decreasing
if i < MAX_VAL / 2:
tmp_score.F1 = (i / MAX_VAL) # Increasing until some point
last_F1 = tmp_score.F1
else:
tmp_score.F1 = last_F1
perf_list.append(perf_datastruct.Perf(score=tmp_score,
threshold=t))
return perf_list
def evaluate_performance(clusters_pairs: List[ClusterMatch],
total_number_element=None) -> List[ClusterMatch]:
"""
Compute statistic about each cluster pairs, for them members. Check the True positive, False positive, etc. rates
:param clusters_pairs: A list of pairs of clusters = [ (cluster1, clusterA), (cluster2, clusterB) ...)
:param total_number_element: Total number of members (not the sum of all members of all clusters, but how many elements is there regardless of their classification) in the "world" considered.
:return: The same List of pairs of cluster, but with the score set-up
"""
# Flush the internal memory of the evaluator and compute statistics over each clusters pair. Store means, etc. in internal memory.
for pair in clusters_pairs:
s = scores.Stats_datastruct()
# Get members of both clusters
truth_set = pair.cluster_1.members
candidate_set = pair.cluster_2.members
# Compute all values of the pair score
s.compute_all(truth_set, candidate_set, total_number_element)
# Store the score
pair.score = s
return clusters_pairs
def compute_score_for_one_threshold(
self, list_results: List, gt_graph: GraphDataStruct,
dist_threshold: float) -> stats_datastruct.Stats_datastruct:
"""
Compute stats about the quality of a result (requests_result), given a specific threshold (dist_threshold)
and compared to a ground truth graph (gt_graph)
:param list_results: Result of a similarity request to server
:param gt_graph: Ground truth file to provide to compute if matches are good or not
:param dist_threshold: threshold to apply to the results to compare to ground truth graph
:return: stats about the quality of a result
"""
# Create ready to go (with 0 valued) score object
tmp_score = stats_datastruct.Stats_datastruct()
tmp_score.reset_basics_values()
# TODO : Construct good datastructure to perform the matching
# Sort cand_graph to mapping [node.id] -> [node.id sorted by distance increasing]
# For each node and its neighbourhood (by distance)
for curr_result in list_results:
# Check if node is correctly formatted
if self.is_correct(curr_result):
# Remove its own occurence from the list if presents.
matches_list = dict_utilities.get_clear_matches(curr_result)
# For all N first matches of the current picture (or below if less matches)
nb_matches_to_process = min(self.cal_conf.NB_TO_CHECK,
len(matches_list))
for i in range(0, nb_matches_to_process):
# fetch the match to process
curr_matched_node = matches_list[i]
# Please note :
# If the two nodes are in the same cluster in gt, then it should be a positive value.
# Then this link is counted as a positive value in the entire dataset.
# The distance and threshold DOES NOT IMPACT the Positive/Negative counts !
if curr_matched_node.get("distance") <= dist_threshold:
# Even if it's request_id, it the current name of the file.
if gt_graph.are_names_in_same_cluster(
curr_result.get("request_id"),
curr_matched_node.get("image_id")):
tmp_score.TP += 1 # Match but good
tmp_score.P += 1 # Should be good
else:
tmp_score.FP += 1 # No match but not good
tmp_score.N += 1 # Should be not good
elif curr_matched_node.get("distance") > dist_threshold:
# Even if it's request_id, it the current name of the file.
if gt_graph.are_names_in_same_cluster(
curr_result.get("request_id"),
curr_matched_node.get("image_id")):
tmp_score.FN += 1 # No match but not good
tmp_score.P += 1 # Should be good
else:
tmp_score.TN += 1 # No match but good
tmp_score.N += 1 # Should be not good
else:
cluster = gt_graph.get_clusters_of(
curr_result.get("request_id"))
if cluster is None or len(cluster.members) <= 1:
# this picture has no cluster OR Only one element in the cluster,
# so it's the node = Good if no match
tmp_score.TN += 1 # No match but good
tmp_score.N += 1 # Should be not good
else:
# No matches, but not alone in the cluster, so should have been one.
tmp_score.FN += 1 # No match but not good
tmp_score.P += 1 # Should be good
tmp_score.total_nb_elements = tmp_score.P + tmp_score.N
tmp_score.compute_in_good_order()
return tmp_score