def evaluate_mask_retriever(query_db_path: str, retriever: str, *, output: str = None):
"""
Evaluates the given mask retriever against all the images in the query dataset.
Parameters
----------
query_db_path : path where que images of the query dataset are located
retriever : mask method to evaluate
Returns
-------
pr: precision
rec: recall
f1: F1-score
results: pandas dataframe that contains image-wise metrics.
"""
if not os.path.exists(query_db_path):
print("[ERROR] Query dataset path is wrong.")
exit()
if output is not None and not os.path.exists(output):
print("[ERROR] Output folder does not exist.")
exit()
# We load the dataset metadata
with open(os.path.join(query_db_path, "frames.pkl"), 'rb') as file:
frames = pkl.load(file)
db_count = len(frames)
#print(f"There are {db_count} images in the query dataset.")
# We load the images and their associated masks
images = [cv2.imread(os.path.join(query_db_path, f"{i:05d}.jpg")) for i in range(db_count)]
masks = [cv2.imread(os.path.join(query_db_path, f"{i:05d}.png")) for i in range(db_count)]
# We generate the masks and compare it with the GT mask, computing several metrics
def to_do(img, retriever, i):
path = os.path.join(output, f"{i:05d}.png")
if os.path.exists(path):
return cv2.imread(path, 0).astype(np.uint8)
m = extract_mask(img, retriever)
if output is not None:
cv2.imwrite(path, m * 255)
return m
generated_masks = [to_do(images[i], retriever, i) for i in range(db_count)]
#if output is not None:
# we save all generated masks
# for i in range(db_count):
# cv2.imwrite(os.path.join(output, f"{i:05d}.png"), generated_masks[i] * 255)
print(f"[INFO] Masks successfully stored in '{output}'")
try:
data = [(i, ) + evaluate_mask(generated_masks[i], masks[i]) for i in range(db_count)]
results = pd.DataFrame(data=data, columns=("index", "precision", "recall", "f1", "tp", "fp", "fn", "tn"))
# We print the average metrics for the whole query dataset.
pr, rec, f1 = results["precision"].mean(), results["recall"].mean(), results["f1"].mean()
return pr, rec, f1, results
except:
print("[INFO] GT Masks not found => evaluation not performed.")
return None
def __function_4vPolygon_F1(m_orig, p1, p2, p3, p4):
# Function that computer the F1 score between the polygon formed by (p1, p2, p3, p4) and m_orig mask.
m = np.zeros(m_orig.shape[:2])
pts = np.array([p1, p2, p3, p4], np.int32)
pts = pts.reshape((-1, 1, 2))
cv2.fillConvexPoly(m, pts, True)
try:
return evaluate_mask(m, m_orig)[2] #compute_iou(m, MASK)
except:
return 0