def get_confusion_tensor(true_labels_tensor, predicted_labels_tensor, n_dim):
"""Creates a confusion matrix for each feature and returns it as an array with shape (n_features, 2, 2)"""
confusion_tensor = np.zeros(shape=(n_dim, 2, 2))
for feature_idx in range(n_dim):
confusion_tensor[feature_idx] = sklearn_confusion_matrix(
true_labels_tensor, predicted_labels_tensor, labels=[0, 1])
return confusion_tensor
def evaluate(self, dataset):
predictions = self.predict(dataset[:,0])
confusion_matrix = sklearn_confusion_matrix(dataset[:,1], predictions, labels=self.__classes)
precisions = []
recalls = []
accuracies = []
for gender in self.__classes:
idx = self.__classes_indexes[gender]
precision = 1
recall = 1
if np.sum(confusion_matrix[idx,:]) > 0:
precision = confusion_matrix[idx][idx]/np.sum(confusion_matrix[idx,:])
if np.sum(confusion_matrix[:, idx]) > 0:
recall = confusion_matrix[idx][idx]/np.sum(confusion_matrix[:, idx])
precisions.append(precision)
recalls.append(recall)
precision = np.mean(precisions)
recall = np.mean(recalls)
f1 = (2*(precision*recall))/float(precision+recall)
accuracy = np.sum(confusion_matrix.diagonal())/float(np.sum(confusion_matrix))
return precision, recall, accuracy, f1
def confusion_matrix(y_true, y_pred, target_names, normalize=False):
if any((val is None for val in (y_true, y_pred))):
raise ValueError("y_true and y_pred are needed to plot confusion "
"matrix")
# calculate how many names you expect
values = set(list(y_true)).union(set(list(y_pred)))
expected_len = len(values)
if target_names is not None:
len_target = len(target_names)
if target_names is not None and (expected_len != len_target):
raise ValueError(
('Data cointains {} different values, but target'
' names contains {} values.'.format(expected_len,
len(target_names))))
# if the user didn't pass target_names, create generic ones
if target_names is not None:
values = list(values)
values.sort()
target_names = ['Class {}'.format(v) for v in values]
cm = sklearn_confusion_matrix(y_true, y_pred)
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
return cm
def confusion_matrix(predictions, labels, columns=None):
if columns is None:
columns = predictions.classALeRCE.unique()
matrix = sklearn_confusion_matrix(labels.classALeRCE,
predictions.classALeRCE,
labels=columns)
confusion_matrix_df = pd.DataFrame(matrix, columns=columns, index=columns)
return confusion_matrix_df
def confusion_matrix(y_pred: Tensor, y_true: Tensor, **kwargs):
try:
y_pred = np.argmax(y_pred.detach().cpu().numpy(), axis=1)
return sklearn_confusion_matrix(
y_true.detach().cpu().numpy(), y_pred, labels=kwargs.get("labels"),
)
except Exception as e:
logger.error(e)
def confusion_matrix(y_test, y_pred):
logger.info('generating confusion matrix')
cm = sklearn_confusion_matrix(y_test, y_pred)
plt.matshow(cm)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()
def confusion_matrix(y_test, y_pred):
logger.info('generating confusion matrix')
cm = sklearn_confusion_matrix(y_test, y_pred)
plt.matshow(cm)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()
def np_to_conf_mat(self):
conf_mat = sklearn_confusion_matrix(self.np_label, self.np_pred)
temp_classes = np.array(list(set(self.np_label)))
temp_classes.sort()
classes = [str(i) for i in temp_classes]
self.conf_mat = conf_mat
self.classes = classes
if self.bin_data:
self.classes = self.intervals
return conf_mat, classes
def confusion_matrix(y_pred: np.array, y: np.array, padding: int) -> np.array:
""" Function that creates a confusion matrix using the Wikipedia convention for the axis.
:param y_pred: predicted tags.
:param y: the ground-truth tags.
:param padding: padding index to be ignored.
Returns:
- Confusion matrix for all the labels + padding label."""
y_pred = np.ma.masked_array(data=y_pred,
mask=(y == padding)).filled(padding)
return sklearn_confusion_matrix(y_pred, y)
def get_detection_metrics(true_labels, predicted_labels):
"""Calculates tp, fp, fn, and tn from a confusion matrix, then get precision, recall, and acc"""
tn, fp, fn, tp = sklearn_confusion_matrix(true_labels,
predicted_labels,
labels=[0, 1]).flatten()
precision = tp / (tp + fp)
recall = tp / (tp + fn)
acc = (tp + tn) / (true_labels.shape[0])
confusion_matrix = np.array([[tn, fp], [fn, tp]])
return {
'tn': tn,
'fp': fp,
'fn': fn,
'tp': tp,
'precision': precision,
'recall': recall,
'acc': acc,
'confusion_matrix': confusion_matrix
}
def get_localization_metrics(true_labels_tensor, predicted_labels_tensor,
n_dim):
"""Creates a confusion matrix for each feature as an array with shape (n_features, 2, 2),
then calculates the micro-precision and micro-recall and returns as a dict"""
confusion_tensor = np.zeros(shape=(n_dim, 2, 2))
for feature_idx in range(n_dim):
confusion_tensor[feature_idx] = sklearn_confusion_matrix(
true_labels_tensor[feature_idx],
predicted_labels_tensor[feature_idx],
labels=[0, 1])
# here we will sum along the feature axis of the confusion_tensor to get the micro-precision and recall
tn, fp, fn, tp = confusion_tensor.sum(axis=0).flatten()
micro_precision = tp / (tp + fp)
micro_recall = tp / (tp + fn)
return {
'tn': tn,
'fp': fp,
'fn': fn,
'tp': tp,
'micro-precision': micro_precision,
'micro-recall': micro_recall,
'confusion_tensor': confusion_tensor
}
def confusion_matrix(y_true, y_predicted, normalize_method='true'):
"""Confusion matrix for binary and multiclass classification.
Arguments:
y_true (ww.DataColumn, pd.Series or np.ndarray): True binary labels.
y_pred (ww.DataColumn, pd.Series or np.ndarray): Predictions from a binary classifier.
normalize_method ({'true', 'pred', 'all', None}): Normalization method to use, if not None. Supported options are: 'true' to normalize by row, 'pred' to normalize by column, or 'all' to normalize by all values. Defaults to 'true'.
Returns:
pd.DataFrame: Confusion matrix. The column header represents the predicted labels while row header represents the actual labels.
"""
y_true = _convert_to_woodwork_structure(y_true)
y_predicted = _convert_to_woodwork_structure(y_predicted)
y_true = _convert_woodwork_types_wrapper(y_true.to_series()).to_numpy()
y_predicted = _convert_woodwork_types_wrapper(
y_predicted.to_series()).to_numpy()
labels = unique_labels(y_true, y_predicted)
conf_mat = sklearn_confusion_matrix(y_true, y_predicted)
conf_mat = pd.DataFrame(conf_mat, index=labels, columns=labels)
if normalize_method is not None:
return normalize_confusion_matrix(conf_mat,
normalize_method=normalize_method)
return conf_mat
f'Current detection_results size: {detection_results.shape[1]} trials'
)
print('Removing trials which ran into an error')
detection_results = detection_results[:, ~exception_array, :]
time_list = time_list[~exception_array]
global_truth = global_truth[~exception_array]
detection = detection[~exception_array]
print(
f'Detection_results size after drop: {detection_results.shape[1]} trials'
)
# Recording Attack Results
confusion_tensor = np.zeros(shape=(n_dim, 2, 2))
for feature_idx, feature_results in enumerate(detection_results):
confusion_tensor[feature_idx] = sklearn_confusion_matrix(
feature_results[:, 1],
feature_results[:, 2],
labels=[0, 1])
# overall detection confusion matrix
global_detection_confusion_matrix = sklearn_confusion_matrix(
global_truth, detection, labels=[0, 1])
# Plotting results
# fig, axes = plt.subplots(sqrtn, sqrtn)
# axes_flat = axes.flatten()
# for feature, axis in enumerate(axes_flat):
# names = ['TN', 'FP', 'FN', 'TP']
# counts = confusion_tensor[feature].astype(np.int).flatten()
# labels = [f'{n}\n{c}' for n, c in zip(names, counts)]
# labels = np.array(labels).reshape(2, 2)
# sn.heatmap(confusion_tensor[feature].astype(np.int), annot=labels, fmt='', xticklabels=False,
# yticklabels=False, linewidth=.5, cbar=False, ax=axis, cmap='Blues')
def eval_model_v2(
context,
xtest,
ytest,
model,
pcurve_bins: int = 10,
pcurve_names: List[str] = ["my classifier"],
plots_artifact_path: str = "",
pred_params: dict = {},
cmap="Blues",
):
"""generate predictions and validation stats
pred_params are non-default, scikit-learn api prediction-function
parameters. For example, a tree-type of model may have a tree depth
limit for its prediction function.
:param xtest: features array type Union(DataItem, DataFrame,
numpy array)
:param ytest: ground-truth labels Union(DataItem, DataFrame,
Series, numpy array, List)
:param model: estimated model
:param pcurve_bins: (10) subdivide [0,1] interval into n bins, x-axis
:param pcurve_names: label for each calibration curve
:param pred_params: (None) dict of predict function parameters
:param cmap: ('Blues') matplotlib color map
"""
import numpy as np
def df_blob(df):
return bytes(df.to_csv(index=False), encoding="utf-8")
if isinstance(ytest, np.ndarray):
unique_labels = np.unique(ytest)
elif isinstance(ytest, list):
unique_labels = set(ytest)
else:
try:
ytest = ytest.values
unique_labels = np.unique(ytest)
except Exception as exc:
raise Exception(f"unrecognized data type for ytest {exc}")
n_classes = len(unique_labels)
is_multiclass = True if n_classes > 2 else False
# INIT DICT...OR SOME OTHER COLLECTOR THAT CAN BE ACCESSED
plots_path = plots_artifact_path or context.artifact_subpath("plots")
extra_data = {}
ypred = model.predict(xtest, **pred_params)
context.log_results({
"accuracy": float(metrics.accuracy_score(ytest, ypred)),
"test-error": np.sum(ytest != ypred) / ytest.shape[0],
})
# PROBABILITIES
if hasattr(model, "predict_proba"):
yprob = model.predict_proba(xtest, **pred_params)
if not is_multiclass:
fraction_of_positives, mean_predicted_value = calibration_curve(
ytest, yprob[:, -1], n_bins=pcurve_bins, strategy="uniform")
cmd = plot_calibration_curve(ytest, [yprob], pcurve_names)
calibration = context.log_artifact(
PlotArtifact(
"probability-calibration",
body=cmd.get_figure(),
title="probability calibration plot",
),
artifact_path=plots_path,
db_key=False,
)
extra_data["probability calibration"] = calibration
# CONFUSION MATRIX
cm = sklearn_confusion_matrix(ytest, ypred, normalize="all")
df = pd.DataFrame(data=cm)
extra_data["confusion matrix table.csv"] = df_blob(df)
cmd = metrics.plot_confusion_matrix(
model,
xtest,
ytest,
normalize="all",
values_format=".2g",
cmap=plt.get_cmap(cmap),
)
confusion = context.log_artifact(
PlotArtifact(
"confusion-matrix",
body=cmd.figure_,
title="Confusion Matrix - Normalized Plot",
),
artifact_path=plots_path,
db_key=False,
)
extra_data["confusion matrix"] = confusion
# LEARNING CURVES
if hasattr(model, "evals_result"):
results = model.evals_result()
train_set = list(results.items())[0]
valid_set = list(results.items())[1]
learning_curves_df = None
if is_multiclass:
if hasattr(train_set[1], "merror"):
learning_curves_df = pd.DataFrame({
"train_error":
train_set[1]["merror"],
"valid_error":
valid_set[1]["merror"],
})
else:
if hasattr(train_set[1], "error"):
learning_curves_df = pd.DataFrame({
"train_error":
train_set[1]["error"],
"valid_error":
valid_set[1]["error"],
})
if learning_curves_df:
extra_data["learning curve table.csv"] = df_blob(
learning_curves_df)
_, ax = plt.subplots()
plt.xlabel("# training examples")
plt.ylabel("error rate")
plt.title("learning curve - error")
ax.plot(learning_curves_df["train_error"], label="train")
ax.plot(learning_curves_df["valid_error"], label="valid")
learning = context.log_artifact(
PlotArtifact("learning-curve",
body=plt.gcf(),
title="Learning Curve - erreur"),
artifact_path=plots_path,
db_key=False,
)
extra_data["learning curve"] = learning
# FEATURE IMPORTANCES
if hasattr(model, "feature_importances_"):
(fi_plot, fi_tbl) = feature_importances(model, xtest.columns)
extra_data["feature importances"] = context.log_artifact(
fi_plot, db_key=False, artifact_path=plots_path)
extra_data["feature importances table.csv"] = df_blob(fi_tbl)
# AUC - ROC - PR CURVES
if is_multiclass:
lb = LabelBinarizer()
ytest_b = lb.fit_transform(ytest)
extra_data["precision_recall_multi"] = context.log_artifact(
precision_recall_multi(ytest_b, yprob, unique_labels),
artifact_path=plots_path,
db_key=False,
)
extra_data["roc_multi"] = context.log_artifact(
roc_multi(ytest_b, yprob, unique_labels),
artifact_path=plots_path,
db_key=False,
)
# AUC multiclass
aucmicro = metrics.roc_auc_score(ytest_b,
yprob,
multi_class="ovo",
average="micro")
aucweighted = metrics.roc_auc_score(ytest_b,
yprob,
multi_class="ovo",
average="weighted")
context.log_results({
"auc-micro": aucmicro,
"auc-weighted": aucweighted
})
# others (todo - macro, micro...)
f1 = metrics.f1_score(ytest, ypred, average="macro")
ps = metrics.precision_score(ytest, ypred, average="macro")
rs = metrics.recall_score(ytest, ypred, average="macro")
context.log_results({
"f1-score": f1,
"precision_score": ps,
"recall_score": rs
})
else:
yprob_pos = yprob[:, 1]
extra_data["precision_recall_bin"] = context.log_artifact(
precision_recall_bin(model, xtest, ytest, yprob_pos),
artifact_path=plots_path,
db_key=False,
)
extra_data["roc_bin"] = context.log_artifact(
roc_bin(ytest, yprob_pos, clear=True),
artifact_path=plots_path,
db_key=False,
)
rocauc = metrics.roc_auc_score(ytest, yprob_pos)
brier_score = metrics.brier_score_loss(ytest,
yprob_pos,
pos_label=ytest.max())
f1 = metrics.f1_score(ytest, ypred)
ps = metrics.precision_score(ytest, ypred)
rs = metrics.recall_score(ytest, ypred)
context.log_results({
"rocauc": rocauc,
"brier_score": brier_score,
"f1-score": f1,
"precision_score": ps,
"recall_score": rs,
})
# return all model metrics and plots
return extra_data
def __compute_unnormalized_confusion_matrices_for_label_group(
true_label_idx: List[Set[int]],
predicted_label_idx: List[Set[int]],
label_group: LabelGroup,
task_labels: List[LabelEntity],
) -> MatrixMetric:
"""
Returns matrix metric for a certain label group
:param true_label_idx:
:param predicted_label_idx:
:param label_group:
:param task_labels:
"""
map_task_labels_idx_to_group_idx = {
task_labels.index(label): i_group
for i_group, label in enumerate(label_group.labels)
}
set_group_labels_idx = set(map_task_labels_idx_to_group_idx.keys())
group_label_names = [
task_labels[label_idx].name for label_idx in set_group_labels_idx
]
if len(group_label_names) == 1:
# Single-class
# we use "not" to make presence of a class to be at index 0, while the absence of it at index 1
y_true = [
int(not set_group_labels_idx.issubset(true_labels))
for true_labels in true_label_idx
]
y_pred = [
int(not set_group_labels_idx.issubset(pred_labels))
for pred_labels in predicted_label_idx
]
group_label_names += [f"~ {group_label_names[0]}"]
column_labels = group_label_names.copy()
remove_last_row = False
else:
# Multiclass
undefined_idx = len(group_label_names) # to define missing value
# find the intersections between GT and task labels, and Prediction and task labels
true_intersections = [
true_labels.intersection(set_group_labels_idx)
for true_labels in true_label_idx
]
pred_intersections = [
pred_labels.intersection(set_group_labels_idx)
for pred_labels in predicted_label_idx
]
# map the intersection to 0-index value
y_true = [
map_task_labels_idx_to_group_idx[list(true_intersection)[0]]
if len(true_intersection) != 0 else undefined_idx
for true_intersection in true_intersections
]
y_pred = [
map_task_labels_idx_to_group_idx[list(pred_intersection)[0]]
if len(pred_intersection) != 0 else undefined_idx
for pred_intersection in pred_intersections
]
column_labels = group_label_names.copy()
column_labels.append("Other")
remove_last_row = True
matrix_data = sklearn_confusion_matrix(y_true,
y_pred,
labels=list(
range(len(column_labels))))
if remove_last_row:
# matrix clean up
matrix_data = np.delete(matrix_data, -1, 0)
if sum(matrix_data[:, -1]) == 0:
# if none of the GT is classified as classes from other groups, clean it up too
matrix_data = np.delete(matrix_data, -1, 1)
column_labels.remove(column_labels[-1])
# Use unnormalized matrix for statistics computation (accuracy, precision, recall)
return MatrixMetric(
name=f"{label_group.name}",
matrix_values=matrix_data,
row_labels=group_label_names,
column_labels=column_labels,
normalize=False,
)
def maskrcnn_eval_ycb_video(model, test_loader):
print('\nevaluating MaskRCNN ..')
model.eval()
# Init folders.
if not os.path.exists(config.YCB_TEST_SAVE_FOLDER):
os.makedirs(config.YCB_TEST_SAVE_FOLDER)
gt_pred_images = glob.glob(config.YCB_TEST_SAVE_FOLDER + '*')
for images in gt_pred_images:
os.remove(images)
APs = []
gt_obj_ids_list, pred_obj_ids_list = [], []
for image_idx, (images, targets) in enumerate(test_loader):
image, target = copy.deepcopy(images), copy.deepcopy(targets)
images = list(image.to(config.DEVICE) for image in images)
with torch.no_grad():
outputs = model(images)
outputs = [{k: v.to(config.CPU_DEVICE)
for k, v in t.items()} for t in outputs]
# Formatting input.
image = image[0]
image = image.to(config.CPU_DEVICE)
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1), dtype=np.uint8)
H, W, C = image.shape
# Formatting targets.
target = target[0]
target = {k: v.to(config.CPU_DEVICE) for k, v in target.items()}
target = ycb_video_dataset_utils.format_target_data(
image.copy(), target.copy())
gt_obj_ids = np.array(target['obj_ids'], dtype=np.int32).flatten()
gt_obj_boxes = np.array(target['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
gt_obj_binary_masks = np.array(target['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# format outputs.
outputs = outputs.pop()
outputs = maskrcnn_format_outputs(image.copy(), outputs.copy())
outputs = maskrcnn_threshold_outputs(image.copy(), outputs.copy())
matched_outputs = maskrcnn_match_pred_to_gt(image.copy(),
target.copy(),
outputs.copy())
scores = np.array(matched_outputs['scores'],
dtype=np.float32).flatten()
obj_ids = np.array(matched_outputs['obj_ids'],
dtype=np.int32).flatten()
obj_boxes = np.array(matched_outputs['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
obj_binary_masks = np.array(matched_outputs['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# confusion matrix.
gt_obj_ids_list.extend(gt_obj_ids.tolist())
pred_obj_ids_list.extend(obj_ids.tolist())
# get average precision.
AP = compute_ap_range(
gt_class_id=gt_obj_ids,
gt_box=gt_obj_boxes,
gt_mask=gt_obj_binary_masks.reshape(H, W, -1),
pred_score=scores,
pred_class_id=obj_ids,
pred_box=obj_boxes,
pred_mask=obj_binary_masks.reshape(H, W, -1),
verbose=False,
)
APs.append(AP)
# create masks.
pred_obj_mask = ycb_video_dataset_utils.get_segmentation_masks(
image=image, obj_ids=obj_ids, binary_masks=obj_binary_masks)
# save masks.
gt_name = config.YCB_TEST_SAVE_FOLDER + str(
image_idx) + config.TEST_GT_EXT
cv2.imwrite(gt_name, target['obj_mask'])
pred_name = config.YCB_TEST_SAVE_FOLDER + str(
image_idx) + config.TEST_PRED_EXT
cv2.imwrite(pred_name, pred_obj_mask)
# Confusion Matrix.
cm = sklearn_confusion_matrix(y_true=gt_obj_ids_list,
y_pred=pred_obj_ids_list)
print(f'\n{cm}')
# mAP
mAP = np.mean(APs)
print(f'\nmAP: {mAP:.5f}')
# getting Fwb
os.chdir(config.MATLAB_SCRIPTS_DIR)
import matlab.engine
eng = matlab.engine.start_matlab()
Fwb = eng.evaluate_arl_affpose_maskrcnn(config.ARL_TEST_SAVE_FOLDER,
nargout=1)
os.chdir(config.ROOT_DIR_PATH)
model.train()
return model, mAP, Fwb
def main():
# if SAVE_AND_EVAL_PRED:
# # Init folders
# print('\neval in .. {}'.format(config.ARL_AFF_EVAL_SAVE_FOLDER))
#
# if not os.path.exists(config.ARL_AFF_EVAL_SAVE_FOLDER):
# os.makedirs(config.ARL_AFF_EVAL_SAVE_FOLDER)
#
# gt_pred_images = glob.glob(config.ARL_AFF_EVAL_SAVE_FOLDER + '*')
# for images in gt_pred_images:
# os.remove(images)
# Load the Model.
print()
model = affnet.ResNetAffNet(pretrained=config.IS_PRETRAINED,
num_classes=config.NUM_CLASSES)
model.to(config.DEVICE)
# Load saved weights.
print(
f"\nrestoring pre-trained MaskRCNN weights: {config.RESTORE_ARL_AFFNET_WEIGHTS} .. "
)
checkpoint = torch.load(config.RESTORE_ARL_AFFNET_WEIGHTS,
map_location=config.DEVICE)
model.load_state_dict(checkpoint["model"])
model.eval()
# Load the dataset.
test_loader = arl_affpose_dataset_loaders.load_arl_affpose_eval_datasets(
random_images=RANDOM_IMAGES,
num_random=NUM_RANDOM,
shuffle_images=SHUFFLE_IMAGES)
# run the predictions.
APs = []
gt_obj_ids_list, pred_obj_ids_list = [], []
for image_idx, (images, targets) in enumerate(test_loader):
print(f'\nImage:{image_idx+1}/{len(test_loader)}')
image, target = copy.deepcopy(images), copy.deepcopy(targets)
images = list(image.to(config.DEVICE) for image in images)
with torch.no_grad():
outputs = model(images)
outputs = [{k: v.to(config.CPU_DEVICE)
for k, v in t.items()} for t in outputs]
# Formatting input.
image = image[0]
image = image.to(config.CPU_DEVICE)
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1), dtype=np.uint8)
H, W, C = image.shape
# Formatting targets.
target = target[0]
target = {k: v.to(config.CPU_DEVICE) for k, v in target.items()}
target = arl_affpose_dataset_utils.format_target_data(
image.copy(), target.copy())
gt_obj_ids = np.array(target['obj_ids'], dtype=np.int32).flatten()
gt_obj_boxes = np.array(target['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
gt_obj_binary_masks = np.array(target['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# Formatting Output.
outputs = outputs.pop()
outputs = eval_utils.affnet_format_outputs(image.copy(),
outputs.copy())
outputs = eval_utils.affnet_threshold_outputs(image.copy(),
outputs.copy())
outputs = eval_utils.maskrcnn_match_pred_to_gt(image.copy(),
target.copy(),
outputs.copy())
scores = np.array(outputs['scores'], dtype=np.float32).flatten()
obj_ids = np.array(outputs['obj_ids'], dtype=np.int32).flatten()
obj_boxes = np.array(outputs['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
obj_binary_masks = np.array(outputs['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
aff_scores = np.array(outputs['aff_scores'],
dtype=np.float32).flatten()
obj_part_ids = np.array(outputs['obj_part_ids'],
dtype=np.int32).flatten()
aff_ids = np.array(outputs['aff_ids'], dtype=np.int32).flatten()
aff_binary_masks = np.array(outputs['aff_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# confusion matrix.
gt_obj_ids_list.extend(gt_obj_ids.tolist())
pred_obj_ids_list.extend(obj_ids.tolist())
# for obj_binary_mask, gt_obj_binary_mask in zip(obj_binary_masks, gt_obj_binary_masks):
# cv2.imshow('obj_binary_mask', obj_binary_mask*20)
# cv2.imshow('gt_obj_binary_mask', gt_obj_binary_mask*20)
# cv2.waitKey(0)
# get average precision.
AP = eval_utils.compute_ap_range(
gt_class_id=gt_obj_ids,
gt_box=gt_obj_boxes,
gt_mask=gt_obj_binary_masks.reshape(H, W, -1),
pred_score=scores,
pred_class_id=obj_ids,
pred_box=obj_boxes,
pred_mask=obj_binary_masks.reshape(H, W, -1),
verbose=False,
)
APs.append(AP)
# print outputs.
for gt_idx, pred_idx in zip(range(len(gt_obj_ids)),
range(len(obj_ids))):
gt_obj_name = "{:<15}".format(
arl_affpose_dataset_utils.map_obj_id_to_name(
gt_obj_ids[gt_idx]))
pred_obj_name = "{:<15}".format(
arl_affpose_dataset_utils.map_obj_id_to_name(
obj_ids[pred_idx]))
score = scores[pred_idx]
bbox_iou = eval_utils.get_iou(obj_boxes[pred_idx],
gt_obj_boxes[gt_idx])
print(
f'GT: {gt_obj_name}',
f'Pred: {pred_obj_name}'
f'Score: {score:.3f},\t\t',
f'IoU: {bbox_iou:.3f},',
)
print("AP @0.5-0.95: {:.5f}".format(AP))
# visualize bbox.
pred_bbox_img = arl_affpose_dataset_utils.draw_bbox_on_img(
image=image, scores=scores, obj_ids=obj_ids, boxes=obj_boxes)
# visualize affordance masks.
pred_aff_mask = arl_affpose_dataset_utils.get_segmentation_masks(
image=image,
obj_ids=aff_ids,
binary_masks=aff_binary_masks,
)
color_aff_mask = arl_affpose_dataset_utils.colorize_aff_mask(
pred_aff_mask)
color_aff_mask = cv2.addWeighted(pred_bbox_img, 0.5, color_aff_mask,
0.5, 0)
# get obj part mask.
pred_obj_part_mask = arl_affpose_dataset_utils.get_obj_part_mask(
image=image,
obj_part_ids=obj_part_ids,
aff_binary_masks=aff_binary_masks,
)
# visualize object masks.
pred_obj_mask = arl_affpose_dataset_utils.convert_obj_part_mask_to_obj_mask(
pred_obj_part_mask)
color_obj_mask = arl_affpose_dataset_utils.colorize_obj_mask(
pred_obj_mask)
color_obj_mask = cv2.addWeighted(pred_bbox_img, 0.5, color_obj_mask,
0.5, 0)
if SAVE_AND_EVAL_PRED:
# saving predictions.
_image_idx = target["image_id"].detach().numpy()[0]
_image_idx = str(1000000 + _image_idx)[1:]
gt_name = config.ARL_AFF_EVAL_SAVE_FOLDER + _image_idx + config.TEST_GT_EXT
pred_name = config.ARL_AFF_EVAL_SAVE_FOLDER + _image_idx + config.TEST_PRED_EXT
obj_part_name = config.ARL_AFF_EVAL_SAVE_FOLDER + _image_idx + config.TEST_OBJ_PART_EXT
cv2.imwrite(gt_name, target['aff_mask'])
cv2.imwrite(pred_name, pred_aff_mask)
cv2.imwrite(obj_part_name, pred_obj_part_mask)
# show plot.
if SHOW_IMAGES:
cv2.imshow('pred_bbox',
cv2.cvtColor(pred_bbox_img, cv2.COLOR_BGR2RGB))
cv2.imshow('pred_aff_mask',
cv2.cvtColor(color_aff_mask, cv2.COLOR_BGR2RGB))
cv2.imshow('pred_obj_part_mask',
cv2.cvtColor(color_obj_mask, cv2.COLOR_BGR2RGB))
cv2.waitKey(0)
# Confusion Matrix.
cm = sklearn_confusion_matrix(y_true=gt_obj_ids_list,
y_pred=pred_obj_ids_list)
print(f'\n{cm}')
# Plot Confusion Matrix.
# eval_utils.plot_confusion_matrix(cm, arl_affpose_dataset_utils.OBJ_NAMES)
# mAP
print("\nmAP @0.5-0.95: over {} test images is {:.3f}".format(
len(APs), np.mean(APs)))
if SAVE_AND_EVAL_PRED:
print()
# getting FwB.
os.chdir(config.MATLAB_SCRIPTS_DIR)
import matlab.engine
eng = matlab.engine.start_matlab()
Fwb = eng.evaluate_arl_affpose_affnet(config.ARL_AFF_EVAL_SAVE_FOLDER,
nargout=1)
os.chdir(config.ROOT_DIR_PATH)
