def plot_bad_rocs_same_person_colors(bad_emb_ids_person_ids_img_names,
emb_id_to_fps_and_tps):
bad_emb_ids = list(
get_every_nth_item(bad_emb_ids_person_ids_img_names, n=0))
@ignore_last_n_args_decorator(n=1)
def is_bad(emb_id):
return emb_id in bad_emb_ids
bad_emb_id_to_fps_and_tps = dict(
starfilter(is_bad, emb_id_to_fps_and_tps.items()))
unique_persons_triplets = list(
unique_everseen(bad_emb_ids_person_ids_img_names,
key=lambda triplet: triplet[1]))
bad_emb_id_to_person_id_dict = dict(
(emb_id, person_id if person_id != 'person_11' else '5 x person_11')
for emb_id, person_id, _ in unique_persons_triplets)
person_ids = get_every_nth_item(unique_persons_triplets, n=1)
colors = ['b', 'g', 'r', 'c', 'm']
person_id_to_color_dict = dict(zip(person_ids, colors))
emb_id_to_equal_colors = dict(
(emb_id, person_id_to_color_dict[person_id])
for emb_id, person_id, _ in bad_emb_ids_person_ids_img_names)
plot_rocs(bad_emb_id_to_fps_and_tps,
bad_emb_id_to_person_id_dict,
emb_id_to_equal_colors,
title='Bad ROCs',
save_path=BAD_ROCS_PERSON_SAME_COLOR_SAVE_PATH)
def get_xs_and_ys(thres_and_met_to_f_measure, thres=False, metric=False):
if thres:
ind = 1
elif metric:
ind = 0
else:
log_error('thres or metric must be true')
return
param_f_measure = [
(thres_met.split('__')[ind], f_measure)
for thres_met, f_measure in thres_and_met_to_f_measure.items()
]
param_f_measure = [(float(param.lstrip('LT')), float(f_measure))
for param, f_measure in param_f_measure]
# TODO: Incorporate variance, too?
# thres_met_f_measures_groups_dict = group_pairs(param_f_measure, ret_dict=True)
# # map_dict_vals(thres_met_f_measures_groups_dict, func=np.mean)
# # return thres_met_f_measures_groups_dict
#
# thres_met_f_measures_groups = sorted(thres_met_f_measures_groups_dict.items())
# grouped_thres_met_f_measures = defaultdict(list)
# for met, f_measures in groupby(met_f_measure, key=lambda mf: mf[0]):
# grouped_thres_met_f_measures[met].append(np.mean(f_measures))
# grouped_thres_met_f_measures = sorted(grouped_thres_met_f_measures.items())
xs = get_every_nth_item(param_f_measure, n=0)
ys = get_every_nth_item(param_f_measure, n=1)
return list(xs), list(ys)
def get_thresholds_and_pos_rate(images_path,
pos_type,
eps=10,
ret_abs_num_pos=False):
compute_pos_rate = compute_tp_rate if pos_type else compute_fp_rate
embeddings_with_ids = list(
EvalDBManager.get_all_embeddings(with_ids=True, as_dict=False))
emb_id_to_img_name_dict = EvalDBManager.get_emb_id_to_name_dict(
images_path)
img_name_to_person_id_dict = get_img_name_to_id_dict(images_path)
num_pos_list = []
thresholds_and_pos = []
for start_emb_id, start_emb in embeddings_with_ids:
compute_dist_to_start_emb = partial(Cluster.compute_dist, start_emb)
def get_emb_id_and_dist_to_start_emb(emb_id, emb):
return emb_id, compute_dist_to_start_emb(emb)
emb_ids_and_dists_to_start_emb = list(
starmap(get_emb_id_and_dist_to_start_emb, embeddings_with_ids))
# compute and sort distances to start emb
sorted_embs_and_dists_to_start_emb = sorted(
emb_ids_and_dists_to_start_emb,
key=lambda emb_and_dist: emb_and_dist[1])
is_start_person = partial(
caltech_are_same_person_func,
start_emb_id,
emb_id_to_img_name_dict=emb_id_to_img_name_dict,
img_name_to_person_id_dict=img_name_to_person_id_dict)
sorted_emb_ids = get_every_nth_item(sorted_embs_and_dists_to_start_emb,
n=0)
matches_with_start_emb = list(map(is_start_person, sorted_emb_ids))
sorted_dists_to_start_emb = list(
get_every_nth_item(sorted_embs_and_dists_to_start_emb, n=1))
max_dist = max(sorted_dists_to_start_emb)
thresholds = np.linspace(0, max_dist, num=max_dist * 100 + eps)
pos_rate = compute_pos_rate(thresholds, sorted_dists_to_start_emb,
matches_with_start_emb)
thresholds_and_pos.append((thresholds, pos_rate))
num_pos_list.append(sum(matches_with_start_emb))
if ret_abs_num_pos:
return thresholds_and_pos, num_pos_list
return thresholds_and_pos
def get_thresholds_and_f_measures(images_path=None,
thresholds_and_tps=None,
thresholds_and_fps=None):
if images_path is None and (thresholds_and_tps is None
or thresholds_and_fps is None):
raise ValueError(
'Either both thresholds or images_path must be provided')
if thresholds_and_tps is None:
thresholds_and_tps = get_thresholds_and_pos_rate(images_path, True)
if thresholds_and_fps is None:
thresholds_and_fps = get_thresholds_and_pos_rate(images_path, False)
thresholds_and_fns = get_thresholds_and_fn_rate(thresholds_and_tps)
get_every_2nd_item = partial(get_every_nth_item, n=1)
tp_rates, fp_rates, fn_rates = (
list(get_every_2nd_item(iterable)) for iterable in
[thresholds_and_tps, thresholds_and_fps, thresholds_and_fns])
precisions = starmap(compute_precision, zip(tp_rates, fp_rates))
recalls = starmap(compute_recall, zip(tp_rates, fn_rates))
f_measures = starmap(compute_f_measure, zip(precisions, recalls))
thresholds = get_every_nth_item(thresholds_and_fns, n=0)
thresholds_and_f_measures = list(zip(thresholds, f_measures))
return thresholds_and_f_measures
def main():
with open(CMD_STATS_FILE, 'r') as file:
lines = file.readlines()
cmd_dict = {'process_images_dir': [], 'reclassify': [], 'clear_data': []}
for line in lines:
cmd_name, n, duration = line.strip('[]\n').split(', ')
cmd_name, n, duration = cmd_name.strip(" '"), int(n), float(duration)
cmd_dict[cmd_name].append((n, round(duration, 3)))
for cmd_name, xs_and_ys in cmd_dict.items():
xs, ys = list(get_every_nth_item(xs_and_ys, 0)), list(
get_every_nth_item(xs_and_ys, 1))
save_path = os.path.join(RESULTS_DIR_PATH, cmd_name +
'.png') if WRITE else None
plot_cmd(cmd_name, xs, ys, save_path)
def print_bad_rocs(images_path, bad_rocs, write_output=True):
emb_id_to_img_name_dict = EvalDBManager.get_emb_id_to_name_dict(
images_path)
img_name_to_person_id_dict = get_img_name_to_id_dict(images_path)
bad_roc_triplets = []
bad_roc_lines = []
for start_emb_id in get_every_nth_item(bad_rocs):
img_name = emb_id_to_img_name_dict[start_emb_id]
person_id = img_name_to_person_id_dict[img_name]
bad_roc_lines.append(
f'emb_id: {start_emb_id} | person_id: {person_id} | img_name: {img_name}'
+ '\n')
bad_roc_triplets.append((start_emb_id, person_id, img_name))
if PRINT_OUTPUT:
print('\n'.join(bad_roc_lines))
if write_output:
with open(BAD_ROCS_TXT_PATH, 'w') as file:
file.writelines(bad_roc_lines)
return bad_roc_triplets
def _get_img_paths_worker(base_dir_path,
recursive=False,
img_extensions=None,
with_abs_paths=False,
rel_dir_path=''):
@ignore_first_n_args_decorator(n=1)
def is_img_known_extensions(obj_path):
return is_img(obj_path, img_extensions)
cur_dir = os.path.join(base_dir_path, rel_dir_path)
# prepend the acquired directory tree of previous recursions
object_rel_paths_in_dir = list(
map(lambda obj: os.path.join(rel_dir_path, obj), os.listdir(cur_dir)))
object_abs_paths_in_dir = list(
map(partial(os.path.join, base_dir_path), object_rel_paths_in_dir))
object_rel_and_abs_paths = zip(object_rel_paths_in_dir,
object_abs_paths_in_dir)
img_rel_and_abs_paths = starfilter(is_img_known_extensions,
object_rel_and_abs_paths)
img_rel_paths_in_dir = get_every_nth_item(img_rel_and_abs_paths, n=0)
if not recursive:
if with_abs_paths:
return img_rel_and_abs_paths
return img_rel_paths_in_dir
def get_subdirs_img_paths(subdir_path):
next_subdir = os.path.split(subdir_path)[-1]
new_rel_dir_path = os.path.join(rel_dir_path, next_subdir)
subdirs_img_paths = _get_img_paths_worker(
base_dir_path,
recursive=True,
img_extensions=img_extensions,
with_abs_paths=with_abs_paths,
rel_dir_path=new_rel_dir_path)
return subdirs_img_paths
subdir_abs_paths = filter(os.path.isdir, object_abs_paths_in_dir)
subdirs_img_paths = map(get_subdirs_img_paths, subdir_abs_paths)
img_objects_iterable = img_rel_and_abs_paths if with_abs_paths else img_rel_paths_in_dir
all_img_objects = chain(img_objects_iterable, *subdirs_img_paths)
return all_img_objects
def sort_dict_by_cols(self, row_dict, only_values=True):
"""
Return a list containing the elements in row_dict, sorted by the order the corresponding keys appear in this
tables' columns. If only_values is True, the dict values rather than items are returned.
Example:
self = TableSchema(first_name, last_name, age)
row_dict = {'last_name': 'Olafson', 'age': 29, 'first_name': 'Lina'}
--> ['Lina', 'Olafson', 29]
:param row_dict:
:param only_values:
:return:
"""
cols = self.get_column_names()
sorted_row_items = sorted(
row_dict.items(),
key=lambda kv_pair: cols.index(kv_pair[0])) # kv = key-value
if only_values:
return list(get_every_nth_item(sorted_row_items, n=1))
return sorted_row_items
def _plot_thresholds_vs_params_helper(thresholds_and_params,
ylabel,
title=None,
x_eps=None,
y_eps=0.05):
thresholds = list(get_every_nth_item(thresholds_and_params, n=0))
min_thres, max_thres = min(map(min, thresholds)), max(map(max, thresholds))
if x_eps is None:
x_eps = (max_thres - min_thres) / 20 # 5% margin
x_axis_limits = [min_thres - x_eps, max_thres + x_eps]
y_axis_limits = [0 - y_eps, 1 + y_eps]
fig, ax = plt.subplots()
xlabel = 'Thresholds'
if title is None:
title = f'Threshold vs. {ylabel}'
plt.title(title)
ax.set_xlim(x_axis_limits)
ax.set_ylim(y_axis_limits)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
return fig, ax