def write_absolute_heatmaps(absolute_heatmap_average, split, instance_problem):
sem_inst_class_list = instance_problem.semantic_instance_class_list
inst_id_list = instance_problem.instance_count_id_list
channel_names = instance_problem.get_channel_labels()
is_background = ['background' in s for s in channel_names]
non_bground_channel_idxs = [
i for i, is_b in enumerate(is_background) if not is_b
]
for channel_idx in non_bground_channel_idxs:
h = plt.figure(0, figsize=FIGSIZE)
plt.clf()
if is_background[channel_idx]:
continue
image = absolute_heatmap_average[channel_idx, :, :]
filename = '{}_absolute_heatmap_{}.png'.format(
split, channel_names[channel_idx])
display_pyutils.display_list_of_images(
[image],
cmap=COLORMAP,
smart_clims=True,
arrange='rows',
list_of_titles=[
'{}, absolute score'.format(channel_names[channel_idx])
])
# filename_base_ext='{}_score_heatmaps_{}.png'
plt.suptitle(filename)
display_pyutils.save_fig_to_workspace(filename, dpi=DPI)
h.clear()
def plot_scatterplot_sq_rq(stats_arrays,
semantic_class_names,
category_idxs_to_display=None,
save_to_workspace=True,
output_dir='/tmp/',
category_colors=None,
xlim=(0, 1),
ylim=(0, 1)):
category_colors = category_colors or [
default_colors[i % len(semantic_class_names)]
for i in range(len(semantic_class_names))
]
category_idxs_to_display = category_idxs_to_display if category_idxs_to_display is not None \
else range(stats_arrays[stats_arrays.keys()[0]].shape[1])
category_names_to_display = [
semantic_class_names[idx] for idx in category_idxs_to_display
]
sqs = [stats_arrays['sq'][:, i] for i in category_idxs_to_display]
rqs = [stats_arrays['rq'][:, i] for i in category_idxs_to_display]
plt.figure(1)
plt.clf()
nanscatter_list_of_xs_and_ys(sqs,
rqs,
labels=category_names_to_display,
xlabel='SQ',
ylabel='RQ',
colors=category_colors)
figname = 'sq_vs_rq' + '.png'
if ylim is not None:
plt.ylim(ylim)
if xlim is not None:
plt.xlim(xlim)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname))
for cat_idx, sq_cat, rq_cat in zip(category_idxs_to_display, sqs, rqs):
plt.figure(1)
plt.clf()
cat_color = category_colors[cat_idx]
cat_name = semantic_class_names[cat_idx]
nanscatter_list_of_xs_and_ys([sq_cat], [rq_cat],
labels=[cat_name],
xlabel='SQ',
ylabel='RQ',
colors=[cat_color])
figname = 'sq_vs_rq_{}'.format(cat_name) + '.png'
if ylim is not None:
plt.ylim(ylim)
if xlim is not None:
plt.xlim(xlim)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname))
def write_relative_heatmaps_by_channel(list_of_relative_heatmap_averages,
instance_problem, split):
channel_names = instance_problem.get_channel_labels()
sem_inst_class_list = instance_problem.semantic_instance_class_list
is_background = ['background' in s for s in channel_names]
non_bground_channel_idxs = [
i for i, is_b in enumerate(is_background) if not is_b
]
inst_id_list = instance_problem.instance_count_id_list
non_bg_sem_classes = list(
np.unique([
s for i, s in enumerate(sem_inst_class_list)
if not is_background[i]
]))
inst_starts_at_1 = 1 - int(
any([
inst_id_list[i] == 0
for i, is_b in enumerate(is_background) if not is_b
]))
# ch_subplot_rc_arrangement = [(non_bg_sem_classes.index(sem_val), inst_id - inst_starts_at_1)
ch_subplot_rc_arrangement = [
(inst_id - inst_starts_at_1, non_bg_sem_classes.index(sem_val))
for sem_val, inst_id, bg in zip(sem_inst_class_list, inst_id_list,
is_background) if not bg
]
# R, C = max([arr[0] for arr in ch_subplot_rc_arrangement]), max([arr[1] for arr in ch_subplot_rc_arrangement])
print('Writing heatmaps relative to each channel')
for channel_idx in non_bground_channel_idxs:
h = plt.figure(0, figsize=FIGSIZE)
plt.clf()
hm = list_of_relative_heatmap_averages[channel_idx]
list_of_subtitles = [
'{} score,\n centered on {} GT'.format(channel_names[channel_idx],
channel_names[rel_ch_idx])
for rel_ch_idx in non_bground_channel_idxs
]
display_pyutils.display_list_of_images(
[hm[rel_ch_idx, :, :] for rel_ch_idx in non_bground_channel_idxs],
list_of_titles=list_of_subtitles,
cmap=COLORMAP,
arrange='custom',
arrangement_rc_list=ch_subplot_rc_arrangement,
smart_clims=True)
filename = '{}_score_heatmaps_rel_by_ch_idx_{}.png'.format(
split, channel_names[channel_idx])
plt.suptitle(filename)
display_pyutils.save_fig_to_workspace(filename, dpi=DPI)
h.clear()
def write_relative_heatmaps_by_sem_cls(
list_of_relative_heatmap_averages_rel_by_semantic, instance_problem,
split):
print('Summing and writing heatmaps relative to each semantic class')
channel_names = instance_problem.get_channel_labels()
sem_class_names = instance_problem.semantic_class_names
n_channels = len(list_of_relative_heatmap_averages_rel_by_semantic)
is_background = ['background' in s for s in channel_names]
non_bg_sem_class_idxs = list(
np.unique([
i for i, s in enumerate(sem_class_names) if 'background' not in s
]))
non_bg_sem_classes = list(
np.unique([
s for i, s in enumerate(sem_class_names) if 'background' not in s
]))
non_bground_channel_idxs = [
i for i, is_b in enumerate(is_background) if not is_b
]
for channel_idx in non_bground_channel_idxs:
h = plt.figure(0, figsize=FIGSIZE)
plt.clf()
hm = list_of_relative_heatmap_averages_rel_by_semantic[channel_idx]
list_of_subtitles = [
'{} score,\n centered on {} GT'.format(channel_names[channel_idx],
sem_class_name)
for rel_sem_idx, sem_class_name in enumerate(non_bg_sem_classes)
]
display_pyutils.display_list_of_images(
[hm[rel_sem_cls, :, :] for rel_sem_cls in non_bg_sem_class_idxs],
list_of_titles=list_of_subtitles,
cmap=COLORMAP,
arrange='rows',
smart_clims=True)
filename = '{}_score_heatmaps_rel_by_sem_cls_{}.png'.format(
split, channel_names[channel_idx])
plt.suptitle(filename)
print('Writing image {}/{}'.format(channel_idx + 1, n_channels))
display_pyutils.save_fig_to_workspace(filename, dpi=DPI)
h.clear()
def write_stats(stats, semantic_class_names, split, metric_name='unknown_stat',
workspace_dir=display_pyutils.WORKSPACE_DIR):
# Write statistics
# Data to plot
display_pyutils.set_my_rc_defaults()
counts_np = stats.numpy()
if counts_np.shape[1] != len(semantic_class_names):
import ipdb;
ipdb.set_trace()
raise ValueError('semantic class names dont match size of counts')
counts_per_class = counts_np.sum(axis=0)
image_counts_per_class = (counts_np > 0).sum(axis=0)
num_class_per_img = {
semantic_class: counts_np[:, semantic_class_names.index(semantic_class)] for
semantic_class in semantic_class_names
}
num_images_with_this_many_cls = {
scn: [np.sum(num_class_per_img[scn] == x) for x in
range(int(num_class_per_img[scn].max()) + 1)]
for scn in num_class_per_img.keys()
}
plt.figure(1, figsize=FIGSIZE)
plt.clf()
ttl = split + '_' + 'total_' + metric_name + '_per_class' + '_across_dataset'
metric_values = counts_per_class
labels = [s + '={}'.format(v) for s, v in zip(semantic_class_names, metric_values)]
pie_chart(metric_values, labels)
plt.title(ttl)
display_pyutils.save_fig_to_workspace(ttl + '.png', workspace_dir=workspace_dir)
plt.figure(2, figsize=FIGSIZE)
plt.clf()
ttl = split + '_' + 'num_images_with_at_least_1' + metric_name + '_per_class'
metric_values = image_counts_per_class
labels = [s + '={}'.format(v) for s, v in zip(semantic_class_names, metric_values)]
pie_chart(metric_values, labels)
plt.title(ttl)
display_pyutils.save_fig_to_workspace(ttl + '.png', workspace_dir=workspace_dir)
for semantic_class in semantic_class_names:
plt.figure(3, figsize=FIGSIZE)
plt.clf()
ttl = split + '_' + 'num_images_per_num_{}_{}'.format(semantic_class, metric_name)
metric_values = num_images_with_this_many_cls[semantic_class]
labels = ['{} {}={}'.format(num, semantic_class, v) for num, v in enumerate(metric_values)]
pie_chart(metric_values, labels)
plt.title(ttl)
display_pyutils.save_fig_to_workspace(ttl + '.png', workspace_dir=workspace_dir)
print('Images written to {}'.format(workspace_dir))
def scatter_loss_vs_pq(loss_npz_file,
pq_npz_file,
save_to_workspace=True,
reverse_x=True):
display_pyutils.set_my_rc_defaults()
output_dir = os.path.dirname(loss_npz_file)
loss_npz_d = np.load(loss_npz_file)
pq_npz_d = np.load(pq_npz_file)
eval_iou_threshold = pq_npz_d['iou_threshold'].item()
loss_arr = loss_npz_d['losses_compiled_per_img_per_cls']
eval_stat_types = ('pq', 'sq', 'rq')
pq_arrs = {
stat_type:
pq_npz_d['collated_stats_per_image_per_cat'].item()[stat_type]
for stat_type in eval_stat_types
}
problem_config = loss_npz_d['problem_config'].item()
assert problem_config.model_channel_semantic_ids == pq_npz_d[
'problem_config'].item().model_channel_semantic_ids, \
'Sanity check failed: Problem configurations should have matched up'
assert problem_config.n_semantic_classes == loss_arr.shape[1]
subplot_idx = 1
sem_names = problem_config.semantic_class_names
sem_idxs_to_visualize = [
i for i, nm in enumerate(sem_names) if nm != 'background'
]
sem_totals_to_visualize = ['total_with_bground', 'total_without_bground']
subR, subC = len(eval_stat_types), len(sem_idxs_to_visualize) + len(
sem_totals_to_visualize)
scale = max(np.ceil(subR / 2), np.ceil(subC / 4))
plt.figure(figsize=[scale * s for s in display_pyutils.BIG_FIGSIZE])
plt.clf()
markers = display_pyutils.MARKERS
default_colors = display_pyutils.GOOD_COLOR_CYCLE
try:
labels_table = problem_config.labels_table
colors_by_idx = {
idx: np.array(labels_table[idx].color)
for idx in range(len(labels_table))
}
except:
raise
size = 30
for eval_stat_idx, eval_stat_type in enumerate(eval_stat_types):
x_arr = pq_arrs[eval_stat_type]
y_arr = loss_arr
assert x_arr.shape == y_arr.shape
eval_identifier = eval_stat_type + '-iou_{}'.format(eval_iou_threshold)
for sem_idx, sem_name in zip(
sem_idxs_to_visualize,
[sem_names[ii] for ii in sem_idxs_to_visualize]):
# pq_d[xlabel]
x, y = x_arr[:, sem_idx], y_arr[:, sem_idx]
plt.subplot(subR, subC, subplot_idx)
ylabel = 'loss'
xlabel = eval_identifier + (' (reversed)' if reverse_x else '')
label = 'loss vs {}, component: {}'.format(eval_stat_type,
sem_name)
scatter(x, y, colors_by_idx[sem_idx], label, markers, size, xlabel,
ylabel)
subplot_idx += 1
plt.xlim([0, 1])
if reverse_x:
plt.gca().invert_xaxis()
for agg_i, aggregate_type in enumerate(sem_totals_to_visualize):
if aggregate_type == 'total_with_bground':
sem_idxs = list(range(len(sem_names)))
elif aggregate_type == 'total_without_bground':
sem_idxs = [
idx for idx, nm in enumerate(sem_names)
if nm != 'background'
]
else:
raise NotImplementedError
plt.subplot(subR, subC, subplot_idx)
ylabel = 'total loss'
xlabel = 'sum ' + eval_identifier + (' (reversed)'
if reverse_x else '')
label = 'loss vs {}, {}'.format(eval_stat_type, aggregate_type)
x, y = x_arr[:, sem_idxs].sum(axis=1), y_arr[:,
sem_idxs].sum(axis=1)
scatter(x, y, default_colors[agg_i], label, markers, size, xlabel,
ylabel)
subplot_idx += 1
plt.xlim([0, max(x)])
if reverse_x:
plt.gca().invert_xaxis()
plt.tight_layout()
onehot_tag = '_onehot' if 'losses_onehot' in loss_npz_file else ''
figname = 'all_loss_vs_eval_iouthresh_{}{}.png'.format(
eval_iou_threshold, onehot_tag)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
figpath = os.path.join(output_dir, figname)
plt.savefig(figpath)
print(TermColors.OKGREEN +
'Scatterplot saved to {}'.format(os.path.abspath(figpath)) +
TermColors.ENDC)
def make_histogram_set(assigned_attributes,
channel_names,
split,
attribute_name,
use_subplots=True):
colors = [
display_pyutils.GOOD_COLOR_CYCLE[np.mod(
i, len(display_pyutils.GOOD_COLOR_CYCLE))]
for i in range(len(channel_names))
]
labels = channel_names
density, global_bins, patches = display_pyutils.histogram(
assigned_attributes,
bins=None,
color=colors,
label=labels,
histtype='stepfilled')
plt.figure(figsize=FIGSIZE)
plt.clf()
if use_subplots:
R = len(channel_names)
plt.subplot(R, 1, 1)
axes_list = []
labels = [
'{} ({} instances)'.format(channel_names[channel_idx],
len(assigned_attributes[channel_idx]))
for channel_idx in range(len(channel_names))
]
for subplot_idx, channel_idx in enumerate(range(len(channel_names))):
ax = plt.subplot(R, 1, subplot_idx + 1)
axes_list.append(ax)
y = assigned_attributes[channel_idx]
channel_name = channel_names[channel_idx]
bins = np.linspace(global_bins[0], global_bins[-1], 100)
label = labels[subplot_idx]
density, bins, patches = display_pyutils.histogram(
y,
bins=bins,
color=colors[subplot_idx],
label=label,
histtype='stepfilled')
# plt.legend(patches, labels, loc='center left', fontsize=8, bbox_to_anchor=(-0.02, 0.5))
if subplot_idx == 0:
title = '{}: {}'.format(split, attribute_name)
plt.title(title, fontsize=16)
# plt.legend(loc='center left', fontsize=8, bbox_to_anchor=(-0.02, 0.5))
plt.xlabel('{} for assigned ground truth instances'.format(
attribute_name),
fontsize=12)
plt.legend(loc='upper right', fontsize=16)
display_pyutils.sync_axes(axes_list, axis='x')
display_pyutils.sync_axes(axes_list, axis='y')
else:
bins = None
density, bins, patches = display_pyutils.histogram(assigned_attributes,
bins=None,
color=colors,
label=labels,
histtype='bar')
title = '{}: {}'.format(split, attribute_name)
plt.xlabel(
'{} for assigned ground truth instances'.format(attribute_name),
fontsize=12)
plt.title(title, fontsize=16)
plt.legend(loc='upper right', fontsize=16)
plt.tight_layout(pad=1.2)
filename = '{}_{}_distributions_{}.png'.format(
split, attribute_name, 'subplots' if use_subplots else 'combined')
display_pyutils.save_fig_to_workspace(filename)
def make_scatterplot3d_set(xs,
ys,
zs,
channel_names,
split,
x_attribute_name,
y_attribute_name,
z_attribute_name,
use_subplots=True):
colors = [
display_pyutils.GOOD_COLOR_CYCLE[np.mod(
i, len(display_pyutils.GOOD_COLOR_CYCLE))]
for i in range(len(channel_names))
]
labels = channel_names
plt.figure(figsize=FIGSIZE)
plt.clf()
if use_subplots:
R = len(channel_names)
plt.subplot(R, 1, 1)
axes_list = []
labels = [
'{} ({} instances)'.format(channel_names[channel_idx],
len(ys[channel_idx]))
for channel_idx in range(len(channel_names))
]
for subplot_idx, channel_idx in enumerate(range(len(channel_names))):
ax = plt.subplot(R, 1, subplot_idx + 1)
axes_list.append(ax)
channel_name = channel_names[channel_idx]
label = labels[subplot_idx]
x = xs[channel_idx]
y = ys[channel_idx]
z = zs[channel_idx]
display_pyutils.scatter3(x,
y,
z,
zdir='z',
label=label,
color=colors[subplot_idx])
# plt.legend(patches, labels, loc='center left', fontsize=8, bbox_to_anchor=(-0.02, 0.5))
if subplot_idx == 0:
title = '{}: {} vs. {}'.format(split, x_attribute_name,
y_attribute_name)
plt.title(title, fontsize=16)
# plt.legend(loc='center left', fontsize=8, bbox_to_anchor=(-0.02, 0.5))
plt.xlabel('{} for assigned ground truth instances'.format(
x_attribute_name),
fontsize=12)
plt.ylabel('{}'.format(y_attribute_name), fontsize=12)
plt.legend(loc='upper right', fontsize=16)
display_pyutils.sync_axes(axes_list, axis='x')
display_pyutils.sync_axes(axes_list, axis='y')
else:
plt.hold(True)
for channel_idx in range(len(channel_names)):
label = labels[channel_idx]
x = xs[channel_idx]
y = ys[channel_idx]
z = zs[channel_idx]
display_pyutils.scatter3(x,
y,
z,
zdir='z',
label=label,
color=colors[channel_idx])
plt.scatter(x, y, label=label, color=colors[channel_idx])
title = '{}: {} vs. {}'.format(split, x_attribute_name,
y_attribute_name)
plt.xlabel(
'{} for assigned ground truth instances'.format(x_attribute_name),
fontsize=12)
plt.ylabel('{}'.format(y_attribute_name), fontsize=12)
plt.title(title, fontsize=16)
plt.legend(loc='upper right', fontsize=16)
plt.tight_layout(pad=1.2)
filename = '{}_scatter3_x_{}_y_{}_z_{}_{}.png'.format(
split, x_attribute_name, y_attribute_name, z_attribute_name,
'subplots' if use_subplots else 'combined')
display_pyutils.save_fig_to_workspace(filename)
def write_stats(train_stats, val_stats, default_train_dataset,
default_val_dataset):
# Write statistics
# Data to plot
for split in ['train', 'val']:
dataset = default_train_dataset if split == 'train' else default_val_dataset
stats = train_stats if split == 'train' else val_stats
instance_counts = stats.instance_counts.numpy()
semantic_class_names = dataset.semantic_class_names if split == 'train' else \
dataset.semantic_class_names
assert instance_counts.shape[0] == len(dataset)
if instance_counts.shape[1] != len(semantic_class_names):
import ipdb
ipdb.set_trace()
raise Exception
instance_counts_per_class = instance_counts.sum(axis=0)
image_counts_per_class = (instance_counts > 0).sum(axis=0)
num_cars_per_image = instance_counts[:,
semantic_class_names.index('car')]
assert int(num_cars_per_image.max()) == num_cars_per_image.max()
num_images_with_this_many_cars = [
np.sum(num_cars_per_image == x)
for x in range(int(num_cars_per_image.max()))
]
plt.figure(1)
plt.clf()
metric_name = split + '_' + 'instance_counts_per_class'
metric_values = instance_counts_per_class
labels = [
s + '={}'.format(v)
for s, v in zip(semantic_class_names, metric_values)
]
pie_chart(metric_values, labels)
plt.title(metric_name)
display_pyutils.save_fig_to_workspace(metric_name + '.png')
plt.figure(2)
plt.clf()
metric_name = split + '_' + 'image_counts_per_class'
metric_values = image_counts_per_class
labels = [
s + '={}'.format(v)
for s, v in zip(semantic_class_names, metric_values)
]
pie_chart(metric_values, labels)
plt.title(metric_name)
display_pyutils.save_fig_to_workspace(metric_name + '.png')
plt.figure(3)
plt.clf()
metric_name = split + '_' + 'num_images_with_this_many_cars'
metric_values = num_images_with_this_many_cars
labels = [
'{} cars={}'.format(num, v) for num, v in enumerate(metric_values)
]
pie_chart(metric_values, labels)
plt.title(metric_name)
display_pyutils.save_fig_to_workspace(metric_name + '.png')
def plot_bar_graph_over_full_dataset(stats_arrays_total,
stats_arrays_per_class,
category_names_by_id,
categories_to_display=None,
save_to_workspace=True,
output_dir='/tmp/',
category_colors_by_id=None):
stat_types = [k for k in stats_arrays_total.keys() if k != 'n']
assert set(stat_types) == set(stats_arrays_per_class.keys()), \
'Keys must be the same: {} vs {}'.format(stats_arrays_total.keys(), stats_arrays_per_class.keys())
example_stat_type = list(stats_arrays_per_class.keys())[0]
categories_available = list(
stats_arrays_per_class[example_stat_type].keys())
assert all(
set(categories_available) == set(
stats_arrays_per_class[stat_type].keys())
for stat_type in stat_types)
total_types_available = list(stats_arrays_total[example_stat_type].keys())
assert all(
set(total_types_available) == set(stats_arrays_total[stat_type].keys())
for stat_type in stat_types)
categories_to_display = categories_to_display if categories_to_display is not None else categories_available
category_colors_by_id = category_colors_by_id or \
{cid: default_colors[cid % len(categories_to_display)]
for cid in range(len(categories_to_display))}
facecolors = [[0.0, 0.0, 0.0] for _ in total_types_available] + \
[category_colors_by_id[cat_id] for cat_id in categories_to_display]
edgecolors = [[0.0, 0.0, 0.0] for _ in total_types_available] + \
[category_colors_by_id[cat_id] for cat_id in categories_to_display]
for stat_type in stat_types:
ylabel = stat_type
y_cat = [
stats_arrays_per_class[stat_type][cat_id]
for cat_id in categories_to_display
]
y_tots = [
stats_arrays_total[stat_type][total_type]
for total_type in total_types_available
]
y = y_tots + y_cat
x_offset = 1
x = [i for i in range(len(y_tots))
] + [i + x_offset + len(y_tots) for i in range(len(y_cat))]
x_labels = total_types_available + [
category_names_by_id[cat_id] for cat_id in categories_to_display
]
plt.figure(1)
plt.clf()
fig, ax = plt.subplots()
rects = ax.bar(x=x,
height=y,
yerr=None,
align='center',
alpha=0.9,
edgecolor=edgecolors,
color=facecolors,
capsize=10)
ax.set_ylabel(ylabel)
ax.set_xticks(x)
ax.set_xticklabels(x_labels, rotation='vertical')
ax.set_title(ylabel)
ax.yaxis.grid(True)
autolabel_above_bars(ax)
figname = 'avg_' + ylabel + '.png'
plt.tight_layout()
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname))
def plot_hists_pq_rq_sq(stats_arrays,
semantic_class_names,
category_idxs_to_display=None,
save_to_workspace=True,
output_dir='/tmp/',
category_colors=None,
ylim=None):
category_colors = category_colors or [
default_colors[i % len(semantic_class_names)]
for i in range(len(semantic_class_names))
]
category_idxs_to_display = category_idxs_to_display if category_idxs_to_display is not None \
else range(stats_arrays[stats_arrays.keys()[0]].shape[1])
category_names_to_display = [
semantic_class_names[idx] for idx in category_idxs_to_display
]
for stat_type, stat_array in stats_arrays.items():
for catidx, catname in zip(category_idxs_to_display,
category_names_to_display):
plt.figure(1)
plt.clf()
display_pyutils.nanhistogram(stat_array[:, catidx],
label=catname,
color=category_colors[catidx])
figname = '{}_{}_hist'.format(stat_type, catname) + '.png'
plt.title(figname.replace('.png', '').replace('_', ' '))
if ylim is not None or stat_type == 'n_inst':
plt.ylim(ylim)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname))
subplot_idx = 1
R, C = len(stats_arrays.keys()), len(category_idxs_to_display)
figsize = (display_pyutils.BIG_FIGSIZE[0] * C / 10,
display_pyutils.BIG_FIGSIZE[1] * R / 3)
fig = plt.figure(1)
plt.clf()
plt.figure(1, figsize=figsize)
r = 0
for stat_type, stat_array in stats_arrays.items():
ax1 = plt.subplot(R, C, subplot_idx)
ax1_subplot_num = subplot_idx
plt.ylabel(stat_type + ' (#images)', labelpad=20)
for catidx, catname in zip(category_idxs_to_display,
category_names_to_display):
if subplot_idx == ax1_subplot_num:
plt.subplot(R, C, subplot_idx)
else:
ax = plt.subplot(R, C, subplot_idx, sharex=ax1, sharey=ax1)
if r == 0:
plt.title(catname, rotation=45, y=1.4)
display_pyutils.nanhistogram(stat_array[:, catidx],
label=catname,
color=category_colors[catidx])
if subplot_idx != ax1_subplot_num:
plt.setp(plt.gca().get_yticklabels(), visible=False)
plottype = '{}_{}'.format(stat_type, catname)
subplotname = '{}_hist'.format(plottype)
subplot_idx += 1
r += 1
# plt.tight_layout()
# if ylim is not None:
# plt.ylim(ylim)
figname = 'all_hist.png'
fig.set_size_inches(figsize)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname), dpi=500)
def plot_scatterplot_rq_ninst(stats_arrays,
semantic_class_names,
category_idxs_to_display=None,
save_to_workspace=True,
output_dir='/tmp/',
category_colors=None,
xlim=None,
ylim=(0, 1)):
category_colors = category_colors or [
default_colors[i % len(semantic_class_names)]
for i in range(len(semantic_class_names))
]
category_idxs_to_display = category_idxs_to_display if category_idxs_to_display is not None \
else range(stats_arrays[stats_arrays.keys()[0]].shape[1])
category_names_to_display = [
semantic_class_names[idx] for idx in category_idxs_to_display
]
rqs = [stats_arrays['rq'][:, i] for i in category_idxs_to_display]
ninst = [stats_arrays['n_inst'][:, i] for i in category_idxs_to_display]
ninst_nan = [np.copy(a) for a in ninst]
for a, b in zip(ninst_nan, ninst):
a[b == 0] = np.nan
plt.figure(1)
plt.clf()
nanscatter_list_of_xs_and_ys(ninst_nan,
rqs,
labels=category_names_to_display,
xlabel='# inst',
ylabel='RQ',
colors=category_colors)
figname = 'ninst_vs_rq' + '.png'
if ylim is not None:
plt.ylim(ylim)
if xlim is not None:
plt.xlim(xlim)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname))
for cat_idx, rq_cat, ninst_cat in zip(category_idxs_to_display, rqs,
ninst_nan):
if not np.all(ninst_cat[~np.isnan(ninst_cat)].astype(int) ==
ninst_cat[~np.isnan(ninst_cat)]):
raise Exception('Number of instances should be integer-valued')
plt.figure(1)
plt.clf()
cat_color = category_colors[cat_idx]
cat_name = semantic_class_names[cat_idx]
nanscatter_list_of_xs_and_ys([ninst_cat], [rq_cat],
labels=[cat_name],
xlabel='# inst',
ylabel='RQ',
colors=[cat_color])
figname = 'ninst_vs_rq_{}'.format(cat_name) + '.png'
if ylim is not None:
plt.ylim(ylim)
if xlim is not None:
plt.xlim(xlim)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname))
def plot_averages_with_error_bars(stats_arrays,
semantic_class_names,
category_idxs_to_display=None,
save_to_workspace=True,
output_dir='/tmp/',
category_colors=None):
"""
Note: we filter n_inst == 0 from the averages.
"""
category_colors = category_colors or [
default_colors[i % len(semantic_class_names)]
for i in range(len(semantic_class_names))
]
category_idxs_to_display = category_idxs_to_display if category_idxs_to_display is not None \
else range(stats_arrays[stats_arrays.keys()[0]].shape[1])
average_per_cat = {}
std_per_cat = {}
for stat_type, stat_array in stats_arrays.items():
average_per_cat[stat_type] = np.nanmean(stat_array, axis=0)
std_per_cat[stat_type] = np.nanstd(stat_array, axis=0)
category_names_to_display = [
semantic_class_names[idx] for idx in category_idxs_to_display
]
stat_types = average_per_cat.keys()
xs = [range(len(category_idxs_to_display)) for _ in range(len(stat_types))]
ys = [
average_per_cat[stat_type][category_idxs_to_display]
for stat_type in stat_types
]
stds = [
std_per_cat[stat_type][category_idxs_to_display]
for stat_type in stat_types
]
for ylabel, x, y, std in zip(stat_types, xs, ys, stds):
plt.figure(1)
plt.clf()
fig, ax = plt.subplots()
ax.bar(x=x,
height=y,
yerr=std,
align='center',
alpha=0.5,
ecolor='black',
color=[category_colors[i] for i in category_idxs_to_display],
capsize=5)
ax.set_ylabel(ylabel)
ax.set_xticks(category_idxs_to_display)
ax.set_xticklabels(list(category_names_to_display),
rotation='vertical')
ax.set_title(ylabel)
ax.yaxis.grid(True)
figname = 'avg_with_err_' + ylabel + '.png'
plt.tight_layout()
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname))
def compare_two_eval_npz_files(npz_1, npz_2, output_dir, identifier_1='1', identifier_2='2', save_to_workspace=True):
ds = {
identifier_1: dict(np.load(npz_1)),
identifier_2: dict(np.load(npz_2))
}
for _, d in ds.items():
for key in d.keys():
d[key] = extract_variable(d, key)
labels_tables = {
k: d['problem_config'].labels_table for k, d in ds.items()
}
category_names_by_col = {}
category_ids_to_name = {}
for identifier, d in ds.items():
ids_by_col = d['categories']
labels_table = d['problem_config'].labels_table
labels_table_ids = [l['id'] for l in labels_table]
labels_table_idxs_by_col = [labels_table_ids.index(cat_id) for cat_id in ids_by_col]
category_names_by_col[identifier] = [labels_table[table_idx]['name'] for table_idx in labels_table_idxs_by_col]
category_ids_to_name[identifier] = {labels_table[idx].id: labels_table[idx].name
for idx in labels_table_idxs_by_col}
overlapping_category_names = \
[c for c in category_names_by_col[identifier_1] if c in category_names_by_col[identifier_2]]
overlapping_labels_table = [labels_tables[identifier_1][category_names_by_col[identifier_1].index(cat_name)]
for cat_name in overlapping_category_names]
stats_arrays_total_dataset_per_subdiv = {}
for identifier, d in ds.items():
tot_dataset = extract_variable(d, 'dataset_totals')
stats_arrays_total_dataset_per_subdiv[identifier] = \
swap_outer_and_inner_keys({k: tot_dataset[k] for k in tot_dataset.keys() if k != 'per_class'})
stats_arrays_per_img_same_cols = {}
stats_arrays_total_dataset_per_class_by_stat_same_cols = {}
for identifier, d in ds.items():
per_image = extract_variable(d, 'collated_stats_per_image_per_cat')
# get rid of nan by image
for stat_type, stat_array_by_type in per_image.items():
n_inst_arrs = per_image['n_inst']
if stat_type != 'n_inst':
stat_array_by_type[n_inst_arrs == 0] = np.nan
stats_arrays_per_img_same_cols[identifier] = {k: subsample_arr_with_categories_by_name(
v, category_names_by_col[identifier], overlapping_category_names) for k, v in
per_image.items()}
per_class = swap_outer_and_inner_keys(extract_variable(d, 'dataset_totals')['per_class'])
stats_arrays_total_dataset_per_class_by_stat_same_cols[identifier] = {
stat_type: assemble_arr_from_dict_with_categories_by_name(
stat_arr, category_ids_to_name[identifier], overlapping_category_names)
for stat_type, stat_arr in per_class.items()
}
stat_types = list(stats_arrays_per_img_same_cols[identifier_1].keys())
R, C = len(stat_types), len(overlapping_category_names)
assert C == stats_arrays_per_img_same_cols[identifier_1][stat_types[0]].shape[1]
scatter_kwargs = dict(
x_arrs_R_C=[[stats_arrays_per_img_same_cols[identifier_1][stat_type][:, col] for col in range(C)]
for stat_type in stat_types],
y_arrs_R_C=[[stats_arrays_per_img_same_cols[identifier_2][stat_type][:, col] for col in range(C)]
for stat_type in stat_types],
x_labels_c=[identifier_1 for _ in range(C)],
y_labels_r=[identifier_2 for _ in range(R)],
colors_c=[l.color for l in overlapping_labels_table],
labels_R_C=[['{} {}'.format(stat_type, cat_name) for cat_name in overlapping_category_names]
for stat_type in stat_types]
)
all_scatters(**scatter_kwargs)
figname = 'per_image_{}_vs_{}.png'.format(identifier_1, identifier_2)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
plt.savefig(os.path.join(output_dir, figname), dpi=500)
fig_names = [figname]
return fig_names