def write_stats(stats, semantic_class_names, split, metric_name='unknown_stat',
workspace_dir=display_pyutils.WORKSPACE_DIR):
# Write statistics
# Data to plot
display_pyutils.set_my_rc_defaults()
counts_np = stats.numpy()
if counts_np.shape[1] != len(semantic_class_names):
import ipdb;
ipdb.set_trace()
raise ValueError('semantic class names dont match size of counts')
counts_per_class = counts_np.sum(axis=0)
image_counts_per_class = (counts_np > 0).sum(axis=0)
num_class_per_img = {
semantic_class: counts_np[:, semantic_class_names.index(semantic_class)] for
semantic_class in semantic_class_names
}
num_images_with_this_many_cls = {
scn: [np.sum(num_class_per_img[scn] == x) for x in
range(int(num_class_per_img[scn].max()) + 1)]
for scn in num_class_per_img.keys()
}
plt.figure(1, figsize=FIGSIZE)
plt.clf()
ttl = split + '_' + 'total_' + metric_name + '_per_class' + '_across_dataset'
metric_values = counts_per_class
labels = [s + '={}'.format(v) for s, v in zip(semantic_class_names, metric_values)]
pie_chart(metric_values, labels)
plt.title(ttl)
display_pyutils.save_fig_to_workspace(ttl + '.png', workspace_dir=workspace_dir)
plt.figure(2, figsize=FIGSIZE)
plt.clf()
ttl = split + '_' + 'num_images_with_at_least_1' + metric_name + '_per_class'
metric_values = image_counts_per_class
labels = [s + '={}'.format(v) for s, v in zip(semantic_class_names, metric_values)]
pie_chart(metric_values, labels)
plt.title(ttl)
display_pyutils.save_fig_to_workspace(ttl + '.png', workspace_dir=workspace_dir)
for semantic_class in semantic_class_names:
plt.figure(3, figsize=FIGSIZE)
plt.clf()
ttl = split + '_' + 'num_images_per_num_{}_{}'.format(semantic_class, metric_name)
metric_values = num_images_with_this_many_cls[semantic_class]
labels = ['{} {}={}'.format(num, semantic_class, v) for num, v in enumerate(metric_values)]
pie_chart(metric_values, labels)
plt.title(ttl)
display_pyutils.save_fig_to_workspace(ttl + '.png', workspace_dir=workspace_dir)
print('Images written to {}'.format(workspace_dir))
def main():
args = parse_args()
logdir = args.logdir
instanceseg.utils.scripts.set_random_seeds()
display_pyutils.set_my_rc_defaults()
cuda = torch.cuda.is_available()
if display_pyutils.check_for_emptied_workspace():
print('Workspace clean.')
else:
print('Workspace not clean, but running anyway.')
# Load directory
cfg, model_pth, out_dir, problem_config, model, my_trainer, optim, dataloaders = \
instanceseg.utils.logs.load_everything_from_logdir(logdir, gpu=args.gpu, packed_as_dict=False)
model.eval()
# Write log directory name to folder
with open(
osp.join(display_pyutils.WORKSPACE_DIR,
osp.basename(osp.normpath(logdir))), 'w') as fid:
fid.write(logdir)
for split in ['train', 'val']:
# NOTE(allie): At the moment, clims is not synced. Need to get the min/max and pass them in.
print('Getting absolute heatmaps')
absolute_heatmap_average = score_heatmaps.get_per_image_per_channel_heatmaps(
model, dataloaders[split], cfg, cuda)
print('Writing images')
write_absolute_heatmaps(absolute_heatmap_average.cpu().numpy(), split,
my_trainer.instance_problem)
print('Computing heatmaps relative to each channel')
list_of_relative_heatmap_averages = score_heatmaps.get_relative_per_image_per_channel_heatmaps(
model, dataloaders[split], cfg, cuda, my_trainer)
print('Writing images')
write_relative_heatmaps_by_channel(
[x.cpu().numpy() for x in list_of_relative_heatmap_averages],
my_trainer.instance_problem, split)
print('Computing heatmaps relative to each channel')
list_of_relative_heatmap_averages_rel_by_semantic = compute_relative_per_sem_class_heatmaps(
list_of_relative_heatmap_averages, my_trainer.instance_problem)
write_relative_heatmaps_by_sem_cls([
x.cpu().numpy()
for x in list_of_relative_heatmap_averages_rel_by_semantic
], my_trainer.instance_problem, split)
def scatter_loss_vs_pq(loss_npz_file,
pq_npz_file,
save_to_workspace=True,
reverse_x=True):
display_pyutils.set_my_rc_defaults()
output_dir = os.path.dirname(loss_npz_file)
loss_npz_d = np.load(loss_npz_file)
pq_npz_d = np.load(pq_npz_file)
eval_iou_threshold = pq_npz_d['iou_threshold'].item()
loss_arr = loss_npz_d['losses_compiled_per_img_per_cls']
eval_stat_types = ('pq', 'sq', 'rq')
pq_arrs = {
stat_type:
pq_npz_d['collated_stats_per_image_per_cat'].item()[stat_type]
for stat_type in eval_stat_types
}
problem_config = loss_npz_d['problem_config'].item()
assert problem_config.model_channel_semantic_ids == pq_npz_d[
'problem_config'].item().model_channel_semantic_ids, \
'Sanity check failed: Problem configurations should have matched up'
assert problem_config.n_semantic_classes == loss_arr.shape[1]
subplot_idx = 1
sem_names = problem_config.semantic_class_names
sem_idxs_to_visualize = [
i for i, nm in enumerate(sem_names) if nm != 'background'
]
sem_totals_to_visualize = ['total_with_bground', 'total_without_bground']
subR, subC = len(eval_stat_types), len(sem_idxs_to_visualize) + len(
sem_totals_to_visualize)
scale = max(np.ceil(subR / 2), np.ceil(subC / 4))
plt.figure(figsize=[scale * s for s in display_pyutils.BIG_FIGSIZE])
plt.clf()
markers = display_pyutils.MARKERS
default_colors = display_pyutils.GOOD_COLOR_CYCLE
try:
labels_table = problem_config.labels_table
colors_by_idx = {
idx: np.array(labels_table[idx].color)
for idx in range(len(labels_table))
}
except:
raise
size = 30
for eval_stat_idx, eval_stat_type in enumerate(eval_stat_types):
x_arr = pq_arrs[eval_stat_type]
y_arr = loss_arr
assert x_arr.shape == y_arr.shape
eval_identifier = eval_stat_type + '-iou_{}'.format(eval_iou_threshold)
for sem_idx, sem_name in zip(
sem_idxs_to_visualize,
[sem_names[ii] for ii in sem_idxs_to_visualize]):
# pq_d[xlabel]
x, y = x_arr[:, sem_idx], y_arr[:, sem_idx]
plt.subplot(subR, subC, subplot_idx)
ylabel = 'loss'
xlabel = eval_identifier + (' (reversed)' if reverse_x else '')
label = 'loss vs {}, component: {}'.format(eval_stat_type,
sem_name)
scatter(x, y, colors_by_idx[sem_idx], label, markers, size, xlabel,
ylabel)
subplot_idx += 1
plt.xlim([0, 1])
if reverse_x:
plt.gca().invert_xaxis()
for agg_i, aggregate_type in enumerate(sem_totals_to_visualize):
if aggregate_type == 'total_with_bground':
sem_idxs = list(range(len(sem_names)))
elif aggregate_type == 'total_without_bground':
sem_idxs = [
idx for idx, nm in enumerate(sem_names)
if nm != 'background'
]
else:
raise NotImplementedError
plt.subplot(subR, subC, subplot_idx)
ylabel = 'total loss'
xlabel = 'sum ' + eval_identifier + (' (reversed)'
if reverse_x else '')
label = 'loss vs {}, {}'.format(eval_stat_type, aggregate_type)
x, y = x_arr[:, sem_idxs].sum(axis=1), y_arr[:,
sem_idxs].sum(axis=1)
scatter(x, y, default_colors[agg_i], label, markers, size, xlabel,
ylabel)
subplot_idx += 1
plt.xlim([0, max(x)])
if reverse_x:
plt.gca().invert_xaxis()
plt.tight_layout()
onehot_tag = '_onehot' if 'losses_onehot' in loss_npz_file else ''
figname = 'all_loss_vs_eval_iouthresh_{}{}.png'.format(
eval_iou_threshold, onehot_tag)
if save_to_workspace:
display_pyutils.save_fig_to_workspace(figname)
figpath = os.path.join(output_dir, figname)
plt.savefig(figpath)
print(TermColors.OKGREEN +
'Scatterplot saved to {}'.format(os.path.abspath(figpath)) +
TermColors.ENDC)
def main():
args = parse_args()
logdir = args.logdir
instanceseg.utils.scripts.set_random_seeds()
display_pyutils.set_my_rc_defaults()
gpu = args.gpu
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
cuda = torch.cuda.is_available()
if display_pyutils.check_for_emptied_workspace():
print('Workspace clean.')
else:
print('Workspace not clean, but running anyway.')
# Load directory
cfg, model_pth, out_dir, problem_config, model, my_trainer, optim, dataloaders = \
instanceseg.utils.logs.load_everything_from_logdir(logdir, gpu=args.gpu, packed_as_dict=False)
model.eval()
# Write log directory name to folder
with open(
osp.join(display_pyutils.WORKSPACE_DIR,
osp.basename(osp.normpath(logdir))), 'w') as fid:
fid.write(logdir)
if cfg['augment_semantic']:
raise NotImplementedError(
'Gotta augment semantic first before running through model.')
# Display histograms of assigned instance sizes
channel_names = problem_config.get_channel_labels()
display_pyutils.set_my_rc_defaults()
for split in ['train', 'val']:
# NOTE(allie): At the moment, clims is not synced. Need to get the min/max and pass them in.
assigned_instance_sizes_2d, xent_losses_by_channel_2d, ious_by_channel_2d, soft_iou_loss_by_channel_2d = \
distribution_assignments.get_per_channel_per_image_sizes_and_losses(
model, dataloaders[split], cuda, my_trainer)
soft_ious_by_channel_2d = 1.0 - soft_iou_loss_by_channel_2d
diff_soft_iou_hard_iou_2d = ious_by_channel_2d - soft_ious_by_channel_2d
for sem_cls_val in range(len(problem_config.semantic_class_names)):
sem_cls_name = problem_config.semantic_class_names[sem_cls_val]
if sem_cls_name == 'background':
continue
sem_cls_channel_idxs = [
i for i, s_val in enumerate(
problem_config.semantic_instance_class_list)
if s_val == sem_cls_val
]
sem_cls_channel_names = [
channel_names[c] for c in sem_cls_channel_idxs
]
plt.figure(0)
plt.clf()
print('{} instances < 10 pixels'.format(
(assigned_instance_sizes_2d < 10).sum()))
attributes_by_channel = OrderedDict([
(name,
convert_arr_to_nested_list_without_zeros(
metric_2d,
zeros_reference_array=assigned_instance_sizes_2d))
for name, metric_2d in
[('instance_size',
assigned_instance_sizes_2d), (
'iou',
ious_by_channel_2d), ('soft_iou',
soft_ious_by_channel_2d),
('soft_iou_loss', soft_iou_loss_by_channel_2d
), ('diff_soft_iou_hard_iou', diff_soft_iou_hard_iou_2d
), ('xent_loss_contribution', xent_losses_by_channel_2d)]
])
# iou values
for i, x_attribute_name in enumerate(attributes_by_channel.keys()):
x = attributes_by_channel[x_attribute_name]
for y_attribute_name in [
k for j, k in enumerate(attributes_by_channel.keys())
if j > i
]:
print(sem_cls_name, x_attribute_name, y_attribute_name)
y = attributes_by_channel[y_attribute_name]
for use_subplots in [False]: # [True, False]:
make_scatterplot_set(
[x[c] for c in sem_cls_channel_idxs],
[y[c] for c in sem_cls_channel_idxs],
sem_cls_channel_names,
split,
x_attribute_name='{}_{}'.format(
x_attribute_name, sem_cls_name),
y_attribute_name='{}_{}'.format(
y_attribute_name, sem_cls_name),
use_subplots=use_subplots)
from tensorboardX import SummaryWriter
from instanceseg.losses.loss import MatchingLossResult
import instanceseg.utils.display as display_pyutils
import instanceseg.utils.export
import instanceseg.utils.imgutils
from instanceseg.analysis import visualization_utils
from instanceseg.datasets import runtime_transformations
from instanceseg.ext.panopticapi.utils import rgb2id, id2rgb
from instanceseg.losses.loss import LossMatchAssignments
from instanceseg.losses.match import GT_VALUE_FOR_FALSE_POSITIVE
from instanceseg.utils import instance_utils
from instanceseg.utils.instance_utils import InstanceProblemConfig
from instanceseg.utils.misc import flatten_dict
display_pyutils.set_my_rc_defaults()
MY_TIMEZONE = 'America/New_York'
def should_write_activations(iteration, epoch):
if iteration < 3000:
return True
else:
return False
TRAIN_FAST = True
DEBUG_ASSERTS = True