def decode_detections(detections, class_mapping):
objs = filter_map(decode_detection, detections)
boxes = pluck('box', objs)
labels = list(map(lookup(class_mapping), pluck('label', objs)))
return table(bbox=torch.FloatTensor(boxes)
if len(boxes) else torch.FloatTensor(0, 4),
label=torch.LongTensor(labels),
confidence=torch.FloatTensor(pluck('confidence', objs)))
def extract_session(session, config):
start = date.parse(session.time)
detections = []
actions = []
detections = annotate.decode_detections(session.open.contents.instances, annotate.class_mapping(config)) \
if session.open.tag == "new" else empty_detections
def previous():
return actions[-1] if len(actions) > 0 else None
def previous_time():
return (actions[-1].time if len(actions) > 0 else 0)
for (datestr, action) in session.history:
t = date.parse(datestr)
action = decode_action(action)
prev = previous()
if prev and action.action in ['transform', 'delete']:
if prev.action == 'confirm' and prev.ids == action.ids:
actions.pop()
time = (t - start).total_seconds()
duration = time - previous_time()
actions.append(action._extend(time = time, duration = min(30, duration), real_duration = duration))
duration = sum (pluck('duration', actions))
end = actions[-1].time
return struct(start = start, detections = detections, actions = actions, \
duration = duration, real_duration = end, type = session.open.tag, threshold=session.threshold)
def image_summaries(history):
summaries = [image_summary(image) for image in history]
durations = pluck('duration', summaries)
cumulative_time = np.cumsum(durations)
return [summary._extend(cumulative_time = t) for summary, t in zip(summaries, cumulative_time)]
def plot_cumulative_line_stacks(x, stacks, keys):
total = torch.Tensor(len(stacks)).zero_()
values = [np.cumsum(pluck(k, stacks, 0)) for k in keys]
plt.stackplot(x, *values, labels=keys)
plt.legend(loc='upper left')
def annotation_summary(dataset):
def count(image):
annotations = image_annotations(image)
n = len(annotations)
categories = struct(
test=n if image.category == 'test' else 0,
validate=n if image.category == 'validate' else 0,
train=n if image.category == 'train' else 0,
new=n if image.category == 'new' else 0,
discard=n if image.category == 'discard' else 0,
)
def box_area(ann):
x1, y1, x2, y2 = ann.box
return (x2 - x1) * (y2 - y1)
def box_length(ann):
x1, y1, x2, y2 = ann.box
return max(x2 - x1, y2 - y1)
box_areas = list(map(box_area, annotations))
box_lengths = list(map(box_length, annotations))
return struct(n=n,
categories=categories,
box_areas=box_areas,
box_lengths=box_lengths,
image_size=image.image_size)
infos = list(map(count, filter_categories(dataset)))
sizes = pluck('image_size', infos)
def image_area(size):
return size[0] * size[1]
totals = reduce(operator.add, infos)
return struct(n_images=len(infos),
categories=totals.categories,
n_annotations=totals.n,
n=stats(pluck('n', infos)),
box_length=stats(totals.box_lengths),
box_area=stats(totals.box_areas),
size_ranges=(min(sizes,
key=image_area), max(sizes,
key=image_area)))
def decode_object_map(annotations, config):
mapping = class_mapping(config)
objs = {k: decode_obj(a) for k, a in annotations.items()}
objs = [
ann._extend(id=int(k)) for k, ann in objs.items() if ann is not None
]
boxes = pluck('box', objs)
labels = list(map(lookup(mapping), pluck('label', objs)))
ids = pluck('id', objs)
return table(bbox=torch.FloatTensor(boxes)
if len(boxes) else torch.FloatTensor(0, 4),
label=torch.LongTensor(labels),
id=torch.LongTensor(ids))
def plot_stacks(x, stacks, keys, width):
total = torch.Tensor(len(stacks)).zero_()
bars = []
for k in keys:
values = pluck(k, stacks, 0)
p = plt.bar(x, values, width, bottom=total.tolist())
bars.append(p[0])
total = total + torch.Tensor(values)
plt.legend(bars, keys)
def image_summary(image):
return struct (
actions = image.actions,
n_actions = len(image.actions),
duration = image.duration,
real_duration = image.real_duration,
instances = image.target._size,
actions_count = count_struct(pluck('action', image.actions), action_types),
correction_count = annotation_corrections(image)
)
def summarize_test(name, results, classes, epoch, log, thresholds=None):
class_names = {c.id: c.name for c in classes}
summary = compute_AP(results, classes, thresholds)
total, class_aps = summary.total, summary.classes
mAP_strs = 'mAP@30: {:.2f}, 50: {:.2f}, 75: {:.2f}'.format(
total.mAP[30], total.mAP[50], total.mAP[75])
train_stats = filter_none(pluck('train_stats', results))
train_summary = summarize_train_stats(name, train_stats, classes, log) \
if len(train_stats) > 0 else ''
print(name + ' epoch: {} AP: {:.2f} mAP@[0.3-0.95]: [{}] {}'.format(
epoch, total.AP * 100, mAP_strs, train_summary))
log.scalars(
name,
struct(AP=total.AP * 100.0,
mAP30=total.mAP[30] * 100.0,
mAP50=total.mAP[50] * 100.0,
mAP75=total.mAP[75] * 100.0))
for k, ap in class_aps.items():
if ap.class_counts is not None:
log.scalars(name + "/counts/" + class_names[k], ap.class_counts)
log.scalars(name + "/thresholds/" + class_names[k], ap.thresholds)
aps = {class_names[k]: ap for k, ap in class_aps.items()}
aps['total'] = total
for k, ap in aps.items():
log.pr_curve(name + "/pr50/" + k, ap.pr50)
log.pr_curve(name + "/pr75/" + k, ap.pr75)
if len(classes) > 1:
log.scalars(name + "/mAP50",
{k: ap.mAP[50] * 100.0
for k, ap in aps.items()})
log.scalars(name + "/mAP75",
{k: ap.mAP[75] * 100.0
for k, ap in aps.items()})
log.scalars(name + "/AP", {k: ap.AP * 100.0 for k, ap in aps.items()})
return total.AP, {k: ap.thresholds for k, ap in class_aps.items()}
def compute_AP(results, classes, conf_thresholds=None):
class_ids = pluck('id', classes)
iou_thresholds = list(range(30, 100, 5))
compute_mAP = evaluate.mAP_classes(results, num_classes=len(class_ids))
info = transpose_structs([compute_mAP(t / 100) for t in iou_thresholds])
info.classes = transpose_lists(info.classes)
assert len(info.classes) == len(class_ids)
target_counts = count_target_classes(results, class_ids)
def summariseAP(ap, class_id=None):
prs = {t: pr for t, pr in zip(iou_thresholds, ap)}
mAP = {t: pr.mAP for t, pr in prs.items()}
class_counts = None
if None not in [conf_thresholds, class_id]:
class_counts = threshold_count(
prs[50].confidence, conf_thresholds[class_id])._extend(
truth=target_counts.get(class_id))
return struct(mAP=mAP,
AP=mean([ap for k, ap in mAP.items() if k >= 50]),
thresholds=compute_thresholds(prs[50]),
pr50=condense_pr(prs[50]),
pr75=condense_pr(prs[75]),
class_counts=class_counts)
return struct(total=summariseAP(info.total),
classes={
id: summariseAP(ap, id)
for id, ap in zip(class_ids, info.classes)
})
def combine_simple(datasets):
images = concat_lists(pluck('images', datasets.values()))
d = next(iter(datasets.values()))
config = d.config._extend(root="/home/oliver/storage/penguins_combined")
return struct(config=config, images=images)
def f(threshold, x_eval, sigma):
weights = gaussian_weights(xs, x_eval, sigma)
results = weighted_mAP(threshold, weights)
return torch.Tensor(pluck('mAP', results))
def plot_noisy(run_path, figure_path):
def read_run(offset, noise, dataset):
logfile = path.join(log_path, run_path, str(noise), str(offset), dataset, 'log.json')
return read_training(logfile)
fig, ax = make_chart(size=(8, 8))
datasets=['penguins', 'scott_base', 'seals', 'apples_lincoln', 'branches']
markers = ['x', 'o', 's', '^', '*']
color_map = plt.get_cmap('viridis')
APs = {dataset:np.zeros((5, 5)) for dataset in datasets}
AP30s = {dataset:np.zeros((5, 5)) for dataset in datasets}
AP50s = {dataset:np.zeros((5, 5)) for dataset in datasets}
AP75s = {dataset:np.zeros((5, 5)) for dataset in datasets}
# dataset_charts = {k: }
levels = [0, 4, 8, 16, 32]
for i, offset in enumerate(levels):
for j, noise in enumerate(levels):
logs = {dataset:read_run(offset, noise, dataset) for dataset in datasets}
for k, log in logs.items():
APs[k][j, i] = log.best_AP.AP
AP30s[k][j, i] = log.best_AP.mAP30
AP50s[k][j, i] = log.best_AP.mAP50
AP75s[k][j, i] = log.best_AP.mAP75
AP = sum(pluck('AP', logs.values())) / len(logs)
epoch = np.arange(0, 40) + 1
ax.plot(epoch, AP, marker = markers[j], color=color_map(i / 4), linewidth=1, markersize=4)
ax.set_xlim(xmin=0, xmax=40)
ax.set_ylim(ymin=0, ymax=100)
ax.set_xlabel("training epoch")
ax.set_ylabel("validation $AP_{COCO}$")
def noise_level(i):
return Line2D([0], [0], color='k', marker = markers[i], label="$\sigma={}\%$".format(levels[i]))
def offset_level(i):
return Line2D([0], [0], color=color_map(i / 4), label="$\Delta={}\%$".format(levels[i]))
legend = [noise_level(i) for i in range(0, 5)] + [offset_level(i) for i in range(0, 5)]
ax.legend(handles=legend, ncol=2, fontsize='x-small', loc='upper left')
ax.grid(True)
def diffs(ap):
baseline = ap[0][0]
ap = -(1.0 - ap / baseline) * 100
ap[0][0] = baseline
return ap
with open(path.join(figure_path, run_path + "_datasets.csv"), mode='w') as csv_file:
degredation = {k:diffs(ap) for k, ap in APs.items()}
for k, d in degredation.items():
csv_file.write(k + ":\n")
np.savetxt(csv_file, d, delimiter=',')
# & $\sigma=0\%$ & $\mathbf{72.8\pm16.4}$ & $-6.3\pm6.4\%$ & $-18.6\pm9.0\%$ & $-39.5\pm17.5\%$ & $-29.2\pm19.2\%$ \\
def tex_table(k, mean, std):
str = k + ": \n"
for j, noise in enumerate(levels):
str += "& $\sigma={}\%$".format(noise)
for i, offset in enumerate(levels):
str += " & "
m, s = mean[j, i], std[j, i]
if i == 0 and j == 0:
str += "$\mathbf{{ {:.1f}\\pm{:.1f} }}$".format(m, s)
else:
str += "${:.1f}\\pm{:.1f}\\%$".format(m, s)
str += " \\\\ \n"
return str
with open(path.join(figure_path, run_path + ".csv"), mode='w') as csv_file:
with open(path.join(figure_path, run_path + ".tex"), mode='w') as tex_file:
for k, aps in {'COCO':APs, '30':AP30s, '50':AP50s, '75':AP75s}.items():
degredation = {k:diffs(ap) for k, ap in aps.items()}
combined = np.stack(degredation.values(), axis=0)
csv_file.write(k + ":\n")
np.savetxt(csv_file, np.mean(combined, 0), delimiter=',')
np.savetxt(csv_file, np.std(combined, 0), delimiter=',')
tex_file.write(tex_table(k, np.mean(combined, 0), np.std(combined, 0)))
fig.savefig(path.join(figure_path, run_path + "_training.pdf"), bbox_inches='tight')
