def decode_detections(detections, class_mapping): objs = filter_map(decode_detection, detections) boxes = pluck('box', objs) labels = list(map(lookup(class_mapping), pluck('label', objs))) return table(bbox=torch.FloatTensor(boxes) if len(boxes) else torch.FloatTensor(0, 4), label=torch.LongTensor(labels), confidence=torch.FloatTensor(pluck('confidence', objs)))
def extract_session(session, config): start = date.parse(session.time) detections = [] actions = [] detections = annotate.decode_detections(session.open.contents.instances, annotate.class_mapping(config)) \ if session.open.tag == "new" else empty_detections def previous(): return actions[-1] if len(actions) > 0 else None def previous_time(): return (actions[-1].time if len(actions) > 0 else 0) for (datestr, action) in session.history: t = date.parse(datestr) action = decode_action(action) prev = previous() if prev and action.action in ['transform', 'delete']: if prev.action == 'confirm' and prev.ids == action.ids: actions.pop() time = (t - start).total_seconds() duration = time - previous_time() actions.append(action._extend(time = time, duration = min(30, duration), real_duration = duration)) duration = sum (pluck('duration', actions)) end = actions[-1].time return struct(start = start, detections = detections, actions = actions, \ duration = duration, real_duration = end, type = session.open.tag, threshold=session.threshold)
def image_summaries(history): summaries = [image_summary(image) for image in history] durations = pluck('duration', summaries) cumulative_time = np.cumsum(durations) return [summary._extend(cumulative_time = t) for summary, t in zip(summaries, cumulative_time)]
def plot_cumulative_line_stacks(x, stacks, keys): total = torch.Tensor(len(stacks)).zero_() values = [np.cumsum(pluck(k, stacks, 0)) for k in keys] plt.stackplot(x, *values, labels=keys) plt.legend(loc='upper left')
def annotation_summary(dataset): def count(image): annotations = image_annotations(image) n = len(annotations) categories = struct( test=n if image.category == 'test' else 0, validate=n if image.category == 'validate' else 0, train=n if image.category == 'train' else 0, new=n if image.category == 'new' else 0, discard=n if image.category == 'discard' else 0, ) def box_area(ann): x1, y1, x2, y2 = ann.box return (x2 - x1) * (y2 - y1) def box_length(ann): x1, y1, x2, y2 = ann.box return max(x2 - x1, y2 - y1) box_areas = list(map(box_area, annotations)) box_lengths = list(map(box_length, annotations)) return struct(n=n, categories=categories, box_areas=box_areas, box_lengths=box_lengths, image_size=image.image_size) infos = list(map(count, filter_categories(dataset))) sizes = pluck('image_size', infos) def image_area(size): return size[0] * size[1] totals = reduce(operator.add, infos) return struct(n_images=len(infos), categories=totals.categories, n_annotations=totals.n, n=stats(pluck('n', infos)), box_length=stats(totals.box_lengths), box_area=stats(totals.box_areas), size_ranges=(min(sizes, key=image_area), max(sizes, key=image_area)))
def decode_object_map(annotations, config): mapping = class_mapping(config) objs = {k: decode_obj(a) for k, a in annotations.items()} objs = [ ann._extend(id=int(k)) for k, ann in objs.items() if ann is not None ] boxes = pluck('box', objs) labels = list(map(lookup(mapping), pluck('label', objs))) ids = pluck('id', objs) return table(bbox=torch.FloatTensor(boxes) if len(boxes) else torch.FloatTensor(0, 4), label=torch.LongTensor(labels), id=torch.LongTensor(ids))
def plot_stacks(x, stacks, keys, width): total = torch.Tensor(len(stacks)).zero_() bars = [] for k in keys: values = pluck(k, stacks, 0) p = plt.bar(x, values, width, bottom=total.tolist()) bars.append(p[0]) total = total + torch.Tensor(values) plt.legend(bars, keys)
def image_summary(image): return struct ( actions = image.actions, n_actions = len(image.actions), duration = image.duration, real_duration = image.real_duration, instances = image.target._size, actions_count = count_struct(pluck('action', image.actions), action_types), correction_count = annotation_corrections(image) )
def summarize_test(name, results, classes, epoch, log, thresholds=None): class_names = {c.id: c.name for c in classes} summary = compute_AP(results, classes, thresholds) total, class_aps = summary.total, summary.classes mAP_strs = 'mAP@30: {:.2f}, 50: {:.2f}, 75: {:.2f}'.format( total.mAP[30], total.mAP[50], total.mAP[75]) train_stats = filter_none(pluck('train_stats', results)) train_summary = summarize_train_stats(name, train_stats, classes, log) \ if len(train_stats) > 0 else '' print(name + ' epoch: {} AP: {:.2f} mAP@[0.3-0.95]: [{}] {}'.format( epoch, total.AP * 100, mAP_strs, train_summary)) log.scalars( name, struct(AP=total.AP * 100.0, mAP30=total.mAP[30] * 100.0, mAP50=total.mAP[50] * 100.0, mAP75=total.mAP[75] * 100.0)) for k, ap in class_aps.items(): if ap.class_counts is not None: log.scalars(name + "/counts/" + class_names[k], ap.class_counts) log.scalars(name + "/thresholds/" + class_names[k], ap.thresholds) aps = {class_names[k]: ap for k, ap in class_aps.items()} aps['total'] = total for k, ap in aps.items(): log.pr_curve(name + "/pr50/" + k, ap.pr50) log.pr_curve(name + "/pr75/" + k, ap.pr75) if len(classes) > 1: log.scalars(name + "/mAP50", {k: ap.mAP[50] * 100.0 for k, ap in aps.items()}) log.scalars(name + "/mAP75", {k: ap.mAP[75] * 100.0 for k, ap in aps.items()}) log.scalars(name + "/AP", {k: ap.AP * 100.0 for k, ap in aps.items()}) return total.AP, {k: ap.thresholds for k, ap in class_aps.items()}
def compute_AP(results, classes, conf_thresholds=None): class_ids = pluck('id', classes) iou_thresholds = list(range(30, 100, 5)) compute_mAP = evaluate.mAP_classes(results, num_classes=len(class_ids)) info = transpose_structs([compute_mAP(t / 100) for t in iou_thresholds]) info.classes = transpose_lists(info.classes) assert len(info.classes) == len(class_ids) target_counts = count_target_classes(results, class_ids) def summariseAP(ap, class_id=None): prs = {t: pr for t, pr in zip(iou_thresholds, ap)} mAP = {t: pr.mAP for t, pr in prs.items()} class_counts = None if None not in [conf_thresholds, class_id]: class_counts = threshold_count( prs[50].confidence, conf_thresholds[class_id])._extend( truth=target_counts.get(class_id)) return struct(mAP=mAP, AP=mean([ap for k, ap in mAP.items() if k >= 50]), thresholds=compute_thresholds(prs[50]), pr50=condense_pr(prs[50]), pr75=condense_pr(prs[75]), class_counts=class_counts) return struct(total=summariseAP(info.total), classes={ id: summariseAP(ap, id) for id, ap in zip(class_ids, info.classes) })
def combine_simple(datasets): images = concat_lists(pluck('images', datasets.values())) d = next(iter(datasets.values())) config = d.config._extend(root="/home/oliver/storage/penguins_combined") return struct(config=config, images=images)
def f(threshold, x_eval, sigma): weights = gaussian_weights(xs, x_eval, sigma) results = weighted_mAP(threshold, weights) return torch.Tensor(pluck('mAP', results))
def plot_noisy(run_path, figure_path): def read_run(offset, noise, dataset): logfile = path.join(log_path, run_path, str(noise), str(offset), dataset, 'log.json') return read_training(logfile) fig, ax = make_chart(size=(8, 8)) datasets=['penguins', 'scott_base', 'seals', 'apples_lincoln', 'branches'] markers = ['x', 'o', 's', '^', '*'] color_map = plt.get_cmap('viridis') APs = {dataset:np.zeros((5, 5)) for dataset in datasets} AP30s = {dataset:np.zeros((5, 5)) for dataset in datasets} AP50s = {dataset:np.zeros((5, 5)) for dataset in datasets} AP75s = {dataset:np.zeros((5, 5)) for dataset in datasets} # dataset_charts = {k: } levels = [0, 4, 8, 16, 32] for i, offset in enumerate(levels): for j, noise in enumerate(levels): logs = {dataset:read_run(offset, noise, dataset) for dataset in datasets} for k, log in logs.items(): APs[k][j, i] = log.best_AP.AP AP30s[k][j, i] = log.best_AP.mAP30 AP50s[k][j, i] = log.best_AP.mAP50 AP75s[k][j, i] = log.best_AP.mAP75 AP = sum(pluck('AP', logs.values())) / len(logs) epoch = np.arange(0, 40) + 1 ax.plot(epoch, AP, marker = markers[j], color=color_map(i / 4), linewidth=1, markersize=4) ax.set_xlim(xmin=0, xmax=40) ax.set_ylim(ymin=0, ymax=100) ax.set_xlabel("training epoch") ax.set_ylabel("validation $AP_{COCO}$") def noise_level(i): return Line2D([0], [0], color='k', marker = markers[i], label="$\sigma={}\%$".format(levels[i])) def offset_level(i): return Line2D([0], [0], color=color_map(i / 4), label="$\Delta={}\%$".format(levels[i])) legend = [noise_level(i) for i in range(0, 5)] + [offset_level(i) for i in range(0, 5)] ax.legend(handles=legend, ncol=2, fontsize='x-small', loc='upper left') ax.grid(True) def diffs(ap): baseline = ap[0][0] ap = -(1.0 - ap / baseline) * 100 ap[0][0] = baseline return ap with open(path.join(figure_path, run_path + "_datasets.csv"), mode='w') as csv_file: degredation = {k:diffs(ap) for k, ap in APs.items()} for k, d in degredation.items(): csv_file.write(k + ":\n") np.savetxt(csv_file, d, delimiter=',') # & $\sigma=0\%$ & $\mathbf{72.8\pm16.4}$ & $-6.3\pm6.4\%$ & $-18.6\pm9.0\%$ & $-39.5\pm17.5\%$ & $-29.2\pm19.2\%$ \\ def tex_table(k, mean, std): str = k + ": \n" for j, noise in enumerate(levels): str += "& $\sigma={}\%$".format(noise) for i, offset in enumerate(levels): str += " & " m, s = mean[j, i], std[j, i] if i == 0 and j == 0: str += "$\mathbf{{ {:.1f}\\pm{:.1f} }}$".format(m, s) else: str += "${:.1f}\\pm{:.1f}\\%$".format(m, s) str += " \\\\ \n" return str with open(path.join(figure_path, run_path + ".csv"), mode='w') as csv_file: with open(path.join(figure_path, run_path + ".tex"), mode='w') as tex_file: for k, aps in {'COCO':APs, '30':AP30s, '50':AP50s, '75':AP75s}.items(): degredation = {k:diffs(ap) for k, ap in aps.items()} combined = np.stack(degredation.values(), axis=0) csv_file.write(k + ":\n") np.savetxt(csv_file, np.mean(combined, 0), delimiter=',') np.savetxt(csv_file, np.std(combined, 0), delimiter=',') tex_file.write(tex_table(k, np.mean(combined, 0), np.std(combined, 0))) fig.savefig(path.join(figure_path, run_path + "_training.pdf"), bbox_inches='tight')