def visualize_activations(gui, data, sepnorm=False, bboxes=None): n_bs, n_maps, n_pixy, n_pixx = data.shape if n_pixy != n_pixx or (n_maps == n_pixx and n_maps == n_pixy): # this is the "weird" format preferred by Alex' convolutions. n_maps, n_pixy, n_pixx, n_bs = data.shape data = data.reshape(n_maps * n_pixy * n_pixx, n_bs).T.reshape(n_bs, n_maps, n_pixy, n_pixx) if gui.sigm: data = 1.0 / (1.0 + np.exp(-data)) if not sepnorm: if gui.center0: data = center_0(data) else: data -= data.min() data /= data.max() + 0.000001 n_plots_y = min(4, n_bs) n_plots_x = min(6, n_maps) if n_maps == 3: n_plots_x += 1 if n_maps == 3: n_plots_x = int(np.ceil(3.0 * np.sqrt(n_bs) / 2.0)) n_plots_y = int(np.ceil(2.0 * np.sqrt(n_bs) / 3.0)) min_transp = 0.02 if not hasattr(gui, "fig"): gui.fig, gui.axes = plt.subplots(n_plots_y, n_plots_x) gui.fig.subplots_adjust(hspace=0.00, wspace=0.00, left=0, top=1, bottom=0.15, right=0.95) gui.fig.canvas.mpl_connect("close_event", lambda e: gui.parent.remove_child(gui)) gui.fig.canvas.set_window_title(gui.title) ax_batch = plt.axes([0.25, 0.10, 0.65, 0.05]) gui.s_batch = DiscreteSlider(ax_batch, "idx in batch", 0, n_bs - 4, valinit=0) gui.s_batch.on_changed(lambda val: gui.parent.set_batch_idx(val)) ax_toggle_sepnorm = plt.axes([0.95, 0.90, 0.05, 0.05]) gui.s_toggle_sepnorm = CheckButtons(ax_toggle_sepnorm, ["sepnorm"], [sepnorm]) gui.s_toggle_sepnorm.on_clicked(lambda val: (gui.toggle_sepnorm(), gui.update())) ax_toggle_sigm = plt.axes([0.95, 0.80, 0.05, 0.05]) gui.s_toggle_sigm = CheckButtons(ax_toggle_sigm, ["Sigm"], [gui.sigm]) gui.s_toggle_sigm.on_clicked(lambda val: (gui.toggle_sigm(), gui.update())) ax_next_train_batch = plt.axes([0.95, 0.70, 0.05, 0.05]) gui.s_next_train_batch = Button(ax_next_train_batch, "Train") gui.s_next_train_batch.on_clicked(lambda val: gui.parent.next_batch_train()) ax_next_test_batch = plt.axes([0.95, 0.60, 0.05, 0.05]) gui.s_next_test_batch = Button(ax_next_test_batch, "Test") gui.s_next_test_batch.on_clicked(lambda val: gui.parent.next_batch_test()) ax_center = plt.axes([0.95, 0.50, 0.05, 0.05]) gui.s_center = CheckButtons(ax_center, ["Center0"], [gui.center0]) gui.s_center.on_clicked(lambda val: (gui.toggle_center0(), gui.update())) ax_toggle_gtbbox = plt.axes([0.95, 0.40, 0.05, 0.05]) gui.s_toggle_gtbbox = CheckButtons(ax_toggle_gtbbox, ["GTBBox"], [gui.gtbbox]) gui.s_toggle_gtbbox.on_clicked(lambda val: (gui.toggle_gtbbox(), gui.update())) ax_toggle_detbbox = plt.axes([0.95, 0.30, 0.05, 0.05]) gui.s_toggle_detbbox = CheckButtons(ax_toggle_detbbox, ["DetBBox"], [gui.detbbox]) gui.s_toggle_detbbox.on_clicked(lambda val: (gui.toggle_detbbox(), gui.update())) ax_map = plt.axes([0.25, 0.05, 0.65, 0.05]) gui.s_map = DiscreteSlider(ax_map, "idx of map", 0, n_maps - 6, valinit=0) gui.s_map.on_changed(lambda val: gui.update()) ax_transp = plt.axes([0.25, 0.00, 0.65, 0.05]) gui.s_transp = Slider(ax_transp, "transparency", 0.0, 1.0, valinit=0.5) gui.s_transp.on_changed(lambda val: gui.update()) gui.labels = {} have_clf = False if hasattr(gui.od, "logreg") and gui.od.logreg is not None: have_clf = True glog.info("Found logistic regression in model") logreg_y_hat = gui.od.logreg.estimator.evaluate().np logreg_y = gui.od.logreg.y.data.np else: glog.info("Found NO logistic regression in model") n_colors = {"green": 0, "blue": 0, "orange": 0, "red": 0} for ax, i in zip(gui.axes.flatten(), xrange(np.prod(gui.axes.shape))): ax.patches = [] # clear patches (=bounding boxes) ax.texts = [] # clear annotations idx_x = i % n_plots_x # map idx_y = i / n_plots_x # batch if False and gui.name not in ["X", "input"] and gui.transp > min_transp: # most likely an output map, which we can now compare to the input map input = gui.parent.input.cache[gui.parent.batch_idx + idx_y].copy() flt = data[gui.parent.batch_idx + idx_y, gui.map_idx + idx_x, :, :].copy() if sepnorm: if gui.center0: flt = center_0(flt) else: flt -= flt.min() flt /= flt.max() + 0.000001 vsi = None try: vsi = pod.get_valid_shape_info(gui.parent.input.op, gui.op) except: glog.warn("Could not create vsi!") pass if vsi is not None: # TODO: incorporate initial margins w.r.t. original image somehow # if `input' is just a ROI in the original image # TODO: determine and draw `valid' margins as well # it can happen that the margins are negative, if the image was padded. if vsi.i_margin_r < 0 or vsi.i_margin_l < 0: dsize = input.shape[0] - min(0, vsi.i_margin_r) - min(0, vsi.i_margin_l) input2 = np.zeros((dsize, dsize, input.shape[2])) input2[ -vsi.i_margin_l : -vsi.i_margin_l + input.shape[0], -vsi.i_margin_l : -vsi.i_margin_l + input.shape[1], ] = input input = input2 final_start = max(0, vsi.i_margin_l) final_end = input.shape[0] + max(0, vsi.i_margin_l) else: final_start = vsi.i_margin_l final_end = input.shape[0] - vsi.i_margin_r input = 1 - input input -= input.min() transp = gui.transp - min_transp roi = input[final_start:final_end, final_start:final_end] flt = flt.reshape(n_pixy, n_pixx) flt = zoom( flt, ( (final_end - final_start + 0.01) / float(n_pixy), (final_end - final_start + 0.01) / float(n_pixx), ), order=0, ) # now change portion of input which is in receptive field flt -= 0.5 # flt is between 0 and 1 if flt.min() >= 0: flt = np.dstack((flt, flt, flt)) roi[:] = transp * roi + (1 - transp) * (roi * flt) else: flt1 = flt.copy() flt1[flt > 0] = 0 flt2 = flt.copy() flt2[flt < 0] = 0 flt = np.dstack((-flt1, flt2, np.zeros_like(flt1))) flt *= 2.0 roi[:] = transp * roi + (1 - transp) * (roi * flt) flt = np.clip(1 - input, 0.0, 1.0) ax.cla() ax.imshow(flt, interpolation="nearest") # elif n_maps != 3 or idx_x != 3: elif n_maps != 3: flt = data[gui.parent.batch_idx + idx_y, gui.map_idx + idx_x, :, :] if sepnorm: if gui.center0: flt = center_0(flt) else: flt -= flt.min() flt /= flt.max() + 0.000001 ax.cla() ax.matshow(flt.reshape(n_pixy, n_pixx), cmap="PuOr_r", vmin=0, vmax=1) if bboxes is not None: if gui.vsi is not None: glog.info("NOT drawine bounding boxes...") # determine the value of the most prominent bbox # v = -1E6 # for b in bboxes[1][gui.parent.batch_idx+idx_y]: # for bb in b: # v = max(v, bb.value) if gui.gtbbox and len(bboxes[0][0]) == n_maps: bb = bboxes[0][gui.parent.batch_idx + idx_y][gui.map_idx + idx_x] draw_bboxes( ax, bb, "yellow", gui.vsi, linewidth=1, ls="dashed", title=classname(gui.map_idx + idx_x) ) if gui.detbbox and len(bboxes[0][0]) == n_maps: bb = bboxes[1][gui.parent.batch_idx + idx_y][gui.map_idx + idx_x] draw_bboxes( ax, bb, "#AAAAFF", gui.vsi, onlyifvalue=None, linewidth=1, ls="solid", title=classname(gui.map_idx + idx_x), ) pass else: glog.debug("No bounding boxes supplied, none drawn.") pass elif gui.parent.batch_idx + idx_y * n_plots_x + idx_x < data.shape[0]: imidx = gui.parent.batch_idx + idx_y * n_plots_x + idx_x flt = data[imidx, :, :, :].reshape(3, n_pixy, n_pixx) flt = np.rollaxis(flt, 0, 3) # move dst axis to end if sepnorm: flt = center_0(flt) # flt -= flt.min() # flt /= flt.max() ax.imshow(flt, interpolation="nearest") if have_clf: # if logreg_y.shape[1] == 2: # names = ['horse', 'cow'] # else: # names = [str(x) for x in xrange(logreg_y.shape[1])] y = logreg_y[imidx] y_hat = logreg_y_hat[imidx, :] y_hat = np.exp(y_hat) / np.sum(np.exp(y_hat)) # softmax sidx = np.argsort(y_hat) correct_class = int(y) cm = mpl.cm.get_cmap("RdYlGn") idx = correct_class s = "*" if idx != correct_class else "" # prec = 1 - abs(y[sidx[-1]] - y_hat[sidx[-1]]) target = 1 if idx == correct_class else 0 prec = 1 - abs(target - y_hat[idx]) prec = min(10, len(y_hat) - np.where(sidx == correct_class)[0][0]) prec = 1.0 - prec / 10.0 tcolor = "black" if abs(prec - 0.5) < 0.2 else "white" color = cm(prec) ax.text( 0.1, 0.91, "%s: %2.3f" % (s + classname(idx), y_hat[idx]), verticalalignment="bottom", horizontalalignment="left", transform=ax.transAxes, color=tcolor, fontsize=7, bbox={"facecolor": color, "pad": 0}, ) if idx == correct_class: idx = sidx[-2] else: idx = sidx[-1] s = "*" if idx != correct_class else "" tcolor = "black" if abs(prec - 0.5) < 0.2 else "white" target = 1 if idx == correct_class else 0 # prec = 1 - abs(target - y_hat[idx]) color = cm(prec) ax.text( 0.1, 0.01, "%s: %2.3f" % (s + classname(idx), y_hat[idx]), verticalalignment="bottom", horizontalalignment="left", transform=ax.transAxes, color=tcolor, fontsize=7, bbox={"facecolor": color, "pad": 0}, ) # if gui.name == "input": # gui.cache[idx_y] = flt.sum(axis=2) # grayscale gui.cache[imidx] = flt.copy() if bboxes is not None and gui.vsi is not None: teachers, predictions, kmeans = bboxes confidences = [p.confidence for p in predictions[imidx]] glog.info("Drawing boundin boxes...") if gui.gtbbox and len(teachers[imidx]) < 25: for m, c in zip(teachers[imidx], get_random_color(42)): # draw_bboxes(ax, m, c, gui.vsi, ls='dashed', title=classname(idx)) # print ("class of bb:", m.klass) draw_bboxes(ax, m, n_pixx, c, gui.vsi, ls="dashed", title=classname(m.klass)) # TODO these are the found bounding boxes if gui.detbbox: # determine the value of the most prominent bbox v = -1e6 for b in predictions[imidx]: pass # print b # for bb in b: # v = max(v, bb.value) # for idx, (t, c) in enumerate(zip(bboxes[3], get_random_color(42))): # #draw_bboxes(ax, m, c, gui.vsi, onlyifvalue=v, ls='solid') # draw_bboxes(ax, t, n_pixx, c, gui.vsi, ls='solid', only_positive=True, confidence=None) try: matched_pred, matched_teach = match_bboxes( teachers[imidx], predictions[imidx], confidences, thresh=gui.transp ) except Exception as e: print "Could not run match_bboxes:" print str(e) print traceback.format_exc() best_val = max(confidences) for idx, (p, c, k, color) in enumerate( zip(predictions[imidx], confidences, kmeans, get_random_color(42)) ): # draw_bboxes(ax, p, c, gui.vsi, onlyifvalue=v, ls='solid') if sigm(c) > gui.transp: color = "red" if matched_pred[idx] == 1: color = "green" elif matched_pred[idx] == 2: color = "blue" draw_bboxes( ax, p, n_pixx, color, gui.vsi, ls="solid", only_positive=True, confidence=str(idx) + (": %1.1f" % sigm(c)), ) # draw_bboxes(ax, k, n_pixx, color, gui.vsi, ls='solid', only_positive=True, confidence=None) n_colors[color] += 1 for idx, t in enumerate(teachers[imidx]): # draw_bboxes(ax, p, c, gui.vsi, onlyifvalue=v, ls='solid') if not matched_teach[idx]: color = "orange" draw_bboxes(ax, t.rect, n_pixx, color, gui.vsi, ls="solid", only_positive=True) # ax.plot((p.x, k.x), (p.y, k.y), color=c) n_colors[color] += 1 else: glog.debug("No bounding boxes supplied, none drawn.") pass cfg(ax) t = 0.0 for c in ["green", "blue", "red", "orange"]: t += n_colors[c] if t > 0: print [(c, "{:>0.2f}".format(n_colors[c] / t)) for c in ["green", "blue", "red", "orange"]]
def show_single_trial(trial, properties0=None, properties1=None, properties2=None): phases = [] for cve_name, cve in trial.cv_events.iteritems(): if not cve_name.startswith("TRAIN"): continue for gd in cve.gds: phases.append(gd) phases = np.unique(phases) for phase in phases: fig = plt.figure(figsize=(12, 14)) fig.suptitle(phase) fig.canvas.set_window_title(phase) for cve_name, cve in trial.cv_events.iteritems(): if not cve_name.startswith("TRAIN"): continue gd = cve.gds[phase] ax = fig.add_subplot(221) X = [x[0] for x in gd.convergence] Y = [x[1] for x in gd.convergence] ax.plot(X, Y, label='convergence') X = [x[0] for x in gd.early_stopping] Y = [x[1] for x in gd.early_stopping] ax.plot(X, Y, label='early_stopping') if hasattr(trial, 'test0_loss') and phase == cve.current_gd.name: plt.axhline(trial.test0_loss, label='test0_loss', color='k') ax.legend() ax = fig.add_subplot(223) for k, v in gd.mon.iteritems(): if properties0 is not None: if k not in properties0: continue for z, l in ((0, ""), (1, "_es")): X = [x[0] for x in v if x[2] == z] Y = [x[1] for x in v if x[2] == z] ax.plot(X, Y, '-', label=k + l) ax.legend() ax.set_title("Monitor") if properties1 is not None: ax = fig.add_subplot(224) D = defaultdict(lambda: {}) for k, v in gd.mon.iteritems(): if k in properties1: X = [x[0] for x in v] Y = [x[1] for x in v] ax.plot(X, Y, '-', label=k) elif k.replace("_mean", "") in properties1: X = [x[0] for x in v] mean = [x[1] for x in v] D[k.replace("_mean", "")]["mean"] = mean D[k.replace("_mean", "")]["x"] = X elif k.replace("_var", "") in properties1: X = [x[0] for x in v] var = [x[1] for x in v] D[k.replace("_var", "")]["var"] = var D[k.replace("_var", "")]["x"] = X else: print "ignoring", k, "in p1" for (k, v), c in zip(D.iteritems(), get_random_color()): mean = v["mean"] va = v["var"] x = v["x"] ax.plot(x, mean, '-', label=k, color=c, linewidth=2) ax.fill_between(X, mean - np.sqrt(var), mean + np.sqrt(var), alpha=.2, color=c) ax.legend() ax.set_title("Monitor") if properties2 is not None: ax = fig.add_subplot(222) for k, v in gd.mon.iteritems(): if k not in properties2: print "ignoring ", k, " in p2" continue X = [x[0] for x in v] Y = [x[1] for x in v] ax.plot(X, Y, '-', label=k) ax.legend() ax.set_title("Monitor") plt.savefig("%s.pdf" % phase)