def run(name: str, shape: List[int], y_lim=None, coord=None): runs = lib.get_runs([name]) trackers = {} for r in runs: for k, s in r.summary.items(): if "/mask_stat/shared/" not in k or ( not (k.endswith("_weight") and k.startswith("permuted_mnist/perm_"))): continue if k not in trackers: trackers[k] = StatTracker() trackers[k].add(s * 100) perm_by_layer = {} for perm in range(1, 10, 2): prefix = f"permuted_mnist/perm_{perm}/mask_stat/shared/" for k, v in trackers.items(): if not k.startswith(prefix): continue layer_name = k.split("/")[-1] if layer_name not in perm_by_layer: perm_by_layer[layer_name] = [] perm_by_layer[layer_name].append(v) fig = plt.figure(figsize=shape) # ax = fig.add_subplot(111,aspect=0.06) n_col = 5 d = n_col + 1 names = OrderedDict() names["Layer 1"] = "layers_0_weight" names["Layer 2"] = "layers_1_weight" names["Layer 3"] = "layers_2_weight" names["Layer 4"] = "layers_3_weight" for j in range(n_col): stats = [perm_by_layer[n][j].get() for n in names.values()] plt.bar([i * d + j for i in range(len(names))], [s.mean for s in stats], yerr=[s.std for s in stats], align='center') plt.xticks([d * x + n_col / 2 - 0.5 for x in range(len(names))], names.keys()) plt.legend([f"T{c*2+2}" for c in range(n_col)], ncol=2, loc="upper center") plt.ylabel("Weights shared [\\%]") if y_lim is not None: plt.ylim(*y_lim) if coord is not None: fig.axes[0].yaxis.set_label_coords(*coord) fig.savefig(f"{basedir}/{name}.pdf", bbox_inches='tight', pad_inches=0.01)
#!/usr/bin/env python3 import lib from lib import StatTracker from lib.common import group import os import matplotlib.pyplot as plt runs = lib.get_runs([ "addmul_feedforward", "addmul_feedforward_big", "addmul_feedforward_huge" ]) os.makedirs("out", exist_ok=True) runs = group(runs, ["layer_sizes"]) print(runs) all_stats = {} for grp, runs in runs.items(): print("----------------------------------- ", grp) for run in runs: tsum = 0 ssum = 0 # print(stats) for k, v in run.summary.items(): kparts = k.split("/") if kparts[-1] != "n_1" or "/all/" in k or not k.startswith( "mask_stat/"): continue print(k, v)
#!/usr/bin/env python3 import os import lib from lib import StatTracker import torch import shutil import matplotlib.pyplot as plt import numpy as np import itertools from mpl_toolkits.axes_grid1 import make_axes_locatable from lib.common import group runs = lib.get_runs(["cifar10"]) BASE_DIR = "out/cifar_drop_stat_confusion/" VER_DIR = f"{BASE_DIR}/cifar10/download/" #shutil.rmtree(VER_DIR, ignore_errors=True) os.makedirs(VER_DIR, exist_ok=True) cifar10_classes = [ 'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck' ] cifar100_classes = [ 'apple', 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle', 'bicycle', 'bottle', 'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel', 'can', 'castle', 'caterpillar', 'cattle', 'chair', 'chimpanzee', 'clock', 'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur',
from collections import OrderedDict import os os.makedirs("out", exist_ok=True) import matplotlib.pyplot as plt run_table = OrderedDict() run_table["Add or sub"] = "dm_math_add_or_sub" run_table["Linear 1D"] = "dm_math_lin1d" run_table["Differentiate"] = "dm_math_diff" run_table["Sort"] = "dm_math_sort" run_table["Poly. Collect"] = "dm_math_polycollect" runs = lib.get_runs(list(run_table.values())) trackers = {} for r in runs: accuracies = {} if r.config["restore_pretrained"]: accuracies["train"] = r.summary["load_validation/hard/accuracy/total"] else: step = r.config["stop_after"] hist = r.scan_history(keys=["validation/hard/accuracy/total"], min_step=step - 1, max_step=step + 1) accuracies["train"] = None for h in hist: assert accuracies["train"] is None accuracies["train"] = h["validation/hard/accuracy/total"]
this_stats = [rnn_stats[f"{plots[n]}{t}"].get() for n in names] row.append(f"Pair {t+1}") for m, s in zip([s.mean * 100 for s in this_stats], [s.std * 100 for s in this_stats]): row.append(f"${m:.0f} \pm {s:.1f}$") print(" & ".join(row) + " \\\\") row = [""] print("\\bottomrule") print("\end{tabular}") def plot_runs(rnn_runs, feedforward_runs): feedforward_runs = group(feedforward_runs, ["layer_sizes"]) rnn_runs = group(rnn_runs, ["tuple.mode"]) plot_all("rnn", rnn_runs) plot_all("feedforward", feedforward_runs) print( f"********************************** all **********************************" ) plot_both(feedforward_runs["layer_sizes_2000,2000,2000,2000"], rnn_runs["tuple.mode_together"]) print("====================== Normal ======================") plot_runs(lib.get_runs(["tuple_rnn"]), lib.get_runs(["tuple_feedforward_big"])) print("====================== First full mask ======================") plot_runs(lib.get_runs(["tuple_rnn_first_full_mask"]), lib.get_runs(["tuple_feedforward_big_first_full_mask"]))
#!/usr/bin/env python3 import lib from lib import StatTracker from lib.common import group import os from mpl_toolkits.axes_grid1 import make_axes_locatable import matplotlib.pyplot as plt runs = lib.get_runs(["addmul_ff_alpha_analysis"]) runs = group(runs, ["mask_loss_weight"]) sharing_stats = {} accuracy_stats = {} for grp, runs in runs.items(): print("----------------------------------- ", grp) for run in runs: print("RUN ID", run.id) tsum = 0 ssum = 0 # print(stats) for k, v in run.summary.items(): kparts = k.split("/") if kparts[-1] != "n_1" or "/all/" in k or not k.startswith( "mask_stat/"): continue print(k, v)
k: baseline_stats[f"scan.train_split_{v}"] [f"validation/{v}/accuracy/total"].get() for k, v in refs.items() } ref_means = {k: v.mean for k, v in ref_stats.items()} ref_std = {k: v.std for k, v in ref_stats.items()} fig = plt.figure(figsize=[3, 1.5]) plt.bar([2.25 * x for x in range(len(names))], [ref_means[n] * 100 for n in names], yerr=[ref_std[n] * 100 for n in names], align='center') plt.bar([2.25 * x + 1 for x in range(len(names))], [means[n] * 100 for n in names], yerr=[std[n] * 100 for n in names], align='center') plt.xticks([2.25 * x + 0.5 for x in range(len(names))], names) plt.ylabel("Test accuracy [\\%]") # plt.legend(["Before", "After"]) fig.savefig(fname, bbox_inches='tight') plot(lib.get_runs(["scan"]), lib.get_runs(["scan_baseline"], filters={"config.task": "scan"}), "out/scan_removed_vs_baseline_lstm.pdf") plot(lib.get_runs(["trafo_scan"]), lib.get_runs(["scan_baseline"], filters={"config.task": "trafo_scan"}), "out/scan_removed_vs_baseline_transformer.pdf")
keys = list(sorted(stats.keys())) if keys[0].startswith("lstm_cells"): for i in range(1, len(keys), 2): keys[i], keys[i - 1] = keys[i - 1], keys[i] # print([friendly_name(k) for k in keys]) names = [friendly_name(k) for k in keys] legend = ["IoU", "IoMin"] plt.bar([2.25 * x for x in range(len(names))], [stats[n].iou.mean for n in keys], yerr=[stats[n].iou.std for n in keys], align='center') plt.bar([2.25 * x + 1 for x in range(len(names))], [stats[n].subsetness.mean for n in keys], yerr=[stats[n].subsetness.std for n in keys], align='center') plt.xticks([2.25 * x + 0.5 for x in range(len(names))], names) plt.ylabel("Proportion") plt.ylim(0, 1) plt.legend(legend) f = f"{prefix}/{grp}.pdf" os.makedirs(os.path.dirname(f), exist_ok=True) fig.savefig(f, bbox_inches='tight') do_plot(lib.get_runs(["tuple_rnn", "tuple_feedforward_big"]), "out/tuple_iou")
#!/usr/bin/env python3 import os import torch import lib from tqdm import tqdm import matplotlib.pyplot as plt import wandb api = wandb.Api() BASE_DIR = "out/scan_jump_output" os.makedirs(BASE_DIR, exist_ok=True) runs = lib.get_runs(["scan"]) for i, r in tqdm(enumerate(runs)): dest = os.path.join(BASE_DIR, "weights", str(i)) os.makedirs(dest, exist_ok=True) r.file( "export/stage_final_masks/stage_1/decoder_output_projection_weight.pth" ).download(root=dest, replace=True) r.file( "export/stage_final_masks/stage_0/decoder_output_projection_weight.pth" ).download(root=dest, replace=True) avg = lib.StatTracker() for i in range(len(runs)): dest = os.path.join(BASE_DIR, "weights", str(i)) m_base = torch.load( f"{BASE_DIR}/weights/{i}/export/stage_final_masks/stage_0/"
print("-------------------- GROUP --------", grp) print(stats.keys()) fig = plt.figure(figsize=[4.5,1.4]) keys = list(sorted(stats.keys())) if keys[0].startswith("lstm_cells"): for i in range(1, len(keys), 2): keys[i], keys[i-1] = keys[i-1], keys[i] # print([friendly_name(k) for k in keys]) names = [friendly_name(k) for k in keys] legend = ["IoU", "IoMin"] plt.bar([2.25 * x for x in range(len(names))], [stats[n].iou.mean for n in keys], yerr=[stats[n].iou.std for n in keys], align='center') plt.bar([2.25 * x + 1 for x in range(len(names))], [stats[n].subsetness.mean for n in keys], yerr=[stats[n].subsetness.std for n in keys], align='center') plt.xticks([2.25 * x + 0.5 for x in range(len(names))], names) plt.ylabel("Proportion") plt.ylim(0,1) plt.legend(legend) f = f"{prefix}/{grp}.pdf" os.makedirs(os.path.dirname(f), exist_ok=True) fig.savefig(f, bbox_inches='tight') do_plot(lib.get_runs(["tuple_rnn", "tuple_feedforward_big"]), "out/tuple_iou") do_plot(lib.get_runs(["tuple_rnn_copyweight"]), "out/tuple_iou_copyweight")
import lib from lib import StatTracker runs = lib.get_runs(["cifar10_mask_stability"]) stat = StatTracker() for r in runs: for k, v in r.summary.items(): if k.startswith("masks_stability/") and "/pair_" in k: stat.add(v) print(stat) print("Count", stat.n)
f.download(root=VER_DIR, replace=True) data = torch.load(full_name) data = data.astype(np.float32) if "confusion_difference" not in f.name: data = data / np.sum(data, axis=1, keepdims=True) data = data * 100 trackers[f.name].add(data) # break # # if i_run >= 2: # break return trackers trackers = create_trackers(runs) for k, v in trackers.items(): s = v.get() figure = draw_confusion(s.mean, s.std) prefix = f"out/cifar10_confusion/{name}/" dir = os.path.join(prefix, os.path.dirname(k)) os.makedirs(dir, exist_ok=True) figure.savefig(f"{prefix}/{k}.pdf", bbox_inches='tight', pad_inches = 0.01) plt.close() draw(lib.get_runs(["cifar10_no_dropout"]), "no_dropout") draw(lib.get_runs(["cifar10"]), "with_dropout") draw(lib.get_runs(["cifar10_resnet"]), "resnet")
assert len(h) == 1 ref_accuracy = h[0][accuracy_key] val = (ref_accuracy - hist[k]) * 100 # Fast hack for editing the plot # val = (1-v)*100 (avg_normal if k.startswith("half_mask_test/normal") else avg_inv).add(val) return avg_normal.get(), avg_inv.get() stats = OrderedDict() stats["SCAN trafo"] = get_stats(lib.get_runs(["trafo_scan_halfmask_inv"])) stats["$+/*$ FNN"] = get_stats(lib.get_runs( ["addmul_feedforward_halfmask_inv"], filters={"config.layer_sizes": "800,800,800,800"}), accuracy_key="validation/iid/accuracy") stats["$+/+$ LSTM"] = get_stats([ r for r in lib.get_runs(["tuple_rnn_halfmask_inv"]) if r.config["tuple.mode"] == "together" ]) stats["$+/+$ FNN Big"] = get_stats( lib.get_runs(["tuple_feedforward_big_halfmask_inv"])) stats["$+/+$ FNN Small"] = get_stats( lib.get_runs(["tuple_feedforward_small_halfmask_inv"])) stats["CIFAR 10 simple"] = get_stats(lib.get_runs(["cifar10_halfmask_inv"]), accuracy_key="validation/iid/accuracy")
rel_drop[i] = 0 trackers[cls].add(min(np.min(rel_drop),0)/target_drop * 100) return [trackers[k].get() for k in cifar10_classes] runs = OrderedDict() runs["Simple"] = "cifar10" runs["No dropout"] = "cifar10_no_dropout" runs["Resnet 110"] = "cifar10_resnet" rel_drops = {} worsts = {} for k, v in runs.items(): r = lib.get_runs([v]) print("Downloading", k) download(v, r) rel_drops[k] = get_relative_drop(v, r) worsts[k] = get_worst(v, r) def draw_stats(stats, shape): fig = plt.figure(figsize=shape) for i, k in enumerate(runs.keys()): stat = stats[k] plt.bar([x * (len(runs) + 1) + i for x in range(len(stat))], [s.mean for s in stat], yerr=[s.std for s in stat], align='center') plt.xticks([(len(runs) + 1) * x + (len(runs) - 1) / 2 for x in range(len(cifar10_classes))], cifar10_classes) for tick in fig.axes[0].xaxis.get_major_ticks()[1::2]: tick.set_pad(15)
for t in range(2): this_rnn_stats = [rnn_stats[f"{plots[n]}{t}"].get() for n in names] means_rnn = [s.mean * 100 for s in this_rnn_stats] std_rnn = [s.std * 100 for s in this_rnn_stats] plt.bar([5.5 * r + 1 + t * 2.5 for r in range(len(names))], means_rnn, yerr=std_rnn, align='center') plt.xticks([5.5 * r + 1.75 for r in range(len(names))], names) plt.ylabel("Accuracy [\\%]") # plt.legend(["F1", "F2", "R1", "R2"], bbox_to_anchor=(1.1, 1.05)) fname = f"{BASE_DIR}/tuple_performance.pdf" fig.axes[0].yaxis.set_label_coords(-0.12, 0.4) fig.savefig(fname, bbox_inches='tight', pad_inches=0.01) rnn_runs = lib.get_runs(["tuple_rnn"]) feedforward_runs = lib.get_runs(["tuple_feedforward_big"]) feedforward_runs = group(feedforward_runs, ["layer_sizes"]) rnn_runs = group(rnn_runs, ["tuple.mode"]) plot_all("rnn", rnn_runs) plot_all("feedforward", feedforward_runs) plot_both(feedforward_runs["layer_sizes_2000,2000,2000,2000"], rnn_runs["tuple.mode_together"])
import itertools from mpl_toolkits.axes_grid1 import make_axes_locatable plt.rcParams['text.usetex'] = True #Let TeX do the typsetting plt.rcParams['text.latex.preamble'] = [ r'\usepackage{sansmath}', r'\sansmath' ] #Force sans-serif math mode (for axes labels) plt.rcParams['font.family'] = 'sans-serif' # ... for regular text plt.rcParams[ 'font.sans-serif'] = 'Helvetica, Avant Garde, Computer Modern Sans serif' # Choose a nice font here TEST = False api = wandb.Api() runs = lib.get_runs(["addmul_feedforward_big", "addmul_rnn"]) BASE_DIR = "out/addmul_confusion_plot/download" shutil.rmtree(BASE_DIR, ignore_errors=True) os.makedirs(BASE_DIR, exist_ok=True) runs = group(runs, ['layer_sizes', "task"]) print(runs.keys()) def draw_confusion(means: np.ndarray, std: np.ndarray): print("MEAN", means) figure = plt.figure(figsize=[2.5, 0.5]) #means.shape) ax = plt.gca() im = plt.imshow(means,
means = {k: stats[v].get().mean for k, v in plots.items()} std = {k: stats[v].get().std for k, v in plots.items()} ref_means = {k: stats[v].get().mean for k, v in refs.items()} ref_std = {k: stats[v].get().std for k, v in refs.items()} return [ref_means[n]*100 for n in names], [ref_std[n]*100 for n in names], \ [means[n]*100 for n in names], [std[n]*100 for n in names] def printres(name, vals, errs): print(name+": "+(" ".join([f"{names[i]}: {v:.1f} ({e:.2f})" for i, (v, e) in enumerate(zip(vals, errs))]))) fig = plt.figure(figsize=[4,1]) ref_m, ref_err, res_m, res_err = get_mean_err(lib.get_runs(["scan"])) t_ref_m, t_ref_err, t_res_m, t_res_err = get_mean_err(lib.get_runs(["trafo_scan"])) printres('LSTM iid', ref_m, ref_err) printres('LSTM splits', res_m, res_err) printres('Trafo iid', t_ref_m, t_ref_err) printres('Trafo splits', t_res_m, t_res_err) print("LSTM - trafo:", " ".join(f"{names[i]}: {l-t:.2f}" for i, (l, t) in enumerate(zip(res_m, t_res_m)))) plt.bar([5*x for x in range(len(names))], ref_m, yerr=ref_err, align='center') plt.bar([5*x+1 for x in range(len(names))], t_ref_m, yerr=t_ref_err, align='center') plt.bar([5*x+2+0.25 for x in range(len(names))], res_m, yerr=res_err, align='center') plt.bar([5*x+3+0.25 for x in range(len(names))], t_res_m, yerr=t_res_err, align='center') plt.xticks([5*x+1.625 for x in range(len(names))], names)