def run(name: str, shape: List[int], y_lim=None, coord=None):
runs = lib.get_runs([name])
trackers = {}
for r in runs:
for k, s in r.summary.items():
if "/mask_stat/shared/" not in k or (
not (k.endswith("_weight")
and k.startswith("permuted_mnist/perm_"))):
continue
if k not in trackers:
trackers[k] = StatTracker()
trackers[k].add(s * 100)
perm_by_layer = {}
for perm in range(1, 10, 2):
prefix = f"permuted_mnist/perm_{perm}/mask_stat/shared/"
for k, v in trackers.items():
if not k.startswith(prefix):
continue
layer_name = k.split("/")[-1]
if layer_name not in perm_by_layer:
perm_by_layer[layer_name] = []
perm_by_layer[layer_name].append(v)
fig = plt.figure(figsize=shape)
# ax = fig.add_subplot(111,aspect=0.06)
n_col = 5
d = n_col + 1
names = OrderedDict()
names["Layer 1"] = "layers_0_weight"
names["Layer 2"] = "layers_1_weight"
names["Layer 3"] = "layers_2_weight"
names["Layer 4"] = "layers_3_weight"
for j in range(n_col):
stats = [perm_by_layer[n][j].get() for n in names.values()]
plt.bar([i * d + j for i in range(len(names))],
[s.mean for s in stats],
yerr=[s.std for s in stats],
align='center')
plt.xticks([d * x + n_col / 2 - 0.5 for x in range(len(names))],
names.keys())
plt.legend([f"T{c*2+2}" for c in range(n_col)], ncol=2, loc="upper center")
plt.ylabel("Weights shared [\\%]")
if y_lim is not None:
plt.ylim(*y_lim)
if coord is not None:
fig.axes[0].yaxis.set_label_coords(*coord)
fig.savefig(f"{basedir}/{name}.pdf", bbox_inches='tight', pad_inches=0.01)
#!/usr/bin/env python3
import lib
from lib import StatTracker
from lib.common import group
import os
import matplotlib.pyplot as plt
runs = lib.get_runs([
"addmul_feedforward", "addmul_feedforward_big", "addmul_feedforward_huge"
])
os.makedirs("out", exist_ok=True)
runs = group(runs, ["layer_sizes"])
print(runs)
all_stats = {}
for grp, runs in runs.items():
print("----------------------------------- ", grp)
for run in runs:
tsum = 0
ssum = 0
# print(stats)
for k, v in run.summary.items():
kparts = k.split("/")
if kparts[-1] != "n_1" or "/all/" in k or not k.startswith(
"mask_stat/"):
continue
print(k, v)
#!/usr/bin/env python3
import os
import lib
from lib import StatTracker
import torch
import shutil
import matplotlib.pyplot as plt
import numpy as np
import itertools
from mpl_toolkits.axes_grid1 import make_axes_locatable
from lib.common import group
runs = lib.get_runs(["cifar10"])
BASE_DIR = "out/cifar_drop_stat_confusion/"
VER_DIR = f"{BASE_DIR}/cifar10/download/"
#shutil.rmtree(VER_DIR, ignore_errors=True)
os.makedirs(VER_DIR, exist_ok=True)
cifar10_classes = [
'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck'
]
cifar100_classes = [
'apple', 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle',
'bicycle', 'bottle', 'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel',
'can', 'castle', 'caterpillar', 'cattle', 'chair', 'chimpanzee', 'clock',
'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur',
from collections import OrderedDict
import os
os.makedirs("out", exist_ok=True)
import matplotlib.pyplot as plt
run_table = OrderedDict()
run_table["Add or sub"] = "dm_math_add_or_sub"
run_table["Linear 1D"] = "dm_math_lin1d"
run_table["Differentiate"] = "dm_math_diff"
run_table["Sort"] = "dm_math_sort"
run_table["Poly. Collect"] = "dm_math_polycollect"
runs = lib.get_runs(list(run_table.values()))
trackers = {}
for r in runs:
accuracies = {}
if r.config["restore_pretrained"]:
accuracies["train"] = r.summary["load_validation/hard/accuracy/total"]
else:
step = r.config["stop_after"]
hist = r.scan_history(keys=["validation/hard/accuracy/total"],
min_step=step - 1,
max_step=step + 1)
accuracies["train"] = None
for h in hist:
assert accuracies["train"] is None
accuracies["train"] = h["validation/hard/accuracy/total"]
this_stats = [rnn_stats[f"{plots[n]}{t}"].get() for n in names]
row.append(f"Pair {t+1}")
for m, s in zip([s.mean * 100 for s in this_stats],
[s.std * 100 for s in this_stats]):
row.append(f"${m:.0f} \pm {s:.1f}$")
print(" & ".join(row) + " \\\\")
row = [""]
print("\\bottomrule")
print("\end{tabular}")
def plot_runs(rnn_runs, feedforward_runs):
feedforward_runs = group(feedforward_runs, ["layer_sizes"])
rnn_runs = group(rnn_runs, ["tuple.mode"])
plot_all("rnn", rnn_runs)
plot_all("feedforward", feedforward_runs)
print(
f"********************************** all **********************************"
)
plot_both(feedforward_runs["layer_sizes_2000,2000,2000,2000"],
rnn_runs["tuple.mode_together"])
print("====================== Normal ======================")
plot_runs(lib.get_runs(["tuple_rnn"]), lib.get_runs(["tuple_feedforward_big"]))
print("====================== First full mask ======================")
plot_runs(lib.get_runs(["tuple_rnn_first_full_mask"]),
lib.get_runs(["tuple_feedforward_big_first_full_mask"]))
#!/usr/bin/env python3
import lib
from lib import StatTracker
from lib.common import group
import os
from mpl_toolkits.axes_grid1 import make_axes_locatable
import matplotlib.pyplot as plt
runs = lib.get_runs(["addmul_ff_alpha_analysis"])
runs = group(runs, ["mask_loss_weight"])
sharing_stats = {}
accuracy_stats = {}
for grp, runs in runs.items():
print("----------------------------------- ", grp)
for run in runs:
print("RUN ID", run.id)
tsum = 0
ssum = 0
# print(stats)
for k, v in run.summary.items():
kparts = k.split("/")
if kparts[-1] != "n_1" or "/all/" in k or not k.startswith(
"mask_stat/"):
continue
print(k, v)
k: baseline_stats[f"scan.train_split_{v}"]
[f"validation/{v}/accuracy/total"].get()
for k, v in refs.items()
}
ref_means = {k: v.mean for k, v in ref_stats.items()}
ref_std = {k: v.std for k, v in ref_stats.items()}
fig = plt.figure(figsize=[3, 1.5])
plt.bar([2.25 * x for x in range(len(names))],
[ref_means[n] * 100 for n in names],
yerr=[ref_std[n] * 100 for n in names],
align='center')
plt.bar([2.25 * x + 1 for x in range(len(names))],
[means[n] * 100 for n in names],
yerr=[std[n] * 100 for n in names],
align='center')
plt.xticks([2.25 * x + 0.5 for x in range(len(names))], names)
plt.ylabel("Test accuracy [\\%]")
# plt.legend(["Before", "After"])
fig.savefig(fname, bbox_inches='tight')
plot(lib.get_runs(["scan"]),
lib.get_runs(["scan_baseline"], filters={"config.task": "scan"}),
"out/scan_removed_vs_baseline_lstm.pdf")
plot(lib.get_runs(["trafo_scan"]),
lib.get_runs(["scan_baseline"], filters={"config.task": "trafo_scan"}),
"out/scan_removed_vs_baseline_transformer.pdf")
keys = list(sorted(stats.keys()))
if keys[0].startswith("lstm_cells"):
for i in range(1, len(keys), 2):
keys[i], keys[i - 1] = keys[i - 1], keys[i]
# print([friendly_name(k) for k in keys])
names = [friendly_name(k) for k in keys]
legend = ["IoU", "IoMin"]
plt.bar([2.25 * x for x in range(len(names))],
[stats[n].iou.mean for n in keys],
yerr=[stats[n].iou.std for n in keys],
align='center')
plt.bar([2.25 * x + 1 for x in range(len(names))],
[stats[n].subsetness.mean for n in keys],
yerr=[stats[n].subsetness.std for n in keys],
align='center')
plt.xticks([2.25 * x + 0.5 for x in range(len(names))], names)
plt.ylabel("Proportion")
plt.ylim(0, 1)
plt.legend(legend)
f = f"{prefix}/{grp}.pdf"
os.makedirs(os.path.dirname(f), exist_ok=True)
fig.savefig(f, bbox_inches='tight')
do_plot(lib.get_runs(["tuple_rnn", "tuple_feedforward_big"]), "out/tuple_iou")
#!/usr/bin/env python3
import os
import torch
import lib
from tqdm import tqdm
import matplotlib.pyplot as plt
import wandb
api = wandb.Api()
BASE_DIR = "out/scan_jump_output"
os.makedirs(BASE_DIR, exist_ok=True)
runs = lib.get_runs(["scan"])
for i, r in tqdm(enumerate(runs)):
dest = os.path.join(BASE_DIR, "weights", str(i))
os.makedirs(dest, exist_ok=True)
r.file(
"export/stage_final_masks/stage_1/decoder_output_projection_weight.pth"
).download(root=dest, replace=True)
r.file(
"export/stage_final_masks/stage_0/decoder_output_projection_weight.pth"
).download(root=dest, replace=True)
avg = lib.StatTracker()
for i in range(len(runs)):
dest = os.path.join(BASE_DIR, "weights", str(i))
m_base = torch.load(
f"{BASE_DIR}/weights/{i}/export/stage_final_masks/stage_0/"
print("-------------------- GROUP --------", grp)
print(stats.keys())
fig = plt.figure(figsize=[4.5,1.4])
keys = list(sorted(stats.keys()))
if keys[0].startswith("lstm_cells"):
for i in range(1, len(keys), 2):
keys[i], keys[i-1] = keys[i-1], keys[i]
# print([friendly_name(k) for k in keys])
names = [friendly_name(k) for k in keys]
legend = ["IoU", "IoMin"]
plt.bar([2.25 * x for x in range(len(names))], [stats[n].iou.mean for n in keys], yerr=[stats[n].iou.std for n in keys], align='center')
plt.bar([2.25 * x + 1 for x in range(len(names))], [stats[n].subsetness.mean for n in keys], yerr=[stats[n].subsetness.std for n in keys], align='center')
plt.xticks([2.25 * x + 0.5 for x in range(len(names))], names)
plt.ylabel("Proportion")
plt.ylim(0,1)
plt.legend(legend)
f = f"{prefix}/{grp}.pdf"
os.makedirs(os.path.dirname(f), exist_ok=True)
fig.savefig(f, bbox_inches='tight')
do_plot(lib.get_runs(["tuple_rnn", "tuple_feedforward_big"]), "out/tuple_iou")
do_plot(lib.get_runs(["tuple_rnn_copyweight"]), "out/tuple_iou_copyweight")
import lib
from lib import StatTracker
runs = lib.get_runs(["cifar10_mask_stability"])
stat = StatTracker()
for r in runs:
for k, v in r.summary.items():
if k.startswith("masks_stability/") and "/pair_" in k:
stat.add(v)
print(stat)
print("Count", stat.n)
f.download(root=VER_DIR, replace=True)
data = torch.load(full_name)
data = data.astype(np.float32)
if "confusion_difference" not in f.name:
data = data / np.sum(data, axis=1, keepdims=True)
data = data * 100
trackers[f.name].add(data)
# break
#
# if i_run >= 2:
# break
return trackers
trackers = create_trackers(runs)
for k, v in trackers.items():
s = v.get()
figure = draw_confusion(s.mean, s.std)
prefix = f"out/cifar10_confusion/{name}/"
dir = os.path.join(prefix, os.path.dirname(k))
os.makedirs(dir, exist_ok=True)
figure.savefig(f"{prefix}/{k}.pdf", bbox_inches='tight', pad_inches = 0.01)
plt.close()
draw(lib.get_runs(["cifar10_no_dropout"]), "no_dropout")
draw(lib.get_runs(["cifar10"]), "with_dropout")
draw(lib.get_runs(["cifar10_resnet"]), "resnet")
assert len(h) == 1
ref_accuracy = h[0][accuracy_key]
val = (ref_accuracy - hist[k]) * 100
# Fast hack for editing the plot
# val = (1-v)*100
(avg_normal
if k.startswith("half_mask_test/normal") else avg_inv).add(val)
return avg_normal.get(), avg_inv.get()
stats = OrderedDict()
stats["SCAN trafo"] = get_stats(lib.get_runs(["trafo_scan_halfmask_inv"]))
stats["$+/*$ FNN"] = get_stats(lib.get_runs(
["addmul_feedforward_halfmask_inv"],
filters={"config.layer_sizes": "800,800,800,800"}),
accuracy_key="validation/iid/accuracy")
stats["$+/+$ LSTM"] = get_stats([
r for r in lib.get_runs(["tuple_rnn_halfmask_inv"])
if r.config["tuple.mode"] == "together"
])
stats["$+/+$ FNN Big"] = get_stats(
lib.get_runs(["tuple_feedforward_big_halfmask_inv"]))
stats["$+/+$ FNN Small"] = get_stats(
lib.get_runs(["tuple_feedforward_small_halfmask_inv"]))
stats["CIFAR 10 simple"] = get_stats(lib.get_runs(["cifar10_halfmask_inv"]),
accuracy_key="validation/iid/accuracy")
rel_drop[i] = 0
trackers[cls].add(min(np.min(rel_drop),0)/target_drop * 100)
return [trackers[k].get() for k in cifar10_classes]
runs = OrderedDict()
runs["Simple"] = "cifar10"
runs["No dropout"] = "cifar10_no_dropout"
runs["Resnet 110"] = "cifar10_resnet"
rel_drops = {}
worsts = {}
for k, v in runs.items():
r = lib.get_runs([v])
print("Downloading", k)
download(v, r)
rel_drops[k] = get_relative_drop(v, r)
worsts[k] = get_worst(v, r)
def draw_stats(stats, shape):
fig = plt.figure(figsize=shape)
for i, k in enumerate(runs.keys()):
stat = stats[k]
plt.bar([x * (len(runs) + 1) + i for x in range(len(stat))], [s.mean for s in stat],
yerr=[s.std for s in stat], align='center')
plt.xticks([(len(runs) + 1) * x + (len(runs) - 1) / 2 for x in range(len(cifar10_classes))], cifar10_classes)
for tick in fig.axes[0].xaxis.get_major_ticks()[1::2]:
tick.set_pad(15)
for t in range(2):
this_rnn_stats = [rnn_stats[f"{plots[n]}{t}"].get() for n in names]
means_rnn = [s.mean * 100 for s in this_rnn_stats]
std_rnn = [s.std * 100 for s in this_rnn_stats]
plt.bar([5.5 * r + 1 + t * 2.5 for r in range(len(names))],
means_rnn,
yerr=std_rnn,
align='center')
plt.xticks([5.5 * r + 1.75 for r in range(len(names))], names)
plt.ylabel("Accuracy [\\%]")
# plt.legend(["F1", "F2", "R1", "R2"], bbox_to_anchor=(1.1, 1.05))
fname = f"{BASE_DIR}/tuple_performance.pdf"
fig.axes[0].yaxis.set_label_coords(-0.12, 0.4)
fig.savefig(fname, bbox_inches='tight', pad_inches=0.01)
rnn_runs = lib.get_runs(["tuple_rnn"])
feedforward_runs = lib.get_runs(["tuple_feedforward_big"])
feedforward_runs = group(feedforward_runs, ["layer_sizes"])
rnn_runs = group(rnn_runs, ["tuple.mode"])
plot_all("rnn", rnn_runs)
plot_all("feedforward", feedforward_runs)
plot_both(feedforward_runs["layer_sizes_2000,2000,2000,2000"],
rnn_runs["tuple.mode_together"])
import itertools
from mpl_toolkits.axes_grid1 import make_axes_locatable
plt.rcParams['text.usetex'] = True #Let TeX do the typsetting
plt.rcParams['text.latex.preamble'] = [
r'\usepackage{sansmath}', r'\sansmath'
] #Force sans-serif math mode (for axes labels)
plt.rcParams['font.family'] = 'sans-serif' # ... for regular text
plt.rcParams[
'font.sans-serif'] = 'Helvetica, Avant Garde, Computer Modern Sans serif' # Choose a nice font here
TEST = False
api = wandb.Api()
runs = lib.get_runs(["addmul_feedforward_big", "addmul_rnn"])
BASE_DIR = "out/addmul_confusion_plot/download"
shutil.rmtree(BASE_DIR, ignore_errors=True)
os.makedirs(BASE_DIR, exist_ok=True)
runs = group(runs, ['layer_sizes', "task"])
print(runs.keys())
def draw_confusion(means: np.ndarray, std: np.ndarray):
print("MEAN", means)
figure = plt.figure(figsize=[2.5, 0.5]) #means.shape)
ax = plt.gca()
im = plt.imshow(means,
means = {k: stats[v].get().mean for k, v in plots.items()}
std = {k: stats[v].get().std for k, v in plots.items()}
ref_means = {k: stats[v].get().mean for k, v in refs.items()}
ref_std = {k: stats[v].get().std for k, v in refs.items()}
return [ref_means[n]*100 for n in names], [ref_std[n]*100 for n in names], \
[means[n]*100 for n in names], [std[n]*100 for n in names]
def printres(name, vals, errs):
print(name+": "+(" ".join([f"{names[i]}: {v:.1f} ({e:.2f})" for i, (v, e) in enumerate(zip(vals, errs))])))
fig = plt.figure(figsize=[4,1])
ref_m, ref_err, res_m, res_err = get_mean_err(lib.get_runs(["scan"]))
t_ref_m, t_ref_err, t_res_m, t_res_err = get_mean_err(lib.get_runs(["trafo_scan"]))
printres('LSTM iid', ref_m, ref_err)
printres('LSTM splits', res_m, res_err)
printres('Trafo iid', t_ref_m, t_ref_err)
printres('Trafo splits', t_res_m, t_res_err)
print("LSTM - trafo:", " ".join(f"{names[i]}: {l-t:.2f}" for i, (l, t) in enumerate(zip(res_m, t_res_m))))
plt.bar([5*x for x in range(len(names))], ref_m, yerr=ref_err, align='center')
plt.bar([5*x+1 for x in range(len(names))], t_ref_m, yerr=t_ref_err, align='center')
plt.bar([5*x+2+0.25 for x in range(len(names))], res_m, yerr=res_err, align='center')
plt.bar([5*x+3+0.25 for x in range(len(names))], t_res_m, yerr=t_res_err, align='center')
plt.xticks([5*x+1.625 for x in range(len(names))], names)
