def nparams(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
run = Experiment.from_dir(args.run, main='model')
model = load_model(run)
print(run)
nparams = sum(p.numel() for p in tqdm(model.parameters()) if p.requires_grad)
print(f'N. Params: {nparams / 10 ** 6:.2g}M ({nparams})')
def retrieval(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
run = Experiment.from_dir(args.run, main='model')
results_file = run.path_to('retrieval.csv')
assert os.path.exists(results_file), f"Results file not found: {results_file}"
results = pd.read_csv(results_file)
# t1s, mean_aps_sym, mean_aps_asym = results.loc[:, ['t1', 'mean_ap_sym', 'mean_ap_asym']]
plt.figure(figsize=(15,5))
ax = plt.gca()
results.plot('t1', 'mean_ap_sym', marker='.', label='sym', ax=ax)
results.plot('t1', 'mean_ap_asym', marker='.', label='asym', ax=ax)
# plt.axhline(mean_ap_res, c='k', label='resnet')
max_diff = (results.mean_ap_asym - results.mean_ap_sym).max()
print(f'Asym-sym max difference: {max_diff:%}')
# plt.plot(t1s, mean_aps_asym - mean_aps_sym, marker='.', label='diff')
plt.title('mAP vs Feature Depth (t) - CIFAR-10')
plt.xlabel('Integration time')
plt.ylabel('mAP')
# plt.ylim([0, 1])
plt.legend(loc='best')
''' NFE sectioning
ns = np.diff(nfe)
xv = np.diff(t1s)/2
xv[1:] += t1s[1:-1]
xv = xv[ns != 0]
for x in xv:
plt.axvline(x, c='k', ls='--')
xv = np.array([0, ] + xv.tolist() + [1,])
xl = np.diff(xv) / 2
xl[1:] += xv[1:-1]
for x, n in zip(xl, np.unique(nfe)):
plt.annotate('NFE = {:.1f}'.format(n), (x, .2), rotation=90, ha='center', va='center')
'''
plt.savefig(args.output, bbox_inches="tight")
def nfe(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
exp = Experiment.from_dir(args.run, main='model')
nfe = pd.read_csv(exp.path_to('nfe.csv.gz')) #, index_col=0)
nfe = nfe[(nfe.t1 == 1) & (nfe.tol == 0.001)].reset_index(drop=True)
nfe.nfe = (nfe.nfe - 2) / 6 # show n. steps instead of NFE
nfe = nfe[nfe.y_true == nfe.y_pred]
dataset = exp.params.dataset.iloc[0]
if dataset == 'mnist':
labels = [str(i) for i in range(10)]
elif dataset == 'cifar10':
labels = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
nfe.y_true = nfe.y_true.apply(lambda x: labels[x])
nfe = nfe.sort_values('y_true')
# g = sns.FacetGrid(nfe, col='y_true')
# g.map(sns.kdeplot, 'nfe')
# ax = sns.boxplot(y='y_true', x='nfe', data=nfe, orient='h')
# ax.set_xlabel('solver steps')
# ax.set_ylabel('image class')
print('{:.2g} \pm {:.2g}'.format(nfe.nfe.mean(), nfe.nfe.std()))
min, max = nfe.nfe.min(), nfe.nfe.max()
values = np.arange(min, max + 1)
plt.xticks(values)
bins = values - .5
plt.xlim(bins[0], bins[-1])
counts, _, _ = plt.hist(nfe.nfe, bins=bins)
plt.grid(b=False, axis='x')
plt.grid(b=True, which='minor', linewidth=0.5, linestyle='--', axis='y')
plt.gca().get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
for v, c in zip(values, counts):
plt.text(v, c, f'{c:g}', ha='center', va='bottom')
plt.savefig(args.output, bbox_inches="tight")
plt.close()
""" Images """
sns.set_style('white')
n = 5
pad = 20
side = 28 if dataset == 'mnist' else 32
side += 2 # make_grid padding
groups = nfe.groupby('y_true')
test_data = load_test_data(exp)
test_data.transform = transforms.ToTensor()
largest_nfe = groups.nfe.nlargest(n)
high_nfe_idxs = largest_nfe.index.get_level_values(1)
high_nfe_images = torch.stack([test_data[i][0] for i in high_nfe_idxs])
high_nfe_grid = make_grid(high_nfe_images, nrow=n)
smallest_nfe = groups.nfe.nsmallest(n).reset_index().sort_values(['y_true', 'nfe'], ascending=[True, False])
low_nfe_idxs = smallest_nfe.level_1 # nsmallest in reverse order
low_nfe_images = torch.stack([test_data[i][0] for i in low_nfe_idxs])
low_nfe_grid = make_grid(low_nfe_images, nrow=n)
smallest_nfe = smallest_nfe.nfe
grid_h = low_nfe_grid.shape[1]
img_pad = torch.zeros((3, grid_h, pad))
grid = torch.cat((high_nfe_grid, img_pad, low_nfe_grid), 2)
plt.imshow(np.transpose(grid.numpy(), (1, 2, 0))) # , interpolation='nearest')
for i, (l, s) in enumerate(zip(largest_nfe, smallest_nfe)):
y, x = (i // n), (i % n)
y, x = (np.array((y, x)) + (0.8, 0.75)) * side
text = plt.text(x, y, str(int(l)), fontsize=5, ha='left', va='top', color='white')
text.set_path_effects([patheffects.Stroke(linewidth=1, foreground='black'), patheffects.Normal()])
disp = side * n + pad + 6
text = plt.text(x + disp, y, str(int(s)), fontsize=5, ha='left', va='top', color='white')
text.set_path_effects([patheffects.Stroke(linewidth=1, foreground='black'), patheffects.Normal()])
ax = plt.gca()
h, _ = ax.get_ylim()
ticks = (np.arange(10) / 10 + 1 / (2 * 10)) * h
plt.yticks(ticks, labels)
plt.xticks([])
h, htxt = -0.03, -0.08
ax.annotate('', xy=(0, h), xycoords='axes fraction', xytext=(1, h), arrowprops=dict(arrowstyle="<->", color='k'))
ax.annotate('high NFE', xy=(0, htxt), xycoords='axes fraction', xytext=(0, htxt))
ax.annotate('low NFE', xy=(1, htxt), xycoords='axes fraction', xytext=(0.87, htxt))
plt.savefig(args.output2, bbox_inches="tight")
def nfe(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
runs = Experiment.from_dir(args.run, main='model')
nfe = pd.read_csv(runs.path_to('nfe.csv'), index_col=0)
sns.boxplot(x='y_true', y='nfe', data=nfe)
plt.savefig(args.output, bbox_inches="tight")
