def _get_results(model):
exp_dir = os.path.join(args.root, f'{args.dataset}_{model}', 'adv-attack')
exps = Experiment.gather(exp_dir)
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'results.csv')
results['model'] = model
return results
def t1(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'tradeoff.csv')
unique_cols = results.apply(pd.Series.nunique) == 1
common_params = results.loc[:, unique_cols].iloc[0]
r = results.loc[:, ~unique_cols]
metric_cols = {'t1', 'test_acc', 'test_loss', 'test_nfe'}
plt.figure(figsize=(10, 6))
ax1 = plt.subplot2grid((2, 2), (0, 0))
ax2 = plt.subplot2grid((2, 2), (1, 0))
ax3 = plt.subplot2grid((2, 2), (1, 1))
title = Experiment.abbreviate(common_params, main='model')
ax1.set_title(title)
ax1.set_ylabel('Test Accuracy')
ax2.set_ylabel('Test NFE')
ax2.set_xlabel('ODE final integration time $t_1$')
ax3.set_ylabel('Test Accuracy')
ax3.set_xlabel('Test NFE')
if set(r.columns) == metric_cols: # single line plot
r = r.sort_values('t1')
ax1.plot(r.t1, r.test_acc, marker='.')
ax2.plot(r.t1, r.test_nfe, marker='.')
else:
# print(r, metric_cols)
grouping_cols = r.columns.difference(metric_cols).tolist()
# print(grouping_cols)
for name, group in r.groupby(grouping_cols):
# print(grouping_cols)
# print(group.reset_index())
params = group.reset_index().loc[0, grouping_cols]
name = Experiment.abbreviate(params, main='model')
r = group.sort_values('t1')
ax1.plot(r.t1, r.test_acc, label=name, marker='.')
ax2.plot(r.t1, r.test_nfe, label=name, marker='.')
ax3.plot(r.test_nfe, r.test_acc, label=name, marker='.')
ax1.legend(bbox_to_anchor=(1, 1), loc="upper left")
plt.minorticks_on()
for ax in (ax1, ax2):
ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
plt.savefig(args.output, bbox_inches="tight")
def clean(args):
runs = Experiment.gather(args.run, main='model')
empty_runs = filter(lambda run: run.log.empty, runs)
dirs = [run.path for run in empty_runs]
n_empty_runs = len(dirs)
print("Empty runs found: {}".format(n_empty_runs))
if n_empty_runs:
print('\n'.join(dirs))
print("Delete them? [y/N] ", end='')
if input().lower() in ('y', 'yes'):
for r in dirs:
command = 'rm -rf {}'.format(r)
print(command)
os.system(command)
def success_rate_single(args):
exps = Experiment.gather(args.run)
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'results.csv')
results = results.rename({'distance_x': 'p', 'distance_y': 'distance'}, axis=1)
results['success_rate'] = np.isfinite(results.distance.values)
natural_errors = results.distance == 0
results = results[~natural_errors] # discard natural errors
rate = pd.pivot_table(results, values='success_rate', index='tol', columns=['epsilon', 'p'])
rate = (rate * 100).round(2)
rate = rate.rename_axis(columns={'p': r'$p$', 'epsilon': r'$\varepsilon$'})
rate = rate.rename(columns={2.0: r'$L_2$', float('inf'): r'$L_\infty$'}, level=1)
print(rate)
def classification_performance(args):
models = ('resnet', 'mixed', 'fullode')
exps = Experiment.gather(args.root, main='model')
exps = Experiment.filter({'dataset': args.dataset}, exps)
results = Experiment.collect_all(exps, 'nfe.csv.gz')
results = results[results.t1 == 1] # keep only complete dynamics
results['accuracy'] = results.y_pred == results.y_true
results = results.rename({'tol_x': 'training_tol', 'tol_y': 'tol'}, axis=1)
rate = pd.pivot_table(results, values='accuracy', index='tol', columns=['model', 'downsample'])
rate = (100 * (1 - rate)).round(2)
rate = rate.rename(mapper=lambda x: r'$10^{{{}}}$'.format(int(round(np.log10(x)))), level=0)
with open(args.output, 'w') as out:
rate.to_latex(out, escape=False, multicolumn_format='c')
print(rate)
def diff(args):
dataset = 'MNIST' if 'mnist' in args.run else 'CIFAR-10'
exps = Experiment.gather(args.run)
exps = Experiment.filter(args.filter, exps)
# exps = Experiment.filter({'distance': float('inf')}, exps)
exps = list(exps)
def _make_plot(d):
results = Experiment.collect_all(exps, f'diff_{d}.csv')
# XXX TO FIX IN diff.py
corrupted = results.loc[:, '0.0':'1.0'].isna().all(axis=1)
if corrupted.any():
results.loc[corrupted, '0.0':'1.0'] = results.loc[corrupted, '0.0.1':'1.0.1'].values
results = results.dropna(axis=1)
results = results.sort_values('tol')
id_cols = [c for c in results.columns if not c.startswith('0') and not c.startswith('1')]
results = results.melt(id_vars=id_cols, var_name='t', value_name='diff')
results['t'] = results.t.astype(np.float32)
results[r'$\tau$'] = results.tol.astype(np.float32).apply(lambda x: rf'$10^{{{np.log10(x).round()}}}$')# .apply(lambda x: rf'$\mathrm{{{x}}}$')
if d == 'cos':
results['diff'] = 1 - results['diff']
plt.figure(figsize=figsize(1))
ax = sns.lineplot(x='t', y='diff', hue=r'$\tau$', ci=None, data=results) # ci='sd'
plt.ylabel(r'$|\mathbf{h}(t) - \mathbf{h}_\text{adv}(t)|_2$')
plt.xlabel(r'$t$')
plt.xlim(0, 1)
sns.despine()
# plt.legend(fontsize='xx-small', title_fontsize='16')
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
for axis in [ax.xaxis, ax.yaxis]:
axis.set_tick_params(direction='out', color=ax.spines['left'].get_ec())
plt.savefig(f'{args.output}_{dataset}.pdf', bbox_inches="tight")
plt.close()
_make_plot('l2')
def train(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
plt.figure(figsize=(10, 6))
ax = plt.gca()
for exp in exps:
# if run.params.loc[0, 'lr'] == 0.1: continue
try:
exp.log.plot('epoch', 'test_acc', label=exp.path, ax=ax)
print(exp.path)
except Exception as e:
print(exp.path)
print(e)
plt.minorticks_on()
ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
plt.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
plt.legend(bbox_to_anchor=(1, 1), loc="upper left")
plt.savefig(args.output, bbox_inches="tight")
def transfer(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results_file = 'finetune.csv' if args.data is None else 'finetune-{}.csv'.format(args.data)
results = Experiment.collect_all(exps, results_file)
perc_scale = 100 if args.percentage else 1
# filter out aggregations for now
results = results[results.t1 >= 0]
results.cv_accuracy *= perc_scale
results['name'] = None
results.loc[(results.downsample == 'residual') & (results.model == 'resnet'), 'name'] = 'Res-Net'
results.loc[(results.downsample == 'residual') & (results.model == 'odenet'), 'name'] = 'Res-ODE'
results.loc[(results.downsample == 'one-shot') & (results.model == 'odenet'), 'name'] = 'ODE-Only'
results['name'] = pd.Categorical(results['name'], ['Res-Net', 'Res-ODE', 'ODE-Only'])
results = results.sort_values('name')
ax = sns.lineplot(x='t1', y='cv_accuracy', hue='name', style='name', markers=('D', 'o', 'o'), dashes=False, data=results)
h, l = ax.get_legend_handles_labels()
h, l = h[1:], l[1:]
ax.lines[0].set_linestyle('--')
h[0].set_linestyle('--')
ax.lines[0].set_color('k')
h[0].set_color('k')
for hi in h:
hi.set_markeredgecolor('w')
plt.xlabel('t')
plt.ylabel('5-fold Accuracy (%)')
plt.xlim(0, 1)
# ax.set_ylim(bottom=0)
plt.legend(handles=h, labels=l, loc='best', ncol=1) # , prop={'size': 8})
plt.savefig(args.output, bbox_inches="tight")
def best(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
if args.l: # search best in logs data
results = Experiment.collect_all(exps, 'log', index=0)
else:
results = Experiment.collect_all(exps, 'results')
metric_cols = {'epoch', 't1', 'test_acc', 'test_loss', 'test_nfe', 'test_tol', 'acc', 'loss', 'nfe-f', 'nfe-b'}
grouping_cols = results.columns.difference(metric_cols).tolist()
idx_acc_max = results.groupby(grouping_cols)['test_acc'].idxmax()
results = results.loc[idx_acc_max]
common_params = results.apply(pd.Series.nunique) == 1
common_values = results.loc[:, common_params].iloc[0]
results = results.loc[:, ~common_params]
with pd.option_context('display.width', None), pd.option_context('max_columns', None):
print(results.sort_values('test_acc', ascending=False).head(args.n))
print(common_values)
def _faiss_gather(s_dir):
faiss_exps = os.path.join(args.run, s_dir)
faiss_exps = Experiment.gather(faiss_exps)
faiss_exps = Experiment.filter(args.filter, faiss_exps)
faiss_exps = list(faiss_exps)
db_sizes = [50000, 100000, 250000, 500000, 750000, 950000]
effec = Experiment.collect_all(faiss_exps, 'metrics*.csv')
effec = effec.query('limit in @db_sizes')
times = Experiment.collect_all(faiss_exps, 'query_times.csv')
times['query_time'] = times.loc[:, 'query_time_run1':
'query_time_run5'].mean(axis=1)
space = Experiment.collect_all(faiss_exps, 'index_stats.csv')
# space['size'] *= 64 / 1024
space['size'] /= 1024**2
space['build_time'] = space.train_time + space.add_time
data = effec.merge(times)
data = data.merge(space)
data['limit'] = data.limit.apply(lambda x: str(x)[:-3] + 'k')
data = pd.pivot_table(data,
values=[
'ap', 'ndcg', 'ndcg@25', 'query_time',
'size', 'build_time'
],
index=['limit', 'n_probes'])
data = data.reset_index().rename(columns={
'limit': 'samples',
'n_probes': 'trade-off'
})
print(data)
return data
def retrieval(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results_file = 'retrieval.csv' if args.data is None else 'retrieval-{}.csv'.format(args.data)
results = Experiment.collect_all(exps, results_file)
perc_scale = 100 if args.percentage else 1
sym_metric = '{}_sym'.format(args.metric)
asym_metric = '{}_asym'.format(args.metric)
assert sym_metric in results.columns, f'Results not available for this run: {sym_metric}'
assert asym_metric in results.columns, f'Results not available for this run: {asym_metric}'
results[[sym_metric, asym_metric]] *= perc_scale
results = results.sort_values('downsample')
is_baseline = (results.downsample == 'residual') & (results.model == 'resnet')
baseline = results[is_baseline]
# baseline = pd.DataFrame()
results = results[~is_baseline]
assert results.dataset.nunique() == 1, "This plot should be drawn with only runs on a single dataset: {}".format(results.dataset.unique())
ci = None
plt.figure()
ax = plt.gca()
eps = 0
if not baseline.empty:
eps = .05
common = dict(xycoords='data', textcoords='offset points', fontsize=8, va='center', ha='center')
mean_aps = baseline.groupby('t1').mean().sort_values('t1')
for block, aps in enumerate(mean_aps.to_dict('records')):
if aps[sym_metric] < .95 * perc_scale:
ax.plot([0, 1 + eps], [aps[sym_metric]]*2, c='k', lw=.8)
ax.annotate(f'#{block}', xy=(1 + eps, aps[sym_metric]), xytext=(8, 0), **common)
if aps[asym_metric] < .95 * perc_scale:
ax.plot([-eps, 1], [aps[asym_metric]]*2, c='k', lw=.8, ls='dashed')
ax.annotate(f'#{block}', xy=(-eps, aps[asym_metric]), xytext=(-8, 0), **common)
sns.lineplot(x='t1', y=sym_metric, hue='downsample', style='downsample', ci=ci, markers=('o', 'o'), dashes=False,
data=results, ax=ax)
sns.lineplot(x='t1', y=asym_metric, hue='downsample', style='downsample', ci=ci, markers=('o', 'o'),
dashes=((2, 2), (2, 2)), data=results, ax=ax)
label_map = {'residual': 'ODE-Net', 'one-shot': 'Full-ODE-Net'}
h, l = ax.get_legend_handles_labels()
h_and_l = zip(h, l)
h_and_l = sorted(h_and_l, key=lambda x: x[1], reverse=True)
h_and_l = (
(h, '{}, {}'.format(label_map[l], 'asymmetric' if h.is_dashed() else 'symmetric'))
for h, l in h_and_l if l in label_map)
h, l = zip(*h_and_l)
if not baseline.empty:
h += (Line2D([], [], c='k', lw=.8), Line2D([], [], c='k', ls='dashed', lw=.8))
l += ('ResNet, symmetric', 'ResNet, asymmetric')
# plt.title(dataset)
plt.xlabel(r'$\mathrm{\mathit{t}}$ (final hidden state time)')
plt.ylabel(r'mean Average Precision {}(%)'.format('@ 10 ' if args.metric == 'ap10' else ''))
plt.xlim(-2*eps, 1 + 2*eps)
y_lim = (0, perc_scale) if args.data is None else (.24 * perc_scale, .68 * perc_scale)
plt.ylim(*y_lim)
major_ticks = np.arange(0, 1.2, 0.2)
minor_ticks = np.arange(0, 1.05, 0.05)
ax.set_xticks(major_ticks)
ax.set_xticks(minor_ticks, minor=True)
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
plt.legend(handles=h, labels=l, loc='lower center', ncol=3, prop={'size': 8})
plt.savefig(args.output, bbox_inches="tight")
def tradeoff(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'nfe.csv.gz')
results = results.sort_values('downsample')
assert results.dataset.nunique() == 1, "This plot should be drawn with only runs on a single dataset"
results['epsilon'] = 1 - results.t1
results['test error'] = 100 * (results.y_pred != results.y_true)
plt.figure()
ax = plt.gca()
handles = []
labels = []
label_map = {'residual': 'ODE-Net', 'one-shot': 'Full-ODE-Net'}
sns.lineplot(x='epsilon', y='test error', hue='downsample', style='downsample', ci='sd', markers=('o', 'o'),
dashes=False, data=results, ax=ax)
ax.set_ylim([0, 100])
ax.set_xlabel(r'$\mathrm{\mathit{\varepsilon}}$ (time anticipation)')
ax.set_ylabel(r'test error %')
h, l = ax.get_legend_handles_labels()
for hi, li in zip(h[1:], l[1:]):
hh = Line2D([], [])
hh.update_from(hi)
hh.set_marker(None)
handles.append(hh)
labels.append(label_map[li])
ax.get_legend().remove()
ax2 = plt.twinx()
sns.lineplot(x='epsilon', y='nfe', hue='downsample', style='downsample', ci='sd', markers=('X', 'X'), dashes=False,
data=results, ax=ax2, legend=False)
# ax2.set_ylabel(r'number of function evaluations (NFE)')
ax2.set_ylabel(r'NFE')
ax2.set_ylim([0, ax2.get_ylim()[1] * .9])
handles.extend([
Line2D([], [], marker='o', markerfacecolor='k', markeredgecolor='w', color='k'),
Line2D([], [], marker='X', markerfacecolor='k', markeredgecolor='w', color='k'),
])
labels.extend(['error', 'nfe'])
handler_map = {h: HandlerLine2D(marker_pad=0) for h in handles[-2:]}
plt.legend(handles=handles, labels=labels, loc='upper center', ncol=2, handler_map=handler_map)
plt.minorticks_on()
ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
# ax2.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax2.set_yticks(np.linspace(ax2.get_yticks()[0], ax2.get_yticks()[-1], len(ax.get_yticks())))
ax.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
ax2.grid(False)
plt.xlim(0, 1)
plt.savefig(args.output, bbox_inches="tight")
def effectiveness_vs_timespace(args):
db_sizes = [50000, 100000, 250000, 500000, 750000, 950000]
def _faiss_gather(s_dir):
faiss_exps = os.path.join(args.run, s_dir)
faiss_exps = Experiment.gather(faiss_exps)
faiss_exps = Experiment.filter(args.filter, faiss_exps)
faiss_exps = list(faiss_exps)
db_sizes = [50000, 100000, 250000, 500000, 750000, 950000]
effec = Experiment.collect_all(faiss_exps, 'metrics*.csv')
effec = effec.query('limit in @db_sizes')
times = Experiment.collect_all(faiss_exps, 'query_times.csv')
times['query_time'] = times.loc[:, 'query_time_run1':
'query_time_run5'].mean(axis=1)
space = Experiment.collect_all(faiss_exps, 'index_stats.csv')
# space['size'] *= 64 / 1024
space['size'] /= 1024**2
space['build_time'] = space.train_time + space.add_time
data = effec.merge(times)
data = data.merge(space)
data['limit'] = data.limit.apply(lambda x: str(x)[:-3] + 'k')
data = pd.pivot_table(data,
values=[
'ap', 'ndcg', 'ndcg@25', 'query_time',
'size', 'build_time'
],
index=['limit', 'n_probes'])
data = data.reset_index().rename(columns={
'limit': 'samples',
'n_probes': 'trade-off'
})
print(data)
return data
# FAISS
fhdata = _faiss_gather('ivfpq_H+M100k')
fhdata['method'] = 'IVFPQ'
# FAISS (T4SA)
ftdata = _faiss_gather('ivfpq_bt4sa')
ftdata['method'] = 'IVFPQ*'
# THR SQ LUCENE
thr_exps = os.path.join(args.run, 'lucene-thr-sq')
thr_exps = Experiment.gather(thr_exps)
thr_exps = Experiment.filter(args.filter, thr_exps)
thr_exps = list(thr_exps)
effec = Experiment.collect_all(thr_exps, 'metrics.csv')
effec = effec.query('limit in @db_sizes')
space = Experiment.collect_all(thr_exps, 'index_stats.csv')
space['size'] /= 10**6
data = effec.merge(space)
data['limit'] = data.limit.apply(lambda x: str(x)[:-3] + 'k')
data['build_time'] = data.add_time
data = pd.pivot_table(
data,
values=['ap', 'ndcg', 'ndcg@25', 'query_time', 'size', 'build_time'],
index=['limit', 'threshold'])
sqdata = data.reset_index().rename(columns={
'limit': 'samples',
'threshold': 'trade-off'
})
print(sqdata)
sqdata['method'] = 'Thr-SQ'
style_order = [str(x)[:-3] + 'k' for x in db_sizes]
data = pd.concat((sqdata, ftdata, fhdata))
for metric in args.metrics:
# EvT PLOT
plt.figure()
ax = sns.lineplot(x='query_time',
y=metric,
hue='method',
style='samples',
markers=True,
style_order=style_order,
data=data)
'''
ax = plt.gca()
common = dict(x='query_time', y=metric, style='samples', markers=True, style_order=style_order, ax=ax)
sns.lineplot(data=sqdata, **common) # size='threshold'
sns.lineplot(data=ftdata, **common) # size='n_probes'
sns.lineplot(data=fhdata, **common) # size='n_probes'
'''
ax.set(xscale='log')
# plt.legend(title='DB size $N$')
plt.xlabel('Query Time (s)')
plt.ylabel('mAP' if metric == 'ap' else metric.upper())
out = f'evt_{metric}.pdf'
plt.savefig(out)
os.system(f'croppdf {out}')
plt.close()
# EvS PLOT
plt.figure()
ax = sns.lineplot(x='size',
y=metric,
hue='method',
style='samples',
markers=True,
style_order=style_order,
estimator=None,
data=data)
'''
ax = plt.gca()
common = dict(x='size', y=metric, style='samples', markers=True, style_order=style_order, ax=ax, estimator=None)
sns.lineplot(data=sqdata, **common) # size='threshold'
sns.lineplot(data=ftdata, **common) # size='n_probes'
sns.lineplot(data=fhdata, **common) # size='n_probes'
'''
ax.set(xscale='log')
plt.xlabel('Index Size (MB)')
plt.ylabel('mAP' if metric == 'ap' else metric.upper())
out = f'evs_{metric}.pdf'
plt.savefig(out)
os.system(f'croppdf {out}')
plt.close()
# EvB PLOT
plt.figure()
ax = sns.lineplot(x='build_time',
y=metric,
hue='method',
style='samples',
markers=True,
style_order=style_order,
estimator=None,
data=data)
'''
ax = plt.gca()
common = dict(x='size', y=metric, style='samples', markers=True, style_order=style_order, ax=ax, estimator=None)
sns.lineplot(data=sqdata, **common) # size='threshold'
sns.lineplot(data=ftdata, **common) # size='n_probes'
sns.lineplot(data=fhdata, **common) # size='n_probes'
'''
# ax.set(xscale='log')
plt.xlabel('Indexing Time (s)')
plt.ylabel('mAP' if metric == 'ap' else metric.upper())
out = f'evb_{metric}.pdf'
plt.savefig(out)
os.system(f'croppdf {out}')
plt.close()