def _get_results(model):
exp_dir = os.path.join(args.root, f'{args.dataset}_{model}', 'adv-attack')
exps = Experiment.gather(exp_dir)
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'results.csv')
results['model'] = model
return results
def nparams(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
run = Experiment.from_dir(args.run, main='model')
model = load_model(run)
print(run)
nparams = sum(p.numel() for p in tqdm(model.parameters()) if p.requires_grad)
print(f'N. Params: {nparams / 10 ** 6:.2g}M ({nparams})')
def t1(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'tradeoff.csv')
unique_cols = results.apply(pd.Series.nunique) == 1
common_params = results.loc[:, unique_cols].iloc[0]
r = results.loc[:, ~unique_cols]
metric_cols = {'t1', 'test_acc', 'test_loss', 'test_nfe'}
plt.figure(figsize=(10, 6))
ax1 = plt.subplot2grid((2, 2), (0, 0))
ax2 = plt.subplot2grid((2, 2), (1, 0))
ax3 = plt.subplot2grid((2, 2), (1, 1))
title = Experiment.abbreviate(common_params, main='model')
ax1.set_title(title)
ax1.set_ylabel('Test Accuracy')
ax2.set_ylabel('Test NFE')
ax2.set_xlabel('ODE final integration time $t_1$')
ax3.set_ylabel('Test Accuracy')
ax3.set_xlabel('Test NFE')
if set(r.columns) == metric_cols: # single line plot
r = r.sort_values('t1')
ax1.plot(r.t1, r.test_acc, marker='.')
ax2.plot(r.t1, r.test_nfe, marker='.')
else:
# print(r, metric_cols)
grouping_cols = r.columns.difference(metric_cols).tolist()
# print(grouping_cols)
for name, group in r.groupby(grouping_cols):
# print(grouping_cols)
# print(group.reset_index())
params = group.reset_index().loc[0, grouping_cols]
name = Experiment.abbreviate(params, main='model')
r = group.sort_values('t1')
ax1.plot(r.t1, r.test_acc, label=name, marker='.')
ax2.plot(r.t1, r.test_nfe, label=name, marker='.')
ax3.plot(r.test_nfe, r.test_acc, label=name, marker='.')
ax1.legend(bbox_to_anchor=(1, 1), loc="upper left")
plt.minorticks_on()
for ax in (ax1, ax2):
ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
plt.savefig(args.output, bbox_inches="tight")
def _make_plot(d):
results = Experiment.collect_all(exps, f'diff_{d}.csv')
# XXX TO FIX IN diff.py
corrupted = results.loc[:, '0.0':'1.0'].isna().all(axis=1)
if corrupted.any():
results.loc[corrupted, '0.0':'1.0'] = results.loc[corrupted, '0.0.1':'1.0.1'].values
results = results.dropna(axis=1)
results = results.sort_values('tol')
id_cols = [c for c in results.columns if not c.startswith('0') and not c.startswith('1')]
results = results.melt(id_vars=id_cols, var_name='t', value_name='diff')
results['t'] = results.t.astype(np.float32)
results[r'$\tau$'] = results.tol.astype(np.float32).apply(lambda x: rf'$10^{{{np.log10(x).round()}}}$')# .apply(lambda x: rf'$\mathrm{{{x}}}$')
if d == 'cos':
results['diff'] = 1 - results['diff']
plt.figure(figsize=figsize(1))
ax = sns.lineplot(x='t', y='diff', hue=r'$\tau$', ci=None, data=results) # ci='sd'
plt.ylabel(r'$|\mathbf{h}(t) - \mathbf{h}_\text{adv}(t)|_2$')
plt.xlabel(r'$t$')
plt.xlim(0, 1)
sns.despine()
# plt.legend(fontsize='xx-small', title_fontsize='16')
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
for axis in [ax.xaxis, ax.yaxis]:
axis.set_tick_params(direction='out', color=ax.spines['left'].get_ec())
plt.savefig(f'{args.output}_{dataset}.pdf', bbox_inches="tight")
plt.close()
def nfe(args):
run = Experiment.from_dir(args.run, main='model')
print(run)
results_file = run.path_to('nfe.csv')
best_ckpt_file = run.ckpt('best')
# check if results exists and are updated, then skip the computation
if os.path.exists(results_file
) and os.path.getctime(results_file) >= os.path.getctime(
best_ckpt_file) and not args.force:
print('Skipping...')
return
test_data = load_test_data(run)
test_loader = DataLoader(test_data, batch_size=1, shuffle=False)
model = load_model(run)
model = model.to(args.device)
model.eval()
model.odeblock.tol = args.tol
def process(datum):
x, y = datum
x = x.to(args.device)
p = model(x)
nfe = model.nfe(reset=True)
pred = p.argmax(dim=1).item()
y = y.item()
return {'y_true': y, 'y_pred': pred, 'nfe': nfe}
data = [process(d) for d in tqdm(test_loader)]
pd.DataFrame(data).to_csv(results_file)
def main():
parser = argparse.ArgumentParser(
description='Plots an experiment in terminal')
parser.add_argument('glob', help='directory of experiment to plot')
parser.add_argument('--window',
help='smoothing window',
type=int,
default=100)
parser.add_argument('--width', help='smoothing window', type=int)
parser.add_argument('--height', help='smoothing window', type=int)
parser.add_argument('-x', help='x axis', default='frames')
parser.add_argument('-y', help='y axis', default='mean_win_rate')
args = parser.parse_args()
exps_and_logs = Experiment.discover_logs(args.glob, JSONLogger())
print('loaded {} experiments'.format(len(exps_and_logs)))
fig = plotille.Figure()
kwargs = {}
for exp, log in exps_and_logs:
if log:
df = pd.DataFrame(log)
fig.plot(X=df[args.x],
Y=df[args.y].rolling(args.window, min_periods=1).mean(),
label=exp.name,
**kwargs)
fig.x_label = args.x
fig.y_label = args.y
if args.height:
fig.height = args.height
if args.width:
fig.width = args.width
print(fig.show(legend=True))
def success_rate_single(args):
exps = Experiment.gather(args.run)
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'results.csv')
results = results.rename({'distance_x': 'p', 'distance_y': 'distance'}, axis=1)
results['success_rate'] = np.isfinite(results.distance.values)
natural_errors = results.distance == 0
results = results[~natural_errors] # discard natural errors
rate = pd.pivot_table(results, values='success_rate', index='tol', columns=['epsilon', 'p'])
rate = (rate * 100).round(2)
rate = rate.rename_axis(columns={'p': r'$p$', 'epsilon': r'$\varepsilon$'})
rate = rate.rename(columns={2.0: r'$L_2$', float('inf'): r'$L_\infty$'}, level=1)
print(rate)
def classification_performance(args):
models = ('resnet', 'mixed', 'fullode')
exps = Experiment.gather(args.root, main='model')
exps = Experiment.filter({'dataset': args.dataset}, exps)
results = Experiment.collect_all(exps, 'nfe.csv.gz')
results = results[results.t1 == 1] # keep only complete dynamics
results['accuracy'] = results.y_pred == results.y_true
results = results.rename({'tol_x': 'training_tol', 'tol_y': 'tol'}, axis=1)
rate = pd.pivot_table(results, values='accuracy', index='tol', columns=['model', 'downsample'])
rate = (100 * (1 - rate)).round(2)
rate = rate.rename(mapper=lambda x: r'$10^{{{}}}$'.format(int(round(np.log10(x)))), level=0)
with open(args.output, 'w') as out:
rate.to_latex(out, escape=False, multicolumn_format='c')
print(rate)
def retrieval(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
run = Experiment.from_dir(args.run, main='model')
results_file = run.path_to('retrieval.csv')
assert os.path.exists(results_file), f"Results file not found: {results_file}"
results = pd.read_csv(results_file)
# t1s, mean_aps_sym, mean_aps_asym = results.loc[:, ['t1', 'mean_ap_sym', 'mean_ap_asym']]
plt.figure(figsize=(15,5))
ax = plt.gca()
results.plot('t1', 'mean_ap_sym', marker='.', label='sym', ax=ax)
results.plot('t1', 'mean_ap_asym', marker='.', label='asym', ax=ax)
# plt.axhline(mean_ap_res, c='k', label='resnet')
max_diff = (results.mean_ap_asym - results.mean_ap_sym).max()
print(f'Asym-sym max difference: {max_diff:%}')
# plt.plot(t1s, mean_aps_asym - mean_aps_sym, marker='.', label='diff')
plt.title('mAP vs Feature Depth (t) - CIFAR-10')
plt.xlabel('Integration time')
plt.ylabel('mAP')
# plt.ylim([0, 1])
plt.legend(loc='best')
''' NFE sectioning
ns = np.diff(nfe)
xv = np.diff(t1s)/2
xv[1:] += t1s[1:-1]
xv = xv[ns != 0]
for x in xv:
plt.axvline(x, c='k', ls='--')
xv = np.array([0, ] + xv.tolist() + [1,])
xl = np.diff(xv) / 2
xl[1:] += xv[1:-1]
for x, n in zip(xl, np.unique(nfe)):
plt.annotate('NFE = {:.1f}'.format(n), (x, .2), rotation=90, ha='center', va='center')
'''
plt.savefig(args.output, bbox_inches="tight")
def finetune(args):
run = Experiment.from_dir(args.run, main='model')
print(run)
features_file = 'features.h5' if args.data is None else 'features-{}.h5'.format(
args.data)
features_file = run.path_to(features_file)
assert os.path.exists(
features_file), f"Features file not found: {features_file}"
results = pd.DataFrame()
results_file = run.path_to('finetune.csv')
if os.path.exists(results_file):
if os.path.getctime(results_file) >= os.path.getctime(
features_file) and not args.force:
results = pd.read_csv(results_file)
params = next(run.params.itertuples())
with h5py.File(features_file, 'r') as f:
features = f['features'][...]
y_true = f['y_true'][...]
t1s = f['t1s'][...]
block = np.zeros_like(t1s, dtype=int)
if params.downsample == "ode":
block = np.concatenate((block, block + 1))
t1s = np.concatenate((t1s, t1s))
svm = LinearSVC()
Cs = np.logspace(-2, 2, 5)
svm = GridSearchCV(svm, {'C': Cs},
scoring='accuracy',
n_jobs=-1,
verbose=10,
cv=5)
for t1, b, fi in tqdm(zip(t1s, block, features)):
if 't1' in results.columns and 'block' in results.columns and (
(results.t1 == t1) & (results.block == b)).any():
print(f'Skipping b={b} t1={t1} ...')
continue
score = svm.fit(fi, y_true).best_score_
print(f'Accuracy: {score:.2%}')
results = results.append({
'block': b,
't1': t1,
'cv_accuracy': score
},
ignore_index=True)
results.to_csv(results_file, index=False)
def train(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
plt.figure(figsize=(10, 6))
ax = plt.gca()
for exp in exps:
# if run.params.loc[0, 'lr'] == 0.1: continue
try:
exp.log.plot('epoch', 'test_acc', label=exp.path, ax=ax)
print(exp.path)
except Exception as e:
print(exp.path)
print(e)
plt.minorticks_on()
ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
plt.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
plt.legend(bbox_to_anchor=(1, 1), loc="upper left")
plt.savefig(args.output, bbox_inches="tight")
def transfer(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results_file = 'finetune.csv' if args.data is None else 'finetune-{}.csv'.format(args.data)
results = Experiment.collect_all(exps, results_file)
perc_scale = 100 if args.percentage else 1
# filter out aggregations for now
results = results[results.t1 >= 0]
results.cv_accuracy *= perc_scale
results['name'] = None
results.loc[(results.downsample == 'residual') & (results.model == 'resnet'), 'name'] = 'Res-Net'
results.loc[(results.downsample == 'residual') & (results.model == 'odenet'), 'name'] = 'Res-ODE'
results.loc[(results.downsample == 'one-shot') & (results.model == 'odenet'), 'name'] = 'ODE-Only'
results['name'] = pd.Categorical(results['name'], ['Res-Net', 'Res-ODE', 'ODE-Only'])
results = results.sort_values('name')
ax = sns.lineplot(x='t1', y='cv_accuracy', hue='name', style='name', markers=('D', 'o', 'o'), dashes=False, data=results)
h, l = ax.get_legend_handles_labels()
h, l = h[1:], l[1:]
ax.lines[0].set_linestyle('--')
h[0].set_linestyle('--')
ax.lines[0].set_color('k')
h[0].set_color('k')
for hi in h:
hi.set_markeredgecolor('w')
plt.xlabel('t')
plt.ylabel('5-fold Accuracy (%)')
plt.xlim(0, 1)
# ax.set_ylim(bottom=0)
plt.legend(handles=h, labels=l, loc='best', ncol=1) # , prop={'size': 8})
plt.savefig(args.output, bbox_inches="tight")
def nfe(args):
run = Experiment.from_dir(args.run, main='model')
print(run)
results_file = run.path_to('nfe.csv.gz')
best_ckpt_file = run.ckpt('best')
results = pd.DataFrame()
# check if results exists and are updated, then skip the computation
if os.path.exists(results_file
) and os.path.getctime(results_file) >= os.path.getctime(
best_ckpt_file) and not args.force:
results = pd.read_csv(results_file,
float_precision='round_trip').round({'t1': 2})
test_data = load_test_data(run)
test_loader = DataLoader(test_data, batch_size=1, shuffle=False)
model = load_model(run)
model = model.to(args.device)
model.eval()
def _nfe(test_loader, model, t1, tol, args):
model.odeblock.t1 = t1
model.odeblock.tol = tol
y_true = []
y_pred = []
nfes = []
for x, y in tqdm(test_loader):
y_true.append(y.item())
y_pred.append(model(x.to(args.device)).argmax(dim=1).item())
nfes.append(model.nfe(reset=True))
return {'y_true': y_true, 'y_pred': y_pred, 'nfe': nfes}
progress = tqdm(itertools.product(args.tol, args.t1))
for tol, t1 in progress:
if 't1' in results.columns and 'tol' in results.columns and (
(results.t1 == t1) & (results.tol == tol)).any():
print(f'Skipping tol={tol} t1={t1} ...')
continue
progress.set_postfix({'tol': tol, 't1': t1})
result = _nfe(test_loader, model, t1, tol, args)
result = pd.DataFrame(result)
result['t1'] = t1
result['tol'] = tol
results = results.append(result, ignore_index=True)
results.to_csv(results_file, index=False)
def best(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
if args.l: # search best in logs data
results = Experiment.collect_all(exps, 'log', index=0)
else:
results = Experiment.collect_all(exps, 'results')
metric_cols = {'epoch', 't1', 'test_acc', 'test_loss', 'test_nfe', 'test_tol', 'acc', 'loss', 'nfe-f', 'nfe-b'}
grouping_cols = results.columns.difference(metric_cols).tolist()
idx_acc_max = results.groupby(grouping_cols)['test_acc'].idxmax()
results = results.loc[idx_acc_max]
common_params = results.apply(pd.Series.nunique) == 1
common_values = results.loc[:, common_params].iloc[0]
results = results.loc[:, ~common_params]
with pd.option_context('display.width', None), pd.option_context('max_columns', None):
print(results.sort_values('test_acc', ascending=False).head(args.n))
print(common_values)
def _faiss_gather(s_dir):
faiss_exps = os.path.join(args.run, s_dir)
faiss_exps = Experiment.gather(faiss_exps)
faiss_exps = Experiment.filter(args.filter, faiss_exps)
faiss_exps = list(faiss_exps)
db_sizes = [50000, 100000, 250000, 500000, 750000, 950000]
effec = Experiment.collect_all(faiss_exps, 'metrics*.csv')
effec = effec.query('limit in @db_sizes')
times = Experiment.collect_all(faiss_exps, 'query_times.csv')
times['query_time'] = times.loc[:, 'query_time_run1':
'query_time_run5'].mean(axis=1)
space = Experiment.collect_all(faiss_exps, 'index_stats.csv')
# space['size'] *= 64 / 1024
space['size'] /= 1024**2
space['build_time'] = space.train_time + space.add_time
data = effec.merge(times)
data = data.merge(space)
data['limit'] = data.limit.apply(lambda x: str(x)[:-3] + 'k')
data = pd.pivot_table(data,
values=[
'ap', 'ndcg', 'ndcg@25', 'query_time',
'size', 'build_time'
],
index=['limit', 'n_probes'])
data = data.reset_index().rename(columns={
'limit': 'samples',
'n_probes': 'trade-off'
})
print(data)
return data
def clean(args):
runs = Experiment.gather(args.run, main='model')
empty_runs = filter(lambda run: run.log.empty, runs)
dirs = [run.path for run in empty_runs]
n_empty_runs = len(dirs)
print("Empty runs found: {}".format(n_empty_runs))
if n_empty_runs:
print('\n'.join(dirs))
print("Delete them? [y/N] ", end='')
if input().lower() in ('y', 'yes'):
for r in dirs:
command = 'rm -rf {}'.format(r)
print(command)
os.system(command)
def accuracy(args):
run = Experiment.from_dir(args.run, main='model')
print(run)
results_file = run.path_to('results')
best_ckpt_file = run.ckpt('best')
all_results = pd.DataFrame()
# check if results exists and are updated, then load them (and probably skip the computation them later)
if os.path.exists(results_file
) and os.path.getctime(results_file) >= os.path.getctime(
best_ckpt_file) and not args.force:
all_results = pd.read_csv(results_file)
params = next(run.params.itertuples())
test_data = load_test_data(run)
test_loader = DataLoader(test_data,
batch_size=params.batch_size,
shuffle=False)
model = load_model(run)
model = model.to(args.device)
model.eval()
t1 = torch.arange(0, 1.05, .05) # from 0 to 1 w/ .05 step
model.odeblock.t1 = t1[1:] # 0 is implicit
model.odeblock.return_last_only = False
if params.downsample == 'ode2':
model.downsample.odeblock.t1 = t1[1:] # 0 is implicit
model.downsample.odeblock.return_last_only = False
model.downsample.odeblock.apply_conv = True
t1 = torch.cat((t1, t1))
T = len(t1)
def _evaluate(loader, model, tol, args):
model.odeblock.tol = tol
if 'ode' in params.downsample:
model.downsample.odeblock.tol = tol
n_batches = 0
n_processed = 0
nfe_forward = 0
n_correct = torch.zeros(T)
tot_losses = torch.zeros(T)
progress = tqdm(loader)
for x, y in progress:
x, y = x.to(args.device), y.to(args.device)
p = model(x) # timestamps (T) x batch (N) x classes (C)
nfe_forward += model.nfe(reset=True)
pp = p.permute(1, 2, 0) # N x C x T
yy = y.unsqueeze(1).expand(-1, T) # N x T
losses = F.cross_entropy(pp, yy, reduction='none') # N x T
tot_losses += losses.sum(0).cpu()
yy = y.unsqueeze(0).expand(T, -1)
n_correct += (yy == p.argmax(dim=-1)).sum(-1).float().cpu()
n_processed += y.shape[0]
n_batches += 1
# logloss = losses.item() / n_processed
# accuracy = n_correct / n_processed
nfe = nfe_forward / n_batches
metrics = {
# 'loss': f'{logloss:4.3f}',
# 'acc': f'{n_correct:4d}/{n_processed:4d} ({accuracy:.2%})',
'nfe': f'{nfe:3.1f}'
}
progress.set_postfix(metrics)
loglosses = tot_losses / n_processed
accuracies = n_correct / n_processed
metrics = {
't1': t1.numpy(),
'test_loss': loglosses.numpy(),
'test_acc': accuracies.numpy(),
'test_nfe': [
nfe,
] * T,
'test_tol': [
tol,
] * T
}
return metrics
progress = tqdm(args.tol)
with torch.no_grad():
for tol in progress:
progress.set_postfix({'tol': tol})
if 'test_tol' in all_results.columns and (all_results.test_tol
== tol).any():
progress.write(f'Skipping: tol={tol:g}')
continue
results = _evaluate(test_loader, model, tol, args)
results = pd.DataFrame(results)
all_results = all_results.append(results, ignore_index=True)
all_results.to_csv(results_file, index=False)
def retrieval(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results_file = 'retrieval.csv' if args.data is None else 'retrieval-{}.csv'.format(args.data)
results = Experiment.collect_all(exps, results_file)
perc_scale = 100 if args.percentage else 1
sym_metric = '{}_sym'.format(args.metric)
asym_metric = '{}_asym'.format(args.metric)
assert sym_metric in results.columns, f'Results not available for this run: {sym_metric}'
assert asym_metric in results.columns, f'Results not available for this run: {asym_metric}'
results[[sym_metric, asym_metric]] *= perc_scale
results = results.sort_values('downsample')
is_baseline = (results.downsample == 'residual') & (results.model == 'resnet')
baseline = results[is_baseline]
# baseline = pd.DataFrame()
results = results[~is_baseline]
assert results.dataset.nunique() == 1, "This plot should be drawn with only runs on a single dataset: {}".format(results.dataset.unique())
ci = None
plt.figure()
ax = plt.gca()
eps = 0
if not baseline.empty:
eps = .05
common = dict(xycoords='data', textcoords='offset points', fontsize=8, va='center', ha='center')
mean_aps = baseline.groupby('t1').mean().sort_values('t1')
for block, aps in enumerate(mean_aps.to_dict('records')):
if aps[sym_metric] < .95 * perc_scale:
ax.plot([0, 1 + eps], [aps[sym_metric]]*2, c='k', lw=.8)
ax.annotate(f'#{block}', xy=(1 + eps, aps[sym_metric]), xytext=(8, 0), **common)
if aps[asym_metric] < .95 * perc_scale:
ax.plot([-eps, 1], [aps[asym_metric]]*2, c='k', lw=.8, ls='dashed')
ax.annotate(f'#{block}', xy=(-eps, aps[asym_metric]), xytext=(-8, 0), **common)
sns.lineplot(x='t1', y=sym_metric, hue='downsample', style='downsample', ci=ci, markers=('o', 'o'), dashes=False,
data=results, ax=ax)
sns.lineplot(x='t1', y=asym_metric, hue='downsample', style='downsample', ci=ci, markers=('o', 'o'),
dashes=((2, 2), (2, 2)), data=results, ax=ax)
label_map = {'residual': 'ODE-Net', 'one-shot': 'Full-ODE-Net'}
h, l = ax.get_legend_handles_labels()
h_and_l = zip(h, l)
h_and_l = sorted(h_and_l, key=lambda x: x[1], reverse=True)
h_and_l = (
(h, '{}, {}'.format(label_map[l], 'asymmetric' if h.is_dashed() else 'symmetric'))
for h, l in h_and_l if l in label_map)
h, l = zip(*h_and_l)
if not baseline.empty:
h += (Line2D([], [], c='k', lw=.8), Line2D([], [], c='k', ls='dashed', lw=.8))
l += ('ResNet, symmetric', 'ResNet, asymmetric')
# plt.title(dataset)
plt.xlabel(r'$\mathrm{\mathit{t}}$ (final hidden state time)')
plt.ylabel(r'mean Average Precision {}(%)'.format('@ 10 ' if args.metric == 'ap10' else ''))
plt.xlim(-2*eps, 1 + 2*eps)
y_lim = (0, perc_scale) if args.data is None else (.24 * perc_scale, .68 * perc_scale)
plt.ylim(*y_lim)
major_ticks = np.arange(0, 1.2, 0.2)
minor_ticks = np.arange(0, 1.05, 0.05)
ax.set_xticks(major_ticks)
ax.set_xticks(minor_ticks, minor=True)
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
plt.legend(handles=h, labels=l, loc='lower center', ncol=3, prop={'size': 8})
plt.savefig(args.output, bbox_inches="tight")
def main(args):
exp = Experiment.from_dir(args.run, main='model')
params = next(exp.params.itertuples())
# data setup
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x.numpy())
])
preproc = utils.PREPROC[params.dataset]
if params.dataset == 'mnist':
data = MNIST('data/mnist', download=True, train=False, transform=transform)
elif params.dataset == 'cifar10':
data = CIFAR10('data/cifar10', download=True, train=False, transform=transform)
preproc = map(lambda x: np.array(x).reshape((3, 1, 1)), preproc) # expand dimensions
preproc = tuple(preproc)
t = np.linspace(0, 1, args.resolution + 1).tolist()
# model setup
model = utils.load_model(exp).eval().cuda()
extractor = utils.load_model(exp).eval().cuda()
extractor.to_features_extractor(keep_pool=False)
extractor.odeblock.t1 = t
if args.tol is None:
args.tol = params.tol
if params.model == 'odenet':
model.odeblock.tol = args.tol
extractor.odeblock.tol = args.tol
fmodel = foolbox.models.PyTorchModel(model, bounds=(0, 1), num_classes=10, preprocessing=preproc)
# attack setup
if args.distance == 2:
attack = foolbox.attacks.L2BasicIterativeAttack
distance = foolbox.distances.MSE
elif args.distance == float('inf'):
attack = foolbox.attacks.LinfinityBasicIterativeAttack
distance = foolbox.distances.Linf
attack = attack(fmodel, distance=distance)
sub_exp_root = exp.path_to('adv-attack')
os.makedirs(sub_exp_root, exist_ok=True)
sub_exp = Experiment(args, root=sub_exp_root, ignore=('run', 'resolution'))
print(sub_exp)
results_file = sub_exp.path_to('results.csv')
diff_l2_file = sub_exp.path_to('diff_l2.csv')
diff_cos_file = sub_exp.path_to('diff_cos.csv')
if not os.path.exists(results_file):
print('No results on attacks found:', results_file)
return
results = pd.read_csv(results_file).set_index('sample_id')
diff_l2 = pd.read_csv(diff_l2_file) if os.path.exists(diff_l2_file) else pd.DataFrame()
diff_cos = pd.read_csv(diff_cos_file) if os.path.exists(diff_cos_file) else pd.DataFrame()
diff_cols = ['sample_id'] + t
progress = tqdm(data)
for i, (image, label) in enumerate(progress):
if (not diff_l2.empty and not diff_cos.empty and
i in diff_l2.sample_id.values and i in diff_cos.sample_id.values):
continue # skipping, already computed
perturbation_distance = results.at[i, 'distance']
if perturbation_distance == 0 or not np.isfinite(perturbation_distance):
continue # skipping natural errors or not-found adversarials
if not isinstance(label, int):
label = label.item()
start = time.time()
adversarial = attack(image, label, unpack=False, binary_search=False, epsilon=args.epsilon)
elapsed = time.time() - start
if adversarial.perturbed is None:
tqdm.write(f'WARN: adversarial not found when reproducing [sample_id = {i}]')
continue
with torch.no_grad():
original_image = torch.from_numpy(adversarial.unperturbed).cuda()
original_traj = extractor(original_image.unsqueeze(0))
adversarial_image = torch.from_numpy(adversarial.perturbed).cuda()
adversarial_traj = extractor(adversarial_image.unsqueeze(0))
adversarial_traj = adversarial_traj.reshape(args.resolution + 1, -1)
original_traj = original_traj.reshape(args.resolution + 1, -1)
""" L2 """
diff_traj = adversarial_traj - original_traj
diff_traj = (diff_traj ** 2).sum(1).sqrt()
diff_traj = diff_traj.cpu().numpy()
tmp = pd.DataFrame([[i] + diff_traj.tolist()], columns=diff_cols)
diff_l2 = diff_l2.append(tmp, ignore_index=True)
diff_l2.to_csv(diff_l2_file, index=False)
""" Cosine similarity """
diff_traj = F.cosine_similarity(adversarial_traj, original_traj)
diff_traj = diff_traj.cpu().numpy()
tmp = pd.DataFrame([[i] + diff_traj.tolist()], columns=diff_cols)
diff_cos = diff_cos.append(tmp, ignore_index=True)
diff_cos.to_csv(diff_cos_file, index=False)
def tradeoff(args):
exps = Experiment.gather(args.run, main='model')
exps = Experiment.filter(args.filter, exps)
results = Experiment.collect_all(exps, 'nfe.csv.gz')
results = results.sort_values('downsample')
assert results.dataset.nunique() == 1, "This plot should be drawn with only runs on a single dataset"
results['epsilon'] = 1 - results.t1
results['test error'] = 100 * (results.y_pred != results.y_true)
plt.figure()
ax = plt.gca()
handles = []
labels = []
label_map = {'residual': 'ODE-Net', 'one-shot': 'Full-ODE-Net'}
sns.lineplot(x='epsilon', y='test error', hue='downsample', style='downsample', ci='sd', markers=('o', 'o'),
dashes=False, data=results, ax=ax)
ax.set_ylim([0, 100])
ax.set_xlabel(r'$\mathrm{\mathit{\varepsilon}}$ (time anticipation)')
ax.set_ylabel(r'test error %')
h, l = ax.get_legend_handles_labels()
for hi, li in zip(h[1:], l[1:]):
hh = Line2D([], [])
hh.update_from(hi)
hh.set_marker(None)
handles.append(hh)
labels.append(label_map[li])
ax.get_legend().remove()
ax2 = plt.twinx()
sns.lineplot(x='epsilon', y='nfe', hue='downsample', style='downsample', ci='sd', markers=('X', 'X'), dashes=False,
data=results, ax=ax2, legend=False)
# ax2.set_ylabel(r'number of function evaluations (NFE)')
ax2.set_ylabel(r'NFE')
ax2.set_ylim([0, ax2.get_ylim()[1] * .9])
handles.extend([
Line2D([], [], marker='o', markerfacecolor='k', markeredgecolor='w', color='k'),
Line2D([], [], marker='X', markerfacecolor='k', markeredgecolor='w', color='k'),
])
labels.extend(['error', 'nfe'])
handler_map = {h: HandlerLine2D(marker_pad=0) for h in handles[-2:]}
plt.legend(handles=handles, labels=labels, loc='upper center', ncol=2, handler_map=handler_map)
plt.minorticks_on()
ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
# ax2.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
ax2.set_yticks(np.linspace(ax2.get_yticks()[0], ax2.get_yticks()[-1], len(ax.get_yticks())))
ax.grid(b=True, which='minor', linewidth=0.5, linestyle='--')
ax2.grid(False)
plt.xlim(0, 1)
plt.savefig(args.output, bbox_inches="tight")
def nfe(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
exp = Experiment.from_dir(args.run, main='model')
nfe = pd.read_csv(exp.path_to('nfe.csv.gz')) #, index_col=0)
nfe = nfe[(nfe.t1 == 1) & (nfe.tol == 0.001)].reset_index(drop=True)
nfe.nfe = (nfe.nfe - 2) / 6 # show n. steps instead of NFE
nfe = nfe[nfe.y_true == nfe.y_pred]
dataset = exp.params.dataset.iloc[0]
if dataset == 'mnist':
labels = [str(i) for i in range(10)]
elif dataset == 'cifar10':
labels = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
nfe.y_true = nfe.y_true.apply(lambda x: labels[x])
nfe = nfe.sort_values('y_true')
# g = sns.FacetGrid(nfe, col='y_true')
# g.map(sns.kdeplot, 'nfe')
# ax = sns.boxplot(y='y_true', x='nfe', data=nfe, orient='h')
# ax.set_xlabel('solver steps')
# ax.set_ylabel('image class')
print('{:.2g} \pm {:.2g}'.format(nfe.nfe.mean(), nfe.nfe.std()))
min, max = nfe.nfe.min(), nfe.nfe.max()
values = np.arange(min, max + 1)
plt.xticks(values)
bins = values - .5
plt.xlim(bins[0], bins[-1])
counts, _, _ = plt.hist(nfe.nfe, bins=bins)
plt.grid(b=False, axis='x')
plt.grid(b=True, which='minor', linewidth=0.5, linestyle='--', axis='y')
plt.gca().get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator())
for v, c in zip(values, counts):
plt.text(v, c, f'{c:g}', ha='center', va='bottom')
plt.savefig(args.output, bbox_inches="tight")
plt.close()
""" Images """
sns.set_style('white')
n = 5
pad = 20
side = 28 if dataset == 'mnist' else 32
side += 2 # make_grid padding
groups = nfe.groupby('y_true')
test_data = load_test_data(exp)
test_data.transform = transforms.ToTensor()
largest_nfe = groups.nfe.nlargest(n)
high_nfe_idxs = largest_nfe.index.get_level_values(1)
high_nfe_images = torch.stack([test_data[i][0] for i in high_nfe_idxs])
high_nfe_grid = make_grid(high_nfe_images, nrow=n)
smallest_nfe = groups.nfe.nsmallest(n).reset_index().sort_values(['y_true', 'nfe'], ascending=[True, False])
low_nfe_idxs = smallest_nfe.level_1 # nsmallest in reverse order
low_nfe_images = torch.stack([test_data[i][0] for i in low_nfe_idxs])
low_nfe_grid = make_grid(low_nfe_images, nrow=n)
smallest_nfe = smallest_nfe.nfe
grid_h = low_nfe_grid.shape[1]
img_pad = torch.zeros((3, grid_h, pad))
grid = torch.cat((high_nfe_grid, img_pad, low_nfe_grid), 2)
plt.imshow(np.transpose(grid.numpy(), (1, 2, 0))) # , interpolation='nearest')
for i, (l, s) in enumerate(zip(largest_nfe, smallest_nfe)):
y, x = (i // n), (i % n)
y, x = (np.array((y, x)) + (0.8, 0.75)) * side
text = plt.text(x, y, str(int(l)), fontsize=5, ha='left', va='top', color='white')
text.set_path_effects([patheffects.Stroke(linewidth=1, foreground='black'), patheffects.Normal()])
disp = side * n + pad + 6
text = plt.text(x + disp, y, str(int(s)), fontsize=5, ha='left', va='top', color='white')
text.set_path_effects([patheffects.Stroke(linewidth=1, foreground='black'), patheffects.Normal()])
ax = plt.gca()
h, _ = ax.get_ylim()
ticks = (np.arange(10) / 10 + 1 / (2 * 10)) * h
plt.yticks(ticks, labels)
plt.xticks([])
h, htxt = -0.03, -0.08
ax.annotate('', xy=(0, h), xycoords='axes fraction', xytext=(1, h), arrowprops=dict(arrowstyle="<->", color='k'))
ax.annotate('high NFE', xy=(0, htxt), xycoords='axes fraction', xytext=(0, htxt))
ax.annotate('low NFE', xy=(1, htxt), xycoords='axes fraction', xytext=(0.87, htxt))
plt.savefig(args.output2, bbox_inches="tight")
def main(args):
root = 'runs_' + args.dataset
exp = Experiment(args,
root=root,
main='model',
ignore=('cuda', 'device', 'epochs', 'resume'))
print(exp)
if os.path.exists(exp.path_to('log')) and not args.resume:
print('Skipping ...')
sys.exit(0)
train_data, test_data, in_ch, out = load_dataset(args)
train_loader = DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True)
test_loader = DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False)
if args.model == 'odenet':
model = ODENet(in_ch,
out=out,
n_filters=args.filters,
downsample=args.downsample,
method=args.method,
tol=args.tol,
adjoint=args.adjoint,
dropout=args.dropout)
else:
model = ResNet(in_ch,
out=out,
n_filters=args.filters,
downsample=args.downsample,
dropout=args.dropout)
model = model.to(args.device)
if args.optim == 'sgd':
optimizer = SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=args.wd)
elif args.optim == 'adam':
optimizer = Adam(model.parameters(), lr=args.lr, weight_decay=args.wd)
# print(train_data)
# print(test_data)
# print(model)
# print(optimizer)
if args.resume:
ckpt = torch.load(exp.ckpt('last'))
print('Loaded: {}'.format(exp.ckpt('last')))
model.load_state_dict(ckpt['model'])
optimizer.load_state_dict(ckpt['optim'])
start_epoch = ckpt['epoch'] + 1
best_accuracy = exp.log['test_acc'].max()
print('Resuming from epoch {}: {}'.format(start_epoch, exp.name))
else:
metrics = evaluate(test_loader, model, args)
best_accuracy = metrics['test_acc']
start_epoch = 1
if args.lrschedule == 'fixed':
scheduler = LambdaLR(
optimizer,
lr_lambda=lambda x: 1) # no-op scheduler, just for cleaner code
elif args.lrschedule == 'plateau':
scheduler = ReduceLROnPlateau(optimizer,
mode='max',
patience=args.patience)
elif args.lrschedule == 'cosine':
scheduler = CosineAnnealingLR(optimizer,
args.lrcycle,
last_epoch=start_epoch - 2)
progress = trange(start_epoch,
args.epochs + 1,
initial=start_epoch,
total=args.epochs)
for epoch in progress:
metrics = {'epoch': epoch}
progress.set_postfix({'Best ACC': f'{best_accuracy:.2%}'})
progress.set_description('TRAIN')
train_metrics = train(train_loader, model, optimizer, args)
progress.set_description('EVAL')
test_metrics = evaluate(test_loader, model, args)
is_best = test_metrics['test_acc'] > best_accuracy
best_accuracy = max(test_metrics['test_acc'], best_accuracy)
metrics.update(train_metrics)
metrics.update(test_metrics)
save_checkpoint(
exp, {
'epoch': epoch,
'params': vars(args),
'model': model.state_dict(),
'optim': optimizer.state_dict(),
'metrics': metrics
}, is_best)
exp.push_log(metrics)
sched_args = metrics[
'test_acc'] if args.lrschedule == 'plateau' else None
scheduler.step(sched_args)
def main(args):
exp = Experiment.from_dir(args.run, main='model')
params = next(exp.params.itertuples())
# data setup
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Lambda(lambda x: x.numpy())])
preproc = utils.PREPROC[params.dataset]
if params.dataset == 'mnist':
data = MNIST('data/mnist',
download=True,
train=False,
transform=transform)
elif params.dataset == 'cifar10':
data = CIFAR10('data/cifar10',
download=True,
train=False,
transform=transform)
preproc = map(lambda x: np.array(x).reshape((3, 1, 1)),
preproc) # expand dimensions
preproc = tuple(preproc)
# model setup
model = utils.load_model(exp).eval().cuda()
if args.tol is None:
args.tol = params.tol
if params.model == 'odenet':
model.odeblock.tol = args.tol
fmodel = foolbox.models.PyTorchModel(model,
bounds=(0, 1),
num_classes=10,
preprocessing=preproc)
# attack setup
if args.distance == 2:
attack = foolbox.attacks.L2BasicIterativeAttack
distance = foolbox.distances.MSE
elif args.distance == float('inf'):
attack = foolbox.attacks.LinfinityBasicIterativeAttack
distance = foolbox.distances.Linf
attack = attack(fmodel, distance=distance)
sub_exp_root = exp.path_to('adv-attack')
os.makedirs(sub_exp_root, exist_ok=True)
sub_exp = Experiment(args, root=sub_exp_root, ignore=('run', ))
print(sub_exp)
results_file = sub_exp.path_to('results.csv')
results = pd.read_csv(results_file) if os.path.exists(
results_file) else pd.DataFrame()
# perform attack
progress = tqdm(data)
for i, (image, label) in enumerate(progress):
if not results.empty and i in results.sample_id.values:
continue
if not isinstance(label, int):
label = label.item()
start = time.time()
adversarial = attack(image,
label,
unpack=False,
binary_search=False,
stepsize=args.stepsize,
epsilon=args.epsilon)
elapsed = time.time() - start
result = pd.DataFrame(dict(
sample_id=i,
label=label,
elapsed_time=elapsed,
distance=adversarial.distance.value,
adversarial_class=adversarial.adversarial_class,
original_class=adversarial.original_class,
),
index=[0])
results = results.append(result, ignore_index=True)
results.to_csv(results_file, index=False)
success = ~results.adversarial_class.isna()
successes = success.sum()
success_rate = success.mean()
progress.set_postfix({
'success_rate':
f'{success_rate:.2%} ({successes}/{len(success)})'
})
def main(args):
exp = Experiment(args, ignore=('epochs', 'resume'))
print(exp)
np.random.seed(args.seed)
tf.random.set_seed(args.seed)
data = load_datasets(args.data)
# TRAIN/VAL/TEST SPLIT
if args.split == 'subjects': # by SUBJECTS
val_subjects = (6, 9, 11, 13, 16, 28, 30, 48, 49)
test_subjects = (3, 4, 19, 38, 45, 46, 51, 52)
train_data = data[~data['sub'].isin(val_subjects + test_subjects)]
val_data = data[data['sub'].isin(val_subjects)]
test_data = data[data['sub'].isin(test_subjects)]
elif args.split == 'random': # 70-20-10 %
train_data, valtest_data = train_test_split(data,
test_size=.3,
shuffle=True)
val_data, test_data = train_test_split(valtest_data, test_size=.33)
lengths = map(len, (data, train_data, val_data, test_data))
print("Total: {} - Train / Val / Test: {} / {} / {}".format(*lengths))
x_shape = (args.resolution, args.resolution, 1)
y_shape = (args.resolution, args.resolution, 1)
train_gen, _ = get_loader(train_data,
batch_size=args.batch_size,
shuffle=True,
augment=True,
x_shape=x_shape)
val_gen, val_categories = get_loader(val_data,
batch_size=args.batch_size,
x_shape=x_shape)
# test_gen, test_categories = get_loader(test_data, batch_size=1, x_shape=x_shape)
log = exp.path_to('log.csv')
# weights_only checkpoints
best_weights_path = exp.path_to('best_weights.h5')
best_mask_weights_path = exp.path_to('best_weights_mask.h5')
# whole model checkpoints
best_ckpt_path = exp.path_to('best_model.h5')
last_ckpt_path = exp.path_to('last_model.h5')
if args.resume and os.path.exists(last_ckpt_path):
custom_objects = {
'AdaBeliefOptimizer': AdaBeliefOptimizer,
'iou_coef': evaluate.iou_coef,
'dice_coef': evaluate.dice_coef,
'hard_swish': hard_swish
}
model = tf.keras.models.load_model(last_ckpt_path,
custom_objects=custom_objects)
optimizer = model.optimizer
initial_epoch = len(pd.read_csv(log))
else:
config = vars(args)
model = build_model(x_shape, y_shape, config)
optimizer = AdaBeliefOptimizer(learning_rate=args.lr,
print_change_log=False)
initial_epoch = 0
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics={
'mask': [evaluate.iou_coef, evaluate.dice_coef],
'tags': 'binary_accuracy'
})
model_stopped_file = exp.path_to('early_stopped.txt')
need_training = not os.path.exists(
model_stopped_file) and initial_epoch < args.epochs
if need_training:
best_checkpointer = ModelCheckpoint(best_weights_path,
monitor='val_loss',
save_best_only=True,
save_weights_only=True)
best_mask_checkpointer = ModelCheckpoint(best_mask_weights_path,
monitor='val_mask_dice_coef',
mode='max',
save_best_only=True,
save_weights_only=True)
last_checkpointer = ModelCheckpoint(last_ckpt_path,
save_best_only=False,
save_weights_only=False)
logger = CSVLogger(log, append=args.resume)
progress = TqdmCallback(verbose=1,
initial=initial_epoch,
dynamic_ncols=True)
early_stop = tf.keras.callbacks.EarlyStopping(
monitor='val_mask_dice_coef', mode='max', patience=100)
callbacks = [
best_checkpointer, best_mask_checkpointer, last_checkpointer,
logger, progress, early_stop
]
model.fit(train_gen,
epochs=args.epochs,
callbacks=callbacks,
initial_epoch=initial_epoch,
steps_per_epoch=len(train_gen),
validation_data=val_gen,
validation_steps=len(val_gen),
verbose=False)
if model.stop_training:
open(model_stopped_file, 'w').close()
tf.keras.models.save_model(model,
best_ckpt_path,
include_optimizer=False)
# evaluation on test set
evaluate.evaluate(exp, force=need_training)
# save best snapshot in SavedModel format
model.load_weights(best_mask_weights_path)
best_savedmodel_path = exp.path_to('best_savedmodel')
model.save(best_savedmodel_path, save_traces=True)
# export to tfjs (Layers model)
tfjs_model_dir = exp.path_to('tfjs')
tfjs.converters.save_keras_model(model, tfjs_model_dir)
def main(args):
dataset, q, x = utils.load_benchmark(args.dataset, args.features)
q = utils.load_features(q, chunks=(2500, 2048))
x = utils.load_features(x, chunks=(2500, 2048))
if args.limit:
x = x[:args.limit]
n_points, dim = x.shape
if args.n_cells is None:
step_k = 2500
min_points_per_centroid = 39.0
max_points_per_centroid = 256.0
# n_train_points = min(n_points, 120000) # train index with less points or it crashes..
min_k = np.ceil(
n_points / (step_k * max_points_per_centroid)).astype(int) * step_k
max_k = np.floor(
n_points / (step_k * min_points_per_centroid)).astype(int) * step_k
args.n_cells = min_k
print('Using min suggested cells:', args.n_cells)
exp = Experiment(args, root=args.output, ignore=('output', 'pretrained'))
print(exp)
# create or load faiss index
index_file = exp.path_to('index.faiss')
if not os.path.exists(index_file):
if args.pretrained:
print('Loading pre-trained empty index ...')
index = faiss.read_index(args.pretrained)
train_time = None
else:
tmp = utils.compute_if_dask(x)
print('Creating index: training ...')
index = faiss.index_factory(
dim, 'IVF{},PQ{}'.format(args.n_cells, args.code_size))
# index = faiss.index_factory(dim, 'IVF{},Flat'.format(args.n_cells))
start = time.time()
index.train(tmp)
train_time = time.time() - start
del tmp
print('Creating index: adding ...')
start = time.time()
bs = 2**14
for i in trange(0, x.shape[0], bs):
batch = utils.compute_if_dask(x[i:i + bs])
index.add(batch)
add_time = time.time() - start
faiss.write_index(index, index_file)
size = os.path.getsize(index_file)
index_stats_file = exp.path_to('index_stats.csv')
index_stats = pd.DataFrame(
{
'size': size,
'train_time': train_time,
'add_time': add_time
},
index=[0])
index_stats.to_csv(index_stats_file, index=False)
else:
print('Loading pre-built index ...')
index = faiss.read_index(index_file)
n_probes = (1, 2, 5, 10, 25) # , 50, 100, 250, 500, 1000, 2500, 5000)
n_probes = filter(lambda x: x <= args.n_cells, n_probes)
params = vars(args)
progress = tqdm(n_probes)
for p in progress:
index.nprobe = p
params['nprobe'] = p
progress.set_postfix(
{k: v
for k, v in params.items() if k != 'output'})
scores = None
scores_file = exp.path_to(f'scores_np{p}.h5')
if not os.path.exists(scores_file):
print('Computing scores:', scores_file)
q = utils.compute_if_dask(q)
# execute kNN search using k = dataset size
ranked_sim, ranked_ids = index.search(q, n_points)
# we need a similarity matrix, we construct it from the ranked results.
# we fill it initially with the lowest score (not recovered IDs has infinity score)
if False: # XXX OPTIMIZED VERSION NOT WORKING!!!!
ranked_ids = np.ma.array(ranked_ids, mask=(ranked_ids < 0))
id_order = ranked_ids.argsort(axis=1)
scores = -ranked_sim[np.arange(q.shape[0]).reshape(-1, 1),
id_order]
del ranked_sim, ranked_ids, id_order
else:
scores = np.full((q.shape[0], n_points), np.inf)
for i, (rsims, rids) in enumerate(zip(ranked_sim, ranked_ids)):
for rsim, rid in zip(rsims, rids):
if rid > 0:
scores[i, rid] = rsim
scores = -scores
utils.save_as_hdf5(scores, scores_file, progress=True)
query_times, query_times_file = exp.require_csv('query_times.csv',
index='n_probes')
for i in trange(1, 6):
if utils.value_missing(query_times, p, f'query_time_run{i}'):
q = utils.compute_if_dask(q)
start = time.time()
index.search(q, n_points)
query_time = time.time() - start
query_times.at[p, f'query_time_run{i}'] = query_time
query_times.to_csv(query_times_file)
metrics, metrics_file = exp.require_csv(f'metrics_np{p}.csv')
if 'ap' not in metrics:
if scores is None:
print('Loading scores...')
scores = utils.load_features(scores_file)
print('Computing mAP...')
metrics['ap'] = dataset.score(scores[...],
reduction=False,
progress=True)
metrics.to_csv(metrics_file, index=False)
if 'ndcg' not in metrics:
dataset._load() # TODO in y_true getter
if scores is None:
print('Loading scores...')
scores = utils.load_features(scores_file)
print('Computing nDCG...')
y_true = dataset.y_true[:, :args.
limit] if args.limit else dataset.y_true
bs = 5
ndcg = []
for i in trange(0, y_true.shape[0], bs):
ndcg.append(
dcg(y_true[i:i + bs], scores[i:i + bs], normalized=True))
ndcg = np.concatenate(ndcg)
# metrics['ndcg'] = dcg(y_true, scores, normalized=True)
metrics['ndcg'] = ndcg
metrics.to_csv(metrics_file, index=False)
if 'ndcg@25' not in metrics:
dataset._load() # TODO in y_true getter
if scores is None:
progress.write('Loading scores...')
scores = utils.load_features(scores_file)[...]
progress.write('Computing nDCG@25...')
y_true = dataset.y_true[:, :args.
limit] if args.limit else dataset.y_true
bs = 50
ndcg = []
for i in trange(0, y_true.shape[0], bs):
ndcg.append(
dcg(y_true[i:i + bs],
scores[i:i + bs],
p=25,
normalized=True))
metrics['ndcg@25'] = np.concatenate(ndcg)
# metrics['ndcg'] = dcg(dataset.y_true, scores, normalized=True)
metrics.to_csv(metrics_file, index=False)
progress.write(f'nDCG@25: {metrics["ndcg@25"].mean()}')
metrics['n_probes'] = p
metrics.to_csv(metrics_file, index=False)
def retrieval(args):
exp = Experiment.from_dir(args.run, main='model')
features_file = exp.path_to('features.h5')
results_file = exp.path_to('retrieval.csv')
assert os.path.exists(
features_file), f"No pre-extracted features found: {features_file}"
all_results = pd.DataFrame()
# check if results exists and are updated, then load them (and probably skip the computation them later)
if os.path.exists(results_file
) and os.path.getctime(results_file) >= os.path.getctime(
features_file) and not args.force:
all_results = pd.read_csv(results_file)
params = next(exp.params.itertuples())
with h5py.File(features_file, 'r') as f:
features = f['features'][...]
y_true = f['y_true'][...]
if params.model == 'odenet':
t1s = f['t1s'][...]
features /= np.linalg.norm(features, axis=-2, keepdims=True)
queries = features # all queries
n_samples = features.shape[
-2] # number of samples, for both models (first dimension might be t1)
n_queries = queries.shape[
-2] # number of queries, for both models (first dimension might be t1)
gt = np.broadcast_to(y_true,
(n_queries, n_samples)) == y_true[:n_queries].reshape(
n_samples, -1) # gt per query in each row
def score(queries, db, gt):
scores = queries.dot(db.T)
aps = [
average_precision_score(gt[i], scores[i])
for i in trange(n_queries)
]
return np.mean(aps)
if params.model == 'odenet':
for i, t1 in enumerate(tqdm(t1s)):
# TODO check and skip
mean_ap_asym = score(queries[i], features[-1], gt) # t1 = 1 for db
mean_ap_sym = score(queries[i], features[i],
gt) # t1 same for queries and db
results = {
't1': t1,
'mean_ap_asym': mean_ap_asym,
'mean_ap_sym': mean_ap_sym
}
all_results = all_results.append(results, ignore_index=True)
all_results.to_csv(results_file, index=False)
else: # resnet
mean_ap = score(features, features, gt)
all_results = all_results.append({'mean_ap': mean_ap},
ignore_index=True)
all_results.to_csv(results_file, index=False)
def nfe(args):
assert Experiment.is_exp_dir(args.run), "Not a run dir: args.run"
runs = Experiment.from_dir(args.run, main='model')
nfe = pd.read_csv(runs.path_to('nfe.csv'), index_col=0)
sns.boxplot(x='y_true', y='nfe', data=nfe)
plt.savefig(args.output, bbox_inches="tight")
def tradeoff(args):
run = Experiment.from_dir(args.run, main='model')
print(run)
results_file = run.path_to('tradeoff.csv')
best_ckpt_file = run.ckpt('best')
results = pd.DataFrame()
# check if results exists and are updated, then load them (and probably skip the computation them later)
if os.path.exists(results_file
) and os.path.getctime(results_file) >= os.path.getctime(
best_ckpt_file) and not args.force:
results = pd.read_csv(results_file)
params = next(run.params.itertuples())
test_data = load_test_data(run)
test_loader = DataLoader(test_data,
batch_size=params.batch_size,
shuffle=False)
model = load_model(run)
model = model.to(args.device)
model.eval()
def _evaluate(loader, model, t1, tol, args):
model.odeblock.t1 = t1
model.odeblock.tol = tol
n_correct = 0
n_batches = 0
n_processed = 0
nfe_forward = 0
progress = tqdm(loader)
for x, y in progress:
x, y = x.to(args.device), y.to(args.device)
p = model(x)
nfe_forward += model.nfe(reset=True)
loss = F.cross_entropy(p, y)
n_correct += (y == p.argmax(dim=1)).sum().item()
n_processed += y.shape[0]
n_batches += 1
logloss = loss.item() / n_processed
accuracy = n_correct / n_processed
nfe = nfe_forward / n_batches
metrics = {
'loss': f'{logloss:4.3f}',
'acc': f'{n_correct:4d}/{n_processed:4d} ({accuracy:.2%})',
'nfe': f'{nfe:3.1f}'
}
progress.set_postfix(metrics)
metrics = {
't1': t1,
'test_loss': logloss,
'test_acc': accuracy,
'test_nfe': nfe,
'test_tol': tol
}
return metrics
progress = tqdm(itertools.product(args.tol, args.t1))
for tol, t1 in progress:
if 't1' in results.columns and 'test_tol' in results.columns and (
(results.t1 == t1) & (results.test_tol == tol)).any():
print(f'Skipping tol={tol} t1={t1} ...')
continue
progress.set_postfix({'tol': tol, 't1': t1})
result = _evaluate(test_loader, model, t1, tol, args)
results = results.append(result, ignore_index=True)
results.to_csv(results_file, index=False)
def main(args):
# do not track lambda param, it can be changed after train
exp = Experiment(args, ignore=('lambda_', ))
print(exp)
if exp.found:
print('Already exists: SKIPPING')
exit(0)
np.random.seed(args.seed)
tf.random.set_seed(args.seed)
# get data
train_dataset = get_train_data(args.category,
image_size=args.image_size,
patch_size=args.patch_size,
batch_size=args.batch_size,
n_batches=args.n_batches,
rotation_range=args.rotation_range,
seed=args.seed)
test_dataset, test_labels = get_test_data(args.category,
image_size=args.image_size,
patch_size=args.patch_size,
batch_size=args.batch_size)
is_object = args.category in objects
# build models
generator = make_generator(args.latent_size,
channels=args.channels,
upsample_first=is_object,
upsample_type=args.ge_up,
bn=args.ge_bn,
act=args.ge_act)
encoder = make_encoder(args.patch_size,
args.latent_size,
channels=args.channels,
bn=args.ge_bn,
act=args.ge_act)
discriminator = make_discriminator(args.patch_size,
args.latent_size,
channels=args.channels,
bn=args.d_bn,
act=args.d_act)
# feature extractor model for evaluation
discriminator_features = get_discriminator_features_model(discriminator)
# build optimizers
generator_encoder_optimizer = O.Adam(args.lr,
beta_1=args.ge_beta1,
beta_2=args.ge_beta2)
discriminator_optimizer = O.Adam(args.lr,
beta_1=args.d_beta1,
beta_2=args.d_beta2)
# reference to the models to use in eval
generator_eval = generator
encoder_eval = encoder
# for smoothing generator and encoder evolution
if args.ge_decay > 0:
ema = tf.train.ExponentialMovingAverage(decay=args.ge_decay)
generator_ema = tf.keras.models.clone_model(generator)
encoder_ema = tf.keras.models.clone_model(encoder)
generator_eval = generator_ema
encoder_eval = encoder_ema
# checkpointer
checkpoint = tf.train.Checkpoint(
generator=generator,
encoder=encoder,
discriminator=discriminator,
generator_encoder_optimizer=generator_encoder_optimizer,
discriminator_optimizer=discriminator_optimizer)
best_ckpt_path = exp.ckpt(f'ckpt_{args.category}_best')
last_ckpt_path = exp.ckpt(f'ckpt_{args.category}_last')
# log stuff
log, log_file = exp.require_csv(f'log_{args.category}.csv.gz')
metrics, metrics_file = exp.require_csv(f'metrics_{args.category}.csv')
best_metric = 0.
best_recon = float('inf')
best_recon_file = exp.path_to(f'best_recon_{args.category}.png')
last_recon_file = exp.path_to(f'last_recon_{args.category}.png')
# animate generation during training
n_preview = 6
train_batch = next(iter(train_dataset))[:n_preview]
test_batch = next(iter(test_dataset))[0][:n_preview]
latent_batch = tf.random.normal([n_preview, args.latent_size])
if not is_object: # take random patches from test images
patch_location = np.random.randint(0,
args.image_size - args.patch_size,
(n_preview, 2))
test_batch = [
x[i:i + args.patch_size, j:j + args.patch_size, :]
for x, (i, j) in zip(test_batch, patch_location)
]
test_batch = K.stack(test_batch)
video_out = exp.path_to(f'{args.category}.mp4')
video_options = dict(fps=30, codec='libx265',
quality=4) # see imageio FFMPEG options
video_saver = VideoSaver(train_batch, test_batch, latent_batch, video_out,
**video_options)
video_saver.generate_and_save(generator, encoder)
# train loop
progress = tqdm(train_dataset, desc=args.category, dynamic_ncols=True)
try:
for step, image_batch in enumerate(progress, start=1):
if step == 1 or args.d_iter == 0: # only for JIT compilation (tf.function) to work
d_train = True
ge_train = True
elif args.d_iter:
n_iter = step % (abs(args.d_iter) + 1) # can be in [0, d_iter]
d_train = (n_iter != 0) if (args.d_iter > 0) else (
n_iter == 0) # True in [1, d_iter]
ge_train = not d_train # True when step == d_iter + 1
else: # d_iter == None: dynamic adjustment
d_train = (scores['fake_score'] > 0) or (scores['real_score'] <
0)
ge_train = (scores['real_score'] > 0) or (scores['fake_score']
< 0)
losses, scores = train_step(image_batch,
generator,
encoder,
discriminator,
generator_encoder_optimizer,
discriminator_optimizer,
d_train,
ge_train,
alpha=args.alpha,
gp_weight=args.gp_weight)
if (args.ge_decay > 0) and (step % 10 == 0):
ge_vars = generator.variables + encoder.variables
ema.apply(ge_vars) # update exponential moving average
# tensor to numpy
losses = {
n: l.numpy() if l is not None else l
for n, l in losses.items()
}
scores = {
n: s.numpy() if s is not None else s
for n, s in scores.items()
}
# log step metrics
entry = {
'step': step,
'timestamp': pd.to_datetime('now'),
**losses,
**scores
}
log = log.append(entry, ignore_index=True)
if step % 100 == 0:
if args.ge_decay > 0:
ge_ema_vars = generator_ema.variables + encoder_ema.variables
for v_ema, v in zip(ge_ema_vars, ge_vars):
v_ema.assign(ema.average(v))
preview = video_saver.generate_and_save(
generator_eval, encoder_eval)
if step % 1000 == 0:
log.to_csv(log_file, index=False)
checkpoint.write(file_prefix=last_ckpt_path)
auc, balanced_accuracy = evaluate(generator_eval,
encoder_eval,
discriminator_features,
test_dataset,
test_labels,
patch_size=args.patch_size,
lambda_=args.lambda_)
entry = {
'step': step,
'auc': auc,
'balanced_accuracy': balanced_accuracy
}
metrics = metrics.append(entry, ignore_index=True)
metrics.to_csv(metrics_file, index=False)
if auc > best_metric:
best_metric = auc
checkpoint.write(file_prefix=best_ckpt_path)
# save last image to inspect it during training
imageio.imwrite(last_recon_file, preview)
recon = losses['images_reconstruction_loss']
if recon < best_recon:
best_recon = recon
imageio.imwrite(best_recon_file, preview)
progress.set_postfix({
'AUC': f'{auc:.1%}',
'BalAcc': f'{balanced_accuracy:.1%}',
'BestAUC': f'{best_metric:.1%}',
})
except KeyboardInterrupt:
checkpoint.write(file_prefix=last_ckpt_path)
finally:
log.to_csv(log_file, index=False)
video_saver.close()
# score the test set
checkpoint.read(best_ckpt_path)
auc, balanced_accuracy = evaluate(generator,
encoder,
discriminator_features,
test_dataset,
test_labels,
patch_size=args.patch_size,
lambda_=args.lambda_)
print(f'{args.category}: AUC={auc}, BalAcc={balanced_accuracy}')
def features(args):
run = Experiment.from_dir(args.run, main='model')
print(run)
params = next(run.params.itertuples())
features_file = 'features.h5' if args.data is None else 'features-{}.h5'.format(
args.data)
features_file = run.path_to(features_file)
results_file = run.path_to('results')
dependecy_file = run.ckpt('best')
if os.path.exists(features_file) and os.path.getctime(
features_file) >= os.path.getctime(
dependecy_file) and not args.force:
print('Skipping...')
sys.exit(0)
if args.data == 'tiny-imagenet-200':
transfer_transform = transforms.Compose([
transforms.Resize(32),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
test_data = TinyImageNet200('data/tiny-imagenet-200',
split='val',
transform=transfer_transform)
else:
test_data = load_test_data(run)
test_loader = DataLoader(test_data,
batch_size=params.batch_size,
shuffle=False)
model = load_model(run)
model = model.to(args.device)
model.eval()
model.to_features_extractor()
if params.model == 'odenet':
if os.path.exists(results_file): # reuse t1s if already tested
results = pd.read_csv(results_file)
results = results[results.t1 <= 1]
t1s = results.t1.sort_values().unique()
else:
t1s = np.arange(.05, 1.05, .05) # from 0 to 1 w/ .05 step
model.odeblock.t1 = t1s.tolist()
if 'ode' in params.downsample:
model.downsample.odeblock.t1 = t1s.tolist()
t1s = np.insert(t1s, 0, 0) # add 0 at the beginning
features = []
y_true = []
with torch.no_grad():
for x, y in tqdm(test_loader):
x = x.to(args.device)
y_true.append(y.numpy())
f = model(x)
f = f.cpu().numpy()
features.append(f)
features = np.concatenate(features, -2) # concat along batch dimension
y_true = np.concatenate(y_true)
with h5py.File(features_file, 'w') as f:
f['features'] = features
f['y_true'] = y_true
if params.model == 'odenet':
f['t1s'] = t1s