def dice_fps(args):
exps = expman.gather(args.run).filter(args.filter)
mask_metrics = exps.collect('test_pred/mask_metrics.csv')
mask_metrics = mask_metrics.groupby('exp_name').dice.max()
time_metrics = exps.collect('timings.csv')
time_metrics = time_metrics.rename({
'Unnamed: 0': 'metrics',
'0': 'value'
},
axis=1)
time_metrics = time_metrics.pivot_table(index='exp_name',
columns='metrics',
values='value')
flops_nparams = exps.collect('flops_nparams.csv')
flops_nparams = flops_nparams.set_index('exp_name')[['flops', 'nparams']]
table = pd.concat(
(time_metrics, mask_metrics, flops_nparams),
axis=1)[['dice', 'fps', 'throughput', 'flops', 'nparams']]
table['dice'] = table.dice.map('{:.1%}'.format)
table['fps'] = table.fps.map('{:.1f}'.format)
table['throughput'] = (table.throughput * 1000).map('{:.1f}ms'.format)
table['flops'] = (table.flops / 10**9).map('{:.1f}G'.format)
table['nparams'] = (table.nparams / 10**6).map('{:.2f}M'.format)
print(table)
def metrics(args):
exps = expman.gather(args.run).filter(args.filter)
mask_metrics = exps.collect('test_pred/mask_metrics.csv')
sns.lineplot(data=mask_metrics,
x='thr',
y='dice',
hue='conv_type',
size='grow_factor',
style='num_filters')
plt.savefig(args.output)
def compare_videos(args):
exps = expman.gather(args.run)
exps = expman.filter(args.filter, exps)
exps = list(exps)
# get params and video paths
params = expman.collect_all(exps)
params['video'] = [glob.glob(e.path_to('*.mp4'))[0] for e in exps]
params = params.sort_values(['alpha', 'd_iter'], ascending=[False, True])
video_paths = params.video.values
video_labels = params.exp_name.values
# find best grid aspect ratio
n = len(video_paths)
square_side = np.ceil(np.sqrt(n)).astype(int)
nrows = np.arange(1, square_side + 1)
ncols = np.ceil(n / nrows)
aspect_ratios_per_nrows = (6 / 7) * (ncols / nrows)
best = np.argmin(np.abs(aspect_ratios_per_nrows - (16 / 9)))
w = ncols[best].astype(int)
h = nrows[best].astype(int)
# build ffmpeg command
input_args = [f'-i {path}' for path in video_paths]
input_args = ' '.join(input_args)
pad_w, pad_h = 10, 30
filter_complex = [
f'[{i}:v] '
f'setpts=PTS-STARTPTS, '
f'pad=width=iw+{pad_w}: height=ih+{pad_h}: x={pad_w // 2}: y={pad_h}, '
f'drawtext=text=\'{label}\': fontcolor=white: fontsize=24: x=(w-tw)/2: y=({pad_h}-th)/2 '
f'[a{i}]' for i, label in enumerate(video_labels)
]
xstack_inputs = ''.join(f'[a{i}]' for i in range(n))
widths = ['0'] + [f'w{i}' for i in range(w - 1)]
heights = ['0'] + [f'h{i}' for i in range(h - 1)]
xstack_layout = [
'+'.join(widths[:i + 1]) + '_' + '+'.join(heights[:j + 1])
for j in range(h) for i in range(w)
]
xstack_layout = '|'.join(xstack_layout)
xstack_filter = f'{xstack_inputs}xstack=inputs={n}: layout={xstack_layout}[out]'
filter_complex += [xstack_filter]
filter_complex = ';'.join(filter_complex)
cmd = f'ffmpeg {input_args} -filter_complex "{filter_complex}" -map "[out]" -c:v hevc -crf 23 -preset fast {args.output}'
print(cmd)
def compare(args):
exps1 = expman.gather(args.run1).filter(args.filter)
exps2 = expman.gather(args.run2).filter(args.filter)
fixed_params1, results1, table1 = _get_scores(exps1)
fixed_params2, results2, table2 = _get_scores(exps2)
with pd.option_context('display.max_rows', None, 'display.max_columns',
None, 'display.width', 1000):
with pd.option_context('display.float_format', '{:.2f}'.format):
print('1]', args.run2)
print(table2)
print()
print('2]', args.run1)
print(table1)
print()
with pd.option_context('display.float_format', '{:.1%}'.format):
print('D] {} - {}'.format(args.run2, args.run1))
print(table2 - table1)
def log(args):
exps = expman.gather(args.run).filter(args.filter)
with PdfPages(args.output) as pdf:
for exp_name, exp in sorted(exps.items()):
print(exp_name)
log = pd.read_csv(exp.path_to('log.csv'), index_col='epoch')
train_cols = [c for c in log.columns if 'val' not in c]
val_cols = [c for c in log.columns if 'val' in c]
test_images = glob(
os.path.join(exp.path_to('test_pred'), '*_samples.png'))
fig = plt.figure(figsize=(14, 10))
fig_shape = (2, 2) if test_images else (2, 1)
ax1 = plt.subplot2grid(fig_shape, (0, 0))
ax2 = plt.subplot2grid(fig_shape, (1, 0))
log[train_cols].plot(ax=ax1)
log[val_cols].plot(ax=ax2)
ax1.legend(loc='center right', bbox_to_anchor=(-0.05, 0.5))
ax2.legend(loc='center right', bbox_to_anchor=(-0.05, 0.5))
ax2.set_ylim((0, 1))
if test_images:
test_images = sorted(test_images)
test_images = list(map(imread, test_images))
max_w = max(i.shape[1] for i in test_images)
pads = [((0, 0), (0, max_w - i.shape[1]), (0, 0))
for i in test_images]
test_images = np.concatenate(
[np.pad(i, pad) for i, pad in zip(test_images, pads)],
axis=0)
ax3 = plt.subplot2grid(fig_shape, (0, 1), rowspan=2)
ax3.imshow(test_images)
ax3.set_axis_off()
log_plot_file = exp.path_to('log_plot.pdf')
plt.suptitle(exp_name)
plt.savefig(log_plot_file, bbox_inches='tight')
pdf.savefig(fig, bbox_inches='tight')
plt.close()
def main(args):
variants = (
('', []),
# ('_qf16', ['--quantize_float16', '*']),
# ('_qu16', ['--quantize_uint16' , '*']),
# ('_qu8' , ['--quantize_uint8' , '*']),
)
converted_models = []
exps = expman.gather(args.run).filter(args.filter)
for exp_name, exp in tqdm(exps.items()):
# ckpt = exp.path_to('best_model.h5')
# ckpt = ckpt if os.path.exists(ckpt) else exp.path_to('last_model.h5')
ckpt = exp.path_to('best_savedmodel/')
for suffix, extra_args in variants:
name = exp_name + suffix
out = os.path.join(
args.output,
name) if args.output else exp.path_to(f'tfjs_graph{suffix}')
if args.force or not os.path.exists(out):
os.makedirs(out, exist_ok=True)
cmd = [
'tensorflowjs_converter', '--input_format',
'tf_saved_model', '--output_format', 'tfjs_graph_model'
] + extra_args + [ckpt, out]
subprocess.call(cmd)
converted_models.append(name)
js_output = 'models = ' + json.dumps(converted_models)
if args.output:
js_filename = os.path.join(args.output, 'models.js')
with open(js_filename, 'w') as f:
f.write(js_output)
else:
print(js_output)
def print_scores(args):
exps = expman.gather(args.run).filter(args.filter)
fixed_params, results, table = _get_scores(exps, args.best)
with pd.option_context('display.max_rows', None, 'display.max_columns',
None, 'display.width', 500):
print('Common Parameters')
print('=================')
print(fixed_params)
print('=================')
print()
print('Run Metrics')
print('=================')
print(results)
print('=================')
print()
print('Best Metrics')
print('=================')
with pd.option_context('display.float_format', '{:.2f}'.format):
print(table)
print('=================')
print()
def main(args):
for exp in expman.gather(args.run).filter(args.filter):
print(exp)
evaluate(exp, force=args.force)
def plot_log(args):
formatter = lambda x, pos: f'{(x // 1000):g}k' if x >= 1000 else f'{x:g}'
formatter = FuncFormatter(formatter)
exps = expman.gather(args.run)
exps = expman.filter(args.filter, exps)
exps = sorted(exps, key=lambda exp: exp.params.category)
with PdfPages(args.output) as pdf:
for exp in tqdm(exps):
category = exp.params.category
train_log = exp.path_to(f'log_{category}.csv.gz')
train_log = pd.read_csv(train_log)
metric_log = exp.path_to(f'metrics_{category}.csv')
metric_log = pd.read_csv(metric_log)
best_recon_file = exp.path_to(f'best_recon_{category}.png')
best_recon = plt.imread(best_recon_file)
last_recon_file = exp.path_to(f'last_recon_{category}.png')
last_recon = plt.imread(last_recon_file)
# prepare figure
zoom = 0.7
fig = plt.figure(figsize=(20 * zoom, 8 * zoom))
# gridspec and axes for plots
gs = fig.add_gridspec(ncols=2,
nrows=2,
hspace=0.05,
wspace=0.05,
right=0.5)
ax1 = fig.add_subplot(gs[0, 0])
ax2 = fig.add_subplot(gs[0, 1])
ax3 = fig.add_subplot(gs[1, 0], sharex=ax1)
ax4 = fig.add_subplot(gs[1, 1], sharex=ax2)
# gridspec and axes for preview images
gs2 = fig.add_gridspec(ncols=1,
nrows=2,
hspace=0,
wspace=0,
left=0.55)
ax5 = fig.add_subplot(gs2[:, 0])
# ticklabels format
ax1.xaxis.set_major_formatter(formatter)
ax2.xaxis.set_major_formatter(formatter)
ax3.xaxis.set_major_formatter(formatter)
ax4.xaxis.set_major_formatter(formatter)
ax5.axis('off')
# minor ticks position
ax3.xaxis.set_minor_locator(AutoMinorLocator())
ax4.xaxis.set_minor_locator(AutoMinorLocator())
# minor grid style
ax1.grid(b=True, which='minor', linewidth=0.5)
ax2.grid(b=True, which='minor', linewidth=0.5)
ax3.grid(b=True, which='minor', linewidth=0.5)
ax4.grid(b=True, which='minor', linewidth=0.5)
# right y-axes
ax2.yaxis.set_label_position("right")
ax2.yaxis.tick_right()
ax4.yaxis.set_label_position("right")
ax4.yaxis.tick_right()
# generator losses
gen = [
'generator_encoder_loss', 'images_reconstruction_loss',
'latent_reconstruction_loss', 'generator_encoder_total_loss'
]
train_log.plot(x='step', y=gen, logy='sym', ax=ax1)
ax1.legend(loc='lower left', bbox_to_anchor=(0.0, 1.0))
# discriminator losses
dis = [
'discriminator_loss', 'gradient_penalty_loss',
'discriminator_total_loss'
]
train_log.plot(x='step', y=dis, logy='sym', ax=ax3)
ax3.legend(loc='upper left', bbox_to_anchor=(0.0, -0.2))
# scores
scores = ['real_score', 'fake_score']
train_log.plot(x='step', y=scores, logy='sym', ax=ax2)
ax2.legend(loc='lower right', bbox_to_anchor=(1.0, 1.0))
# metrics
metric_log.plot(x='step', y=['auc', 'balanced_accuracy'], ax=ax4)
best_recon_step = train_log[(
train_log.step %
1000) == 0].images_reconstruction_loss.idxmin()
best_recon_step = train_log.loc[best_recon_step, 'step']
ax4.axvline(best_recon_step, color='black', lw=1)
ax4.legend(loc='upper right', bbox_to_anchor=(1.0, -0.2))
# best/last reconstruction
hw = best_recon.shape[1] // 2
recon = np.hstack((best_recon[:, :hw, :], last_recon[:, :hw, :]))
ax5.imshow(recon)
ax5.margins(0)
# params
params_str = exp.params.to_string()
plt.figtext(0.08,
0.5,
params_str,
ha='right',
va='center',
family='monospace')
# save sigle figure in exp folder
log_pdf = exp.path_to(f'log_{category}.pdf')
fig.savefig(log_pdf, bbox_inches='tight')
# add as page in PDF
pdf.savefig(fig, bbox_inches='tight')
plt.close(fig)
def bd(args):
exps = expman.gather(args.run).filter(args.filter)
blink_metrics = exps.collect('test_pred/all_blink_roc_metrics.csv')
blink_metrics = blink_metrics.iloc[3::4].rename({'0': 'auc'}, axis=1)
aucs = blink_metrics.auc.values
print(f'{aucs.mean()} +- {aucs.std()}')
def ee(args):
sns.set_theme(context='notebook', style='whitegrid')
exps = expman.gather(args.run).filter(args.filter)
mask_metrics = exps.collect('test_pred/mask_metrics.csv').groupby(
'exp_id')[['dice', 'iou']].max()
flops_nparams = exps.collect('flops_nparams.csv')
data = pd.merge(mask_metrics, flops_nparams, on='exp_id')
data['dice'] *= 100
named_data = data.rename(
{
'nparams': '# Params',
'dice': 'mean Dice Coeff. (%)',
'conv_type': '$t$ (Conv. Type)',
'grow_factor': r'$\gamma$',
'num_filters': '$k$ (# Filters)',
'flops': 'FLOPs',
'num_stages': '$s$ (# Stages)',
},
axis=1).replace({
'bn-conv': 'conv-bn',
'sep-bn-conv': 'sep-conv-bn'
})
g = sns.relplot(data=named_data,
x='FLOPs',
y='mean Dice Coeff. (%)',
hue='$t$ (Conv. Type)',
hue_order=['conv', 'conv-bn', 'sep-conv', 'sep-conv-bn'],
col='$s$ (# Stages)',
style='$k$ (# Filters)',
markers=True,
markersize=9,
kind='line',
dashes=True,
facet_kws=dict(despine=False, legend_out=False),
legend=True,
height=3.8,
aspect=1.3,
markeredgecolor='white')
b_formatter = ticker.FuncFormatter(
lambda x, pos: '{:.2f}'.format(x / 10**9) + 'B')
h, l = g.axes.flatten()[0].get_legend_handles_labels()
for hi in h:
hi.set_markeredgecolor('white')
g.axes.flatten()[0].legend_.remove()
g.fig.legend(h,
l,
ncol=2,
bbox_to_anchor=(0.53, 0.53),
fancybox=False,
columnspacing=0,
framealpha=1,
handlelength=1.2)
for ax in g.axes.flatten():
ax.yaxis.set_minor_locator(ticker.AutoMinorLocator())
ax.set_ylim(bottom=40, top=90)
ax.set_xscale('symlog')
ax.set_xlim(left=0.04 * 10**9, right=2 * 10**9)
ax.xaxis.set_minor_locator(
ticker.SymmetricalLogLocator(base=10,
linthresh=2,
subs=[1.5, 2, 3, 4, 5, 6, 8]))
ax.xaxis.set_minor_formatter(b_formatter)
ax.grid(which='minor', linestyle='--', color='#eeeeee')
ax.xaxis.set_major_formatter(b_formatter)
ax.tick_params(axis="x", which="both", rotation=90)
plt.savefig(args.output, bbox_inches='tight')