def make_legend_img(classname_to_rgb, dpi=96, shape=(200, 200), mode='line',
transparent=False):
"""
Makes an image of a categorical legend
CommandLine:
python -m netharn.util.mplutil make_legend_img
Example:
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> import kwplot
>>> classname_to_rgb = {
>>> 'blue': kwplot.Color('blue').as01(),
>>> 'red': kwplot.Color('red').as01(),
>>> }
>>> img = make_legend_img(classname_to_rgb)
>>> # xdoctest: +REQUIRES(--show)
>>> kwplot.autompl()
>>> kwplot.imshow(img)
>>> kwplot.show_if_requested()
"""
import kwplot
plt = kwplot.autoplt()
def append_phantom_legend_label(label, color, type_='line', alpha=1.0, ax=None):
if ax is None:
ax = plt.gca()
_phantom_legend_list = getattr(ax, '_phantom_legend_list', None)
if _phantom_legend_list is None:
_phantom_legend_list = []
setattr(ax, '_phantom_legend_list', _phantom_legend_list)
if type_ == 'line':
phantom_actor = plt.Line2D((0, 0), (1, 1), color=color, label=label,
alpha=alpha)
else:
phantom_actor = plt.Circle((0, 0), 1, fc=color, label=label,
alpha=alpha)
_phantom_legend_list.append(phantom_actor)
fig = plt.figure(dpi=dpi)
w, h = shape[1] / dpi, shape[0] / dpi
fig.set_size_inches(w, h)
ax = fig.add_subplot(1, 1, 1)
for label, color in classname_to_rgb.items():
append_phantom_legend_label(label, color, type_=mode, ax=ax)
_phantom_legend_list = getattr(ax, '_phantom_legend_list', None)
if _phantom_legend_list is None:
_phantom_legend_list = []
setattr(ax, '_phantom_legend_list', _phantom_legend_list)
ax.legend(handles=_phantom_legend_list)
ax.grid(False)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.axis('off')
legend_img = render_figure_to_image(fig, dpi=dpi, transparent=transparent)
plt.close(fig)
return legend_img
def main():
import ubelt as ub
dis_results = [
dis2(loop_first),
' | ',
dis2(loop_first2),
]
print(ub.hzcat(dis_results))
rows = benchmark_loop_first_variants()
import pandas as pd
df = pd.DataFrame(rows)
import kwplot
plt = kwplot.autoplt()
# import matplotlib as mpl
# mpl.use('Qt5Agg')
# from matplotlib import pyplot as plt
import seaborn as sns
sns.set()
sns.lineplot(data=df, x='num_items', y='mean', hue='method_name',
markers='method_name')
plt.show()
def mwe_check_before_select():
import ubelt as ub
import numpy as np
results = []
ns = np.logspace(1, 7, 100).astype(np.int)
for n in ub.ProgIter(ns, desc='time-tradeoff', verbose=3):
print('n = {!r}'.format(n))
y_true = np.random.randint(0, 100, n).astype(np.int64)
y_pred = np.random.randint(0, 100, n).astype(np.int64)
sample_weight = np.random.rand(n)
isvalid = np.random.rand(n) > 0.5
import timerit
ti = timerit.Timerit(9, bestof=3, verbose=2)
for timer in ti.reset('all-check'):
with timer:
np.all(isvalid)
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
for timer in ti.reset('all-index'):
with timer:
y_true[isvalid]
y_pred[isvalid]
sample_weight[isvalid]
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
import pandas as pd
df = pd.DataFrame(results)
import kwplot
import seaborn as sns
kwplot.autoplt()
sns.set()
ax = sns.lineplot(data=df, x='n', y='time', hue='label')
ax.set_yscale('log')
ax.set_xscale('log')
pass
def test_last_digits():
mapping = {}
tail_edge_hist = ub.ddict(lambda: 0)
for x in ub.ProgIter(range(1, int(1e5))):
import ubelt as ub
for y in collatz(x):
if x in mapping:
break
x_tail = x - (10 * (x // 10))
y_tail = y - (10 * (y // 10))
mapping[x] = y
tail_edge_hist[(str(x_tail), str(y_tail))] += 1
x = y
import kwarray
print(kwarray.stats_dict(np.array(list(tail_edge_hist.values()))))
import networkx as nx
tail_g = nx.DiGraph()
tail_g.add_edges_from(tail_edge_hist.keys())
for cycle in nx.simple_cycles(tail_g):
print('cycle = {!r}'.format(cycle))
print('tail_g.adj = {!r}'.format(tail_g.adj))
for n in sorted(tail_g.nodes, key=int):
pred = tuple(tail_g.pred[n].keys())
succ = tuple(tail_g.succ[n].keys())
in_d = tail_g.in_degree(n)
out_d = tail_g.out_degree(n)
print(
f'{str(pred):>15} -> {n:>2} -> {str(succ):<15} : {out_d:<2} {in_d:<2}'
)
import kwplot
import graphid
plt = kwplot.autoplt()
sccs = list(nx.strongly_connected_components(tail_g))
nx.draw_networkx(tail_g)
# nx.draw_circular(tail_g)
ax = plt.gca()
ax.cla()
# nx.draw_networkx(tail_g)
graphid.util.util_graphviz.show_nx(tail_g)
CCs = list(nx.connected_components(tail_g.to_undirected()))
def draw():
import kwplot
plt = kwplot.autoplt()
plotkw = dict(
xlabel='learning-rate', ylabel='loss', xscale=scale,
ymin=0, xmin='data', xmax=high, fnum=1,
)
R = 2 if 'vali' in records else 1
for i, tag in enumerate(['train', 'vali']):
if tag in records:
kwplot.multi_plot(
xdata=records[tag]['lr'], ydata=records[tag]['loss'],
ymax=1.2 * np.percentile(records[tag]['loss'], 60) / .6,
pnum=(1, R, i), title=tag + ' lr-scan', **plotkw)
plt.plot(best_lr, best_loss, '*')
def colorbar_image(domain,
cmap='plasma',
dpi=96,
shape=(200, 20),
transparent=False):
"""
Notes:
shape is approximate
Ignore:
domain = np.linspace(-30, 200)
cmap='plasma'
dpi = 80
dsize = (20, 200)
util.imwrite('foo.png', util.colorbar_image(np.arange(0, 1)), shape=(400, 80))
import plottool as pt
pt.qtensure()
import matplotlib as mpl
mpl.style.use('ggplot')
util.imwrite('foo.png', util.colorbar_image(np.linspace(0, 1, 100), dpi=200, shape=(1000, 40), transparent=1))
ub.startfile('foo.png')
"""
import kwplot
plt = kwplot.autoplt()
fig = plt.figure(dpi=dpi)
w, h = shape[1] / dpi, shape[0] / dpi
# w, h = 1, 10
fig.set_size_inches(w, h)
ax = fig.add_subplot('111')
sm = plt.cm.ScalarMappable(cmap=plt.get_cmap(cmap))
sm.set_array(domain)
plt.colorbar(sm, cax=ax)
cb_img = render_figure_to_image(fig, dpi=dpi, transparent=transparent)
plt.close(fig)
return cb_img
def benchmark_ondisk_crop():
import kwplot
plt = kwplot.autoplt()
region = 'small_random'
dim = 3
# xdata = [64, 128, 256, 512]
# xdata = [64, 128, 256, 320, 512, 640, 768, 896, 1024]
# xdata = np.linspace(64, 4096, num=8).astype(np.int)
# xdata = np.linspace(64, 2048, num=8).astype(np.int)
# xdata = np.linspace(64, 1024, num=8).astype(np.int)
xdata = [256, 1024, 4096, 8192, 16384]
# xdata = [256, 1024, 4096, 8192]
xdata = [256, 1024, 2048]
# xdata = [256]
ydata = ub.ddict(list)
# for size in [64, 128, 256, 512, 1024, 2048, 4096]:
for size in xdata:
result = time_ondisk_crop(size, dim=dim, region=region, num=5)
for key, val in result.items():
min, mean, std = val
ydata[key].append(mean * 1e6)
# Sort legend by descending time taken on the largest image
ydata = ub.odict(sorted(ydata.items(), key=lambda i: -i[1][-1]))
kwplot.multi_plot(
xdata,
ydata,
ylabel='micro-seconds (us)',
xlabel='image size',
title='Chip region={} benchmark for {}D image data'.format(
region, dim),
# yscale='log',
ymin=1,
)
plt.show()
def main():
sns = kwplot.autosns() # NOQA
plt = kwplot.autoplt() # NOQA
if 1:
array = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
values = [-1, 0, 1, 4, 6, 6.1, 6.5, 7, 10.3, 10.5, 10.7, 15, 16]
if 0:
array = np.linspace(0, 30)
values = np.linspace(0, 30)
if 0:
xscale = 20
num = 20
array = np.array(sorted(np.random.rand(num) * xscale)).round()
values = np.hstack([
np.unique(np.random.choice(array, 3)),
np.random.rand(num // 2) * xscale
])
fig = kwplot.figure(fnum=1, doclf=1, pnum=(2, 1, 1))
ax = fig.gca()
plot_searchsorted_visualization(array, values, side='left', ax=ax)
ax.set_title('association = searchsorted(array, values, side=left)')
fig = kwplot.figure(fnum=1, doclf=0, pnum=(2, 1, 2))
ax = fig.gca()
plot_searchsorted_visualization(array, values, side='right', ax=ax)
ax.set_title('association = searchsorted(array, values, side=right)')
import ubelt as ub
fig.suptitle(
ub.codeblock('''
Notice:
side=left and side=right have the same result except when the value is
already in the array.
'''))
def main():
"""
Main function for the generic classification example with an undocumented
hack for the lrtest.
"""
harn = setup_harn()
harn.initialize()
if ub.argflag('--lrtest'):
# Undocumented hidden feature,
# Perform an LR-test, then resetup the harness. Optionally draw the
# results using matplotlib.
from netharn.prefit.lr_tests import lr_range_test
result = lr_range_test(harn,
init_value=1e-4,
final_value=0.5,
beta=0.3,
explode_factor=10,
num_iters=200)
if ub.argflag('--show'):
import kwplot
plt = kwplot.autoplt()
result.draw()
plt.show()
# Recreate a new version of the harness with the recommended LR.
config = harn.script_config.asdict()
config['lr'] = (result.recommended_lr * 10)
harn = setup_harn(**config)
harn.initialize()
# This starts the main loop which will run until the monitor's terminator
# criterion is satisfied. If the initialize step loaded a checkpointed that
# already met the termination criterion, then this will simply return.
deploy_fpath = harn.run()
# The returned deploy_fpath is the path to an exported netharn model.
# This model is the on with the best weights according to the monitor.
print('deploy_fpath = {!r}'.format(deploy_fpath))
return harn
def convert_camvid_raw_to_coco(camvid_raw_info):
"""
Converts the raw camvid format to an MSCOCO based format, ( which lets use
use kwcoco's COCO backend).
Example:
>>> # xdoctest: +REQUIRES(--download)
>>> camvid_raw_info = grab_raw_camvid()
>>> # test with a reduced set of data
>>> del camvid_raw_info['img_paths'][2:]
>>> del camvid_raw_info['mask_paths'][2:]
>>> dset = convert_camvid_raw_to_coco(camvid_raw_info)
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> kwplot.figure(fnum=1, pnum=(1, 2, 1))
>>> dset.show_image(gid=1)
>>> kwplot.figure(fnum=1, pnum=(1, 2, 2))
>>> dset.show_image(gid=2)
"""
import re
import kwimage
import kwcoco
print('Converting CamVid to MS-COCO format')
dset_root, img_paths, label_path, mask_paths = ub.take(
camvid_raw_info,
'dset_root, img_paths, label_path, mask_paths'.split(', '))
img_infos = {
'img_fname': img_paths,
'mask_fname': mask_paths,
}
keys = list(img_infos.keys())
next_vals = list(zip(*img_infos.values()))
image_items = [{k: v for k, v in zip(keys, vals)} for vals in next_vals]
dataset = {
'img_root': dset_root,
'images': [],
'categories': [],
'annotations': [],
}
lines = ub.readfrom(label_path).split('\n')
lines = [line for line in lines if line]
for line in lines:
color_text, name = re.split('\t+', line)
r, g, b = map(int, color_text.split(' '))
color = (r, g, b)
# Parse the special camvid format
cid = (r << 16) + (g << 8) + (b << 0)
cat = {
'id': cid,
'name': name,
'color': color,
}
dataset['categories'].append(cat)
for gid, img_item in enumerate(image_items, start=1):
img = {
'id': gid,
'file_name': img_item['img_fname'],
# nonstandard image field
'segmentation': img_item['mask_fname'],
}
dataset['images'].append(img)
dset = kwcoco.CocoDataset(dataset)
dset.rename_categories({'Void': 'background'})
assert dset.name_to_cat['background']['id'] == 0
dset.name_to_cat['background'].setdefault('alias', []).append('Void')
if False:
_define_camvid_class_hierarcy(dset)
if 1:
# TODO: Binarize CCs (and efficiently encode if possible)
import numpy as np
bad_info = []
once = False
# Add images
dset.remove_annotations(list(dset.index.anns.keys()))
for gid, img in ub.ProgIter(dset.imgs.items(),
desc='parse label masks'):
mask_fpath = join(dset_root, img['segmentation'])
rgb_mask = kwimage.imread(mask_fpath, space='rgb')
r, g, b = rgb_mask.T.astype(np.int64)
cid_mask = np.ascontiguousarray(rgb_to_cid(r, g, b).T)
cids = set(np.unique(cid_mask)) - {0}
for cid in cids:
if cid not in dset.cats:
if gid == 618:
# Handle a known issue with image 618
c_mask = (cid == cid_mask).astype(np.uint8)
total_bad = c_mask.sum()
if total_bad < 32:
if not once:
print(
'gid 618 has a few known bad pixels, ignoring them'
)
once = True
continue
else:
raise Exception('more bad pixels than expected')
else:
raise Exception(
'UNKNOWN cid = {!r} in gid={!r}'.format(cid, gid))
# bad_rgb = cid_to_rgb(cid)
# print('bad_rgb = {!r}'.format(bad_rgb))
# print('WARNING UNKNOWN cid = {!r} in gid={!r}'.format(cid, gid))
# bad_info.append({
# 'gid': gid,
# 'cid': cid,
# })
else:
ann = {
'category_id': cid,
'image_id': gid
# 'segmentation': mask.to_coco()
}
assert cid in dset.cats
c_mask = (cid == cid_mask).astype(np.uint8)
mask = kwimage.Mask(c_mask, 'c_mask')
box = kwimage.Boxes([mask.get_xywh()], 'xywh')
# box = mask.to_boxes()
ann['bbox'] = ub.peek(box.to_coco())
ann['segmentation'] = mask.to_coco()
dset.add_annotation(**ann)
if 0:
bad_cids = [i['cid'] for i in bad_info]
print(sorted([c['color'] for c in dataset['categories']]))
print(sorted(set([cid_to_rgb(i['cid']) for i in bad_info])))
gid = 618
img = dset.imgs[gid]
mask_fpath = join(dset_root, img['segmentation'])
rgb_mask = kwimage.imread(mask_fpath, space='rgb')
r, g, b = rgb_mask.T.astype(np.int64)
cid_mask = np.ascontiguousarray(rgb_to_cid(r, g, b).T)
cid_hist = ub.dict_hist(cid_mask.ravel())
bad_cid_hist = {}
for cid in bad_cids:
bad_cid_hist[cid] = cid_hist.pop(cid)
import kwplot
kwplot.autompl()
kwplot.imshow(rgb_mask)
if 0:
import kwplot
plt = kwplot.autoplt()
plt.clf()
dset.show_image(1)
import xdev
gid_list = list(dset.imgs)
for gid in xdev.InteractiveIter(gid_list):
dset.show_image(gid)
xdev.InteractiveIter.draw()
dset._build_index()
dset._build_hashid()
return dset
def mwe_check_sample_weight():
import ubelt as ub
from sklearn.utils.validation import _check_sample_weight
import numpy as np
results = []
ns = np.logspace(1, 6, 100).astype(np.int)
for n in ub.ProgIter(ns, desc='time-tradeoff', verbose=3):
print('n = {!r}'.format(n))
y_true = np.random.randint(0, 100, n).astype(np.int64)
y_pred = np.random.randint(0, 100, n).astype(np.int64)
sample_weight = np.random.rand(n)
import timerit
ti = timerit.Timerit(9, bestof=3, verbose=2)
for timer in ti.reset('old-sample-weight-given'):
with timer:
np.asarray(sample_weight)
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
for timer in ti.reset('new-sample-weight-given'):
with timer:
_check_sample_weight(sample_weight, y_true, dtype=np.int64)
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
for timer in ti.reset('old-sample-weight-default'):
with timer:
np.ones(y_true.shape[0], dtype=np.int64)
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
for timer in ti.reset('new-sample-weight-default'):
with timer:
_check_sample_weight(None, y_true, dtype=np.int64)
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
import pandas as pd
df = pd.DataFrame(results)
import kwplot
import seaborn as sns
kwplot.autoplt()
sns.set()
ax = sns.lineplot(data=df, x='n', y='time', hue='label')
ax.set_yscale('log')
ax.set_xscale('log')
from sklearn.utils.validation import _check_sample_weight
import numpy as np
results = []
ns = np.logspace(1, 6, 100).astype(np.int)
for n in ub.ProgIter(ns, desc='time-tradeoff', verbose=3):
print('n = {!r}'.format(n))
n_labels = 100
y_true = np.random.randint(0, n_labels, n).astype(np.int64)
y_pred = np.random.randint(0, n_labels, n).astype(np.int64)
sample_weight = np.ones(y_true.shape[0], dtype=np.int64)
for timer in ti.reset('use-old-uint8-sample-weight-default'):
with timer:
if sample_weight.dtype.kind in {'i', 'u', 'b'}:
dtype = np.int64
else:
dtype = np.float64
cm = coo_matrix((sample_weight, (y_true, y_pred)),
shape=(n_labels, n_labels), dtype=dtype,
).toarray()
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
sample_weight = _check_sample_weight(None, y_true, dtype=np.int64)
for timer in ti.reset('use-new-float64-sample-weight-default'):
with timer:
if sample_weight.dtype.kind in {'i', 'u', 'b'}:
dtype = np.int64
else:
dtype = np.float64
cm = coo_matrix((sample_weight, (y_true, y_pred)),
shape=(n_labels, n_labels), dtype=dtype,
).toarray()
results.append({
'n': n,
'label': ti.label,
'time': ti.mean(),
})
import pandas as pd
df = pd.DataFrame(results)
import kwplot
import seaborn as sns
kwplot.autoplt()
sns.set()
ax = sns.lineplot(data=df, x='n', y='time', hue='label')
ax.set_yscale('log')
ax.set_xscale('log')
def eval_detections_cli(**kw):
"""
CommandLine:
xdoctest -m ~/code/netharn/netharn/metrics/detect_metrics.py eval_detections_cli
"""
import scriptconfig as scfg
import ndsampler
class EvalDetectionCLI(scfg.Config):
default = {
'true': scfg.Path(None, help='true coco dataset'),
'pred': scfg.Path(None, help='predicted coco dataset'),
'out_dpath': scfg.Path('./out', help='output directory')
}
pass
config = EvalDetectionCLI()
cmdline = kw.pop('cmdline', True)
config.load(kw, cmdline=cmdline)
true_coco = ndsampler.CocoDataset(config['true'])
pred_coco = ndsampler.CocoDataset(config['pred'])
from netharn.metrics.detect_metrics import DetectionMetrics
dmet = DetectionMetrics.from_coco(true_coco, pred_coco)
voc_info = dmet.score_voc()
cls_info = voc_info['perclass'][0]
tp = cls_info['tp']
fp = cls_info['fp']
fn = cls_info['fn']
tpr = cls_info['tpr']
ppv = cls_info['ppv']
fp = cls_info['fp']
# Compute the MCC as TN->inf
thresh = cls_info['thresholds']
# https://erotemic.wordpress.com/2019/10/23/closed-form-of-the-mcc-when-tn-inf/
mcc_lim = tp / (np.sqrt(fn + tp) * np.sqrt(fp + tp))
f1 = 2 * (ppv * tpr) / (ppv + tpr)
draw = False
if draw:
mcc_idx = mcc_lim.argmax()
f1_idx = f1.argmax()
import kwplot
plt = kwplot.autoplt()
kwplot.multi_plot(
xdata=thresh,
ydata=mcc_lim,
xlabel='threshold',
ylabel='mcc*',
fnum=1,
pnum=(1, 4, 1),
title='MCC*',
color=['blue'],
)
plt.plot(thresh[mcc_idx], mcc_lim[mcc_idx], 'r*', markersize=20)
plt.plot(thresh[f1_idx], mcc_lim[f1_idx], 'k*', markersize=20)
kwplot.multi_plot(
xdata=fp,
ydata=tpr,
xlabel='fp (fa)',
ylabel='tpr (pd)',
fnum=1,
pnum=(1, 4, 2),
title='ROC',
color=['blue'],
)
plt.plot(fp[mcc_idx], tpr[mcc_idx], 'r*', markersize=20)
plt.plot(fp[f1_idx], tpr[f1_idx], 'k*', markersize=20)
kwplot.multi_plot(
xdata=tpr,
ydata=ppv,
xlabel='tpr (recall)',
ylabel='ppv (precision)',
fnum=1,
pnum=(1, 4, 3),
title='PR',
color=['blue'],
)
plt.plot(tpr[mcc_idx], ppv[mcc_idx], 'r*', markersize=20)
plt.plot(tpr[f1_idx], ppv[f1_idx], 'k*', markersize=20)
kwplot.multi_plot(
xdata=thresh,
ydata=f1,
xlabel='threshold',
ylabel='f1',
fnum=1,
pnum=(1, 4, 4),
title='F1',
color=['blue'],
)
plt.plot(thresh[mcc_idx], f1[mcc_idx], 'r*', markersize=20)
plt.plot(thresh[f1_idx], f1[f1_idx], 'k*', markersize=20)
def _precompute_class_weights(dset, mode='median-idf'):
"""
Example:
>>> # xdoctest: +REQUIRES(--download)
>>> import sys, ubelt
>>> sys.path.append(ubelt.expandpath('~/code/netharn/examples'))
>>> from sseg_camvid import * # NOQA
>>> harn = setup_harn(0, workers=0, xpu='cpu').initialize()
>>> dset = harn.datasets['train']
"""
assert mode in ['median-idf', 'log-median-idf']
total_freq = _cached_class_frequency(dset)
def logb(arr, base):
if base == 'e':
return np.log(arr)
elif base == 2:
return np.log2(arr)
elif base == 10:
return np.log10(arr)
else:
out = np.log(arr)
out /= np.log(base)
return out
_min, _max = np.percentile(total_freq, [5, 95])
is_valid = (_min <= total_freq) & (total_freq <= _max)
if np.any(is_valid):
middle_value = np.median(total_freq[is_valid])
else:
middle_value = np.median(total_freq)
# variant of median-inverse-frequency
nonzero_freq = total_freq[total_freq != 0]
if len(nonzero_freq):
total_freq[total_freq == 0] = nonzero_freq.min() / 2
if mode == 'median-idf':
weights = (middle_value / total_freq)
weights[~np.isfinite(weights)] = 1.0
elif mode == 'log-median-idf':
weights = (middle_value / total_freq)
weights[~np.isfinite(weights)] = 1.0
base = 2
base = np.exp(1)
weights = logb(weights + (base - 1), base)
weights = np.maximum(weights, .1)
weights = np.minimum(weights, 10)
else:
raise KeyError('mode = {!r}'.format(mode))
weights = np.round(weights, 2)
cname_to_weight = ub.dzip(dset.classes, weights)
print('weights: ' + ub.repr2(cname_to_weight))
if False:
# Inspect the weights
import kwplot
kwplot.autoplt()
cname_to_weight = ub.dzip(dset.classes, weights)
cname_to_weight = ub.dict_subset(cname_to_weight, ub.argsort(cname_to_weight))
kwplot.multi_plot(
ydata=list(cname_to_weight.values()),
kind='bar',
xticklabels=list(cname_to_weight.keys()),
xtick_rotation=90,
fnum=2, doclf=True)
return weights
def main():
import kwplot
plt = kwplot.autoplt()
sns = kwplot.autosns()
alias = {
'3090': 'nvctrl GeForce GTX 1080 Ti 1 temp',
'1080ti': 'nvctrl GeForce RTX 3090 0 temp',
# 'cpu': 'lmsensor coretemp-isa-0000 Package id 0',
}
all_df = read_psensor_log()
unique_rawdevs = all_df.device.unique()
for rawdev in unique_rawdevs:
cpu_prefix = 'lmsensor coretemp-isa'
if rawdev.startswith(cpu_prefix):
suffix = rawdev[len(cpu_prefix):].split(' ', 1)[1].strip()
alias['CPU ' + suffix] = rawdev
if 'nvctrl' in rawdev and 'temp' in rawdev:
alias['GPU ' + rawdev[7:-5]] = rawdev
mapper = ub.invert_dict(alias)
all_df['device'] = all_df['device'].apply(lambda x: mapper.get(x, None))
all_df = all_df[all_df['device'].apply(lambda x: x is not None)]
hours = int(ub.argval('--hours', default=48))
delta = datetime.timedelta(hours=hours)
min_time = datetime.datetime.now() - delta
is_recent = all_df.datetime > min_time
recent_df = all_df[is_recent]
chosen = recent_df
# chosen = all_df
if 0:
pivtbl = recent_df.pivot('unix_timestamp', 'device', 'temp')
pivtbl = pivtbl.sort_index()
smoothed_rows = []
for window_idxs in ub.iter_window(list(range(len(pivtbl))), size=10):
window = pivtbl.iloc[list(window_idxs)]
max_val = window.max(axis=0, skipna=True)
for k, v in max_val.to_dict().items():
smoothed_rows.append({
'unix_timestamp': window.index[1],
'device': k,
'temp': v,
})
max_extra = pd.DataFrame(smoothed_rows)
sns.lineplot(data=max_extra,
x='unix_timestamp',
y='temp',
hue='device')
df = recent_df.copy()
df['device'] = df['device'].apply(lambda x: 'Core'
if x.startswith('Core') else x)
df['time'] = df['unix_timestamp'].apply(
datetime.datetime.fromtimestamp)
plt.gcf().clf()
# sns.lineplot(data=chosen, x='unix_timestamp', y='temp', hue='device')
for xx, (sess, group) in enumerate(chosen.groupby('session_x')):
# ax.cla()
ax = plt.gca()
sns.lineplot(data=group,
x='unix_timestamp',
y='temp',
hue='device',
legend=xx == 0)
label_xaxis_dates(ax)
ax.figure.subplots_adjust(bottom=0.2)
ax.set_ylim(0, 100)
plt.locator_params(axis='y', nbins=10)
# import matplotlib as mpl
# Draw shutdown time as black lines
end_times = []
for sx, group in chosen.groupby('session_x'):
shutdown_time = group['unix_timestamp'].max()
end_times.append(shutdown_time)
for shutdown_time in sorted(end_times)[:-1]:
ax.plot((shutdown_time, shutdown_time), [0, 100], color='k')
# ci_df = pd.concat([max_extra, recent_df])
# ci_df['device'] = ci_df['device'].apply(lambda x: 'Core' if x.startswith('Core') else x)
# sns.lineplot(data=ci_df, x='unix_timestamp', y='temp', hue='device')
# from matplotlib.dates import date2num
# all_df['date_ord'] = all_df['datetime'].map(lambda a: date2num(a))
# sns.lineplot(data=pt)
# sns.lineplot(data=recent_df, x='unix_timestamp', y='temp', hue='device')
# sns.regplot(data=recent_df, x='unix_timestamp', y='temp', hue='device')
plt.show()
def plot_weight_scatter(harn):
"""
Draw a scatter plot of the initial weights versus the final weights of a
network.
Example:
>>> import netharn as nh
>>> harn = nh.FitHarn.demo()
>>> harn.run()
Ignore:
>>> from netharn.plots.weight_scatter import * # NOQA
>>> from netharn.examples import mnist
>>> import kwplot
>>> harn = mnist.setup_harn()
>>> harn.preferences['timeout'] = 60 * 1
>>> kwplot.autompl(force='agg')
>>> harn.run()
>>> kwplot.autompl(force='auto')
>>> plot_weight_scatter(harn)
"""
import netharn as nh
cpu = nh.XPU.coerce('cpu')
path1 = join(harn.train_dpath, 'initial_state', 'initial_state.pt')
state1 = cpu.load(path1)
weights1 = state1['model_state_dict']
path2 = harn.best_snapshot()
state2 = cpu.load(path2)
weights2 = state2['model_state_dict']
keys1 = set(weights1.keys())
keys2 = set(weights2.keys())
keys = keys1 & keys2
assert keys == keys2
accum1 = []
accum2 = []
for key in keys:
w1 = weights1[key]
w2 = weights2[key]
accum1.append(w1.numpy().ravel())
accum2.append(w2.numpy().ravel())
points1 = np.hstack(accum1)
points2 = np.hstack(accum2)
# Find cosine of angle between the vectors
import scipy
cosangle = scipy.spatial.distance.cosine(points1, points2)
print('cosangle = {!r}'.format(cosangle))
import kwplot
import seaborn
seaborn.set()
plt = kwplot.autoplt()
plt.clf()
x = points1[::1]
y = points2[::1]
ax = plt.gca()
ax.figure.clf()
# seaborn.kdeplot(x, y, shade=True, gridsize=50)
ax = plt.gca()
ax.scatter(x, y, s=1, alpha=0.1, c='blue')
ax.set_xlabel('initial weights')
ax.set_ylabel('trained weights')
def benchmark_video_readers():
"""
"On My Machine" I get:
ti.measures = {
'mean' : {
'cv2 sequential access' : 0.0137,
'decord sequential access' : 0.0175,
'cv2 open + first access' : 0.0222,
'decord open + first access' : 0.0565,
'vi3o sequential access' : 0.0642,
'cv2 open + one random access' : 0.0723,
'decord open + one random access': 0.0946,
'vi3o open + first access' : 0.1045,
'cv2 random access' : 0.3316,
'decord random access' : 0.3472,
'decord random batch access' : 0.3482,
'vi3o open + one random access' : 0.3590,
'vi3o random access' : 1.6660,
},
'mean+std': {
'cv2 sequential access' : 0.0145,
'decord sequential access' : 0.0182,
'cv2 open + first access' : 0.0230,
'vi3o sequential access' : 0.0881,
'decord open + first access' : 0.1038,
'vi3o open + first access' : 0.1059,
'cv2 open + one random access' : 0.1151,
'decord open + one random access': 0.1329,
'cv2 random access' : 0.3334,
'decord random access' : 0.3496,
'decord random batch access' : 0.3511,
'vi3o open + one random access' : 0.5215,
'vi3o random access' : 1.6890,
},
'mean-std': {
'decord open + first access' : 0.0091,
'cv2 sequential access' : 0.0130,
'decord sequential access' : 0.0168,
'cv2 open + first access' : 0.0214,
'cv2 open + one random access' : 0.0295,
'vi3o sequential access' : 0.0403,
'decord open + one random access': 0.0563,
'vi3o open + first access' : 0.1032,
'vi3o open + one random access' : 0.1965,
'cv2 random access' : 0.3299,
'decord random access' : 0.3448,
'decord random batch access' : 0.3452,
'vi3o random access' : 1.6429,
},
'min' : {
'cv2 sequential access' : 0.0128,
'decord sequential access' : 0.0166,
'cv2 open + first access' : 0.0210,
'vi3o sequential access' : 0.0233,
'decord open + first access' : 0.0251,
'cv2 open + one random access' : 0.0282,
'decord open + one random access': 0.0527,
'vi3o open + one random access' : 0.1013,
'vi3o open + first access' : 0.1026,
'cv2 random access' : 0.3299,
'decord random access' : 0.3433,
'decord random batch access' : 0.3452,
'vi3o random access' : 1.6423,
},
}
"""
# video_fpath = ub.grabdata('https://download.blender.org/peach/bigbuckbunny_movies/big_buck_bunny_720p_h264.mov')
try:
import vi3o
except Exception:
vi3o = None
# video_fpath = ub.grabdata('https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4')
video_fpath = ub.grabdata('https://file-examples-com.github.io/uploads/2018/04/file_example_MOV_1280_1_4MB.mov')
ti = timerit.Timerit(10, bestof=3, verbose=3, unit='ms')
video_length = len(CV2VideoReader(video_fpath))
num_frames = min(5, video_length)
rng = kwarray.ensure_rng(0)
random_indices = rng.randint(0, video_length, size=num_frames).tolist()
if True:
with timerit.Timer(label='open cv2') as cv2_open_timer:
cv2_video = CV2VideoReader(video_fpath)
for timer in ti.reset('cv2 sequential access'):
cv2_video.seek(0)
with timer:
for frame, _ in zip(cv2_video, range(num_frames)):
pass
for timer in ti.reset('cv2 random access'):
with timer:
for index in random_indices:
cv2_video[index]
if vi3o is not None:
with timerit.Timer(label='open vi3o') as vi3o_open_timer:
vi3o_video = vi3o.Video(video_fpath)
for timer in ti.reset('vi3o sequential access'):
with timer:
for frame, _ in zip(vi3o_video, range(num_frames)):
pass
for timer in ti.reset('vi3o random access'):
with timer:
for index in random_indices:
vi3o_video[index]
if True:
import decord
with timerit.Timer(label='open decord') as decord_open_timer:
decord_video = decord.VideoReader(video_fpath)
for timer in ti.reset('decord sequential access'):
with timer:
for frame, _ in zip(decord_video, range(num_frames)):
pass
for timer in ti.reset('decord random access'):
with timer:
for index in random_indices:
decord_video[index]
for timer in ti.reset('decord random batch access'):
with timer:
decord_video.get_batch(random_indices)
if True:
# One Random Access Case
def _work_to_clear_io_caches():
import kwimage
# Let some caches be cleared
for i in range(10):
for key in kwimage.grab_test_image.keys():
kwimage.grab_test_image(key)
rng = kwarray.ensure_rng(0)
for timer in ti.reset('cv2 open + one random access'):
_work_to_clear_io_caches()
with timer:
_cv2_video = CV2VideoReader(video_fpath)
index = rng.randint(0, video_length, size=1)[0]
_cv2_video[index]
if vi3o is not None:
rng = kwarray.ensure_rng(0)
for timer in ti.reset('vi3o open + one random access'):
_work_to_clear_io_caches()
with timer:
_vi3o_video = vi3o.Video(video_fpath)
index = rng.randint(0, video_length, size=1)[0]
_vi3o_video[index]
rng = kwarray.ensure_rng(0)
for timer in ti.reset('decord open + one random access'):
_work_to_clear_io_caches()
with timer:
_decord_video = decord.VideoReader(video_fpath)
index = rng.randint(0, video_length, size=1)[0]
_decord_video[index]
for timer in ti.reset('cv2 open + first access'):
_work_to_clear_io_caches()
with timer:
_cv2_video = CV2VideoReader(video_fpath)
_cv2_video[0]
if vi3o is not None:
for timer in ti.reset('vi3o open + first access'):
_work_to_clear_io_caches()
with timer:
_vi3o_video = vi3o.Video(video_fpath)
_vi3o_video[0]
for timer in ti.reset('decord open + first access'):
_work_to_clear_io_caches()
with timer:
_decord_video = decord.VideoReader(video_fpath)
_decord_video[0]
measures = ub.map_vals(ub.sorted_vals, ti.measures)
print('ti.measures = {}'.format(ub.repr2(measures, nl=2, align=':', precision=4)))
print('cv2_open_timer.elapsed = {!r}'.format(cv2_open_timer.elapsed))
print('decord_open_timer.elapsed = {!r}'.format(decord_open_timer.elapsed))
if vi3o:
print('vi3o_open_timer.elapsed = {!r}'.format(vi3o_open_timer.elapsed))
import kwplot
import seaborn as sns
sns.set()
plt = kwplot.autoplt()
df = pd.DataFrame(ti.measures)
df['key'] = df.index
df['expt'] = df['key'].apply(lambda k: ' '.join(k.split(' ')[1:]))
df['module'] = df['key'].apply(lambda k: k.split(' ')[0])
# relmod = 'decord'
relmod = 'cv2'
for k, group in df.groupby('expt'):
measure = 'mean'
relval = group[group['module'] == relmod][measure].values.ravel()
if len(relval) > 0:
assert len(relval) == 1
df.loc[group.index, measure + '_rel'] = group[measure] / relval
df.loc[group.index, measure + '_slower_than_' + relmod] = group[measure] / relval
df.loc[group.index, measure + '_faster_than_' + relmod] = relval / group[measure]
fig = kwplot.figure(fnum=1, doclf=True)
ax = fig.gca()
y_key = "mean_faster_than_" + relmod
sub_df = df.loc[~df[y_key].isnull()]
sns.barplot(
x="expt", y=y_key, data=sub_df, hue='module', ax=ax)
ax.set_title('cpu video reading benchmarks')
plt.show()
def make_legend_img(label_to_color,
dpi=96,
shape=(200, 200),
mode='line',
transparent=False):
"""
Makes an image of a categorical legend
Args:
label_to_color (Dict[str, Color]): mapping from string label to the
color.
CommandLine:
xdoctest -m kwplot.mpl_make make_legend_img --show
Example:
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> import kwplot
>>> import kwimage
>>> label_to_color = {
>>> 'blue': kwimage.Color('blue').as01(),
>>> 'red': kwimage.Color('red').as01(),
>>> 'green': 'green',
>>> 'yellow': 'yellow',
>>> 'orangered': 'orangered',
>>> }
>>> img = make_legend_img(label_to_color, transparent=True)
>>> # xdoctest: +REQUIRES(--show)
>>> kwplot.autompl()
>>> kwplot.imshow(img)
>>> kwplot.show_if_requested()
"""
import kwplot
import kwimage
plt = kwplot.autoplt()
def append_phantom_legend_label(label,
color,
type_='line',
alpha=1.0,
ax=None):
if ax is None:
ax = plt.gca()
_phantom_legend_list = getattr(ax, '_phantom_legend_list', None)
if _phantom_legend_list is None:
_phantom_legend_list = []
setattr(ax, '_phantom_legend_list', _phantom_legend_list)
if type_ == 'line':
phantom_actor = plt.Line2D((0, 0), (1, 1),
color=color,
label=label,
alpha=alpha)
else:
phantom_actor = plt.Circle((0, 0),
1,
fc=color,
label=label,
alpha=alpha)
_phantom_legend_list.append(phantom_actor)
fig = plt.figure(dpi=dpi)
w, h = shape[1] / dpi, shape[0] / dpi
fig.set_size_inches(w, h)
# ax = fig.add_subplot('111')
ax = fig.add_subplot(1, 1, 1)
for label, color in label_to_color.items():
color = kwimage.Color(color).as01()
append_phantom_legend_label(label, color, type_=mode, ax=ax)
_phantom_legend_list = getattr(ax, '_phantom_legend_list', None)
if _phantom_legend_list is None:
_phantom_legend_list = []
setattr(ax, '_phantom_legend_list', _phantom_legend_list)
ax.legend(handles=_phantom_legend_list)
ax.grid(False)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.axis('off')
legend_img = render_figure_to_image(fig, dpi=dpi, transparent=transparent)
legend_img = kwimage.convert_colorspace(legend_img,
src_space='bgr',
dst_space='rgb')
legend_img = crop_border_by_color(legend_img)
plt.close(fig)
return legend_img
def ford_circles():
"""
Draw Ford Circles
This is a Ford Circle diagram of the Rationals and Float32 numbers.
Only 163 of the 32608 rationals I generated can be exactly represented by a float32.
[MF 14]
[MF 95]
[MF 14] https://www.youtube.com/watch?v=83ZjYvkdzYI&list=PL5A714C94D40392AB&index=14
[MF 95] https://www.youtube.com/watch?v=gATEJ3f3FBM&list=PL5A714C94D40392AB&index=95
Examples:
import kwplot
kwplot.autompl()
"""
import kwplot
import ubelt as ub
import matplotlib as mpl
plt = kwplot.autoplt()
sns = kwplot.autosns() # NOQA
limit = 256 * 256
print('limit = {!r}'.format(limit))
rats_to_plot = set()
maxx = 1
_iter = Rational.members(limit=limit)
_genrat = set(ub.ProgIter(_iter, total=limit, desc='gen rats'))
rats_to_plot |= _genrat
rats_to_plot2 = sorted({Rational(r % maxx) for r in rats_to_plot} | {maxx})
floats = sorted(
ub.unique(map(float, rats_to_plot2),
key=lambda f: f.as_integer_ratio()))
print(f'{len(rats_to_plot) = }')
print(f'{len(rats_to_plot2) = }')
print(f'{len(floats) = }')
import numpy as np
ax = kwplot.figure(fnum=1, doclf=True).gca()
prog = ub.ProgIter(sorted(rats_to_plot2), verbose=1)
dtype = np.float32
patches = ub.ddict(list)
errors = []
for rat in prog:
denominator = rat.denominator
radius = 1 / (2 * (denominator * denominator))
point = (rat, radius)
flt = dtype(rat)
a, b = flt.as_integer_ratio()
flt_as_rat = Rational(a, b)
error = abs(rat - flt_as_rat)
if error == 0:
new_circle = plt.Circle(point,
radius,
facecolor='dodgerblue',
edgecolor='none',
linewidth=0,
alpha=0.5)
patches['good'].append(new_circle)
else:
errors.append(error)
# Plot a line for error
new_circle = plt.Circle(point,
radius,
facecolor='orangered',
edgecolor='none',
linewidth=0,
alpha=0.5)
patches['bad'].append(new_circle)
ax.plot((rat - error, rat + error), (radius, radius),
'x-',
color='darkgray')
print(ub.map_vals(len, patches))
total = float(sum(errors))
print('total = {!r}'.format(total))
print(max(errors))
print(min(errors))
for v in patches.values():
first = ub.peek(v)
prop = ub.dict_isect(first.properties(),
['facecolor', 'linewidth', 'alpha', 'edgecolor'])
col = mpl.collections.PatchCollection(v, **prop)
ax.add_collection(col)
# Lets look for the holes in IEEE float
# for flt in ub.ProgIter(sorted(floats), verbose=1):
kwplot.phantom_legend({
f'rationals without a {dtype}':
'orangered',
f'rationals with a {dtype}':
'dodgerblue',
f'x-x indicates {dtype} approximation error':
'darkgray',
})
ax.set_title('Holes in IEEE 754 Float64')
ax.set_xlabel('A rational number')
ax.set_ylabel('The squared rational denominator')
# import numpy as np
# points = np.array([c.center for c in _circles])
# maxx, maxy = points.max(axis=0)
# print('maxx = {!r}'.format(maxx))
# print('maxy = {!r}'.format(maxy))
# maxx, maxy = maxx // 2, maxy // 2
# ax.set_xlim(0, np.sqrt(int(maxx)))
# ax.set_ylim(0, np.sqrt(int(maxy)))
# ax.set_aspect('equal')
# ax.set_xlim(0.2, 0.22)
ax.set_xlim(0, 1)
ax.set_ylim(0, 0.1)
def main(cls, cmdline=True, **kw):
"""
TODO:
- [ ] Visualize auxiliary data
Example:
>>> # xdoctest: +SKIP
>>> kw = {'src': 'special:shapes8'}
>>> cmdline = False
>>> cls = CocoShowCLI
>>> cls.main(cmdline, **kw)
"""
import kwcoco
import kwimage
import kwplot
config = cls.CLIConfig(kw, cmdline=cmdline)
print('config = {}'.format(ub.repr2(dict(config), nl=1)))
if config['src'] is None:
raise Exception('must specify source: {}'.format(config['src']))
dset = kwcoco.CocoDataset.coerce(config['src'])
print('dset.fpath = {!r}'.format(dset.fpath))
plt = kwplot.autoplt()
gid = config['gid']
aid = config['aid']
out_fpath = config['dst']
if gid is None and aid is None:
gid = ub.peek(dset.imgs)
if config['mode'] == 'matplotlib':
show_kw = {
'show_annots': config['show_annots'],
'channels': config['channels'],
}
if config['show_annots'] == 'both':
show_kw.pop('show_annots')
show_kw['title'] = ''
ax = dset.show_image(gid=gid,
aid=aid,
pnum=(1, 2, 1),
show_annots=False,
**show_kw)
ax = dset.show_image(gid=gid,
aid=aid,
pnum=(1, 2, 2),
show_annots=True,
**show_kw)
else:
ax = dset.show_image(gid=gid, aid=aid, **show_kw)
if out_fpath is None:
if 1:
try:
import xdev
except Exception:
pass
else:
gids = [gid] + list(set(dset.imgs.keys()) - {gid})
for gid in xdev.InteractiveIter(gids):
ax = dset.show_image(gid=gid, aid=aid, **show_kw)
xdev.InteractiveIter.draw()
plt.show(block=False)
plt.show()
else:
ax.figure.savefig(out_fpath)
elif config['mode'] == 'opencv':
canvas = dset.draw_image(gid, channels=config['channels'])
if out_fpath is None:
kwplot.imshow(canvas)
plt.show()
else:
kwimage.imwrite(out_fpath, canvas)
else:
raise KeyError(config['mode'])
def main():
# Regress a 3D surface as in https://stackoverflow.com/a/53151677/887074
# Mapping from 2D coordinates to the elevation in the 3rd dimension
import netharn as nh
import numpy as np
import torch
import ubelt as ub
num_train = 100
TRAIN_SURFACE = 'rosenbrock'
if TRAIN_SURFACE == 'random':
train_points = torch.rand(num_train, 3)
train_XY = train_points[:, 0:2]
train_X = train_points[:, 0:1]
train_Y = train_points[:, 1:2]
train_Z = train_points[:, 2:3]
elif TRAIN_SURFACE == 'rosenbrock':
# Train with the Rosenbrock function
# https://en.wikipedia.org/wiki/Rosenbrock_function
train_points = torch.rand(num_train, 2)
train_XY = train_points[:, 0:2]
train_X = train_points[:, 0:1]
train_Y = train_points[:, 1:2]
a = 1
b = 100
train_Z = (a - train_X)**2 + b * (train_Y - train_X**2)**2 + 2
np_train_X = train_X.data.cpu().numpy()
np_train_Y = train_Y.data.cpu().numpy()
np_train_Z = train_Z.data.cpu().numpy()
test_resolution = 100
xbasis = np.linspace(0, 1, test_resolution).astype(np.float32)
ybasis = np.linspace(0, 1, test_resolution).astype(np.float32)
X, Y = np.meshgrid(xbasis, ybasis)
test_X = X.ravel()[:, None]
test_Y = Y.ravel()[:, None]
test_XY = np.concatenate([test_X, test_Y], axis=1)
test_XY = torch.from_numpy(test_XY)
import kwplot
plt = kwplot.autoplt()
from mpl_toolkits.mplot3d import Axes3D # NOQA
ax = plt.gca(projection='3d')
ax.cla()
# Plot the training data
train_data_pc = ax.scatter3D(np_train_X,
np_train_Y,
np_train_Z,
color='red')
model = nh.layers.MultiLayerPerceptronNd(dim=0,
in_channels=2,
hidden_channels=[100] * 10,
out_channels=1,
bias=False)
optim = torch.optim.Adam(model.parameters(), lr=1e-2)
max_iters = 100
if 0:
iter_ = ub.ProgIter(range(max_iters), desc='iterate')
else:
import xdev
iter_ = xdev.InteractiveIter(list(range(max_iters)))
poly3d_pc = None
for iter_idx in iter_:
optim.zero_grad()
pred_Z = model.forward(train_XY)
loss = torch.nn.functional.mse_loss(pred_Z, train_Z)
loss.backward()
optim.step()
test_Z = model.forward(test_XY).data.cpu().numpy()
param_total = sum(p.sum() for p in model.parameters())
print('param_total = {!r}'.format(param_total))
if hasattr(iter_, 'draw'):
num = test_X.shape[0]
s = np.sqrt(num)
assert s % 1 == 0
s = int(s)
X = test_X.reshape(s, s)
Y = test_Y.reshape(s, s)
Z = test_Z.reshape(s, s)
if poly3d_pc is not None:
# Remove previous surface
poly3d_pc.remove()
poly3d_pc = ax.plot_surface(X, Y, Z, color='blue')
iter_.draw()
def _devcheck_corner():
self = DelayedWarp.random(rng=0)
print(self.nesting())
region_slices = (slice(40, 90), slice(20, 62))
region_box = kwimage.Boxes.from_slice(region_slices, shape=self.shape)
region_bounds = region_box.to_polygons()[0]
for leaf in self._optimize_paths():
pass
tf_leaf_to_root = leaf['transform']
tf_root_to_leaf = np.linalg.inv(tf_leaf_to_root)
leaf_region_bounds = region_bounds.warp(tf_root_to_leaf)
leaf_region_box = leaf_region_bounds.bounding_box().to_ltrb()
leaf_crop_box = leaf_region_box.quantize()
lt_x, lt_y, rb_x, rb_y = leaf_crop_box.data[0, 0:4]
root_crop_corners = leaf_crop_box.to_polygons()[0].warp(tf_leaf_to_root)
# leaf_crop_slices = (slice(lt_y, rb_y), slice(lt_x, rb_x))
crop_offset = leaf_crop_box.data[0, 0:2]
corner_offset = leaf_region_box.data[0, 0:2]
offset_xy = crop_offset - corner_offset
tf_root_to_leaf
# NOTE:
# Cropping applies a translation in whatever space we do it in
# We need to save the bounds of the crop.
# But now we need to adjust the transform so it points to the
# cropped-leaf-space not just the leaf-space, so we invert the implicit
# crop
tf_crop_to_leaf = Affine.affine(offset=crop_offset)
# tf_newroot_to_root = Affine.affine(offset=region_box.data[0, 0:2])
tf_root_to_newroot = Affine.affine(offset=region_box.data[0, 0:2]).inv()
tf_crop_to_leaf = Affine.affine(offset=crop_offset)
tf_crop_to_newroot = tf_root_to_newroot @ tf_leaf_to_root @ tf_crop_to_leaf
tf_newroot_to_crop = tf_crop_to_newroot.inv()
# tf_leaf_to_crop
# tf_corner_offset = Affine.affine(offset=offset_xy)
subpixel_offset = Affine.affine(offset=offset_xy).matrix
tf_crop_to_leaf = subpixel_offset
# tf_crop_to_root = tf_leaf_to_root @ tf_crop_to_leaf
# tf_root_to_crop = np.linalg.inv(tf_crop_to_root)
if 1:
import kwplot
kwplot.autoplt()
lw, lh = leaf['sub_data_shape'][0:2]
leaf_box = kwimage.Boxes([[0, 0, lw, lh]], 'xywh')
root_box = kwimage.Boxes([[0, 0, self.dsize[0], self.dsize[1]]],
'xywh')
ax1 = kwplot.figure(fnum=1, pnum=(2, 2, 1), doclf=1).gca()
ax2 = kwplot.figure(fnum=1, pnum=(2, 2, 2)).gca()
ax3 = kwplot.figure(fnum=1, pnum=(2, 2, 3)).gca()
ax4 = kwplot.figure(fnum=1, pnum=(2, 2, 4)).gca()
root_box.draw(setlim=True, ax=ax1)
leaf_box.draw(setlim=True, ax=ax2)
region_bounds.draw(ax=ax1, color='green', alpha=.4)
leaf_region_bounds.draw(ax=ax2, color='green', alpha=.4)
leaf_crop_box.draw(ax=ax2, color='purple')
root_crop_corners.draw(ax=ax1, color='purple', alpha=.4)
new_w = region_box.to_xywh().data[0, 2]
new_h = region_box.to_xywh().data[0, 3]
ax3.set_xlim(0, new_w)
ax3.set_ylim(0, new_h)
crop_w = leaf_crop_box.to_xywh().data[0, 2]
crop_h = leaf_crop_box.to_xywh().data[0, 3]
ax4.set_xlim(0, crop_w)
ax4.set_ylim(0, crop_h)
pts3_ = kwimage.Points.random(3).scale((new_w, new_h))
pts3 = kwimage.Points(
xy=np.vstack([[[0, 0], [5, 5], [0, 49], [40, 45]], pts3_.xy]))
pts4 = pts3.warp(tf_newroot_to_crop.matrix)
pts3.draw(ax=ax3)
pts4.draw(ax=ax4)
probs /= energy.sum()
# Do something with probs
# Get probabilities for the next state and update
updates = np.random.rand(D)
energy *= updates
def renormalize_demo_v4(D, T):
import numpy as np
import kwarray
energy = kwarray.fast_rand.uniform32(size=D)
probs = np.empty_like(energy)
for _ in range(T):
probs[:] = energy
probs /= energy.sum()
# Do something with probs
# Get probabilities for the next state and update
updates = kwarray.fast_rand.uniform32(size=D)
energy *= updates
if __name__ == '__main__':
"""
CommandLine:
python ~/misc/tests/python/bench_renormalization.py
"""
import kwplot
plt = kwplot.autoplt()
run_benchmark_renormalization()
plt.show()
def benchmark_template():
import ubelt as ub
import pandas as pd
import timerit
def method1(x, y, z):
ret = []
for i in range((x + y) * z):
ret.append(i)
return ret
def method2(x, y, z):
ret = [i for i in range((x + y) * z)]
return ret
method_lut = locals() # can populate this some other way
# Change params here to modify number of trials
ti = timerit.Timerit(100, bestof=10, verbose=1)
# if True, record every trail run and show variance in seaborn
# if False, use the standard timerit min/mean measures
RECORD_ALL = True
# These are the parameters that we benchmark over
basis = {
'method': ['method1', 'method2'],
'x': list(range(7)),
'y': [0, 100],
'z': [2, 3]
# 'param_name': [param values],
}
xlabel = 'x'
# Set these to param labels that directly transfer to method kwargs
kw_labels = ['x', 'y', 'z']
# Set these to empty lists if they are not used
group_labels = {
'style': ['y'],
'size': ['z'],
}
group_labels['hue'] = list((ub.oset(basis) - {xlabel}) -
set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(ub.dict_isect(
params, labels),
compact=1,
si=1)
key = ub.repr2(params, compact=1, si=1)
# Make any modifications you need to compute input kwargs for each
# method here.
kwargs = ub.dict_isect(params.copy(), kw_labels)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
with timer:
# Put the logic you want to time here
method(**kwargs)
if RECORD_ALL:
# Seaborn will show the variance if this is enabled, otherwise
# use the robust timerit mean / min times
chunk_iter = ub.chunks(ti.times, ti.bestof)
times = list(map(min, chunk_iter)) # TODO: timerit method for this
for time in times:
row = {
# 'mean': ti.mean(),
'time': time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
else:
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
time_key = 'time' if RECORD_ALL else 'min'
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values(time_key)
if RECORD_ALL:
# Show the min / mean if we record all
min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1)
mean_times = data.groupby('key')[['time'
]].mean().rename({'time': 'mean'},
axis=1)
stats_data = pd.concat([min_times, mean_times], axis=1)
stats_data = stats_data.sort_values('min')
else:
stats_data = data
USE_OPENSKILL = 1
if USE_OPENSKILL:
# Lets try a real ranking method
# https://github.com/OpenDebates/openskill.py
import openskill
method_ratings = {m: openskill.Rating() for m in basis['method']}
other_keys = sorted(
set(stats_data.columns) -
{'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'})
for params, variants in stats_data.groupby(other_keys):
variants = variants.sort_values('mean')
ranking = variants['method'].reset_index(drop=True)
mean_speedup = variants['mean'].max() / variants['mean']
stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup
min_speedup = variants['min'].max() / variants['min']
stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup
if USE_OPENSKILL:
# The idea is that each setting of parameters is a game, and each
# "method" is a player. We rank the players by which is fastest,
# and update their ranking according to the Weng-Lin Bayes ranking
# model. This does not take the fact that some "games" (i.e.
# parameter settings) are more important than others, but it should
# be fairly robust on average.
old_ratings = [[r] for r in ub.take(method_ratings, ranking)]
new_values = openskill.rate(old_ratings) # Not inplace
new_ratings = [openskill.Rating(*new[0]) for new in new_values]
method_ratings.update(ub.dzip(ranking, new_ratings))
print('Statistics:')
print(stats_data)
if USE_OPENSKILL:
from openskill import predict_win
win_prob = predict_win([[r] for r in method_ratings.values()])
skill_agg = pd.Series(ub.dzip(method_ratings.keys(),
win_prob)).sort_values(ascending=False)
print('Aggregated Rankings =\n{}'.format(skill_agg))
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plt = kwplot.autoplt()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data,
x=xlabel,
y=time_key,
marker='o',
ax=ax,
**plotkw)
ax.set_title('Benchmark Name')
ax.set_xlabel('Size (todo: A better x-variable description)')
ax.set_ylabel('Time (todo: A better y-variable description)')
# ax.set_xscale('log')
# ax.set_yscale('log')
try:
__IPYTHON__
except NameError:
plt.show()
def compute_move_effect(mon1, mon2, move, charge=1.0, rng=None):
"""
Compute damage and other effects caused by a move
Args:
mon1: (Pokemon): attacker
mon2: (Pokemon): defender
move: (dict): move dictionary
charge: (float): between 0 and 1, if this move is a charge move,
this is the percent of bubbles popped.
rng (RandomState):
Example:
>>> from pypogo.battle import * # NOQA
>>> mon1 = Pokemon.random('machamp', moves=['counter', 'cross chop', 'rock slide'], shadow=True).maximize(1500)
>>> mon1 = Pokemon.random('articuno', moves=['Ice Shard', 'Icy Wind', 'Hurricane'], shadow=True).maximize(1500)
>>> mon2 = Pokemon.random('snorlax', moves=['lick', 'super_power'], shadow=True).maximize(1500)
>>> mon2 = Pokemon.random('magikarp', shadow=True).maximize(1500)
>>> move = mon1.pvp_charge_moves[0]
>>> effect = compute_move_effect(mon1, mon2, move)
>>> print('effect = {}'.format(ub.repr2(effect, nl=2)))
>>> move = mon2.pvp_charge_moves[0]
>>> effect = compute_move_effect(mon2, mon1, move)
>>> print('effect = {}'.format(ub.repr2(effect, nl=2)))
"""
import math
move_type = move['type']
move_power = move['power']
effectiveness_lut = mon1.api.data['type_effectiveness']
effectiveness = 1
for def_type in mon2.typing:
effectiveness *= effectiveness_lut[move_type][def_type]
# Attack and defense after taking IVs and level into account
adjusted_attack = mon1.adjusted['attack']
adjusted_defense = max(mon2.adjusted['defense'], 1e-5)
stab = 1.2 if (move_type in mon1.typing) else 1.0
def modifier_factor(delta):
if delta > 0:
return 1 + (delta / 4)
else:
return 1 / (1 + (-delta / 4))
attack_modifier_factor = modifier_factor(mon1.modifiers['attack'])
defense_modifier_factor = modifier_factor(mon2.modifiers['defense'])
attack_shadow_factor = 1.2 if mon1.shadow else 1.0
defense_shadow_factor = (1 / 1.2) if mon2.shadow else 1.0
pvp_bonus_multiplier = 1.3 # Hard coded in game, See [3].
attack_power = (pvp_bonus_multiplier * charge * stab * adjusted_attack *
attack_modifier_factor * attack_shadow_factor *
effectiveness)
defense_power = (adjusted_defense * defense_modifier_factor *
defense_shadow_factor)
half = 0.5 # not sure why a half is in the formula
damage = math.floor(half * move_power * attack_power / defense_power) + 1
if rng is None:
rng = random.Random()
# Determine if any buffs / debuffs occur
buffs = move.get('buffs', None)
modifiers = []
if buffs is not None:
chance = move['buffs']['activation_chance']
if chance == 1 or chance > rng.random():
attacker_att_delta = buffs.get('attacker_attack_stat_stage_change',
0)
attacker_def_delta = buffs.get(
'attacker_defense_stat_stage_change', 0)
target_att_delta = buffs.get('target_attack_stat_stage_change', 0)
target_def_delta = buffs.get('target_defense_stat_stage_change', 0)
if attacker_att_delta != 0:
modifiers.append({
'target': mon1,
'stat': 'attack',
'delta': attacker_att_delta,
})
if attacker_def_delta != 0:
modifiers.append({
'target': mon1,
'stat': 'defense',
'delta': attacker_def_delta,
})
if target_att_delta != 0:
modifiers.append({
'target': mon2,
'stat': 'attack',
'delta': target_att_delta,
})
if target_def_delta != 0:
modifiers.append({
'target': mon2,
'stat': 'defense',
'delta': target_def_delta,
})
# Buff factors can be [1, 1.25, 1.5, 1.75, 2.0]
# DeBuff factors can be [1, 0.8, 0.66, 0.57, 0.5]
# This formula might make more sense, but its not what it is.
# 2 ** np.linspace(-1, 1, 9)
# np.logspace(-1, 1, 9, base=2)
# Is there a natural way to express?
# I don't think so, the right half is linear and the left half is
# non-linear.
if 0:
import kwplot
import numpy as np
plt = kwplot.autoplt()
from scipy.optimize import curve_fit
x = np.array([-4, -3, -2, -1, 0, 1, 2, 3, 4])
y = np.array([0.5, 0.57, 0.66, 0.8, 1, 1.25, 1.5, 1.75, 2.0])
def func(x, c0, c1, c2, c3, c4):
return (c4 * x**4 + c3 * x**3 + c2 * x**2 + c1 * x**1 + c0 * x**0 +
# b1 ** (x * e1)
0)
left_y = 1 / (1 + (-x / 4))
right_y = 1 + (x / 4)
popt, pcov = curve_fit(func, x, y)
y_nat = 2.**(np.array(x) / 4)
plt.clf()
plt.plot(x, y, 'b-o', label='real')
plt.plot(x, y_nat, 'r-o', label='natural')
plt.plot(x, np.abs(y - y_nat), 'r-o', label='nat_abs_delta')
plt.plot(x, left_y, '--', label='left-y')
plt.plot(x, right_y, '--', label='right-y')
x_hat = np.linspace(x[0], x[-1], 100)
y_hat = func(x_hat, *popt)
plt.plot(x_hat, y_hat, 'g--', label='fit')
plt.legend()
effect = {
'desc':
f'{mon1.name!r} (hp={mon1.hp}, energy={mon1.energy}) used {move["name"]!r} against {mon2.name!r} (hp={mon2.hp}, energy={mon2.energy})',
'attacker': mon1,
'target': mon2,
'damage': damage,
'stab': stab,
'pvp_bonus_multiplier': pvp_bonus_multiplier,
'adjusted_attack': adjusted_attack,
'attack_shadow_factor': attack_shadow_factor,
'effectiveness': effectiveness,
'adjusted_defense': adjusted_defense,
'defense_modifier_factor': defense_modifier_factor,
'attack_modifier_factor': attack_modifier_factor,
'defense_shadow_factor': defense_shadow_factor,
'attack_power': attack_power,
'defense_power': defense_power,
'energy_delta': move['energy_delta'],
'turn_duration': move['turn_duration'],
'modifiers': modifiers,
}
return effect
def main():
"""
Run password security analysis
Example:
>>> import sys, ubelt
>>> sys.path.append(ubelt.expandpath('~/misc/notes'))
>>> from password_model import * # NOQA
>>> main()
"""
import itertools as it
from fractions import Fraction
import pandas as pd
# Build our adversary and our strategies
devices, scales = build_threat_models()
password_schemes = build_password_strategy()
# Other estimates or assumptions
estimates = {
# estimated cost of using a kilowatt for an hour
# http://www.wrecc.com/what-uses-watts-in-your-home/
# https://www.coinwarz.com/mining/ethereum/calculator
'dollars_per_kwh': 0.10,
}
rows = []
for device, scheme, scale in it.product(devices, password_schemes, scales):
for benchmark in device['benchmarks']:
states = Fraction(scheme['states'])
num_devices = Fraction(scale['num_devices'])
dollars_per_kwh = Fraction(estimates['dollars_per_kwh'])
hashmode_attempts_per_second = benchmark['attempts_per_second']
attempts_per_second = num_devices * Fraction(
int(hashmode_attempts_per_second))
seconds = states / Fraction(attempts_per_second)
hours = seconds / Fraction(3600)
device_kilowatts = Fraction(device['watts']) / Fraction(1000)
device_dollars_per_hour = device_kilowatts * dollars_per_kwh
dollars_per_device = device_dollars_per_hour * hours
dollars = dollars_per_device * num_devices
total_kilowatts = device_kilowatts * num_devices * hours
row = {
'scheme': scheme['name'],
'entropy': scheme['entropy'],
'hashmode': benchmark['hashmode'],
'hashmode_attempts_per_second':
int(hashmode_attempts_per_second),
'device': device['name'],
'scale': scale['name'],
'num_devices': scale['num_devices'],
'seconds': seconds,
'dollars': dollars,
'kilowatts': total_kilowatts,
'hours': hours,
'dollars_per_kwh': estimates['dollars_per_kwh'],
}
rows.append(row)
df = pd.DataFrame(rows)
df = df.sort_values('entropy')
chosen_device = 'RTX_3090'
df = df[df['device'] == chosen_device]
df['time'] = df['seconds'].apply(humanize_seconds)
df['cost'] = df['dollars'].apply(partial(humanize_dollars, colored=1))
df['entropy'] = df['entropy'].round(2)
df['num_devices'] = df['num_devices'].apply(int)
hashmodes = sorted([d['hashmode'] for d in device['benchmarks']])
# https://github.com/pandas-dev/pandas/issues/18066
monkeypatch_pandas_colored_stdout()
# Output our assumptions
print('\n---')
print('Assumptions:')
device_info = ub.group_items(devices,
lambda x: x['name'])[chosen_device][0]
print('estimates = {!r}'.format(estimates))
print('device_info = {}'.format(ub.repr2(device_info, nl=2)))
# For each hashmode, print the scheme-vs-num_devices-vs-time matrix
hashmode_to_pivots = {}
for hashmode in hashmodes:
subdf = df
subdf = subdf[subdf['hashmode'] == hashmode]
subdf = subdf.sort_values(['entropy', 'num_devices'])
piv = subdf.pivot(['entropy', 'cost', 'scheme'],
['num_devices', 'scale'], 'time')
# piv.style.applymap(color_cases)
hashmode_to_pivots[hashmode] = piv
for hashmode in hashmodes:
print('\n---')
print('hashmode = {!r}'.format(hashmode))
piv = hashmode_to_pivots[hashmode]
print(piv)
# Print the scheme-vs-hashmode-vs-cost matrix
print('\n---')
print('Cost Matrix:')
subdf = df[df['scale'] == df['scale'].iloc[0]]
piv = subdf.pivot(['entropy', 'scheme'],
['hashmode_attempts_per_second', 'hashmode'], 'cost')
piv = piv.sort_index(axis=1, ascending=False)
piv.columns = piv.columns.droplevel(0)
print(piv)
# Make the visualizations
if ub.argflag('--show'):
import kwplot
from matplotlib.colors import LogNorm
import matplotlib as mpl
plt = kwplot.autoplt()
sns = kwplot.autosns()
use_latex = ub.argflag('--latex')
if use_latex:
mpl.rcParams['text.usetex'] = True
def time_labelize(x):
text = humanize_seconds(x, colored=False, named=True, precision=2)
parts = text.split(' ')
if use_latex:
text = r'{\huge ' + parts[0] + '}' + '\n' + ' '.join(parts[1:])
else:
text = parts[0] + '\n' + ' '.join(parts[1:])
return text
def dollar_labelize(dollars):
cost = humanize_dollars(dollars, named=(dollars > 1))
if use_latex:
cost = cost.replace('$', r'\$')
return cost
hashmode_to_notes = {}
for dev in devices[0]['benchmarks']:
hashmode_to_notes[dev['hashmode']] = dev['notes']
if 1:
# Independent of the adversary scale we can plot cost versus scheme
# cost vs hashmod?
subdf = df[df['scale'] == df['scale'].iloc[0]]
piv = subdf.pivot(['entropy', 'scheme'],
['hashmode_attempts_per_second', 'hashmode'],
'dollars')
piv = piv.sort_index(axis=1, ascending=False)
# https://stackoverflow.com/questions/64234474/cust-y-lbls-seaborn
ax: mpl.axes.Axes = plt.subplots(figsize=(15, 10))[1]
annot = piv.applymap(dollar_labelize)
piv = piv.applymap(float)
sns.heatmap(piv,
annot=annot,
ax=ax,
fmt='s',
norm=LogNorm(vmin=1, vmax=100_000_000_000_000_000),
annot_kws={'size': 16},
cmap='cividis',
cbar_kws={
'label': 'dollars',
'pad': 0.001
})
# Find colorbar
for subax in ax.figure.axes:
if subax.get_label() == '<colorbar>':
subax.set_ylabel('dollars', labelpad=0)
break
new_ytick_labels = []
for ent, scheme in piv.index.to_list():
if use_latex:
scheme = r'{\LARGE ' + scheme + '}'
_ = '{scheme}\nEntropy={ent}bits'.format(scheme=scheme,
ent=ent)
new_ytick_labels.append(_)
new_xtick_labels = []
for _, hashmode in piv.columns.to_list():
notes = ''
if hashmode in hashmode_to_notes:
notes = '\n(' + hashmode_to_notes[hashmode] + ')'
new_xtick_labels.append(hashmode + notes)
ax.set_xticklabels(new_xtick_labels, rotation=0)
ax.set_yticklabels(new_ytick_labels, rotation=0)
ax.set_ylabel('Password Scheme, Entropy', labelpad=24)
ax.set_xlabel('Hashmode', labelpad=16)
if use_latex:
title = '{{\\Huge Password Cost Security}}'
ax.set_title(title)
else:
ax.set_title('Password Cost Security')
ax.figure.subplots_adjust(bottom=0.1,
left=0.20,
right=1.0,
top=0.90,
wspace=0.001)
if ub.argflag('--save'):
fname = 'passwd_cost_security.png'
ax.figure.savefig(fname)
if 1:
# For each hashmode plot (scheme versus adversary scale)
for hashmode in ub.ProgIter(hashmodes, desc='plotting'):
subdf = df
subdf = subdf[subdf['hashmode'] == hashmode]
subdf = subdf.sort_values(['entropy', 'num_devices'])
piv = subdf.pivot(['entropy', 'dollars', 'scheme'],
['num_devices', 'scale'], 'seconds')
piv = piv.applymap(float)
# https://stackoverflow.com/questions/64234474/cust-y-lbls-seaborn
ax: mpl.axes.Axes = plt.subplots(figsize=(15, 10))[1]
annot = piv.applymap(time_labelize)
sns.heatmap(piv,
annot=annot,
ax=ax,
fmt='s',
norm=LogNorm(vmin=1, vmax=8640000000),
annot_kws={'size': 10},
cbar_kws={
'label': 'seconds',
'pad': 0.001
})
# Find colorbar
for subax in ax.figure.axes:
if subax.get_label() == '<colorbar>':
subax.set_ylabel('seconds', labelpad=0)
break
new_ytick_labels = []
for ent, dollars, scheme in piv.index.to_list():
cost = dollar_labelize(dollars)
if use_latex:
scheme = r'{\LARGE ' + scheme + '}'
_ = '{scheme}\nEntropy={ent}bits\nCost={cost}'.format(
scheme=scheme, cost=cost, ent=ent)
new_ytick_labels.append(_)
new_xtick_labels = []
for n, name in piv.columns.to_list():
if use_latex:
name = r'{\LARGE ' + name + '}'
_ = name + '\n' + named_large_number(n,
precision=0) + ' GPUs'
new_xtick_labels.append(_)
ax.set_xticklabels(new_xtick_labels, rotation=0)
# ax.set_yticklabels(new_ytick_labels, horizontalalignment='left', pad=30)
ax.set_yticklabels(new_ytick_labels)
ax.set_ylabel('Password Scheme, Entropy, and Cost to Crack',
labelpad=24)
ax.set_xlabel('Adversary Resources', labelpad=16)
notes = ''
if hashmode in hashmode_to_notes:
notes = ' (' + hashmode_to_notes[hashmode] + ')'
if use_latex:
title = '{{\\Huge Password Time Security}}\nhashmode={}{}'.format(
hashmode, notes)
ax.set_title(title)
else:
ax.set_title(
'Password Time Security\n(hashmode={}{})'.format(
hashmode, notes))
ax.figure.subplots_adjust(bottom=0.1,
left=0.20,
right=1.0,
top=0.90,
wspace=0.001)
if ub.argflag('--save'):
fname = 'passwd_robustness_{}.png'.format(hashmode)
ax.figure.savefig(fname)
plt.show()
def benchmark_nested_break():
"""
There are several ways to do a nested break, but which one is best?
https://twitter.com/nedbat/status/1515345787563220996
"""
import ubelt as ub
import pandas as pd
import timerit
import itertools as it
def method1_itertools(iter1, iter2):
for i, j in it.product(iter1, iter2):
if i == 20 and j == 20:
break
def method2_except(iter1, iter2):
class Found(Exception):
pass
try:
for i in iter1:
for j in iter2:
if i == 20 and j == 20:
raise Found
except Found:
pass
class FoundPredef(Exception):
pass
def method2_5_except_predef(iter1, iter2):
try:
for i in iter1:
for j in iter2:
if i == 20 and j == 20:
raise FoundPredef
except FoundPredef:
pass
def method3_gendef(iter1, iter2):
def genfunc():
for i in iter1:
for j in iter2:
yield i, j
for i, j in genfunc():
if i == 20 and j == 20:
break
def method4_genexp(iter1, iter2):
genexpr = ((i, j) for i in iter1 for j in iter2)
for i, j in genexpr:
if i == 20 and j == 20:
break
method_lut = locals() # can populate this some other way
# Change params here to modify number of trials
ti = timerit.Timerit(1000, bestof=10, verbose=1)
# if True, record every trail run and show variance in seaborn
# if False, use the standard timerit min/mean measures
RECORD_ALL = True
# These are the parameters that we benchmark over
import numpy as np
basis = {
'method': ['method1_itertools', 'method2_except', 'method2_5_except_predef', 'method3_gendef', 'method4_genexp'],
# 'n1': np.logspace(1, np.log2(100), 30, base=2).astype(int),
# 'n2': np.logspace(1, np.log2(100), 30, base=2).astype(int),
'size': np.logspace(1, np.log2(10000), 30, base=2).astype(int),
'input_style': ['range', 'list', 'customized_iter'],
# 'param_name': [param values],
}
xlabel = 'size'
xinput_labels = ['n1', 'n2', 'size']
# Set these to param labels that directly transfer to method kwargs
kw_labels = []
# Set these to empty lists if they are not used
group_labels = {
'style': ['input_style'],
'size': [],
}
group_labels['hue'] = list(
(ub.oset(basis) - {xlabel} - xinput_labels) - set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
def make_input(params):
# Given the parameterization make the benchmark function input
# n1 = params['n1']
# n2 = params['n2']
size = params['size']
n1 = int(np.sqrt(size))
n2 = int(np.sqrt(size))
if params['input_style'] == 'list':
iter1 = list(range(n1))
iter2 = list(range(n1))
elif params['input_style'] == 'range':
iter1 = range(n1)
iter2 = range(n2)
elif params['input_style'] == 'customized_iter':
import random
def rando1():
rng1 = random.Random(0)
for _ in range(n1):
yield rng1.randint(0, n2)
def rando2():
rng2 = random.Random(1)
for _ in range(n1):
yield rng2.randint(0, n2)
iter1 = rando1()
iter2 = rando2()
else:
raise KeyError
return {'iter1': iter1, 'iter2': iter2}
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
# size = params['n1'] * params['n2']
# params['size'] = size
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(
ub.dict_isect(params, labels), compact=1, si=1)
key = ub.repr2(params, compact=1, si=1)
# Make any modifications you need to compute input kwargs for each
# method here.
kwargs = ub.dict_isect(params.copy(), kw_labels)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
kwargs.update(make_input(params))
with timer:
# Put the logic you want to time here
method(**kwargs)
if RECORD_ALL:
# Seaborn will show the variance if this is enabled, otherwise
# use the robust timerit mean / min times
# chunk_iter = ub.chunks(ti.times, ti.bestof)
# times = list(map(min, chunk_iter)) # TODO: timerit method for this
times = ti.robust_times()
for time in times:
row = {
# 'mean': ti.mean(),
'time': time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
else:
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
time_key = 'time' if RECORD_ALL else 'min'
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values(time_key)
if RECORD_ALL:
# Show the min / mean if we record all
min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1)
mean_times = data.groupby('key')[['time']].mean().rename({'time': 'mean'}, axis=1)
stats_data = pd.concat([min_times, mean_times], axis=1)
stats_data = stats_data.sort_values('min')
else:
stats_data = data
USE_OPENSKILL = 1
if USE_OPENSKILL:
# Lets try a real ranking method
# https://github.com/OpenDebates/openskill.py
import openskill
method_ratings = {m: openskill.Rating() for m in basis['method']}
other_keys = sorted(set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'})
for params, variants in stats_data.groupby(other_keys):
variants = variants.sort_values('mean')
ranking = variants['method'].reset_index(drop=True)
mean_speedup = variants['mean'].max() / variants['mean']
stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup
min_speedup = variants['min'].max() / variants['min']
stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup
if USE_OPENSKILL:
# The idea is that each setting of parameters is a game, and each
# "method" is a player. We rank the players by which is fastest,
# and update their ranking according to the Weng-Lin Bayes ranking
# model. This does not take the fact that some "games" (i.e.
# parameter settings) are more important than others, but it should
# be fairly robust on average.
old_ratings = [[r] for r in ub.take(method_ratings, ranking)]
new_values = openskill.rate(old_ratings) # Not inplace
new_ratings = [openskill.Rating(*new[0]) for new in new_values]
method_ratings.update(ub.dzip(ranking, new_ratings))
print('Statistics:')
print(stats_data)
if USE_OPENSKILL:
from openskill import predict_win
win_prob = predict_win([[r] for r in method_ratings.values()])
skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False)
print('method_ratings = {}'.format(ub.repr2(method_ratings, nl=1)))
print('Aggregated Rankings =\n{}'.format(skill_agg))
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plt = kwplot.autoplt()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw)
ax.set_title(f'Benchmark Nested Breaks: #Trials {ti.num}, bestof {ti.bestof}')
ax.set_xlabel(f'{xlabel}')
ax.set_ylabel('Time')
ax.set_xscale('log')
ax.set_yscale('log')
try:
__IPYTHON__
except NameError:
plt.show()
def bench_bbox_iou_method():
"""
On my system the torch impl was fastest (when the data was on the GPU).
"""
from kwimage.structs.boxes import _box_ious_torch, _box_ious_py, _bbox_ious_c
ydata = ub.ddict(list)
xdata = [
10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 2000, 4000
]
bias = 0
if _bbox_ious_c is None:
print('CYTHON IMPLEMENATION IS NOT AVAILABLE')
for num in xdata:
results = {}
# Setup Timer
N = max(20, int(1000 / num))
ti = ub.Timerit(N, bestof=10)
# Setup input dat
boxes1 = kwimage.Boxes.random(num, scale=10.0, rng=0, format='ltrb')
boxes2 = kwimage.Boxes.random(num + 1,
scale=10.0,
rng=1,
format='ltrb')
ltrb1 = boxes1.tensor().data
ltrb2 = boxes2.tensor().data
for timer in ti.reset('iou-torch-cpu'):
with timer:
out = _box_ious_torch(ltrb1, ltrb2, bias)
results[ti.label] = out.data.cpu().numpy()
ydata[ti.label].append(ti.mean())
gpu = torch.device(0)
ltrb1 = boxes1.tensor().data.to(gpu)
ltrb2 = boxes2.tensor().data.to(gpu)
for timer in ti.reset('iou-torch-gpu'):
with timer:
out = _box_ious_torch(ltrb1, ltrb2, bias)
torch.cuda.synchronize()
results[ti.label] = out.data.cpu().numpy()
ydata[ti.label].append(ti.mean())
ltrb1 = boxes1.numpy().data
ltrb2 = boxes2.numpy().data
for timer in ti.reset('iou-numpy'):
with timer:
out = _box_ious_py(ltrb1, ltrb2, bias)
results[ti.label] = out
ydata[ti.label].append(ti.mean())
if _bbox_ious_c:
ltrb1 = boxes1.numpy().data.astype(np.float32)
ltrb2 = boxes2.numpy().data.astype(np.float32)
for timer in ti.reset('iou-cython'):
with timer:
out = _bbox_ious_c(ltrb1, ltrb2, bias)
results[ti.label] = out
ydata[ti.label].append(ti.mean())
eq = partial(np.allclose, atol=1e-07)
passed = ub.allsame(results.values(), eq)
if passed:
print(
'All methods produced the same answer for num={}'.format(num))
else:
for k1, k2 in it.combinations(results.keys(), 2):
v1 = results[k1]
v2 = results[k2]
if eq(v1, v2):
print('pass: {} == {}'.format(k1, k2))
else:
diff = np.abs(v1 - v2)
print(
'FAIL: {} != {}: diff(max={}, mean={}, sum={})'.format(
k1, k2, diff.max(), diff.mean(), diff.sum()))
raise AssertionError('different methods report different results')
print('num = {!r}'.format(num))
print('ti.measures = {}'.format(
ub.repr2(ub.map_vals(ub.sorted_vals, ti.measures),
align=':',
nl=2,
precision=6)))
import kwplot
kwplot.autoplt()
kwplot.multi_plot(xdata, ydata, xlabel='num boxes', ylabel='seconds')
kwplot.show_if_requested()
def benchmark_mul_vs_pow():
import ubelt as ub
import pandas as pd
import timerit
from functools import reduce
import operator as op
import itertools as it
def method_pow_via_mul_raw(n):
""" Construct a function that does multiplication of a value n times """
return eval('lambda v: ' + ' * '.join(['v'] * n))
def method_pow_via_mul_for(v, n):
ret = v
for _ in range(1, n):
ret = ret * v
return ret
def method_pow_via_mul_reduce(v, n):
""" Alternative way to multiply a value n times """
return reduce(op.mul, it.repeat(v, n))
def method_pow_via_pow(v, n):
return v ** n
method_lut = locals() # can populate this some other way
ti = timerit.Timerit(500000, bestof=1000, verbose=2)
basis = {
'method': ['method_pow_via_mul_raw', 'method_pow_via_pow'],
'n': list(range(1, 20)),
'v': ['random-int', 'random-float'],
# 'param_name': [param values],
}
xlabel = 'n'
kw_labels = ['v', 'n']
group_labels = {
'style': ['v'],
'size': [],
}
group_labels['hue'] = list(
(ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(
ub.dict_isect(params, labels), compact=1, si=1)
key = ub.repr2(params, compact=1, si=1)
kwargs = ub.dict_isect(params.copy(), kw_labels)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
if params['method'] == 'method_pow_via_mul_raw':
method = method(kwargs.pop('n'))
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
import random
if kwargs['v'] == 'random':
kwargs['v'] = random.randint(1, 31000) if random.random() > 0.5 else random.random()
elif kwargs['v'] == 'random-int':
kwargs['v'] = random.randint(1, 31000)
elif kwargs['v'] == 'random-float':
kwargs['v'] = random.random()
with timer:
# Put the logic you want to time here
method(**kwargs)
for time in map(min, ub.chunks(ti.times, ti.bestof)):
row = {
# 'mean': ti.mean(),
'time': time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
# data = data.sort_values('time')
print(data)
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plt = kwplot.autoplt()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y='time', marker='o', ax=ax, **plotkw)
ax.set_title('Benchmark')
ax.set_xlabel('N')
ax.set_ylabel('Time')
ax.set_yscale('log')
plt.show()
