def benchmark_loop_first_variants():
import ubelt as ub
basis = {
'num_items': [10, 100, 1000, 10000, 100000],
}
data_grid = ub.named_product(**basis)
rows = []
import timerit
ti = timerit.Timerit(100, bestof=10, verbose=2)
for data_kw in data_grid:
items = list(range(data_kw['num_items']))
method_name = 'loop_first'
for timer in ti.reset(method_name):
with timer:
method_loop_first(items)
rows.append({
'num_items': data_kw['num_items'],
'method_name': method_name,
'mean': ti.mean()
})
method_name = 'loop_first2'
for timer in ti.reset(method_name):
with timer:
method_loop_first2(items)
rows.append({
'num_items': data_kw['num_items'],
'method_name': method_name,
'mean': ti.mean()
})
method_name = 'enumerate'
for timer in ti.reset(method_name):
with timer:
method_enumerate(items)
rows.append({
'num_items': data_kw['num_items'],
'method_name': method_name,
'mean': ti.mean()
})
print('ti.rankings = {}'.format(ub.repr2(ti.rankings, nl=2, align=':')))
return rows
def run_benchmark_renormalization():
"""
See if we can renormalize probabilities after update with a faster method
that maintains memory a bit better
Example:
>>> import sys, ubelt
>>> sys.path.append(ubelt.expandpath('~/misc/tests/python'))
>>> from bench_renormalization import * # NOQA
>>> run_benchmark_renormalization()
"""
import ubelt as ub
import xdev
import pathlib
import timerit
fpath = pathlib.Path('~/misc/tests/python/renormalize_cython.pyx').expanduser()
renormalize_cython = xdev.import_module_from_pyx(fpath, annotate=True,
verbose=3, recompile=True)
xdev.profile_now(renormalize_demo_v1)(1000, 100)
xdev.profile_now(renormalize_demo_v2)(1000, 100)
xdev.profile_now(renormalize_demo_v3)(1000, 100)
xdev.profile_now(renormalize_demo_v4)(1000, 100)
func_list = [
# renormalize_demo_v1,
renormalize_demo_v2,
# renormalize_demo_v3,
# renormalize_demo_v4,
renormalize_cython.renormalize_demo_cython_v1,
renormalize_cython.renormalize_demo_cython_v2,
renormalize_cython.renormalize_demo_cython_v3,
]
methods = {f.__name__: f for f in func_list}
for key, method in methods.items():
with timerit.Timer(label=key, verbose=0) as t:
method(1000, 100)
print(f'{key:<30} {t.toc():0.6f}')
arg_basis = {
'T': [10, 20, 30, 50],
'D': [10, 50, 100, 300],
}
args_grid = []
for argkw in list(ub.named_product(arg_basis)):
if argkw['T'] <= argkw['D']:
arg_basis['size'] = argkw['T'] * argkw['D']
args_grid.append(argkw)
ti = timerit.Timerit(100, bestof=10, verbose=2)
measures = []
for method_name, method in methods.items():
for argkw in args_grid:
row = ub.dict_union({'method': method_name}, argkw)
key = ub.repr2(row, compact=1)
argkey = ub.repr2(argkw, compact=1)
kwargs = ub.dict_subset(argkw, ['T', 'D'])
for timer in ti.reset('time'):
with timer:
method(**kwargs)
row['mean'] = ti.mean()
row['min'] = ti.min()
row['key'] = key
row['argkey'] = argkey
measures.append(row)
import pandas as pd
df = pd.DataFrame(measures)
import kwplot
sns = kwplot.autosns()
kwplot.figure(fnum=1, pnum=(1, 2, 1), docla=True)
sns.lineplot(data=df, x='D', y='min', hue='method', style='method')
kwplot.figure(fnum=1, pnum=(1, 2, 2), docla=True)
sns.lineplot(data=df, x='T', y='min', hue='method', style='method')
p = (df.pivot(['method'], ['argkey'], ['mean']))
print(p.mean(axis=1).sort_values())
def benchmark_template():
import ubelt as ub
import pandas as pd
import timerit
def method1(x, y, z):
ret = []
for i in range((x + y) * z):
ret.append(i)
return ret
def method2(x, y, z):
ret = [i for i in range((x + y) * z)]
return ret
method_lut = locals() # can populate this some other way
# Change params here to modify number of trials
ti = timerit.Timerit(100, bestof=10, verbose=1)
# if True, record every trail run and show variance in seaborn
# if False, use the standard timerit min/mean measures
RECORD_ALL = True
# These are the parameters that we benchmark over
basis = {
'method': ['method1', 'method2'],
'x': list(range(7)),
'y': [0, 100],
'z': [2, 3]
# 'param_name': [param values],
}
xlabel = 'x'
# Set these to param labels that directly transfer to method kwargs
kw_labels = ['x', 'y', 'z']
# Set these to empty lists if they are not used
group_labels = {
'style': ['y'],
'size': ['z'],
}
group_labels['hue'] = list((ub.oset(basis) - {xlabel}) -
set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(ub.dict_isect(
params, labels),
compact=1,
si=1)
key = ub.repr2(params, compact=1, si=1)
# Make any modifications you need to compute input kwargs for each
# method here.
kwargs = ub.dict_isect(params.copy(), kw_labels)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
with timer:
# Put the logic you want to time here
method(**kwargs)
if RECORD_ALL:
# Seaborn will show the variance if this is enabled, otherwise
# use the robust timerit mean / min times
chunk_iter = ub.chunks(ti.times, ti.bestof)
times = list(map(min, chunk_iter)) # TODO: timerit method for this
for time in times:
row = {
# 'mean': ti.mean(),
'time': time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
else:
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
time_key = 'time' if RECORD_ALL else 'min'
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values(time_key)
if RECORD_ALL:
# Show the min / mean if we record all
min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1)
mean_times = data.groupby('key')[['time'
]].mean().rename({'time': 'mean'},
axis=1)
stats_data = pd.concat([min_times, mean_times], axis=1)
stats_data = stats_data.sort_values('min')
else:
stats_data = data
USE_OPENSKILL = 1
if USE_OPENSKILL:
# Lets try a real ranking method
# https://github.com/OpenDebates/openskill.py
import openskill
method_ratings = {m: openskill.Rating() for m in basis['method']}
other_keys = sorted(
set(stats_data.columns) -
{'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'})
for params, variants in stats_data.groupby(other_keys):
variants = variants.sort_values('mean')
ranking = variants['method'].reset_index(drop=True)
mean_speedup = variants['mean'].max() / variants['mean']
stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup
min_speedup = variants['min'].max() / variants['min']
stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup
if USE_OPENSKILL:
# The idea is that each setting of parameters is a game, and each
# "method" is a player. We rank the players by which is fastest,
# and update their ranking according to the Weng-Lin Bayes ranking
# model. This does not take the fact that some "games" (i.e.
# parameter settings) are more important than others, but it should
# be fairly robust on average.
old_ratings = [[r] for r in ub.take(method_ratings, ranking)]
new_values = openskill.rate(old_ratings) # Not inplace
new_ratings = [openskill.Rating(*new[0]) for new in new_values]
method_ratings.update(ub.dzip(ranking, new_ratings))
print('Statistics:')
print(stats_data)
if USE_OPENSKILL:
from openskill import predict_win
win_prob = predict_win([[r] for r in method_ratings.values()])
skill_agg = pd.Series(ub.dzip(method_ratings.keys(),
win_prob)).sort_values(ascending=False)
print('Aggregated Rankings =\n{}'.format(skill_agg))
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plt = kwplot.autoplt()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data,
x=xlabel,
y=time_key,
marker='o',
ax=ax,
**plotkw)
ax.set_title('Benchmark Name')
ax.set_xlabel('Size (todo: A better x-variable description)')
ax.set_ylabel('Time (todo: A better y-variable description)')
# ax.set_xscale('log')
# ax.set_yscale('log')
try:
__IPYTHON__
except NameError:
plt.show()
def benchmark_repeat_vs_reduce_mul():
import ubelt as ub
import pandas as pd
import timerit
def reduce_daq_rec(func, arrs):
if len(arrs) == 1:
return arrs[0]
if len(arrs) == 2:
return func(arrs[0], arrs[1])
elif len(arrs) == 3:
return func(func(arrs[0], arrs[1]), arrs[3])
else:
arrs1 = arrs[0::2]
arrs2 = arrs[1::2]
res1 = reduce_daq_rec(func, arrs1)
res2 = reduce_daq_rec(func, arrs2)
res = func(res1, res2)
return res
def reduce_daq_iter(func, arrs):
"""
https://www.baeldung.com/cs/convert-recursion-to-iteration
https://stackoverflow.com/questions/159590/way-to-go-from-recursion-to-iteration
arrs = [2, 3, 5, 7, 11, 13, 17, 21]
"""
raise NotImplementedError
# TODO: make the iterative version
from collections import deque
empty_result = None
stack = deque([(arrs, empty_result)])
idx = 0
while stack:
print('----')
print('stack = {}'.format(ub.repr2(list(stack), nl=1)))
arrs0, result = stack.pop()
if len(arrs0) == 0:
raise Exception
if result is not None:
# raise Exception
results = [result]
while stack:
next_arrs0, next_result = stack.pop()
if next_result is None:
break
else:
results.append(next_result)
if results:
if len(results) == 1:
stack.append((results, results[0]))
else:
stack.append((results, None))
if next_result is None:
stack.append((next_arrs0, None))
elif result is None:
if len(arrs0) == 1:
result = arrs0[0]
stack.append((arrs0, result))
# return arrs0[0]
if len(arrs0) == 2:
result = func(arrs0[0], arrs0[1])
stack.append((arrs0, result))
elif len(arrs0) == 3:
result = func(func(arrs0[0], arrs0[1]), arrs0[3])
stack.append((arrs0, result))
else:
arrs01 = arrs0[0::2]
arrs02 = arrs0[1::2]
stack.append((arrs0, empty_result))
stack.append((arrs01, empty_result))
stack.append((arrs02, empty_result))
# res1 = reduce_daq_rec(func, arrs01)
# res2 = reduce_daq_rec(func, arrs2)
# res = func(res1, res2)
idx += 1
if idx > 10:
raise Exception
return res
def method_daq_rec(arrs):
return reduce_daq_rec(np.multiply, arrs)
def method_repeat(arrs):
"""
helper code:
arr_names = ['a{:02d}'.format(idx) for idx in range(1, 32 + 1)]
lhs = ', '.join(arr_names)
rhs = ' * '.join(arr_names)
print(f'{lhs} = arrs')
print(f'ret = {rhs}')
"""
# Hard coded pure python syntax for multiplying
if len(arrs) == 4:
a01, a02, a03, a04 = arrs
ret = a01 * a02 * a03 * a04
elif len(arrs) == 8:
a01, a02, a03, a04, a05, a06, a07, a08 = arrs
ret = a01 * a02 * a03 * a04 * a05 * a06 * a07 * a08
elif len(arrs) == 32:
a01, a02, a03, a04, a05, a06, a07, a08, a09, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28, a29, a30, a31, a32 = arrs
ret = a01 * a02 * a03 * a04 * a05 * a06 * a07 * a08 * a09 * a10 * a11 * a12 * a13 * a14 * a15 * a16 * a17 * a18 * a19 * a20 * a21 * a22 * a23 * a24 * a25 * a26 * a27 * a28 * a29 * a30 * a31 * a32
return ret
def method_reduce(arrs):
ret = np.multiply.reduce(arrs)
return ret
def method_stack(arrs):
stacked = np.stack(arrs)
ret = stacked.prod(axis=0)
return ret
method_lut = locals() # can populate this some other way
ti = timerit.Timerit(10000, bestof=10, verbose=2)
basis = {
'method':
['method_repeat', 'method_reduce', 'method_stack', 'method_daq_rec'],
'arr_size': [10, 100, 1000, 10000],
'num_arrs': [4, 8, 32],
}
xlabel = 'arr_size'
kw_labels = []
group_labels = {
'style': ['num_arrs'],
'size': [],
}
group_labels['hue'] = list((ub.oset(basis) - {xlabel}) -
set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(ub.dict_isect(
params, labels),
compact=1,
si=1)
key = ub.repr2(params, compact=1, si=1)
kwargs = ub.dict_isect(params.copy(), kw_labels)
arr_size = params['arr_size']
num_arrs = params['num_arrs']
arrs = []
for _ in range(num_arrs):
arr = np.random.rand(arr_size)
arrs.append(arr)
kwargs['arrs'] = arrs
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
with timer:
# Put the logic you want to time here
method(**kwargs)
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values('min')
print(data)
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw)
ax.set_title('Benchmark')
ax.set_xlabel('Array Size')
ax.set_ylabel('Time')
def benchmark_nested_break():
"""
There are several ways to do a nested break, but which one is best?
https://twitter.com/nedbat/status/1515345787563220996
"""
import ubelt as ub
import pandas as pd
import timerit
import itertools as it
def method1_itertools(iter1, iter2):
for i, j in it.product(iter1, iter2):
if i == 20 and j == 20:
break
def method2_except(iter1, iter2):
class Found(Exception):
pass
try:
for i in iter1:
for j in iter2:
if i == 20 and j == 20:
raise Found
except Found:
pass
class FoundPredef(Exception):
pass
def method2_5_except_predef(iter1, iter2):
try:
for i in iter1:
for j in iter2:
if i == 20 and j == 20:
raise FoundPredef
except FoundPredef:
pass
def method3_gendef(iter1, iter2):
def genfunc():
for i in iter1:
for j in iter2:
yield i, j
for i, j in genfunc():
if i == 20 and j == 20:
break
def method4_genexp(iter1, iter2):
genexpr = ((i, j) for i in iter1 for j in iter2)
for i, j in genexpr:
if i == 20 and j == 20:
break
method_lut = locals() # can populate this some other way
# Change params here to modify number of trials
ti = timerit.Timerit(1000, bestof=10, verbose=1)
# if True, record every trail run and show variance in seaborn
# if False, use the standard timerit min/mean measures
RECORD_ALL = True
# These are the parameters that we benchmark over
import numpy as np
basis = {
'method': ['method1_itertools', 'method2_except', 'method2_5_except_predef', 'method3_gendef', 'method4_genexp'],
# 'n1': np.logspace(1, np.log2(100), 30, base=2).astype(int),
# 'n2': np.logspace(1, np.log2(100), 30, base=2).astype(int),
'size': np.logspace(1, np.log2(10000), 30, base=2).astype(int),
'input_style': ['range', 'list', 'customized_iter'],
# 'param_name': [param values],
}
xlabel = 'size'
xinput_labels = ['n1', 'n2', 'size']
# Set these to param labels that directly transfer to method kwargs
kw_labels = []
# Set these to empty lists if they are not used
group_labels = {
'style': ['input_style'],
'size': [],
}
group_labels['hue'] = list(
(ub.oset(basis) - {xlabel} - xinput_labels) - set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
def make_input(params):
# Given the parameterization make the benchmark function input
# n1 = params['n1']
# n2 = params['n2']
size = params['size']
n1 = int(np.sqrt(size))
n2 = int(np.sqrt(size))
if params['input_style'] == 'list':
iter1 = list(range(n1))
iter2 = list(range(n1))
elif params['input_style'] == 'range':
iter1 = range(n1)
iter2 = range(n2)
elif params['input_style'] == 'customized_iter':
import random
def rando1():
rng1 = random.Random(0)
for _ in range(n1):
yield rng1.randint(0, n2)
def rando2():
rng2 = random.Random(1)
for _ in range(n1):
yield rng2.randint(0, n2)
iter1 = rando1()
iter2 = rando2()
else:
raise KeyError
return {'iter1': iter1, 'iter2': iter2}
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
# size = params['n1'] * params['n2']
# params['size'] = size
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(
ub.dict_isect(params, labels), compact=1, si=1)
key = ub.repr2(params, compact=1, si=1)
# Make any modifications you need to compute input kwargs for each
# method here.
kwargs = ub.dict_isect(params.copy(), kw_labels)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
kwargs.update(make_input(params))
with timer:
# Put the logic you want to time here
method(**kwargs)
if RECORD_ALL:
# Seaborn will show the variance if this is enabled, otherwise
# use the robust timerit mean / min times
# chunk_iter = ub.chunks(ti.times, ti.bestof)
# times = list(map(min, chunk_iter)) # TODO: timerit method for this
times = ti.robust_times()
for time in times:
row = {
# 'mean': ti.mean(),
'time': time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
else:
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
time_key = 'time' if RECORD_ALL else 'min'
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values(time_key)
if RECORD_ALL:
# Show the min / mean if we record all
min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1)
mean_times = data.groupby('key')[['time']].mean().rename({'time': 'mean'}, axis=1)
stats_data = pd.concat([min_times, mean_times], axis=1)
stats_data = stats_data.sort_values('min')
else:
stats_data = data
USE_OPENSKILL = 1
if USE_OPENSKILL:
# Lets try a real ranking method
# https://github.com/OpenDebates/openskill.py
import openskill
method_ratings = {m: openskill.Rating() for m in basis['method']}
other_keys = sorted(set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'})
for params, variants in stats_data.groupby(other_keys):
variants = variants.sort_values('mean')
ranking = variants['method'].reset_index(drop=True)
mean_speedup = variants['mean'].max() / variants['mean']
stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup
min_speedup = variants['min'].max() / variants['min']
stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup
if USE_OPENSKILL:
# The idea is that each setting of parameters is a game, and each
# "method" is a player. We rank the players by which is fastest,
# and update their ranking according to the Weng-Lin Bayes ranking
# model. This does not take the fact that some "games" (i.e.
# parameter settings) are more important than others, but it should
# be fairly robust on average.
old_ratings = [[r] for r in ub.take(method_ratings, ranking)]
new_values = openskill.rate(old_ratings) # Not inplace
new_ratings = [openskill.Rating(*new[0]) for new in new_values]
method_ratings.update(ub.dzip(ranking, new_ratings))
print('Statistics:')
print(stats_data)
if USE_OPENSKILL:
from openskill import predict_win
win_prob = predict_win([[r] for r in method_ratings.values()])
skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False)
print('method_ratings = {}'.format(ub.repr2(method_ratings, nl=1)))
print('Aggregated Rankings =\n{}'.format(skill_agg))
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plt = kwplot.autoplt()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw)
ax.set_title(f'Benchmark Nested Breaks: #Trials {ti.num}, bestof {ti.bestof}')
ax.set_xlabel(f'{xlabel}')
ax.set_ylabel('Time')
ax.set_xscale('log')
ax.set_yscale('log')
try:
__IPYTHON__
except NameError:
plt.show()
data_kwkeys = ub.compatible(basis, generate_data)
func_kwkeys = ub.compatible(basis, method_lut[basis['method'][0]])
# These variable influence what is plotted on the x-asis y-axis and
# with different line types
xlabel = 'size'
ylabel = 'time'
group_labels = {
'size': ['subsize'],
'style': ['subsize'],
}
hue_labels = ub.oset(basis) - {xlabel}
if group_labels:
hue_labels = hue_labels - set.union(*map(set, group_labels.values()))
group_labels['hue'] = list(hue_labels)
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(ub.dict_isect(
params, labels),
compact=1,
si=1)
key = ub.repr2(params, compact=1, si=1)
data_kwargs = ub.dict_isect(params.copy(), data_kwkeys)
func_kwargs = generate_data(**data_kwargs)
method = method_lut[params['method']]
def benchmark_reversed_range():
import ubelt as ub
import pandas as pd
import timerit
import itertools as it
methods = []
def custom_reversed_range_v1(start, stop):
final = stop - 1
for idx in range(stop - start):
yield final - idx
def custom_reversed_range_v2(start, stop):
yield from it.islice(it.count(stop - 1, step=-1), stop - start)
@methods.append
def reversed_builtin(x):
start = 10
stop = x + start
ret = list(reversed(range(start, stop)))
return ret
@methods.append
def negative_range(x):
start = 10
stop = x + start
ret = list(range(stop - 1, start - 1, -1))
return ret
# @methods.append
# def custom_v1(x):
# start = 10
# stop = x + start
# ret = list(custom_reversed_range_v1(start, stop))
# return ret
# @methods.append
# def custom_v2(x):
# start = 10
# stop = x + start
# ret = list(custom_reversed_range_v2(start, stop))
# return ret
method_lut = {f.__name__: f for f in methods}
results = {k: func(10) for k, func in method_lut.items()}
print('results = {}'.format(ub.repr2(results, nl=1, align=':')))
if not ub.allsame(results.values()):
raise AssertionError('Failed consistency check')
ti = timerit.Timerit(1000, bestof=10, verbose=2)
basis = {
'method': list(method_lut.keys()),
'x': [2 ** i for i in range(14)],
}
grid_iter = ub.named_product(basis)
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
key = ub.repr2(params, compact=1, si=1)
kwargs = params.copy()
method_key = kwargs.pop('method')
method = method_lut[method_key]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
with timer:
# Put the logic you want to time here
method(**kwargs)
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**params,
}
rows.append(row)
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
print(data)
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x='x', y='min', hue='method', marker='o', ax=ax)
# ax.set_xscale('log')
ax.set_title('Benchmark Reveral Methods ')
ax.set_xlabel('A better x-variable description')
ax.set_ylabel('A better y-variable description')
def benchmark_unpack():
"""
What is faster unpacking items with slice syntax or tuple-unpacking
Slice unpacking seems to be a tad faster.
"""
import ubelt as ub
import random
import pandas as pd
import timerit
import string
def tuple_unpack(items):
*prefix, key = items
return prefix, key
def slice_unpack(items):
prefix, key = items[:-1], items[-1]
return prefix, key
method_lut = locals() # can populate this some other way
ti = timerit.Timerit(5000, bestof=3, verbose=2)
basis = {
'method': ['tuple_unpack', 'slice_unpack'],
'size': list(range(1, 64 + 1)),
'type': ['string', 'float'],
}
xlabel = 'size'
kw_labels = []
group_labels = {
'style': ['type'],
'size': [],
}
group_labels['hue'] = list((ub.oset(basis) - {xlabel}) -
set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(ub.dict_isect(
params, labels),
compact=1,
si=1)
key = ub.repr2(params, compact=1, si=1)
size = params['size']
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
if type == 'string':
items = [
''.join(random.choices(string.printable, k=5))
for _ in range(size)
]
elif type == 'float':
items = [random.random() for _ in range(size)]
with timer:
method(items)
for time in ti.times:
row = {
'time': time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values('time')
summary_rows = []
for method, group in data.groupby('method'):
row = {}
row['method'] = method
row['mean'] = group['time'].mean()
row['std'] = group['time'].std()
row['min'] = group['time'].min()
row['max'] = group['time'].max()
summary_rows.append(row)
print(pd.DataFrame(summary_rows).sort_values('mean'))
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data,
x=xlabel,
y='time',
marker='o',
ax=ax,
**plotkw)
ax.set_title('Benchmark')
ax.set_xlabel('Execution time')
ax.set_ylabel('Size of slices')
def benchmark_pathlib_vs_fspath():
import ubelt as ub
import pathlib
import pandas as pd
import random
import timerit
import os
def method_pathlib(inputs):
p = pathlib.Path(*inputs)
def method_ospath(inputs):
p = os.path.join(*inputs)
method_lut = locals() # can populate this some other way
ti = timerit.Timerit(10000, bestof=10, verbose=2)
basis = {
'method': ['method_pathlib', 'method_ospath'],
'num_parts': [2, 4, 8, 12, 16],
}
xlabel = 'num_parts'
kw_labels = []
group_labels = {
'style': [],
'size': [],
}
group_labels['hue'] = list((ub.oset(basis) - {xlabel}) -
set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(ub.dict_isect(
params, labels),
compact=1,
si=1)
key = ub.repr2(params, compact=1, si=1)
kwargs = ub.dict_isect(params.copy(), kw_labels)
n = params['num_parts']
inputs = [chr(random.randint(97, 120)) for _ in range(n)]
kwargs['inputs'] = inputs
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
with timer:
# Put the logic you want to time here
method(**kwargs)
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values('min')
print(data)
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw)
ax.set_title('Benchmark')
ax.set_xlabel('Time')
ax.set_ylabel('Number of parts')
def benchmark_dict_diff_impl():
import ubelt as ub
import pandas as pd
import timerit
import random
def method_diffkeys(*args):
first_dict = args[0]
keys = set(first_dict)
keys.difference_update(*map(set, args[1:]))
new0 = dict((k, first_dict[k]) for k in keys)
return new0
def method_diffkeys_list(*args):
first_dict = args[0]
remove_keys = set.union(*map(set, args[1:]))
keep_keys = [k for k in first_dict.keys() if k not in remove_keys]
new = dict((k, first_dict[k]) for k in keep_keys)
return new
def method_diffkeys_oset(*args):
first_dict = args[0]
keys = ub.oset(first_dict)
keys.difference_update(*map(set, args[1:]))
new0 = dict((k, first_dict[k]) for k in keys)
return new0
def method_ifkeys_setcomp(*args):
first_dict = args[0]
remove_keys = {k for ks in args[1:] for k in ks}
new1 = dict((k, v) for k, v in first_dict.items() if k not in remove_keys)
return new1
def method_ifkeys_setunion(*args):
first_dict = args[0]
remove_keys = set.union(*map(set, args[1:]))
new2 = dict((k, v) for k, v in first_dict.items() if k not in remove_keys)
return new2
def method_ifkeys_getitem(*args):
first_dict = args[0]
remove_keys = set.union(*map(set, args[1:]))
new3 = dict((k, first_dict[k]) for k in first_dict.keys() if k not in remove_keys)
return new3
def method_ifkeys_dictcomp(*args):
# Cannot use until 3.6 is dropped (it is faster)
first_dict = args[0]
remove_keys = set.union(*map(set, args[1:]))
new4 = {k: v for k, v in first_dict.items() if k not in remove_keys}
return new4
def method_ifkeys_dictcomp_getitem(*args):
# Cannot use until 3.6 is dropped (it is faster)
first_dict = args[0]
remove_keys = set.union(*map(set, args[1:]))
new4 = {k: first_dict[k] for k in first_dict.keys() if k not in remove_keys}
return new4
method_lut = locals() # can populate this some other way
def make_data(num_items, num_other, remove_fraction, keytype):
if keytype == 'str':
keytype = str
if keytype == 'int':
keytype = int
first_keys = [random.randint(0, 1000) for _ in range(num_items)]
k = int(remove_fraction * len(first_keys))
remove_sets = [list(ub.unique(random.choices(first_keys, k=k) + [random.randint(0, 1000) for _ in range(num_items)])) for _ in range(num_other)]
first_dict = {keytype(k): k for k in first_keys}
args = [first_dict] + [{keytype(k): k for k in ks} for ks in remove_sets]
return args
ti = timerit.Timerit(200, bestof=1, verbose=2)
basis = {
'method': [
# Cant use because unordered
# 'method_diffkeys',
# Cant use because python 3.6
'method_ifkeys_dictcomp',
'method_ifkeys_dictcomp_getitem',
'method_ifkeys_setunion',
'method_ifkeys_getitem',
'method_diffkeys_list',
# Probably not good
# 'method_ifkeys_setcomp',
# 'method_diffkeys_oset',
],
'num_items': [10, 100, 1000],
'num_other': [1, 3, 5],
# 'num_other': [1],
'remove_fraction': [0, 0.2, 0.5, 0.7, 1.0],
# 'remove_fraction': [0.2, 0.8],
'keytype': ['str', 'int'],
# 'keytype': ['str'],
# 'param_name': [param values],
}
xlabel = 'num_items'
kw_labels = ['num_items', 'num_other', 'remove_fraction', 'keytype']
group_labels = {
'style': ['num_other', 'keytype'],
'size': ['remove_fraction'],
}
group_labels['hue'] = list(
(ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(
ub.dict_isect(params, labels), compact=1, si=1)
key = ub.repr2(params, compact=1, si=1)
kwargs = ub.dict_isect(params.copy(), kw_labels)
args = make_data(**kwargs)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
with timer:
# Put the logic you want to time here
method(*args)
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values('min')
print(data)
# for each parameter setting, group all methods with that used those exact
# comparable params. Then rank how good each method did. That will be a
# preference profile. We will give that preference profile a weight (e.g.
# based on the fastest method in the bunch) and then aggregate them with
# some voting method.
USE_OPENSKILL = 1
if USE_OPENSKILL:
# Lets try a real ranking method
# https://github.com/OpenDebates/openskill.py
import openskill
method_ratings = {m: openskill.Rating() for m in basis['method']}
weighted_rankings = ub.ddict(lambda: ub.ddict(float))
for params, variants in data.groupby(['num_other', 'keytype', 'remove_fraction', 'num_items']):
variants = variants.sort_values('mean')
ranking = variants['method'].reset_index(drop=True)
if USE_OPENSKILL:
# The idea is that each setting of parameters is a game, and each
# "method" is a player. We rank the players by which is fastest,
# and update their ranking according to the Weng-Lin Bayes ranking
# model. This does not take the fact that some "games" (i.e.
# parameter settings) are more important than others, but it should
# be fairly robust on average.
old_ratings = [[r] for r in ub.take(method_ratings, ranking)]
new_values = openskill.rate(old_ratings) # Not inplace
new_ratings = [openskill.Rating(*new[0]) for new in new_values]
method_ratings.update(ub.dzip(ranking, new_ratings))
# Choose a ranking weight scheme
weight = variants['mean'].min()
# weight = 1
for rank, method in enumerate(ranking):
weighted_rankings[method][rank] += weight
weighted_rankings[method]['total'] += weight
# Probably a more robust voting method to do this
weight_rank_rows = []
for method_name, ranks in weighted_rankings.items():
weights = ub.dict_diff(ranks, ['total'])
p_rank = ub.map_vals(lambda w: w / ranks['total'], weights)
for rank, w in p_rank.items():
weight_rank_rows.append({'rank': rank, 'weight': w, 'name': method_name})
weight_rank_df = pd.DataFrame(weight_rank_rows)
piv = weight_rank_df.pivot(['name'], ['rank'], ['weight'])
print(piv)
if USE_OPENSKILL:
from openskill import predict_win
win_prob = predict_win([[r] for r in method_ratings.values()])
skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False)
print('skill_agg =\n{}'.format(skill_agg))
aggregated = (piv * piv.columns.levels[1].values).sum(axis=1).sort_values()
print('weight aggregated =\n{}'.format(aggregated))
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw)
ax.set_title('Benchmark')
ax.set_xlabel('A better x-variable description')
ax.set_ylabel('A better y-variable description')
def benchmark_mul_vs_pow():
import ubelt as ub
import pandas as pd
import timerit
from functools import reduce
import operator as op
import itertools as it
def method_pow_via_mul_raw(n):
""" Construct a function that does multiplication of a value n times """
return eval('lambda v: ' + ' * '.join(['v'] * n))
def method_pow_via_mul_for(v, n):
ret = v
for _ in range(1, n):
ret = ret * v
return ret
def method_pow_via_mul_reduce(v, n):
""" Alternative way to multiply a value n times """
return reduce(op.mul, it.repeat(v, n))
def method_pow_via_pow(v, n):
return v ** n
method_lut = locals() # can populate this some other way
ti = timerit.Timerit(500000, bestof=1000, verbose=2)
basis = {
'method': ['method_pow_via_mul_raw', 'method_pow_via_pow'],
'n': list(range(1, 20)),
'v': ['random-int', 'random-float'],
# 'param_name': [param values],
}
xlabel = 'n'
kw_labels = ['v', 'n']
group_labels = {
'style': ['v'],
'size': [],
}
group_labels['hue'] = list(
(ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(
ub.dict_isect(params, labels), compact=1, si=1)
key = ub.repr2(params, compact=1, si=1)
kwargs = ub.dict_isect(params.copy(), kw_labels)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
if params['method'] == 'method_pow_via_mul_raw':
method = method(kwargs.pop('n'))
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
import random
if kwargs['v'] == 'random':
kwargs['v'] = random.randint(1, 31000) if random.random() > 0.5 else random.random()
elif kwargs['v'] == 'random-int':
kwargs['v'] = random.randint(1, 31000)
elif kwargs['v'] == 'random-float':
kwargs['v'] = random.random()
with timer:
# Put the logic you want to time here
method(**kwargs)
for time in map(min, ub.chunks(ti.times, ti.bestof)):
row = {
# 'mean': ti.mean(),
'time': time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
# data = data.sort_values('time')
print(data)
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plt = kwplot.autoplt()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y='time', marker='o', ax=ax, **plotkw)
ax.set_title('Benchmark')
ax.set_xlabel('N')
ax.set_ylabel('Time')
ax.set_yscale('log')
plt.show()
