def make_legend_img(classname_to_rgb, dpi=96, shape=(200, 200), mode='line', transparent=False): """ Makes an image of a categorical legend CommandLine: python -m netharn.util.mplutil make_legend_img Example: >>> # xdoctest: +REQUIRES(module:kwplot) >>> import kwplot >>> classname_to_rgb = { >>> 'blue': kwplot.Color('blue').as01(), >>> 'red': kwplot.Color('red').as01(), >>> } >>> img = make_legend_img(classname_to_rgb) >>> # xdoctest: +REQUIRES(--show) >>> kwplot.autompl() >>> kwplot.imshow(img) >>> kwplot.show_if_requested() """ import kwplot plt = kwplot.autoplt() def append_phantom_legend_label(label, color, type_='line', alpha=1.0, ax=None): if ax is None: ax = plt.gca() _phantom_legend_list = getattr(ax, '_phantom_legend_list', None) if _phantom_legend_list is None: _phantom_legend_list = [] setattr(ax, '_phantom_legend_list', _phantom_legend_list) if type_ == 'line': phantom_actor = plt.Line2D((0, 0), (1, 1), color=color, label=label, alpha=alpha) else: phantom_actor = plt.Circle((0, 0), 1, fc=color, label=label, alpha=alpha) _phantom_legend_list.append(phantom_actor) fig = plt.figure(dpi=dpi) w, h = shape[1] / dpi, shape[0] / dpi fig.set_size_inches(w, h) ax = fig.add_subplot(1, 1, 1) for label, color in classname_to_rgb.items(): append_phantom_legend_label(label, color, type_=mode, ax=ax) _phantom_legend_list = getattr(ax, '_phantom_legend_list', None) if _phantom_legend_list is None: _phantom_legend_list = [] setattr(ax, '_phantom_legend_list', _phantom_legend_list) ax.legend(handles=_phantom_legend_list) ax.grid(False) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) plt.axis('off') legend_img = render_figure_to_image(fig, dpi=dpi, transparent=transparent) plt.close(fig) return legend_img
def main(): import ubelt as ub dis_results = [ dis2(loop_first), ' | ', dis2(loop_first2), ] print(ub.hzcat(dis_results)) rows = benchmark_loop_first_variants() import pandas as pd df = pd.DataFrame(rows) import kwplot plt = kwplot.autoplt() # import matplotlib as mpl # mpl.use('Qt5Agg') # from matplotlib import pyplot as plt import seaborn as sns sns.set() sns.lineplot(data=df, x='num_items', y='mean', hue='method_name', markers='method_name') plt.show()
def mwe_check_before_select(): import ubelt as ub import numpy as np results = [] ns = np.logspace(1, 7, 100).astype(np.int) for n in ub.ProgIter(ns, desc='time-tradeoff', verbose=3): print('n = {!r}'.format(n)) y_true = np.random.randint(0, 100, n).astype(np.int64) y_pred = np.random.randint(0, 100, n).astype(np.int64) sample_weight = np.random.rand(n) isvalid = np.random.rand(n) > 0.5 import timerit ti = timerit.Timerit(9, bestof=3, verbose=2) for timer in ti.reset('all-check'): with timer: np.all(isvalid) results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) for timer in ti.reset('all-index'): with timer: y_true[isvalid] y_pred[isvalid] sample_weight[isvalid] results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) import pandas as pd df = pd.DataFrame(results) import kwplot import seaborn as sns kwplot.autoplt() sns.set() ax = sns.lineplot(data=df, x='n', y='time', hue='label') ax.set_yscale('log') ax.set_xscale('log') pass
def test_last_digits(): mapping = {} tail_edge_hist = ub.ddict(lambda: 0) for x in ub.ProgIter(range(1, int(1e5))): import ubelt as ub for y in collatz(x): if x in mapping: break x_tail = x - (10 * (x // 10)) y_tail = y - (10 * (y // 10)) mapping[x] = y tail_edge_hist[(str(x_tail), str(y_tail))] += 1 x = y import kwarray print(kwarray.stats_dict(np.array(list(tail_edge_hist.values())))) import networkx as nx tail_g = nx.DiGraph() tail_g.add_edges_from(tail_edge_hist.keys()) for cycle in nx.simple_cycles(tail_g): print('cycle = {!r}'.format(cycle)) print('tail_g.adj = {!r}'.format(tail_g.adj)) for n in sorted(tail_g.nodes, key=int): pred = tuple(tail_g.pred[n].keys()) succ = tuple(tail_g.succ[n].keys()) in_d = tail_g.in_degree(n) out_d = tail_g.out_degree(n) print( f'{str(pred):>15} -> {n:>2} -> {str(succ):<15} : {out_d:<2} {in_d:<2}' ) import kwplot import graphid plt = kwplot.autoplt() sccs = list(nx.strongly_connected_components(tail_g)) nx.draw_networkx(tail_g) # nx.draw_circular(tail_g) ax = plt.gca() ax.cla() # nx.draw_networkx(tail_g) graphid.util.util_graphviz.show_nx(tail_g) CCs = list(nx.connected_components(tail_g.to_undirected()))
def draw(): import kwplot plt = kwplot.autoplt() plotkw = dict( xlabel='learning-rate', ylabel='loss', xscale=scale, ymin=0, xmin='data', xmax=high, fnum=1, ) R = 2 if 'vali' in records else 1 for i, tag in enumerate(['train', 'vali']): if tag in records: kwplot.multi_plot( xdata=records[tag]['lr'], ydata=records[tag]['loss'], ymax=1.2 * np.percentile(records[tag]['loss'], 60) / .6, pnum=(1, R, i), title=tag + ' lr-scan', **plotkw) plt.plot(best_lr, best_loss, '*')
def colorbar_image(domain, cmap='plasma', dpi=96, shape=(200, 20), transparent=False): """ Notes: shape is approximate Ignore: domain = np.linspace(-30, 200) cmap='plasma' dpi = 80 dsize = (20, 200) util.imwrite('foo.png', util.colorbar_image(np.arange(0, 1)), shape=(400, 80)) import plottool as pt pt.qtensure() import matplotlib as mpl mpl.style.use('ggplot') util.imwrite('foo.png', util.colorbar_image(np.linspace(0, 1, 100), dpi=200, shape=(1000, 40), transparent=1)) ub.startfile('foo.png') """ import kwplot plt = kwplot.autoplt() fig = plt.figure(dpi=dpi) w, h = shape[1] / dpi, shape[0] / dpi # w, h = 1, 10 fig.set_size_inches(w, h) ax = fig.add_subplot('111') sm = plt.cm.ScalarMappable(cmap=plt.get_cmap(cmap)) sm.set_array(domain) plt.colorbar(sm, cax=ax) cb_img = render_figure_to_image(fig, dpi=dpi, transparent=transparent) plt.close(fig) return cb_img
def benchmark_ondisk_crop(): import kwplot plt = kwplot.autoplt() region = 'small_random' dim = 3 # xdata = [64, 128, 256, 512] # xdata = [64, 128, 256, 320, 512, 640, 768, 896, 1024] # xdata = np.linspace(64, 4096, num=8).astype(np.int) # xdata = np.linspace(64, 2048, num=8).astype(np.int) # xdata = np.linspace(64, 1024, num=8).astype(np.int) xdata = [256, 1024, 4096, 8192, 16384] # xdata = [256, 1024, 4096, 8192] xdata = [256, 1024, 2048] # xdata = [256] ydata = ub.ddict(list) # for size in [64, 128, 256, 512, 1024, 2048, 4096]: for size in xdata: result = time_ondisk_crop(size, dim=dim, region=region, num=5) for key, val in result.items(): min, mean, std = val ydata[key].append(mean * 1e6) # Sort legend by descending time taken on the largest image ydata = ub.odict(sorted(ydata.items(), key=lambda i: -i[1][-1])) kwplot.multi_plot( xdata, ydata, ylabel='micro-seconds (us)', xlabel='image size', title='Chip region={} benchmark for {}D image data'.format( region, dim), # yscale='log', ymin=1, ) plt.show()
def main(): sns = kwplot.autosns() # NOQA plt = kwplot.autoplt() # NOQA if 1: array = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] values = [-1, 0, 1, 4, 6, 6.1, 6.5, 7, 10.3, 10.5, 10.7, 15, 16] if 0: array = np.linspace(0, 30) values = np.linspace(0, 30) if 0: xscale = 20 num = 20 array = np.array(sorted(np.random.rand(num) * xscale)).round() values = np.hstack([ np.unique(np.random.choice(array, 3)), np.random.rand(num // 2) * xscale ]) fig = kwplot.figure(fnum=1, doclf=1, pnum=(2, 1, 1)) ax = fig.gca() plot_searchsorted_visualization(array, values, side='left', ax=ax) ax.set_title('association = searchsorted(array, values, side=left)') fig = kwplot.figure(fnum=1, doclf=0, pnum=(2, 1, 2)) ax = fig.gca() plot_searchsorted_visualization(array, values, side='right', ax=ax) ax.set_title('association = searchsorted(array, values, side=right)') import ubelt as ub fig.suptitle( ub.codeblock(''' Notice: side=left and side=right have the same result except when the value is already in the array. '''))
def main(): """ Main function for the generic classification example with an undocumented hack for the lrtest. """ harn = setup_harn() harn.initialize() if ub.argflag('--lrtest'): # Undocumented hidden feature, # Perform an LR-test, then resetup the harness. Optionally draw the # results using matplotlib. from netharn.prefit.lr_tests import lr_range_test result = lr_range_test(harn, init_value=1e-4, final_value=0.5, beta=0.3, explode_factor=10, num_iters=200) if ub.argflag('--show'): import kwplot plt = kwplot.autoplt() result.draw() plt.show() # Recreate a new version of the harness with the recommended LR. config = harn.script_config.asdict() config['lr'] = (result.recommended_lr * 10) harn = setup_harn(**config) harn.initialize() # This starts the main loop which will run until the monitor's terminator # criterion is satisfied. If the initialize step loaded a checkpointed that # already met the termination criterion, then this will simply return. deploy_fpath = harn.run() # The returned deploy_fpath is the path to an exported netharn model. # This model is the on with the best weights according to the monitor. print('deploy_fpath = {!r}'.format(deploy_fpath)) return harn
def convert_camvid_raw_to_coco(camvid_raw_info): """ Converts the raw camvid format to an MSCOCO based format, ( which lets use use kwcoco's COCO backend). Example: >>> # xdoctest: +REQUIRES(--download) >>> camvid_raw_info = grab_raw_camvid() >>> # test with a reduced set of data >>> del camvid_raw_info['img_paths'][2:] >>> del camvid_raw_info['mask_paths'][2:] >>> dset = convert_camvid_raw_to_coco(camvid_raw_info) >>> # xdoctest: +REQUIRES(--show) >>> import kwplot >>> plt = kwplot.autoplt() >>> kwplot.figure(fnum=1, pnum=(1, 2, 1)) >>> dset.show_image(gid=1) >>> kwplot.figure(fnum=1, pnum=(1, 2, 2)) >>> dset.show_image(gid=2) """ import re import kwimage import kwcoco print('Converting CamVid to MS-COCO format') dset_root, img_paths, label_path, mask_paths = ub.take( camvid_raw_info, 'dset_root, img_paths, label_path, mask_paths'.split(', ')) img_infos = { 'img_fname': img_paths, 'mask_fname': mask_paths, } keys = list(img_infos.keys()) next_vals = list(zip(*img_infos.values())) image_items = [{k: v for k, v in zip(keys, vals)} for vals in next_vals] dataset = { 'img_root': dset_root, 'images': [], 'categories': [], 'annotations': [], } lines = ub.readfrom(label_path).split('\n') lines = [line for line in lines if line] for line in lines: color_text, name = re.split('\t+', line) r, g, b = map(int, color_text.split(' ')) color = (r, g, b) # Parse the special camvid format cid = (r << 16) + (g << 8) + (b << 0) cat = { 'id': cid, 'name': name, 'color': color, } dataset['categories'].append(cat) for gid, img_item in enumerate(image_items, start=1): img = { 'id': gid, 'file_name': img_item['img_fname'], # nonstandard image field 'segmentation': img_item['mask_fname'], } dataset['images'].append(img) dset = kwcoco.CocoDataset(dataset) dset.rename_categories({'Void': 'background'}) assert dset.name_to_cat['background']['id'] == 0 dset.name_to_cat['background'].setdefault('alias', []).append('Void') if False: _define_camvid_class_hierarcy(dset) if 1: # TODO: Binarize CCs (and efficiently encode if possible) import numpy as np bad_info = [] once = False # Add images dset.remove_annotations(list(dset.index.anns.keys())) for gid, img in ub.ProgIter(dset.imgs.items(), desc='parse label masks'): mask_fpath = join(dset_root, img['segmentation']) rgb_mask = kwimage.imread(mask_fpath, space='rgb') r, g, b = rgb_mask.T.astype(np.int64) cid_mask = np.ascontiguousarray(rgb_to_cid(r, g, b).T) cids = set(np.unique(cid_mask)) - {0} for cid in cids: if cid not in dset.cats: if gid == 618: # Handle a known issue with image 618 c_mask = (cid == cid_mask).astype(np.uint8) total_bad = c_mask.sum() if total_bad < 32: if not once: print( 'gid 618 has a few known bad pixels, ignoring them' ) once = True continue else: raise Exception('more bad pixels than expected') else: raise Exception( 'UNKNOWN cid = {!r} in gid={!r}'.format(cid, gid)) # bad_rgb = cid_to_rgb(cid) # print('bad_rgb = {!r}'.format(bad_rgb)) # print('WARNING UNKNOWN cid = {!r} in gid={!r}'.format(cid, gid)) # bad_info.append({ # 'gid': gid, # 'cid': cid, # }) else: ann = { 'category_id': cid, 'image_id': gid # 'segmentation': mask.to_coco() } assert cid in dset.cats c_mask = (cid == cid_mask).astype(np.uint8) mask = kwimage.Mask(c_mask, 'c_mask') box = kwimage.Boxes([mask.get_xywh()], 'xywh') # box = mask.to_boxes() ann['bbox'] = ub.peek(box.to_coco()) ann['segmentation'] = mask.to_coco() dset.add_annotation(**ann) if 0: bad_cids = [i['cid'] for i in bad_info] print(sorted([c['color'] for c in dataset['categories']])) print(sorted(set([cid_to_rgb(i['cid']) for i in bad_info]))) gid = 618 img = dset.imgs[gid] mask_fpath = join(dset_root, img['segmentation']) rgb_mask = kwimage.imread(mask_fpath, space='rgb') r, g, b = rgb_mask.T.astype(np.int64) cid_mask = np.ascontiguousarray(rgb_to_cid(r, g, b).T) cid_hist = ub.dict_hist(cid_mask.ravel()) bad_cid_hist = {} for cid in bad_cids: bad_cid_hist[cid] = cid_hist.pop(cid) import kwplot kwplot.autompl() kwplot.imshow(rgb_mask) if 0: import kwplot plt = kwplot.autoplt() plt.clf() dset.show_image(1) import xdev gid_list = list(dset.imgs) for gid in xdev.InteractiveIter(gid_list): dset.show_image(gid) xdev.InteractiveIter.draw() dset._build_index() dset._build_hashid() return dset
def mwe_check_sample_weight(): import ubelt as ub from sklearn.utils.validation import _check_sample_weight import numpy as np results = [] ns = np.logspace(1, 6, 100).astype(np.int) for n in ub.ProgIter(ns, desc='time-tradeoff', verbose=3): print('n = {!r}'.format(n)) y_true = np.random.randint(0, 100, n).astype(np.int64) y_pred = np.random.randint(0, 100, n).astype(np.int64) sample_weight = np.random.rand(n) import timerit ti = timerit.Timerit(9, bestof=3, verbose=2) for timer in ti.reset('old-sample-weight-given'): with timer: np.asarray(sample_weight) results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) for timer in ti.reset('new-sample-weight-given'): with timer: _check_sample_weight(sample_weight, y_true, dtype=np.int64) results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) for timer in ti.reset('old-sample-weight-default'): with timer: np.ones(y_true.shape[0], dtype=np.int64) results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) for timer in ti.reset('new-sample-weight-default'): with timer: _check_sample_weight(None, y_true, dtype=np.int64) results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) import pandas as pd df = pd.DataFrame(results) import kwplot import seaborn as sns kwplot.autoplt() sns.set() ax = sns.lineplot(data=df, x='n', y='time', hue='label') ax.set_yscale('log') ax.set_xscale('log') from sklearn.utils.validation import _check_sample_weight import numpy as np results = [] ns = np.logspace(1, 6, 100).astype(np.int) for n in ub.ProgIter(ns, desc='time-tradeoff', verbose=3): print('n = {!r}'.format(n)) n_labels = 100 y_true = np.random.randint(0, n_labels, n).astype(np.int64) y_pred = np.random.randint(0, n_labels, n).astype(np.int64) sample_weight = np.ones(y_true.shape[0], dtype=np.int64) for timer in ti.reset('use-old-uint8-sample-weight-default'): with timer: if sample_weight.dtype.kind in {'i', 'u', 'b'}: dtype = np.int64 else: dtype = np.float64 cm = coo_matrix((sample_weight, (y_true, y_pred)), shape=(n_labels, n_labels), dtype=dtype, ).toarray() results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) sample_weight = _check_sample_weight(None, y_true, dtype=np.int64) for timer in ti.reset('use-new-float64-sample-weight-default'): with timer: if sample_weight.dtype.kind in {'i', 'u', 'b'}: dtype = np.int64 else: dtype = np.float64 cm = coo_matrix((sample_weight, (y_true, y_pred)), shape=(n_labels, n_labels), dtype=dtype, ).toarray() results.append({ 'n': n, 'label': ti.label, 'time': ti.mean(), }) import pandas as pd df = pd.DataFrame(results) import kwplot import seaborn as sns kwplot.autoplt() sns.set() ax = sns.lineplot(data=df, x='n', y='time', hue='label') ax.set_yscale('log') ax.set_xscale('log')
def eval_detections_cli(**kw): """ CommandLine: xdoctest -m ~/code/netharn/netharn/metrics/detect_metrics.py eval_detections_cli """ import scriptconfig as scfg import ndsampler class EvalDetectionCLI(scfg.Config): default = { 'true': scfg.Path(None, help='true coco dataset'), 'pred': scfg.Path(None, help='predicted coco dataset'), 'out_dpath': scfg.Path('./out', help='output directory') } pass config = EvalDetectionCLI() cmdline = kw.pop('cmdline', True) config.load(kw, cmdline=cmdline) true_coco = ndsampler.CocoDataset(config['true']) pred_coco = ndsampler.CocoDataset(config['pred']) from netharn.metrics.detect_metrics import DetectionMetrics dmet = DetectionMetrics.from_coco(true_coco, pred_coco) voc_info = dmet.score_voc() cls_info = voc_info['perclass'][0] tp = cls_info['tp'] fp = cls_info['fp'] fn = cls_info['fn'] tpr = cls_info['tpr'] ppv = cls_info['ppv'] fp = cls_info['fp'] # Compute the MCC as TN->inf thresh = cls_info['thresholds'] # https://erotemic.wordpress.com/2019/10/23/closed-form-of-the-mcc-when-tn-inf/ mcc_lim = tp / (np.sqrt(fn + tp) * np.sqrt(fp + tp)) f1 = 2 * (ppv * tpr) / (ppv + tpr) draw = False if draw: mcc_idx = mcc_lim.argmax() f1_idx = f1.argmax() import kwplot plt = kwplot.autoplt() kwplot.multi_plot( xdata=thresh, ydata=mcc_lim, xlabel='threshold', ylabel='mcc*', fnum=1, pnum=(1, 4, 1), title='MCC*', color=['blue'], ) plt.plot(thresh[mcc_idx], mcc_lim[mcc_idx], 'r*', markersize=20) plt.plot(thresh[f1_idx], mcc_lim[f1_idx], 'k*', markersize=20) kwplot.multi_plot( xdata=fp, ydata=tpr, xlabel='fp (fa)', ylabel='tpr (pd)', fnum=1, pnum=(1, 4, 2), title='ROC', color=['blue'], ) plt.plot(fp[mcc_idx], tpr[mcc_idx], 'r*', markersize=20) plt.plot(fp[f1_idx], tpr[f1_idx], 'k*', markersize=20) kwplot.multi_plot( xdata=tpr, ydata=ppv, xlabel='tpr (recall)', ylabel='ppv (precision)', fnum=1, pnum=(1, 4, 3), title='PR', color=['blue'], ) plt.plot(tpr[mcc_idx], ppv[mcc_idx], 'r*', markersize=20) plt.plot(tpr[f1_idx], ppv[f1_idx], 'k*', markersize=20) kwplot.multi_plot( xdata=thresh, ydata=f1, xlabel='threshold', ylabel='f1', fnum=1, pnum=(1, 4, 4), title='F1', color=['blue'], ) plt.plot(thresh[mcc_idx], f1[mcc_idx], 'r*', markersize=20) plt.plot(thresh[f1_idx], f1[f1_idx], 'k*', markersize=20)
def _precompute_class_weights(dset, mode='median-idf'): """ Example: >>> # xdoctest: +REQUIRES(--download) >>> import sys, ubelt >>> sys.path.append(ubelt.expandpath('~/code/netharn/examples')) >>> from sseg_camvid import * # NOQA >>> harn = setup_harn(0, workers=0, xpu='cpu').initialize() >>> dset = harn.datasets['train'] """ assert mode in ['median-idf', 'log-median-idf'] total_freq = _cached_class_frequency(dset) def logb(arr, base): if base == 'e': return np.log(arr) elif base == 2: return np.log2(arr) elif base == 10: return np.log10(arr) else: out = np.log(arr) out /= np.log(base) return out _min, _max = np.percentile(total_freq, [5, 95]) is_valid = (_min <= total_freq) & (total_freq <= _max) if np.any(is_valid): middle_value = np.median(total_freq[is_valid]) else: middle_value = np.median(total_freq) # variant of median-inverse-frequency nonzero_freq = total_freq[total_freq != 0] if len(nonzero_freq): total_freq[total_freq == 0] = nonzero_freq.min() / 2 if mode == 'median-idf': weights = (middle_value / total_freq) weights[~np.isfinite(weights)] = 1.0 elif mode == 'log-median-idf': weights = (middle_value / total_freq) weights[~np.isfinite(weights)] = 1.0 base = 2 base = np.exp(1) weights = logb(weights + (base - 1), base) weights = np.maximum(weights, .1) weights = np.minimum(weights, 10) else: raise KeyError('mode = {!r}'.format(mode)) weights = np.round(weights, 2) cname_to_weight = ub.dzip(dset.classes, weights) print('weights: ' + ub.repr2(cname_to_weight)) if False: # Inspect the weights import kwplot kwplot.autoplt() cname_to_weight = ub.dzip(dset.classes, weights) cname_to_weight = ub.dict_subset(cname_to_weight, ub.argsort(cname_to_weight)) kwplot.multi_plot( ydata=list(cname_to_weight.values()), kind='bar', xticklabels=list(cname_to_weight.keys()), xtick_rotation=90, fnum=2, doclf=True) return weights
def main(): import kwplot plt = kwplot.autoplt() sns = kwplot.autosns() alias = { '3090': 'nvctrl GeForce GTX 1080 Ti 1 temp', '1080ti': 'nvctrl GeForce RTX 3090 0 temp', # 'cpu': 'lmsensor coretemp-isa-0000 Package id 0', } all_df = read_psensor_log() unique_rawdevs = all_df.device.unique() for rawdev in unique_rawdevs: cpu_prefix = 'lmsensor coretemp-isa' if rawdev.startswith(cpu_prefix): suffix = rawdev[len(cpu_prefix):].split(' ', 1)[1].strip() alias['CPU ' + suffix] = rawdev if 'nvctrl' in rawdev and 'temp' in rawdev: alias['GPU ' + rawdev[7:-5]] = rawdev mapper = ub.invert_dict(alias) all_df['device'] = all_df['device'].apply(lambda x: mapper.get(x, None)) all_df = all_df[all_df['device'].apply(lambda x: x is not None)] hours = int(ub.argval('--hours', default=48)) delta = datetime.timedelta(hours=hours) min_time = datetime.datetime.now() - delta is_recent = all_df.datetime > min_time recent_df = all_df[is_recent] chosen = recent_df # chosen = all_df if 0: pivtbl = recent_df.pivot('unix_timestamp', 'device', 'temp') pivtbl = pivtbl.sort_index() smoothed_rows = [] for window_idxs in ub.iter_window(list(range(len(pivtbl))), size=10): window = pivtbl.iloc[list(window_idxs)] max_val = window.max(axis=0, skipna=True) for k, v in max_val.to_dict().items(): smoothed_rows.append({ 'unix_timestamp': window.index[1], 'device': k, 'temp': v, }) max_extra = pd.DataFrame(smoothed_rows) sns.lineplot(data=max_extra, x='unix_timestamp', y='temp', hue='device') df = recent_df.copy() df['device'] = df['device'].apply(lambda x: 'Core' if x.startswith('Core') else x) df['time'] = df['unix_timestamp'].apply( datetime.datetime.fromtimestamp) plt.gcf().clf() # sns.lineplot(data=chosen, x='unix_timestamp', y='temp', hue='device') for xx, (sess, group) in enumerate(chosen.groupby('session_x')): # ax.cla() ax = plt.gca() sns.lineplot(data=group, x='unix_timestamp', y='temp', hue='device', legend=xx == 0) label_xaxis_dates(ax) ax.figure.subplots_adjust(bottom=0.2) ax.set_ylim(0, 100) plt.locator_params(axis='y', nbins=10) # import matplotlib as mpl # Draw shutdown time as black lines end_times = [] for sx, group in chosen.groupby('session_x'): shutdown_time = group['unix_timestamp'].max() end_times.append(shutdown_time) for shutdown_time in sorted(end_times)[:-1]: ax.plot((shutdown_time, shutdown_time), [0, 100], color='k') # ci_df = pd.concat([max_extra, recent_df]) # ci_df['device'] = ci_df['device'].apply(lambda x: 'Core' if x.startswith('Core') else x) # sns.lineplot(data=ci_df, x='unix_timestamp', y='temp', hue='device') # from matplotlib.dates import date2num # all_df['date_ord'] = all_df['datetime'].map(lambda a: date2num(a)) # sns.lineplot(data=pt) # sns.lineplot(data=recent_df, x='unix_timestamp', y='temp', hue='device') # sns.regplot(data=recent_df, x='unix_timestamp', y='temp', hue='device') plt.show()
def plot_weight_scatter(harn): """ Draw a scatter plot of the initial weights versus the final weights of a network. Example: >>> import netharn as nh >>> harn = nh.FitHarn.demo() >>> harn.run() Ignore: >>> from netharn.plots.weight_scatter import * # NOQA >>> from netharn.examples import mnist >>> import kwplot >>> harn = mnist.setup_harn() >>> harn.preferences['timeout'] = 60 * 1 >>> kwplot.autompl(force='agg') >>> harn.run() >>> kwplot.autompl(force='auto') >>> plot_weight_scatter(harn) """ import netharn as nh cpu = nh.XPU.coerce('cpu') path1 = join(harn.train_dpath, 'initial_state', 'initial_state.pt') state1 = cpu.load(path1) weights1 = state1['model_state_dict'] path2 = harn.best_snapshot() state2 = cpu.load(path2) weights2 = state2['model_state_dict'] keys1 = set(weights1.keys()) keys2 = set(weights2.keys()) keys = keys1 & keys2 assert keys == keys2 accum1 = [] accum2 = [] for key in keys: w1 = weights1[key] w2 = weights2[key] accum1.append(w1.numpy().ravel()) accum2.append(w2.numpy().ravel()) points1 = np.hstack(accum1) points2 = np.hstack(accum2) # Find cosine of angle between the vectors import scipy cosangle = scipy.spatial.distance.cosine(points1, points2) print('cosangle = {!r}'.format(cosangle)) import kwplot import seaborn seaborn.set() plt = kwplot.autoplt() plt.clf() x = points1[::1] y = points2[::1] ax = plt.gca() ax.figure.clf() # seaborn.kdeplot(x, y, shade=True, gridsize=50) ax = plt.gca() ax.scatter(x, y, s=1, alpha=0.1, c='blue') ax.set_xlabel('initial weights') ax.set_ylabel('trained weights')
def benchmark_video_readers(): """ "On My Machine" I get: ti.measures = { 'mean' : { 'cv2 sequential access' : 0.0137, 'decord sequential access' : 0.0175, 'cv2 open + first access' : 0.0222, 'decord open + first access' : 0.0565, 'vi3o sequential access' : 0.0642, 'cv2 open + one random access' : 0.0723, 'decord open + one random access': 0.0946, 'vi3o open + first access' : 0.1045, 'cv2 random access' : 0.3316, 'decord random access' : 0.3472, 'decord random batch access' : 0.3482, 'vi3o open + one random access' : 0.3590, 'vi3o random access' : 1.6660, }, 'mean+std': { 'cv2 sequential access' : 0.0145, 'decord sequential access' : 0.0182, 'cv2 open + first access' : 0.0230, 'vi3o sequential access' : 0.0881, 'decord open + first access' : 0.1038, 'vi3o open + first access' : 0.1059, 'cv2 open + one random access' : 0.1151, 'decord open + one random access': 0.1329, 'cv2 random access' : 0.3334, 'decord random access' : 0.3496, 'decord random batch access' : 0.3511, 'vi3o open + one random access' : 0.5215, 'vi3o random access' : 1.6890, }, 'mean-std': { 'decord open + first access' : 0.0091, 'cv2 sequential access' : 0.0130, 'decord sequential access' : 0.0168, 'cv2 open + first access' : 0.0214, 'cv2 open + one random access' : 0.0295, 'vi3o sequential access' : 0.0403, 'decord open + one random access': 0.0563, 'vi3o open + first access' : 0.1032, 'vi3o open + one random access' : 0.1965, 'cv2 random access' : 0.3299, 'decord random access' : 0.3448, 'decord random batch access' : 0.3452, 'vi3o random access' : 1.6429, }, 'min' : { 'cv2 sequential access' : 0.0128, 'decord sequential access' : 0.0166, 'cv2 open + first access' : 0.0210, 'vi3o sequential access' : 0.0233, 'decord open + first access' : 0.0251, 'cv2 open + one random access' : 0.0282, 'decord open + one random access': 0.0527, 'vi3o open + one random access' : 0.1013, 'vi3o open + first access' : 0.1026, 'cv2 random access' : 0.3299, 'decord random access' : 0.3433, 'decord random batch access' : 0.3452, 'vi3o random access' : 1.6423, }, } """ # video_fpath = ub.grabdata('https://download.blender.org/peach/bigbuckbunny_movies/big_buck_bunny_720p_h264.mov') try: import vi3o except Exception: vi3o = None # video_fpath = ub.grabdata('https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4') video_fpath = ub.grabdata('https://file-examples-com.github.io/uploads/2018/04/file_example_MOV_1280_1_4MB.mov') ti = timerit.Timerit(10, bestof=3, verbose=3, unit='ms') video_length = len(CV2VideoReader(video_fpath)) num_frames = min(5, video_length) rng = kwarray.ensure_rng(0) random_indices = rng.randint(0, video_length, size=num_frames).tolist() if True: with timerit.Timer(label='open cv2') as cv2_open_timer: cv2_video = CV2VideoReader(video_fpath) for timer in ti.reset('cv2 sequential access'): cv2_video.seek(0) with timer: for frame, _ in zip(cv2_video, range(num_frames)): pass for timer in ti.reset('cv2 random access'): with timer: for index in random_indices: cv2_video[index] if vi3o is not None: with timerit.Timer(label='open vi3o') as vi3o_open_timer: vi3o_video = vi3o.Video(video_fpath) for timer in ti.reset('vi3o sequential access'): with timer: for frame, _ in zip(vi3o_video, range(num_frames)): pass for timer in ti.reset('vi3o random access'): with timer: for index in random_indices: vi3o_video[index] if True: import decord with timerit.Timer(label='open decord') as decord_open_timer: decord_video = decord.VideoReader(video_fpath) for timer in ti.reset('decord sequential access'): with timer: for frame, _ in zip(decord_video, range(num_frames)): pass for timer in ti.reset('decord random access'): with timer: for index in random_indices: decord_video[index] for timer in ti.reset('decord random batch access'): with timer: decord_video.get_batch(random_indices) if True: # One Random Access Case def _work_to_clear_io_caches(): import kwimage # Let some caches be cleared for i in range(10): for key in kwimage.grab_test_image.keys(): kwimage.grab_test_image(key) rng = kwarray.ensure_rng(0) for timer in ti.reset('cv2 open + one random access'): _work_to_clear_io_caches() with timer: _cv2_video = CV2VideoReader(video_fpath) index = rng.randint(0, video_length, size=1)[0] _cv2_video[index] if vi3o is not None: rng = kwarray.ensure_rng(0) for timer in ti.reset('vi3o open + one random access'): _work_to_clear_io_caches() with timer: _vi3o_video = vi3o.Video(video_fpath) index = rng.randint(0, video_length, size=1)[0] _vi3o_video[index] rng = kwarray.ensure_rng(0) for timer in ti.reset('decord open + one random access'): _work_to_clear_io_caches() with timer: _decord_video = decord.VideoReader(video_fpath) index = rng.randint(0, video_length, size=1)[0] _decord_video[index] for timer in ti.reset('cv2 open + first access'): _work_to_clear_io_caches() with timer: _cv2_video = CV2VideoReader(video_fpath) _cv2_video[0] if vi3o is not None: for timer in ti.reset('vi3o open + first access'): _work_to_clear_io_caches() with timer: _vi3o_video = vi3o.Video(video_fpath) _vi3o_video[0] for timer in ti.reset('decord open + first access'): _work_to_clear_io_caches() with timer: _decord_video = decord.VideoReader(video_fpath) _decord_video[0] measures = ub.map_vals(ub.sorted_vals, ti.measures) print('ti.measures = {}'.format(ub.repr2(measures, nl=2, align=':', precision=4))) print('cv2_open_timer.elapsed = {!r}'.format(cv2_open_timer.elapsed)) print('decord_open_timer.elapsed = {!r}'.format(decord_open_timer.elapsed)) if vi3o: print('vi3o_open_timer.elapsed = {!r}'.format(vi3o_open_timer.elapsed)) import kwplot import seaborn as sns sns.set() plt = kwplot.autoplt() df = pd.DataFrame(ti.measures) df['key'] = df.index df['expt'] = df['key'].apply(lambda k: ' '.join(k.split(' ')[1:])) df['module'] = df['key'].apply(lambda k: k.split(' ')[0]) # relmod = 'decord' relmod = 'cv2' for k, group in df.groupby('expt'): measure = 'mean' relval = group[group['module'] == relmod][measure].values.ravel() if len(relval) > 0: assert len(relval) == 1 df.loc[group.index, measure + '_rel'] = group[measure] / relval df.loc[group.index, measure + '_slower_than_' + relmod] = group[measure] / relval df.loc[group.index, measure + '_faster_than_' + relmod] = relval / group[measure] fig = kwplot.figure(fnum=1, doclf=True) ax = fig.gca() y_key = "mean_faster_than_" + relmod sub_df = df.loc[~df[y_key].isnull()] sns.barplot( x="expt", y=y_key, data=sub_df, hue='module', ax=ax) ax.set_title('cpu video reading benchmarks') plt.show()
def make_legend_img(label_to_color, dpi=96, shape=(200, 200), mode='line', transparent=False): """ Makes an image of a categorical legend Args: label_to_color (Dict[str, Color]): mapping from string label to the color. CommandLine: xdoctest -m kwplot.mpl_make make_legend_img --show Example: >>> # xdoctest: +REQUIRES(module:kwplot) >>> import kwplot >>> import kwimage >>> label_to_color = { >>> 'blue': kwimage.Color('blue').as01(), >>> 'red': kwimage.Color('red').as01(), >>> 'green': 'green', >>> 'yellow': 'yellow', >>> 'orangered': 'orangered', >>> } >>> img = make_legend_img(label_to_color, transparent=True) >>> # xdoctest: +REQUIRES(--show) >>> kwplot.autompl() >>> kwplot.imshow(img) >>> kwplot.show_if_requested() """ import kwplot import kwimage plt = kwplot.autoplt() def append_phantom_legend_label(label, color, type_='line', alpha=1.0, ax=None): if ax is None: ax = plt.gca() _phantom_legend_list = getattr(ax, '_phantom_legend_list', None) if _phantom_legend_list is None: _phantom_legend_list = [] setattr(ax, '_phantom_legend_list', _phantom_legend_list) if type_ == 'line': phantom_actor = plt.Line2D((0, 0), (1, 1), color=color, label=label, alpha=alpha) else: phantom_actor = plt.Circle((0, 0), 1, fc=color, label=label, alpha=alpha) _phantom_legend_list.append(phantom_actor) fig = plt.figure(dpi=dpi) w, h = shape[1] / dpi, shape[0] / dpi fig.set_size_inches(w, h) # ax = fig.add_subplot('111') ax = fig.add_subplot(1, 1, 1) for label, color in label_to_color.items(): color = kwimage.Color(color).as01() append_phantom_legend_label(label, color, type_=mode, ax=ax) _phantom_legend_list = getattr(ax, '_phantom_legend_list', None) if _phantom_legend_list is None: _phantom_legend_list = [] setattr(ax, '_phantom_legend_list', _phantom_legend_list) ax.legend(handles=_phantom_legend_list) ax.grid(False) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) plt.axis('off') legend_img = render_figure_to_image(fig, dpi=dpi, transparent=transparent) legend_img = kwimage.convert_colorspace(legend_img, src_space='bgr', dst_space='rgb') legend_img = crop_border_by_color(legend_img) plt.close(fig) return legend_img
def ford_circles(): """ Draw Ford Circles This is a Ford Circle diagram of the Rationals and Float32 numbers. Only 163 of the 32608 rationals I generated can be exactly represented by a float32. [MF 14] [MF 95] [MF 14] https://www.youtube.com/watch?v=83ZjYvkdzYI&list=PL5A714C94D40392AB&index=14 [MF 95] https://www.youtube.com/watch?v=gATEJ3f3FBM&list=PL5A714C94D40392AB&index=95 Examples: import kwplot kwplot.autompl() """ import kwplot import ubelt as ub import matplotlib as mpl plt = kwplot.autoplt() sns = kwplot.autosns() # NOQA limit = 256 * 256 print('limit = {!r}'.format(limit)) rats_to_plot = set() maxx = 1 _iter = Rational.members(limit=limit) _genrat = set(ub.ProgIter(_iter, total=limit, desc='gen rats')) rats_to_plot |= _genrat rats_to_plot2 = sorted({Rational(r % maxx) for r in rats_to_plot} | {maxx}) floats = sorted( ub.unique(map(float, rats_to_plot2), key=lambda f: f.as_integer_ratio())) print(f'{len(rats_to_plot) = }') print(f'{len(rats_to_plot2) = }') print(f'{len(floats) = }') import numpy as np ax = kwplot.figure(fnum=1, doclf=True).gca() prog = ub.ProgIter(sorted(rats_to_plot2), verbose=1) dtype = np.float32 patches = ub.ddict(list) errors = [] for rat in prog: denominator = rat.denominator radius = 1 / (2 * (denominator * denominator)) point = (rat, radius) flt = dtype(rat) a, b = flt.as_integer_ratio() flt_as_rat = Rational(a, b) error = abs(rat - flt_as_rat) if error == 0: new_circle = plt.Circle(point, radius, facecolor='dodgerblue', edgecolor='none', linewidth=0, alpha=0.5) patches['good'].append(new_circle) else: errors.append(error) # Plot a line for error new_circle = plt.Circle(point, radius, facecolor='orangered', edgecolor='none', linewidth=0, alpha=0.5) patches['bad'].append(new_circle) ax.plot((rat - error, rat + error), (radius, radius), 'x-', color='darkgray') print(ub.map_vals(len, patches)) total = float(sum(errors)) print('total = {!r}'.format(total)) print(max(errors)) print(min(errors)) for v in patches.values(): first = ub.peek(v) prop = ub.dict_isect(first.properties(), ['facecolor', 'linewidth', 'alpha', 'edgecolor']) col = mpl.collections.PatchCollection(v, **prop) ax.add_collection(col) # Lets look for the holes in IEEE float # for flt in ub.ProgIter(sorted(floats), verbose=1): kwplot.phantom_legend({ f'rationals without a {dtype}': 'orangered', f'rationals with a {dtype}': 'dodgerblue', f'x-x indicates {dtype} approximation error': 'darkgray', }) ax.set_title('Holes in IEEE 754 Float64') ax.set_xlabel('A rational number') ax.set_ylabel('The squared rational denominator') # import numpy as np # points = np.array([c.center for c in _circles]) # maxx, maxy = points.max(axis=0) # print('maxx = {!r}'.format(maxx)) # print('maxy = {!r}'.format(maxy)) # maxx, maxy = maxx // 2, maxy // 2 # ax.set_xlim(0, np.sqrt(int(maxx))) # ax.set_ylim(0, np.sqrt(int(maxy))) # ax.set_aspect('equal') # ax.set_xlim(0.2, 0.22) ax.set_xlim(0, 1) ax.set_ylim(0, 0.1)
def main(cls, cmdline=True, **kw): """ TODO: - [ ] Visualize auxiliary data Example: >>> # xdoctest: +SKIP >>> kw = {'src': 'special:shapes8'} >>> cmdline = False >>> cls = CocoShowCLI >>> cls.main(cmdline, **kw) """ import kwcoco import kwimage import kwplot config = cls.CLIConfig(kw, cmdline=cmdline) print('config = {}'.format(ub.repr2(dict(config), nl=1))) if config['src'] is None: raise Exception('must specify source: {}'.format(config['src'])) dset = kwcoco.CocoDataset.coerce(config['src']) print('dset.fpath = {!r}'.format(dset.fpath)) plt = kwplot.autoplt() gid = config['gid'] aid = config['aid'] out_fpath = config['dst'] if gid is None and aid is None: gid = ub.peek(dset.imgs) if config['mode'] == 'matplotlib': show_kw = { 'show_annots': config['show_annots'], 'channels': config['channels'], } if config['show_annots'] == 'both': show_kw.pop('show_annots') show_kw['title'] = '' ax = dset.show_image(gid=gid, aid=aid, pnum=(1, 2, 1), show_annots=False, **show_kw) ax = dset.show_image(gid=gid, aid=aid, pnum=(1, 2, 2), show_annots=True, **show_kw) else: ax = dset.show_image(gid=gid, aid=aid, **show_kw) if out_fpath is None: if 1: try: import xdev except Exception: pass else: gids = [gid] + list(set(dset.imgs.keys()) - {gid}) for gid in xdev.InteractiveIter(gids): ax = dset.show_image(gid=gid, aid=aid, **show_kw) xdev.InteractiveIter.draw() plt.show(block=False) plt.show() else: ax.figure.savefig(out_fpath) elif config['mode'] == 'opencv': canvas = dset.draw_image(gid, channels=config['channels']) if out_fpath is None: kwplot.imshow(canvas) plt.show() else: kwimage.imwrite(out_fpath, canvas) else: raise KeyError(config['mode'])
def main(): # Regress a 3D surface as in https://stackoverflow.com/a/53151677/887074 # Mapping from 2D coordinates to the elevation in the 3rd dimension import netharn as nh import numpy as np import torch import ubelt as ub num_train = 100 TRAIN_SURFACE = 'rosenbrock' if TRAIN_SURFACE == 'random': train_points = torch.rand(num_train, 3) train_XY = train_points[:, 0:2] train_X = train_points[:, 0:1] train_Y = train_points[:, 1:2] train_Z = train_points[:, 2:3] elif TRAIN_SURFACE == 'rosenbrock': # Train with the Rosenbrock function # https://en.wikipedia.org/wiki/Rosenbrock_function train_points = torch.rand(num_train, 2) train_XY = train_points[:, 0:2] train_X = train_points[:, 0:1] train_Y = train_points[:, 1:2] a = 1 b = 100 train_Z = (a - train_X)**2 + b * (train_Y - train_X**2)**2 + 2 np_train_X = train_X.data.cpu().numpy() np_train_Y = train_Y.data.cpu().numpy() np_train_Z = train_Z.data.cpu().numpy() test_resolution = 100 xbasis = np.linspace(0, 1, test_resolution).astype(np.float32) ybasis = np.linspace(0, 1, test_resolution).astype(np.float32) X, Y = np.meshgrid(xbasis, ybasis) test_X = X.ravel()[:, None] test_Y = Y.ravel()[:, None] test_XY = np.concatenate([test_X, test_Y], axis=1) test_XY = torch.from_numpy(test_XY) import kwplot plt = kwplot.autoplt() from mpl_toolkits.mplot3d import Axes3D # NOQA ax = plt.gca(projection='3d') ax.cla() # Plot the training data train_data_pc = ax.scatter3D(np_train_X, np_train_Y, np_train_Z, color='red') model = nh.layers.MultiLayerPerceptronNd(dim=0, in_channels=2, hidden_channels=[100] * 10, out_channels=1, bias=False) optim = torch.optim.Adam(model.parameters(), lr=1e-2) max_iters = 100 if 0: iter_ = ub.ProgIter(range(max_iters), desc='iterate') else: import xdev iter_ = xdev.InteractiveIter(list(range(max_iters))) poly3d_pc = None for iter_idx in iter_: optim.zero_grad() pred_Z = model.forward(train_XY) loss = torch.nn.functional.mse_loss(pred_Z, train_Z) loss.backward() optim.step() test_Z = model.forward(test_XY).data.cpu().numpy() param_total = sum(p.sum() for p in model.parameters()) print('param_total = {!r}'.format(param_total)) if hasattr(iter_, 'draw'): num = test_X.shape[0] s = np.sqrt(num) assert s % 1 == 0 s = int(s) X = test_X.reshape(s, s) Y = test_Y.reshape(s, s) Z = test_Z.reshape(s, s) if poly3d_pc is not None: # Remove previous surface poly3d_pc.remove() poly3d_pc = ax.plot_surface(X, Y, Z, color='blue') iter_.draw()
def _devcheck_corner(): self = DelayedWarp.random(rng=0) print(self.nesting()) region_slices = (slice(40, 90), slice(20, 62)) region_box = kwimage.Boxes.from_slice(region_slices, shape=self.shape) region_bounds = region_box.to_polygons()[0] for leaf in self._optimize_paths(): pass tf_leaf_to_root = leaf['transform'] tf_root_to_leaf = np.linalg.inv(tf_leaf_to_root) leaf_region_bounds = region_bounds.warp(tf_root_to_leaf) leaf_region_box = leaf_region_bounds.bounding_box().to_ltrb() leaf_crop_box = leaf_region_box.quantize() lt_x, lt_y, rb_x, rb_y = leaf_crop_box.data[0, 0:4] root_crop_corners = leaf_crop_box.to_polygons()[0].warp(tf_leaf_to_root) # leaf_crop_slices = (slice(lt_y, rb_y), slice(lt_x, rb_x)) crop_offset = leaf_crop_box.data[0, 0:2] corner_offset = leaf_region_box.data[0, 0:2] offset_xy = crop_offset - corner_offset tf_root_to_leaf # NOTE: # Cropping applies a translation in whatever space we do it in # We need to save the bounds of the crop. # But now we need to adjust the transform so it points to the # cropped-leaf-space not just the leaf-space, so we invert the implicit # crop tf_crop_to_leaf = Affine.affine(offset=crop_offset) # tf_newroot_to_root = Affine.affine(offset=region_box.data[0, 0:2]) tf_root_to_newroot = Affine.affine(offset=region_box.data[0, 0:2]).inv() tf_crop_to_leaf = Affine.affine(offset=crop_offset) tf_crop_to_newroot = tf_root_to_newroot @ tf_leaf_to_root @ tf_crop_to_leaf tf_newroot_to_crop = tf_crop_to_newroot.inv() # tf_leaf_to_crop # tf_corner_offset = Affine.affine(offset=offset_xy) subpixel_offset = Affine.affine(offset=offset_xy).matrix tf_crop_to_leaf = subpixel_offset # tf_crop_to_root = tf_leaf_to_root @ tf_crop_to_leaf # tf_root_to_crop = np.linalg.inv(tf_crop_to_root) if 1: import kwplot kwplot.autoplt() lw, lh = leaf['sub_data_shape'][0:2] leaf_box = kwimage.Boxes([[0, 0, lw, lh]], 'xywh') root_box = kwimage.Boxes([[0, 0, self.dsize[0], self.dsize[1]]], 'xywh') ax1 = kwplot.figure(fnum=1, pnum=(2, 2, 1), doclf=1).gca() ax2 = kwplot.figure(fnum=1, pnum=(2, 2, 2)).gca() ax3 = kwplot.figure(fnum=1, pnum=(2, 2, 3)).gca() ax4 = kwplot.figure(fnum=1, pnum=(2, 2, 4)).gca() root_box.draw(setlim=True, ax=ax1) leaf_box.draw(setlim=True, ax=ax2) region_bounds.draw(ax=ax1, color='green', alpha=.4) leaf_region_bounds.draw(ax=ax2, color='green', alpha=.4) leaf_crop_box.draw(ax=ax2, color='purple') root_crop_corners.draw(ax=ax1, color='purple', alpha=.4) new_w = region_box.to_xywh().data[0, 2] new_h = region_box.to_xywh().data[0, 3] ax3.set_xlim(0, new_w) ax3.set_ylim(0, new_h) crop_w = leaf_crop_box.to_xywh().data[0, 2] crop_h = leaf_crop_box.to_xywh().data[0, 3] ax4.set_xlim(0, crop_w) ax4.set_ylim(0, crop_h) pts3_ = kwimage.Points.random(3).scale((new_w, new_h)) pts3 = kwimage.Points( xy=np.vstack([[[0, 0], [5, 5], [0, 49], [40, 45]], pts3_.xy])) pts4 = pts3.warp(tf_newroot_to_crop.matrix) pts3.draw(ax=ax3) pts4.draw(ax=ax4)
probs /= energy.sum() # Do something with probs # Get probabilities for the next state and update updates = np.random.rand(D) energy *= updates def renormalize_demo_v4(D, T): import numpy as np import kwarray energy = kwarray.fast_rand.uniform32(size=D) probs = np.empty_like(energy) for _ in range(T): probs[:] = energy probs /= energy.sum() # Do something with probs # Get probabilities for the next state and update updates = kwarray.fast_rand.uniform32(size=D) energy *= updates if __name__ == '__main__': """ CommandLine: python ~/misc/tests/python/bench_renormalization.py """ import kwplot plt = kwplot.autoplt() run_benchmark_renormalization() plt.show()
def benchmark_template(): import ubelt as ub import pandas as pd import timerit def method1(x, y, z): ret = [] for i in range((x + y) * z): ret.append(i) return ret def method2(x, y, z): ret = [i for i in range((x + y) * z)] return ret method_lut = locals() # can populate this some other way # Change params here to modify number of trials ti = timerit.Timerit(100, bestof=10, verbose=1) # if True, record every trail run and show variance in seaborn # if False, use the standard timerit min/mean measures RECORD_ALL = True # These are the parameters that we benchmark over basis = { 'method': ['method1', 'method2'], 'x': list(range(7)), 'y': [0, 100], 'z': [2, 3] # 'param_name': [param values], } xlabel = 'x' # Set these to param labels that directly transfer to method kwargs kw_labels = ['x', 'y', 'z'] # Set these to empty lists if they are not used group_labels = { 'style': ['y'], 'size': ['z'], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) # Make any modifications you need to compute input kwargs for each # method here. kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(**kwargs) if RECORD_ALL: # Seaborn will show the variance if this is enabled, otherwise # use the robust timerit mean / min times chunk_iter = ub.chunks(ti.times, ti.bestof) times = list(map(min, chunk_iter)) # TODO: timerit method for this for time in times: row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) else: row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) time_key = 'time' if RECORD_ALL else 'min' # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values(time_key) if RECORD_ALL: # Show the min / mean if we record all min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1) mean_times = data.groupby('key')[['time' ]].mean().rename({'time': 'mean'}, axis=1) stats_data = pd.concat([min_times, mean_times], axis=1) stats_data = stats_data.sort_values('min') else: stats_data = data USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} other_keys = sorted( set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'}) for params, variants in stats_data.groupby(other_keys): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) mean_speedup = variants['mean'].max() / variants['mean'] stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup min_speedup = variants['min'].max() / variants['min'] stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) print('Statistics:') print(stats_data) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('Aggregated Rankings =\n{}'.format(skill_agg)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw) ax.set_title('Benchmark Name') ax.set_xlabel('Size (todo: A better x-variable description)') ax.set_ylabel('Time (todo: A better y-variable description)') # ax.set_xscale('log') # ax.set_yscale('log') try: __IPYTHON__ except NameError: plt.show()
def compute_move_effect(mon1, mon2, move, charge=1.0, rng=None): """ Compute damage and other effects caused by a move Args: mon1: (Pokemon): attacker mon2: (Pokemon): defender move: (dict): move dictionary charge: (float): between 0 and 1, if this move is a charge move, this is the percent of bubbles popped. rng (RandomState): Example: >>> from pypogo.battle import * # NOQA >>> mon1 = Pokemon.random('machamp', moves=['counter', 'cross chop', 'rock slide'], shadow=True).maximize(1500) >>> mon1 = Pokemon.random('articuno', moves=['Ice Shard', 'Icy Wind', 'Hurricane'], shadow=True).maximize(1500) >>> mon2 = Pokemon.random('snorlax', moves=['lick', 'super_power'], shadow=True).maximize(1500) >>> mon2 = Pokemon.random('magikarp', shadow=True).maximize(1500) >>> move = mon1.pvp_charge_moves[0] >>> effect = compute_move_effect(mon1, mon2, move) >>> print('effect = {}'.format(ub.repr2(effect, nl=2))) >>> move = mon2.pvp_charge_moves[0] >>> effect = compute_move_effect(mon2, mon1, move) >>> print('effect = {}'.format(ub.repr2(effect, nl=2))) """ import math move_type = move['type'] move_power = move['power'] effectiveness_lut = mon1.api.data['type_effectiveness'] effectiveness = 1 for def_type in mon2.typing: effectiveness *= effectiveness_lut[move_type][def_type] # Attack and defense after taking IVs and level into account adjusted_attack = mon1.adjusted['attack'] adjusted_defense = max(mon2.adjusted['defense'], 1e-5) stab = 1.2 if (move_type in mon1.typing) else 1.0 def modifier_factor(delta): if delta > 0: return 1 + (delta / 4) else: return 1 / (1 + (-delta / 4)) attack_modifier_factor = modifier_factor(mon1.modifiers['attack']) defense_modifier_factor = modifier_factor(mon2.modifiers['defense']) attack_shadow_factor = 1.2 if mon1.shadow else 1.0 defense_shadow_factor = (1 / 1.2) if mon2.shadow else 1.0 pvp_bonus_multiplier = 1.3 # Hard coded in game, See [3]. attack_power = (pvp_bonus_multiplier * charge * stab * adjusted_attack * attack_modifier_factor * attack_shadow_factor * effectiveness) defense_power = (adjusted_defense * defense_modifier_factor * defense_shadow_factor) half = 0.5 # not sure why a half is in the formula damage = math.floor(half * move_power * attack_power / defense_power) + 1 if rng is None: rng = random.Random() # Determine if any buffs / debuffs occur buffs = move.get('buffs', None) modifiers = [] if buffs is not None: chance = move['buffs']['activation_chance'] if chance == 1 or chance > rng.random(): attacker_att_delta = buffs.get('attacker_attack_stat_stage_change', 0) attacker_def_delta = buffs.get( 'attacker_defense_stat_stage_change', 0) target_att_delta = buffs.get('target_attack_stat_stage_change', 0) target_def_delta = buffs.get('target_defense_stat_stage_change', 0) if attacker_att_delta != 0: modifiers.append({ 'target': mon1, 'stat': 'attack', 'delta': attacker_att_delta, }) if attacker_def_delta != 0: modifiers.append({ 'target': mon1, 'stat': 'defense', 'delta': attacker_def_delta, }) if target_att_delta != 0: modifiers.append({ 'target': mon2, 'stat': 'attack', 'delta': target_att_delta, }) if target_def_delta != 0: modifiers.append({ 'target': mon2, 'stat': 'defense', 'delta': target_def_delta, }) # Buff factors can be [1, 1.25, 1.5, 1.75, 2.0] # DeBuff factors can be [1, 0.8, 0.66, 0.57, 0.5] # This formula might make more sense, but its not what it is. # 2 ** np.linspace(-1, 1, 9) # np.logspace(-1, 1, 9, base=2) # Is there a natural way to express? # I don't think so, the right half is linear and the left half is # non-linear. if 0: import kwplot import numpy as np plt = kwplot.autoplt() from scipy.optimize import curve_fit x = np.array([-4, -3, -2, -1, 0, 1, 2, 3, 4]) y = np.array([0.5, 0.57, 0.66, 0.8, 1, 1.25, 1.5, 1.75, 2.0]) def func(x, c0, c1, c2, c3, c4): return (c4 * x**4 + c3 * x**3 + c2 * x**2 + c1 * x**1 + c0 * x**0 + # b1 ** (x * e1) 0) left_y = 1 / (1 + (-x / 4)) right_y = 1 + (x / 4) popt, pcov = curve_fit(func, x, y) y_nat = 2.**(np.array(x) / 4) plt.clf() plt.plot(x, y, 'b-o', label='real') plt.plot(x, y_nat, 'r-o', label='natural') plt.plot(x, np.abs(y - y_nat), 'r-o', label='nat_abs_delta') plt.plot(x, left_y, '--', label='left-y') plt.plot(x, right_y, '--', label='right-y') x_hat = np.linspace(x[0], x[-1], 100) y_hat = func(x_hat, *popt) plt.plot(x_hat, y_hat, 'g--', label='fit') plt.legend() effect = { 'desc': f'{mon1.name!r} (hp={mon1.hp}, energy={mon1.energy}) used {move["name"]!r} against {mon2.name!r} (hp={mon2.hp}, energy={mon2.energy})', 'attacker': mon1, 'target': mon2, 'damage': damage, 'stab': stab, 'pvp_bonus_multiplier': pvp_bonus_multiplier, 'adjusted_attack': adjusted_attack, 'attack_shadow_factor': attack_shadow_factor, 'effectiveness': effectiveness, 'adjusted_defense': adjusted_defense, 'defense_modifier_factor': defense_modifier_factor, 'attack_modifier_factor': attack_modifier_factor, 'defense_shadow_factor': defense_shadow_factor, 'attack_power': attack_power, 'defense_power': defense_power, 'energy_delta': move['energy_delta'], 'turn_duration': move['turn_duration'], 'modifiers': modifiers, } return effect
def main(): """ Run password security analysis Example: >>> import sys, ubelt >>> sys.path.append(ubelt.expandpath('~/misc/notes')) >>> from password_model import * # NOQA >>> main() """ import itertools as it from fractions import Fraction import pandas as pd # Build our adversary and our strategies devices, scales = build_threat_models() password_schemes = build_password_strategy() # Other estimates or assumptions estimates = { # estimated cost of using a kilowatt for an hour # http://www.wrecc.com/what-uses-watts-in-your-home/ # https://www.coinwarz.com/mining/ethereum/calculator 'dollars_per_kwh': 0.10, } rows = [] for device, scheme, scale in it.product(devices, password_schemes, scales): for benchmark in device['benchmarks']: states = Fraction(scheme['states']) num_devices = Fraction(scale['num_devices']) dollars_per_kwh = Fraction(estimates['dollars_per_kwh']) hashmode_attempts_per_second = benchmark['attempts_per_second'] attempts_per_second = num_devices * Fraction( int(hashmode_attempts_per_second)) seconds = states / Fraction(attempts_per_second) hours = seconds / Fraction(3600) device_kilowatts = Fraction(device['watts']) / Fraction(1000) device_dollars_per_hour = device_kilowatts * dollars_per_kwh dollars_per_device = device_dollars_per_hour * hours dollars = dollars_per_device * num_devices total_kilowatts = device_kilowatts * num_devices * hours row = { 'scheme': scheme['name'], 'entropy': scheme['entropy'], 'hashmode': benchmark['hashmode'], 'hashmode_attempts_per_second': int(hashmode_attempts_per_second), 'device': device['name'], 'scale': scale['name'], 'num_devices': scale['num_devices'], 'seconds': seconds, 'dollars': dollars, 'kilowatts': total_kilowatts, 'hours': hours, 'dollars_per_kwh': estimates['dollars_per_kwh'], } rows.append(row) df = pd.DataFrame(rows) df = df.sort_values('entropy') chosen_device = 'RTX_3090' df = df[df['device'] == chosen_device] df['time'] = df['seconds'].apply(humanize_seconds) df['cost'] = df['dollars'].apply(partial(humanize_dollars, colored=1)) df['entropy'] = df['entropy'].round(2) df['num_devices'] = df['num_devices'].apply(int) hashmodes = sorted([d['hashmode'] for d in device['benchmarks']]) # https://github.com/pandas-dev/pandas/issues/18066 monkeypatch_pandas_colored_stdout() # Output our assumptions print('\n---') print('Assumptions:') device_info = ub.group_items(devices, lambda x: x['name'])[chosen_device][0] print('estimates = {!r}'.format(estimates)) print('device_info = {}'.format(ub.repr2(device_info, nl=2))) # For each hashmode, print the scheme-vs-num_devices-vs-time matrix hashmode_to_pivots = {} for hashmode in hashmodes: subdf = df subdf = subdf[subdf['hashmode'] == hashmode] subdf = subdf.sort_values(['entropy', 'num_devices']) piv = subdf.pivot(['entropy', 'cost', 'scheme'], ['num_devices', 'scale'], 'time') # piv.style.applymap(color_cases) hashmode_to_pivots[hashmode] = piv for hashmode in hashmodes: print('\n---') print('hashmode = {!r}'.format(hashmode)) piv = hashmode_to_pivots[hashmode] print(piv) # Print the scheme-vs-hashmode-vs-cost matrix print('\n---') print('Cost Matrix:') subdf = df[df['scale'] == df['scale'].iloc[0]] piv = subdf.pivot(['entropy', 'scheme'], ['hashmode_attempts_per_second', 'hashmode'], 'cost') piv = piv.sort_index(axis=1, ascending=False) piv.columns = piv.columns.droplevel(0) print(piv) # Make the visualizations if ub.argflag('--show'): import kwplot from matplotlib.colors import LogNorm import matplotlib as mpl plt = kwplot.autoplt() sns = kwplot.autosns() use_latex = ub.argflag('--latex') if use_latex: mpl.rcParams['text.usetex'] = True def time_labelize(x): text = humanize_seconds(x, colored=False, named=True, precision=2) parts = text.split(' ') if use_latex: text = r'{\huge ' + parts[0] + '}' + '\n' + ' '.join(parts[1:]) else: text = parts[0] + '\n' + ' '.join(parts[1:]) return text def dollar_labelize(dollars): cost = humanize_dollars(dollars, named=(dollars > 1)) if use_latex: cost = cost.replace('$', r'\$') return cost hashmode_to_notes = {} for dev in devices[0]['benchmarks']: hashmode_to_notes[dev['hashmode']] = dev['notes'] if 1: # Independent of the adversary scale we can plot cost versus scheme # cost vs hashmod? subdf = df[df['scale'] == df['scale'].iloc[0]] piv = subdf.pivot(['entropy', 'scheme'], ['hashmode_attempts_per_second', 'hashmode'], 'dollars') piv = piv.sort_index(axis=1, ascending=False) # https://stackoverflow.com/questions/64234474/cust-y-lbls-seaborn ax: mpl.axes.Axes = plt.subplots(figsize=(15, 10))[1] annot = piv.applymap(dollar_labelize) piv = piv.applymap(float) sns.heatmap(piv, annot=annot, ax=ax, fmt='s', norm=LogNorm(vmin=1, vmax=100_000_000_000_000_000), annot_kws={'size': 16}, cmap='cividis', cbar_kws={ 'label': 'dollars', 'pad': 0.001 }) # Find colorbar for subax in ax.figure.axes: if subax.get_label() == '<colorbar>': subax.set_ylabel('dollars', labelpad=0) break new_ytick_labels = [] for ent, scheme in piv.index.to_list(): if use_latex: scheme = r'{\LARGE ' + scheme + '}' _ = '{scheme}\nEntropy={ent}bits'.format(scheme=scheme, ent=ent) new_ytick_labels.append(_) new_xtick_labels = [] for _, hashmode in piv.columns.to_list(): notes = '' if hashmode in hashmode_to_notes: notes = '\n(' + hashmode_to_notes[hashmode] + ')' new_xtick_labels.append(hashmode + notes) ax.set_xticklabels(new_xtick_labels, rotation=0) ax.set_yticklabels(new_ytick_labels, rotation=0) ax.set_ylabel('Password Scheme, Entropy', labelpad=24) ax.set_xlabel('Hashmode', labelpad=16) if use_latex: title = '{{\\Huge Password Cost Security}}' ax.set_title(title) else: ax.set_title('Password Cost Security') ax.figure.subplots_adjust(bottom=0.1, left=0.20, right=1.0, top=0.90, wspace=0.001) if ub.argflag('--save'): fname = 'passwd_cost_security.png' ax.figure.savefig(fname) if 1: # For each hashmode plot (scheme versus adversary scale) for hashmode in ub.ProgIter(hashmodes, desc='plotting'): subdf = df subdf = subdf[subdf['hashmode'] == hashmode] subdf = subdf.sort_values(['entropy', 'num_devices']) piv = subdf.pivot(['entropy', 'dollars', 'scheme'], ['num_devices', 'scale'], 'seconds') piv = piv.applymap(float) # https://stackoverflow.com/questions/64234474/cust-y-lbls-seaborn ax: mpl.axes.Axes = plt.subplots(figsize=(15, 10))[1] annot = piv.applymap(time_labelize) sns.heatmap(piv, annot=annot, ax=ax, fmt='s', norm=LogNorm(vmin=1, vmax=8640000000), annot_kws={'size': 10}, cbar_kws={ 'label': 'seconds', 'pad': 0.001 }) # Find colorbar for subax in ax.figure.axes: if subax.get_label() == '<colorbar>': subax.set_ylabel('seconds', labelpad=0) break new_ytick_labels = [] for ent, dollars, scheme in piv.index.to_list(): cost = dollar_labelize(dollars) if use_latex: scheme = r'{\LARGE ' + scheme + '}' _ = '{scheme}\nEntropy={ent}bits\nCost={cost}'.format( scheme=scheme, cost=cost, ent=ent) new_ytick_labels.append(_) new_xtick_labels = [] for n, name in piv.columns.to_list(): if use_latex: name = r'{\LARGE ' + name + '}' _ = name + '\n' + named_large_number(n, precision=0) + ' GPUs' new_xtick_labels.append(_) ax.set_xticklabels(new_xtick_labels, rotation=0) # ax.set_yticklabels(new_ytick_labels, horizontalalignment='left', pad=30) ax.set_yticklabels(new_ytick_labels) ax.set_ylabel('Password Scheme, Entropy, and Cost to Crack', labelpad=24) ax.set_xlabel('Adversary Resources', labelpad=16) notes = '' if hashmode in hashmode_to_notes: notes = ' (' + hashmode_to_notes[hashmode] + ')' if use_latex: title = '{{\\Huge Password Time Security}}\nhashmode={}{}'.format( hashmode, notes) ax.set_title(title) else: ax.set_title( 'Password Time Security\n(hashmode={}{})'.format( hashmode, notes)) ax.figure.subplots_adjust(bottom=0.1, left=0.20, right=1.0, top=0.90, wspace=0.001) if ub.argflag('--save'): fname = 'passwd_robustness_{}.png'.format(hashmode) ax.figure.savefig(fname) plt.show()
def benchmark_nested_break(): """ There are several ways to do a nested break, but which one is best? https://twitter.com/nedbat/status/1515345787563220996 """ import ubelt as ub import pandas as pd import timerit import itertools as it def method1_itertools(iter1, iter2): for i, j in it.product(iter1, iter2): if i == 20 and j == 20: break def method2_except(iter1, iter2): class Found(Exception): pass try: for i in iter1: for j in iter2: if i == 20 and j == 20: raise Found except Found: pass class FoundPredef(Exception): pass def method2_5_except_predef(iter1, iter2): try: for i in iter1: for j in iter2: if i == 20 and j == 20: raise FoundPredef except FoundPredef: pass def method3_gendef(iter1, iter2): def genfunc(): for i in iter1: for j in iter2: yield i, j for i, j in genfunc(): if i == 20 and j == 20: break def method4_genexp(iter1, iter2): genexpr = ((i, j) for i in iter1 for j in iter2) for i, j in genexpr: if i == 20 and j == 20: break method_lut = locals() # can populate this some other way # Change params here to modify number of trials ti = timerit.Timerit(1000, bestof=10, verbose=1) # if True, record every trail run and show variance in seaborn # if False, use the standard timerit min/mean measures RECORD_ALL = True # These are the parameters that we benchmark over import numpy as np basis = { 'method': ['method1_itertools', 'method2_except', 'method2_5_except_predef', 'method3_gendef', 'method4_genexp'], # 'n1': np.logspace(1, np.log2(100), 30, base=2).astype(int), # 'n2': np.logspace(1, np.log2(100), 30, base=2).astype(int), 'size': np.logspace(1, np.log2(10000), 30, base=2).astype(int), 'input_style': ['range', 'list', 'customized_iter'], # 'param_name': [param values], } xlabel = 'size' xinput_labels = ['n1', 'n2', 'size'] # Set these to param labels that directly transfer to method kwargs kw_labels = [] # Set these to empty lists if they are not used group_labels = { 'style': ['input_style'], 'size': [], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel} - xinput_labels) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) def make_input(params): # Given the parameterization make the benchmark function input # n1 = params['n1'] # n2 = params['n2'] size = params['size'] n1 = int(np.sqrt(size)) n2 = int(np.sqrt(size)) if params['input_style'] == 'list': iter1 = list(range(n1)) iter2 = list(range(n1)) elif params['input_style'] == 'range': iter1 = range(n1) iter2 = range(n2) elif params['input_style'] == 'customized_iter': import random def rando1(): rng1 = random.Random(0) for _ in range(n1): yield rng1.randint(0, n2) def rando2(): rng2 = random.Random(1) for _ in range(n1): yield rng2.randint(0, n2) iter1 = rando1() iter2 = rando2() else: raise KeyError return {'iter1': iter1, 'iter2': iter2} # For each variation of your experiment, create a row. rows = [] for params in grid_iter: # size = params['n1'] * params['n2'] # params['size'] = size group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) # Make any modifications you need to compute input kwargs for each # method here. kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... kwargs.update(make_input(params)) with timer: # Put the logic you want to time here method(**kwargs) if RECORD_ALL: # Seaborn will show the variance if this is enabled, otherwise # use the robust timerit mean / min times # chunk_iter = ub.chunks(ti.times, ti.bestof) # times = list(map(min, chunk_iter)) # TODO: timerit method for this times = ti.robust_times() for time in times: row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) else: row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) time_key = 'time' if RECORD_ALL else 'min' # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values(time_key) if RECORD_ALL: # Show the min / mean if we record all min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1) mean_times = data.groupby('key')[['time']].mean().rename({'time': 'mean'}, axis=1) stats_data = pd.concat([min_times, mean_times], axis=1) stats_data = stats_data.sort_values('min') else: stats_data = data USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} other_keys = sorted(set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'}) for params, variants in stats_data.groupby(other_keys): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) mean_speedup = variants['mean'].max() / variants['mean'] stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup min_speedup = variants['min'].max() / variants['min'] stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) print('Statistics:') print(stats_data) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('method_ratings = {}'.format(ub.repr2(method_ratings, nl=1))) print('Aggregated Rankings =\n{}'.format(skill_agg)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw) ax.set_title(f'Benchmark Nested Breaks: #Trials {ti.num}, bestof {ti.bestof}') ax.set_xlabel(f'{xlabel}') ax.set_ylabel('Time') ax.set_xscale('log') ax.set_yscale('log') try: __IPYTHON__ except NameError: plt.show()
def bench_bbox_iou_method(): """ On my system the torch impl was fastest (when the data was on the GPU). """ from kwimage.structs.boxes import _box_ious_torch, _box_ious_py, _bbox_ious_c ydata = ub.ddict(list) xdata = [ 10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 2000, 4000 ] bias = 0 if _bbox_ious_c is None: print('CYTHON IMPLEMENATION IS NOT AVAILABLE') for num in xdata: results = {} # Setup Timer N = max(20, int(1000 / num)) ti = ub.Timerit(N, bestof=10) # Setup input dat boxes1 = kwimage.Boxes.random(num, scale=10.0, rng=0, format='ltrb') boxes2 = kwimage.Boxes.random(num + 1, scale=10.0, rng=1, format='ltrb') ltrb1 = boxes1.tensor().data ltrb2 = boxes2.tensor().data for timer in ti.reset('iou-torch-cpu'): with timer: out = _box_ious_torch(ltrb1, ltrb2, bias) results[ti.label] = out.data.cpu().numpy() ydata[ti.label].append(ti.mean()) gpu = torch.device(0) ltrb1 = boxes1.tensor().data.to(gpu) ltrb2 = boxes2.tensor().data.to(gpu) for timer in ti.reset('iou-torch-gpu'): with timer: out = _box_ious_torch(ltrb1, ltrb2, bias) torch.cuda.synchronize() results[ti.label] = out.data.cpu().numpy() ydata[ti.label].append(ti.mean()) ltrb1 = boxes1.numpy().data ltrb2 = boxes2.numpy().data for timer in ti.reset('iou-numpy'): with timer: out = _box_ious_py(ltrb1, ltrb2, bias) results[ti.label] = out ydata[ti.label].append(ti.mean()) if _bbox_ious_c: ltrb1 = boxes1.numpy().data.astype(np.float32) ltrb2 = boxes2.numpy().data.astype(np.float32) for timer in ti.reset('iou-cython'): with timer: out = _bbox_ious_c(ltrb1, ltrb2, bias) results[ti.label] = out ydata[ti.label].append(ti.mean()) eq = partial(np.allclose, atol=1e-07) passed = ub.allsame(results.values(), eq) if passed: print( 'All methods produced the same answer for num={}'.format(num)) else: for k1, k2 in it.combinations(results.keys(), 2): v1 = results[k1] v2 = results[k2] if eq(v1, v2): print('pass: {} == {}'.format(k1, k2)) else: diff = np.abs(v1 - v2) print( 'FAIL: {} != {}: diff(max={}, mean={}, sum={})'.format( k1, k2, diff.max(), diff.mean(), diff.sum())) raise AssertionError('different methods report different results') print('num = {!r}'.format(num)) print('ti.measures = {}'.format( ub.repr2(ub.map_vals(ub.sorted_vals, ti.measures), align=':', nl=2, precision=6))) import kwplot kwplot.autoplt() kwplot.multi_plot(xdata, ydata, xlabel='num boxes', ylabel='seconds') kwplot.show_if_requested()
def benchmark_mul_vs_pow(): import ubelt as ub import pandas as pd import timerit from functools import reduce import operator as op import itertools as it def method_pow_via_mul_raw(n): """ Construct a function that does multiplication of a value n times """ return eval('lambda v: ' + ' * '.join(['v'] * n)) def method_pow_via_mul_for(v, n): ret = v for _ in range(1, n): ret = ret * v return ret def method_pow_via_mul_reduce(v, n): """ Alternative way to multiply a value n times """ return reduce(op.mul, it.repeat(v, n)) def method_pow_via_pow(v, n): return v ** n method_lut = locals() # can populate this some other way ti = timerit.Timerit(500000, bestof=1000, verbose=2) basis = { 'method': ['method_pow_via_mul_raw', 'method_pow_via_pow'], 'n': list(range(1, 20)), 'v': ['random-int', 'random-float'], # 'param_name': [param values], } xlabel = 'n' kw_labels = ['v', 'n'] group_labels = { 'style': ['v'], 'size': [], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit if params['method'] == 'method_pow_via_mul_raw': method = method(kwargs.pop('n')) for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... import random if kwargs['v'] == 'random': kwargs['v'] = random.randint(1, 31000) if random.random() > 0.5 else random.random() elif kwargs['v'] == 'random-int': kwargs['v'] = random.randint(1, 31000) elif kwargs['v'] == 'random-float': kwargs['v'] = random.random() with timer: # Put the logic you want to time here method(**kwargs) for time in map(min, ub.chunks(ti.times, ti.bestof)): row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) # data = data.sort_values('time') print(data) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='time', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('N') ax.set_ylabel('Time') ax.set_yscale('log') plt.show()