def __call__(self, *args, **kw): # newbase = base | (kw & options) # imagine all the syntax if self._magic: kw = self._magic(kw) newbase = ub.dict_union(self._base, ub.dict_isect(kw, self._options)) new = Element(newbase, self._options, self._magic) return new
def binarize_peritem(cfsn_vecs, negative_classes=None): """ Creates a binary representation useful for measuring the performance of detectors. It is assumed that scores of "positive" classes should be high and "negative" clases should be low. Args: negative_classes (List[str | int]): list of negative class names or idxs, by default chooses any class with a true class index of -1. These classes should ideally have low scores. Example: >>> # xdoctest: +REQUIRES(module:ndsampler) >>> from netharn.metrics import DetectionMetrics >>> dmet = DetectionMetrics.demo( >>> nimgs=10, nboxes=(0, 10), n_fp=(0, 1), nclasses=3) >>> cfsn_vecs = dmet.confusion_vectors() >>> class_idxs = list(dmet.classes.node_to_idx.values()) >>> binvecs = cfsn_vecs.binarize_peritem() """ import kwarray # import warnings # warnings.warn('binarize_peritem DOES NOT PRODUCE CORRECT RESULTS') negative_cidxs = {-1} if negative_classes is not None: @ub.memoize def _lower_classes(): if cfsn_vecs.classes is None: raise Exception( 'classes must be known if negative_classes are strings' ) return [c.lower() for c in cfsn_vecs.classes] for c in negative_classes: import six if isinstance(c, six.string_types): classes = _lower_classes() try: cidx = classes.index(c) except Exception: continue else: cidx = int(c) negative_cidxs.add(cidx) is_false = kwarray.isect_flags(cfsn_vecs.data['true'], negative_cidxs) _data = { 'is_true': ~is_false, 'pred_score': cfsn_vecs.data['score'], } extra = ub.dict_isect(_data, ['txs', 'pxs', 'gid', 'weight']) _data.update(extra) bin_data = kwarray.DataFrameArray(_data) binvecs = BinaryConfusionVectors(bin_data) return binvecs
def bench_dict_isect(): import ubelt as ub def random_dict(n): import random keys = set(random.randint(0, n) for _ in range(n)) return {k: k for k in keys} d1 = random_dict(1000) d2 = random_dict(1000) import xdev xdev.profile_now(ub.dict_isect)(d1, d2) xdev.profile_now(dict_isect_variant0)(d1, d2) xdev.profile_now(dict_isect_variant1)(d1, d2) xdev.profile_now(dict_isect_variant2)(d1, d2) xdev.profile_now(dict_isect_variant3)(d1, d2) import timerit ti = timerit.Timerit(100, bestof=10, verbose=2) for timer in ti.reset('current'): with timer: ub.dict_isect(d1, d2) for timer in ti.reset('inline'): with timer: {k: v for k, v in d1.items() if k in d2} for timer in ti.reset('dict_isect_variant0'): with timer: dict_isect_variant0(d1, d2) for timer in ti.reset('dict_isect_variant1'): with timer: dict_isect_variant1(d1, d2) for timer in ti.reset('dict_isect_variant2'): with timer: dict_isect_variant1(d1, d2) for timer in ti.reset('dict_isect_variant3'): with timer: dict_isect_variant3(d1, d2) print('ti.rankings = {}'.format(ub.repr2(ti.rankings['min'], precision=8, align=':', nl=1, sort=0)))
def coerce(cls, data=None, **kwargs): """ Attempt to coerce the data into an affine object Args: data : some data we attempt to coerce to an Affine matrix **kwargs : some data we attempt to coerce to an Affine matrix, mutually exclusive with `data`. Returns: Affine Example: >>> import kwimage >>> kwimage.Affine.coerce({'type': 'affine', 'matrix': [[1, 0, 0], [0, 1, 0]]}) >>> kwimage.Affine.coerce({'scale': 2}) >>> kwimage.Affine.coerce({'offset': 3}) >>> kwimage.Affine.coerce(np.eye(3)) >>> kwimage.Affine.coerce(None) >>> kwimage.Affine.coerce(skimage.transform.AffineTransform(scale=30)) """ if data is None and not kwargs: return cls(matrix=None) if data is None: data = kwargs if isinstance(data, np.ndarray): self = cls(matrix=data) elif isinstance(data, cls): self = data elif isinstance(data, skimage.transform.AffineTransform): self = cls(matrix=data.params) elif data.__class__.__name__ == cls.__name__: self = data elif isinstance(data, dict): keys = set(data.keys()) known_params = {'scale', 'shear', 'offset', 'theta', 'type'} params = ub.dict_isect(data, known_params) if 'matrix' in keys: self = cls(matrix=np.array(data['matrix'])) elif len(known_params & keys): params.pop('type', None) self = cls.affine(**params) else: raise KeyError(', '.join(list(data.keys()))) else: raise TypeError(type(data)) return self
def coarsen(cfsn_vecs, cxs): """ Creates a coarsened set of vectors """ import ndsampler import kwarray assert cfsn_vecs.probs is not None, 'need probs' if not isinstance(cfsn_vecs.classes, ndsampler.CategoryTree): raise TypeError('classes must be a ndsampler.CategoryTree') descendent_map = cfsn_vecs.classes.idx_to_descendants_idxs( include_cfsn_vecs=True) valid_descendant_mapping = ub.dict_isect(descendent_map, cxs) # mapping from current category indexes to the new coarse ones # Anything without an explicit key will be mapped to background bg_idx = cfsn_vecs.classes.index('background') mapping = { v: k for k, vs in valid_descendant_mapping.items() for v in vs } new_true = np.array( [mapping.get(x, bg_idx) for x in cfsn_vecs.data['true']]) new_pred = np.array( [mapping.get(x, bg_idx) for x in cfsn_vecs.data['pred']]) new_score = np.array([p[x] for x, p in zip(new_pred, cfsn_vecs.probs)]) new_y_df = { 'true': new_true, 'pred': new_pred, 'score': new_score, 'weight': cfsn_vecs.data['weight'], 'txs': cfsn_vecs.data['txs'], 'pxs': cfsn_vecs.data['pxs'], 'gid': cfsn_vecs.data['gid'], } new_y_df = kwarray.DataFrameArray(new_y_df) coarse_cfsn_vecs = ConfusionVectors(new_y_df, cfsn_vecs.classes, cfsn_vecs.probs) return coarse_cfsn_vecs
def __init__(self, num, dims=2, rng=None, **kwargs): self.rng = kwarray.ensure_rng(rng) self.num = num self.dims = dims # self.config = { # 'perception_thresh': 0.08, # 'max_speed': 0.005, # 'max_force': 0.0003, # } self.config = { 'perception_thresh': 0.2, 'max_speed': 0.01, 'max_force': 0.001, 'damping': 0.99, } self.config.update(ub.dict_isect(kwargs, self.config)) self.pos = None self.vel = None self.acc = None
def populate_from(self, dset): from sqlalchemy import inspect session = self.session inspector = inspect(self.engine) for key in self.engine.table_names(): colinfo = inspector.get_columns(key) colnames = {c['name'] for c in colinfo} # TODO: is there a better way to grab this information? cls = TBLNAME_TO_CLASS[key] for item in dset.dataset.get(key, []): item_ = ub.dict_isect(item, colnames) # Everything else is a foreign key item['foreign'] = ub.dict_diff(item, item_) if key == 'annotations': # Need custom code to translate list-based properties x, y, w, h = item['bbox'] item_['bbox_x'] = x item_['bbox_y'] = y item_['bbox_w'] = w item_['bbox_h'] = h row = cls(**item_) session.add(row) session.commit()
def demo(cfsn_vecs): """ Example: >>> # xdoctest: +REQUIRES(module:ndsampler) >>> cfsn_vecs = ConfusionVectors.demo() >>> print('cfsn_vecs = {!r}'.format(cfsn_vecs)) >>> cx_to_binvecs = cfsn_vecs.binarize_ovr() >>> print('cx_to_binvecs = {!r}'.format(cx_to_binvecs)) """ from netharn.metrics import DetectionMetrics dmet = DetectionMetrics.demo(nimgs=10, nboxes=(0, 10), n_fp=(0, 1), nclasses=3) # print('dmet = {!r}'.format(dmet)) cfsn_vecs = dmet.confusion_vectors() cfsn_vecs.data._data = ub.dict_isect(cfsn_vecs.data._data, [ 'true', 'pred', 'score', 'weight', ]) return cfsn_vecs
def coerce(config={}, **kw): """ Accepts keywords: optimizer / optim : can be sgd, adam, adamw, rmsprop learning_rate / lr : a float weight_decay / decay : a float momentum: a float, only used if the optimizer accepts it params: This is a SPECIAL keyword that is handled differently. It is interpreted by `netharn.hyper.Hyperparams.make_optimizer`. In this simplest case you can pass "params" as a list of torch parameter objects or a list of dictionaries containing param groups and special group options (just as you would when constructing an optimizer from scratch). We don't recommend this while using netharn unless you know what you are doing (Note, that params will correctly change device if the model is mounted). In the case where you do not want to group parameters with different options, it is best practice to simply not specify params. In the case where you want to group parameters set params to either a List[Dict] or a Dict[str, Dict]. The items / values of this collection should be a dictionary. The keys / values of this dictionary should be the per-group optimizer options. Additionally, there should be a key "params" (note this is a nested per-group params not to be confused with the top-level "params"). Each per-group "params" should be either (1) a list of parameter names (preferred), (2) a string that specifies a regular expression (matching layer names will be included in this group), or (3) a list of parameter objects. For example, the top-level params might look like: params={ 'head': {'lr': 0.003, 'params': '.*head.*'}, 'backbone': {'lr': 0.001, 'params': '.*backbone.*'}, 'preproc': {'lr': 0.0, 'params': [ 'model.conv1', 'model.norm1', , 'model.relu1']} } Note that head and backbone specify membership via regular expression whereas preproc explicitly specifies a list of parameter names. Notes: pip install torch-optimizer Returns: Tuple[type, dict]: a type and arguments to construct it References: https://datascience.stackexchange.com/questions/26792/difference-between-rmsprop-with-momentum-and-adam-optimizers https://github.com/jettify/pytorch-optimizer CommandLine: xdoctest -m /home/joncrall/code/netharn/netharn/api.py Optimizer.coerce Example: >>> config = {'optimizer': 'sgd', 'params': [ >>> {'lr': 3e-3, 'params': '.*head.*'}, >>> {'lr': 1e-3, 'params': '.*backbone.*'}, >>> ]} >>> optim_ = Optimizer.coerce(config) >>> # xdoctest: +REQUIRES(module:torch_optimizer) >>> from netharn.api import * # NOQA >>> config = {'optimizer': 'DiffGrad'} >>> optim_ = Optimizer.coerce(config, lr=1e-5) >>> print('optim_ = {!r}'.format(optim_)) >>> assert optim_[1]['lr'] == 1e-5 >>> config = {'optimizer': 'Yogi'} >>> optim_ = Optimizer.coerce(config) >>> print('optim_ = {!r}'.format(optim_)) >>> from netharn.api import * # NOQA >>> Optimizer.coerce({'optimizer': 'ASGD'}) TODO: - [ ] https://pytorch.org/blog/stochastic-weight-averaging-in-pytorch/ """ import netharn as nh config = _update_defaults(config, kw) key = config.get('optimizer', config.get('optim', 'sgd')).lower() lr = config.get('learning_rate', config.get('lr', 3e-3)) decay = config.get('weight_decay', config.get('decay', 0)) momentum = config.get('momentum', 0.9) params = config.get('params', None) # TODO: allow for "discriminative fine-tuning" if key == 'sgd': cls = torch.optim.SGD kw = { 'lr': lr, 'weight_decay': decay, 'momentum': momentum, 'nesterov': True, } elif key == 'adam': cls = torch.optim.Adam kw = { 'lr': lr, 'weight_decay': decay, # 'betas': (0.9, 0.999), # 'eps': 1e-8, # 'amsgrad': False } elif key == 'adamw': if _TORCH_IS_GE_1_2_0: from torch.optim import AdamW cls = AdamW else: cls = nh.optimizers.AdamW kw = { 'lr': lr, # 'betas': (0.9, 0.999), # 'eps': 1e-8, # 'amsgrad': False } elif key == 'rmsprop': cls = torch.optim.RMSprop kw = { 'lr': lr, 'weight_decay': decay, 'momentum': momentum, 'alpha': 0.9, } else: from netharn.util import util_inspect _lut = {} optim_modules = [ torch.optim, ] try: # Allow coerce to use torch_optimizer package if available import torch_optimizer except Exception: torch_optimizer = None else: optim_modules.append(torch_optimizer) _lut.update({ k.lower(): c.__name__ for k, c in torch_optimizer._NAME_OPTIM_MAP.items()}) _lut.update({ k.lower(): k for k in dir(torch.optim) if not k.startswith('_')}) key = _lut[key] cls = None for module in optim_modules: cls = getattr(module, key, None) if cls is not None: defaultkw = util_inspect.default_kwargs(cls) kw = defaultkw.copy() kw.update(ub.dict_isect(config, kw)) break if cls is None: raise KeyError(key) kw['params'] = params optim_ = (cls, kw) return optim_
def binarize_ovr(cfsn_vecs, mode=1, keyby='name', ignore_classes={'ignore'}): """ Transforms cfsn_vecs into one-vs-rest BinaryConfusionVectors for each category. Args: mode (int, default=1): 0 for heirarchy aware or 1 for voc like. MODE 0 IS PROBABLY BROKEN keyby (int | str) : can be cx or name ignore_classes (Set[str]): category names to ignore Returns: OneVsRestConfusionVectors: which behaves like Dict[int, BinaryConfusionVectors]: cx_to_binvecs Example: >>> # xdoctest: +REQUIRES(module:ndsampler) >>> cfsn_vecs = ConfusionVectors.demo() >>> print('cfsn_vecs = {!r}'.format(cfsn_vecs)) >>> catname_to_binvecs = cfsn_vecs.binarize_ovr(keyby='name') >>> print('catname_to_binvecs = {!r}'.format(catname_to_binvecs)) Notes: Consider we want to measure how well we can classify beagles. Given a multiclass confusion vector, we need to carefully select a subset. We ignore any truth that is coarser than our current label. We also ignore any background predictions on irrelevant classes y_true | y_pred | score ------------------------------- dog | dog <- ignore coarser truths dog | cat <- ignore coarser truths dog | beagle <- ignore coarser truths cat | dog cat | cat cat | background <- ignore failures to predict unrelated classes cat | maine-coon beagle | beagle beagle | dog beagle | background beagle | cat Snoopy | beagle Snoopy | cat maine-coon | background <- ignore failures to predict unrelated classes maine-coon | beagle maine-coon | cat Anything not marked as ignore is counted. We count anything marked as beagle or a finer grained class (e.g. Snoopy) as a positive case. All other cases are negative. The scores come from the predicted probability of beagle, which must be remembered outside the dataframe. """ import kwarray classes = cfsn_vecs.classes data = cfsn_vecs.data if mode == 0: if cfsn_vecs.probs is None: raise ValueError('cannot binarize in mode=0 without probs') pdist = classes.idx_pairwise_distance() cx_to_binvecs = {} for cx in range(len(classes)): if classes[cx] == 'background' or classes[cx] in ignore_classes: continue if mode == 0: import warnings warnings.warn( 'THIS CALCLUATION MIGHT BE WRONG. MANY OTHERS ' 'IN THIS FILE WERE, AND I HAVENT CHECKED THIS ONE YET') # Lookup original probability predictions for the class of interest new_scores = cfsn_vecs.probs[:, cx] # Determine which truth items have compatible classes # Note: we ignore any truth-label that is COARSER than the # class-of-interest. # E.g: how well do we classify Beagle? -> we should ignore any truth # label marked as Dog because it may or may not be a Beagle? with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=RuntimeWarning) dist = pdist[cx] coarser_cxs = np.where(dist < 0)[0] finer_eq_cxs = np.where(dist >= 0)[0] is_finer_eq = kwarray.isect_flags(data['true'], finer_eq_cxs) is_coarser = kwarray.isect_flags(data['true'], coarser_cxs) # Construct a binary data frame to pass to sklearn functions. bin_data = { 'is_true': is_finer_eq.astype(np.uint8), 'pred_score': new_scores, 'weight': data['weight'] * (np.float32(1.0) - is_coarser), 'txs': cfsn_vecs.data['txs'], 'pxs': cfsn_vecs.data['pxs'], 'gid': cfsn_vecs.data['gid'], } bin_data = kwarray.DataFrameArray(bin_data) # Ignore cases where we failed to predict an irrelevant class flags = (data['pred'] == -1) & (bin_data['is_true'] == 0) bin_data['weight'][flags] = 0 # bin_data = bin_data.compress(~flags) bin_cfsn = BinaryConfusionVectors(bin_data, cx, classes) elif mode == 1: # More VOC-like, not heirarchy friendly if cfsn_vecs.probs is not None: # We know the actual score predicted for this category in # this case. is_true = cfsn_vecs.data['true'] == cx pred_score = cfsn_vecs.probs[:, cx] else: import warnings warnings.warn( 'Binarize ovr is only approximate if not all probabilities are known' ) # If we don't know the probabilities for non-predicted # categories then we have to guess. is_true = cfsn_vecs.data['true'] == cx # do we know the actual predicted score for this category? score_is_unknown = data['pred'] != cx pred_score = data['score'].copy() # These scores were for a different class, so assume # other classes were predicted with a uniform prior approx_score = (1 - pred_score[score_is_unknown]) / ( len(classes) - 1) # Except in the case where predicted class is -1. In this # case no prediction was actually made (above a threshold) # so the assumed score should be significantly lower, we # conservatively choose zero. unknown_preds = data['pred'][score_is_unknown] approx_score[unknown_preds == -1] = 0 pred_score[score_is_unknown] = approx_score bin_data = { # is_true denotes if the true class of the item is the # category of interest. 'is_true': is_true, 'pred_score': pred_score, } extra = ub.dict_isect(data._data, ['txs', 'pxs', 'gid', 'weight']) bin_data.update(extra) bin_data = kwarray.DataFrameArray(bin_data) bin_cfsn = BinaryConfusionVectors(bin_data, cx, classes) cx_to_binvecs[cx] = bin_cfsn if keyby == 'cx': cx_to_binvecs = cx_to_binvecs elif keyby == 'name': cx_to_binvecs = ub.map_keys(cfsn_vecs.classes, cx_to_binvecs) else: raise KeyError(keyby) ovr_cfns = OneVsRestConfusionVectors(cx_to_binvecs, cfsn_vecs.classes) return ovr_cfns
def benchmark_dict_diff_impl(): import ubelt as ub import pandas as pd import timerit import random def method_diffkeys(*args): first_dict = args[0] keys = set(first_dict) keys.difference_update(*map(set, args[1:])) new0 = dict((k, first_dict[k]) for k in keys) return new0 def method_diffkeys_list(*args): first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) keep_keys = [k for k in first_dict.keys() if k not in remove_keys] new = dict((k, first_dict[k]) for k in keep_keys) return new def method_diffkeys_oset(*args): first_dict = args[0] keys = ub.oset(first_dict) keys.difference_update(*map(set, args[1:])) new0 = dict((k, first_dict[k]) for k in keys) return new0 def method_ifkeys_setcomp(*args): first_dict = args[0] remove_keys = {k for ks in args[1:] for k in ks} new1 = dict((k, v) for k, v in first_dict.items() if k not in remove_keys) return new1 def method_ifkeys_setunion(*args): first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new2 = dict((k, v) for k, v in first_dict.items() if k not in remove_keys) return new2 def method_ifkeys_getitem(*args): first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new3 = dict((k, first_dict[k]) for k in first_dict.keys() if k not in remove_keys) return new3 def method_ifkeys_dictcomp(*args): # Cannot use until 3.6 is dropped (it is faster) first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new4 = {k: v for k, v in first_dict.items() if k not in remove_keys} return new4 def method_ifkeys_dictcomp_getitem(*args): # Cannot use until 3.6 is dropped (it is faster) first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new4 = {k: first_dict[k] for k in first_dict.keys() if k not in remove_keys} return new4 method_lut = locals() # can populate this some other way def make_data(num_items, num_other, remove_fraction, keytype): if keytype == 'str': keytype = str if keytype == 'int': keytype = int first_keys = [random.randint(0, 1000) for _ in range(num_items)] k = int(remove_fraction * len(first_keys)) remove_sets = [list(ub.unique(random.choices(first_keys, k=k) + [random.randint(0, 1000) for _ in range(num_items)])) for _ in range(num_other)] first_dict = {keytype(k): k for k in first_keys} args = [first_dict] + [{keytype(k): k for k in ks} for ks in remove_sets] return args ti = timerit.Timerit(200, bestof=1, verbose=2) basis = { 'method': [ # Cant use because unordered # 'method_diffkeys', # Cant use because python 3.6 'method_ifkeys_dictcomp', 'method_ifkeys_dictcomp_getitem', 'method_ifkeys_setunion', 'method_ifkeys_getitem', 'method_diffkeys_list', # Probably not good # 'method_ifkeys_setcomp', # 'method_diffkeys_oset', ], 'num_items': [10, 100, 1000], 'num_other': [1, 3, 5], # 'num_other': [1], 'remove_fraction': [0, 0.2, 0.5, 0.7, 1.0], # 'remove_fraction': [0.2, 0.8], 'keytype': ['str', 'int'], # 'keytype': ['str'], # 'param_name': [param values], } xlabel = 'num_items' kw_labels = ['num_items', 'num_other', 'remove_fraction', 'keytype'] group_labels = { 'style': ['num_other', 'keytype'], 'size': ['remove_fraction'], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) args = make_data(**kwargs) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(*args) row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('min') print(data) # for each parameter setting, group all methods with that used those exact # comparable params. Then rank how good each method did. That will be a # preference profile. We will give that preference profile a weight (e.g. # based on the fastest method in the bunch) and then aggregate them with # some voting method. USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} weighted_rankings = ub.ddict(lambda: ub.ddict(float)) for params, variants in data.groupby(['num_other', 'keytype', 'remove_fraction', 'num_items']): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) # Choose a ranking weight scheme weight = variants['mean'].min() # weight = 1 for rank, method in enumerate(ranking): weighted_rankings[method][rank] += weight weighted_rankings[method]['total'] += weight # Probably a more robust voting method to do this weight_rank_rows = [] for method_name, ranks in weighted_rankings.items(): weights = ub.dict_diff(ranks, ['total']) p_rank = ub.map_vals(lambda w: w / ranks['total'], weights) for rank, w in p_rank.items(): weight_rank_rows.append({'rank': rank, 'weight': w, 'name': method_name}) weight_rank_df = pd.DataFrame(weight_rank_rows) piv = weight_rank_df.pivot(['name'], ['rank'], ['weight']) print(piv) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('skill_agg =\n{}'.format(skill_agg)) aggregated = (piv * piv.columns.levels[1].values).sum(axis=1).sort_values() print('weight aggregated =\n{}'.format(aggregated)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('A better x-variable description') ax.set_ylabel('A better y-variable description')
def benchmark_mul_vs_pow(): import ubelt as ub import pandas as pd import timerit from functools import reduce import operator as op import itertools as it def method_pow_via_mul_raw(n): """ Construct a function that does multiplication of a value n times """ return eval('lambda v: ' + ' * '.join(['v'] * n)) def method_pow_via_mul_for(v, n): ret = v for _ in range(1, n): ret = ret * v return ret def method_pow_via_mul_reduce(v, n): """ Alternative way to multiply a value n times """ return reduce(op.mul, it.repeat(v, n)) def method_pow_via_pow(v, n): return v ** n method_lut = locals() # can populate this some other way ti = timerit.Timerit(500000, bestof=1000, verbose=2) basis = { 'method': ['method_pow_via_mul_raw', 'method_pow_via_pow'], 'n': list(range(1, 20)), 'v': ['random-int', 'random-float'], # 'param_name': [param values], } xlabel = 'n' kw_labels = ['v', 'n'] group_labels = { 'style': ['v'], 'size': [], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit if params['method'] == 'method_pow_via_mul_raw': method = method(kwargs.pop('n')) for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... import random if kwargs['v'] == 'random': kwargs['v'] = random.randint(1, 31000) if random.random() > 0.5 else random.random() elif kwargs['v'] == 'random-int': kwargs['v'] = random.randint(1, 31000) elif kwargs['v'] == 'random-float': kwargs['v'] = random.random() with timer: # Put the logic you want to time here method(**kwargs) for time in map(min, ub.chunks(ti.times, ti.bestof)): row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) # data = data.sort_values('time') print(data) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='time', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('N') ax.set_ylabel('Time') ax.set_yscale('log') plt.show()
def __init__(api): api.base = 'https://pogoapi.net/api/v1/' api.routes = { 'pokemon_stats': api.base + 'pokemon_stats.json', 'current_pokemon_moves': api.base + 'current_pokemon_moves.json', 'pokemon_evolutions': api.base + 'pokemon_evolutions.json', 'cp_multiplier': api.base + 'cp_multiplier.json', 'pokemon_types': api.base + 'pokemon_types.json', 'charged_moves': api.base + 'charged_moves.json', 'fast_moves': api.base + 'fast_moves.json', 'type_effectiveness': api.base + 'type_effectiveness.json', 'pokemon_powerup_requirements': api.base + 'pokemon_powerup_requirements.json', 'pokemon_candy_to_evolve': api.base + 'pokemon_candy_to_evolve.json', 'pokemon_buddy_distances': api.base + 'pokemon_buddy_distances.json', 'shadow_pokemon': api.base + 'shadow_pokemon.json', 'pokemon_forms': api.base + 'pokemon_forms.json', 'pvp_exclusive_pokemon': api.base + 'pvp_exclusive_pokemon.json', 'galarian_pokemon': api.base + 'galarian_pokemon.json', 'alolan_pokemon': api.base + 'alolan_pokemon.json', 'shiny_pokemon': api.base + 'shiny_pokemon.json', 'mega_pokemon': api.base + 'mega_pokemon.json', 'baby_pokemon': api.base + 'baby_pokemon.json', 'nesting_pokemon': api.base + 'nesting_pokemon.json', 'released_pokemon': api.base + 'released_pokemon.json', 'pokemon_names': api.base + 'pokemon_names.json', 'api_hashes': api.base + 'api_hashes.json', 'pvp_fast_moves': api.base + 'pvp_fast_moves.json', 'pvp_charged_moves': api.base + 'pvp_charged_moves.json', } # TODO: determine when to redownload api.data = {} for key, url in api.routes.items(): redo = 0 data_fpath = ub.grabdata(url, verbose=1, redo=redo, expires=24 * 60 * 60) with open(data_fpath, 'r') as file: data = json.load(file) api.data[key] = data # Make the API global for now pokemon_stats = api.data['pokemon_stats'] _name_to_stats = ub.group_items( pokemon_stats, lambda item: item['pokemon_name'].lower()) _name_to_stats = dict(_name_to_stats) api.name_to_stats = _name_to_stats _name_to_moves = ub.group_items( api.data['current_pokemon_moves'], lambda item: item['pokemon_name'].lower()) _name_to_moves.default_factory = None _name_to_moves = dict(_name_to_moves) # base = 'http://pokeapi.co/api/v2/pokemon/' api.name_to_moves = _name_to_moves evolutions = api.data['pokemon_evolutions'] _name_to_evolutions = ub.group_items( evolutions, lambda item: item['pokemon_name'].lower()) _name_to_evolutions = dict(_name_to_evolutions) for key, form_stats in api.name_to_stats.items(): if key not in _name_to_evolutions: noevos = [] for s in form_stats: empty = ub.dict_isect( s, {'form', 'pokemon_name', 'pokemon_id'}) empty['evolutions'] = [] noevos.append(empty) _name_to_evolutions[key] = noevos _name_to_types = ub.group_items( api.data['pokemon_types'], lambda item: item['pokemon_name'].lower()) _name_to_types = dict(_name_to_types) api.name_to_type = _name_to_types evo_graph = nx.DiGraph() for name, form_evo_list in _name_to_evolutions.items(): for form_evo in form_evo_list: u = form_evo['pokemon_name'].lower() evo_graph.add_node(u) for evo in form_evo['evolutions']: v = evo['pokemon_name'].lower() evo_graph.add_edge(u, v) api.name_to_family = {} api.name_to_base = {} evo_graph.remove_edges_from(nx.selfloop_edges(evo_graph)) api.evo_graph = evo_graph for cc in list(nx.connected_components(api.evo_graph.to_undirected())): bases = [n for n in cc if len(evo_graph.pred[n]) == 0] base = bases[0] for n in cc: api.name_to_family[n] = cc api.name_to_base[n] = base api.name_to_evolutions = _name_to_evolutions api.pve_fast_moves = ub.group_items( api.data['fast_moves'], lambda item: normalize(item['name'].lower())) api.pve_fast_moves.default_factory = None api.pve_charged_moves = ub.group_items( api.data['charged_moves'], lambda item: normalize(item['name'].lower())) api.pve_charged_moves.default_factory = None api.pvp_fast_moves = ub.group_items( api.data['pvp_fast_moves'], lambda item: normalize(item['name'].lower())) api.pvp_fast_moves.default_factory = None api.pvp_charged_moves = ub.group_items( api.data['pvp_charged_moves'], lambda item: normalize(item['name'].lower())) api.pvp_charged_moves.default_factory = None if 0: ub.map_vals(len, api.pve_fast_moves) ub.map_vals(len, api.pve_charged_moves) api.learnable = { # TODO: remove 'stunfisk_galarian': { 'fast': [ 'MUD_SHOT', 'METAL_CLAW', ], 'charge': [ 'EARTHQUAKE', 'FLASH_CANNON', 'MUDDY_WATER', 'ROCK_SLIDE', ] } }
def benchmark_template(): import ubelt as ub import pandas as pd import timerit def method1(x, y, z): ret = [] for i in range((x + y) * z): ret.append(i) return ret def method2(x, y, z): ret = [i for i in range((x + y) * z)] return ret method_lut = locals() # can populate this some other way # Change params here to modify number of trials ti = timerit.Timerit(100, bestof=10, verbose=1) # if True, record every trail run and show variance in seaborn # if False, use the standard timerit min/mean measures RECORD_ALL = True # These are the parameters that we benchmark over basis = { 'method': ['method1', 'method2'], 'x': list(range(7)), 'y': [0, 100], 'z': [2, 3] # 'param_name': [param values], } xlabel = 'x' # Set these to param labels that directly transfer to method kwargs kw_labels = ['x', 'y', 'z'] # Set these to empty lists if they are not used group_labels = { 'style': ['y'], 'size': ['z'], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) # Make any modifications you need to compute input kwargs for each # method here. kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(**kwargs) if RECORD_ALL: # Seaborn will show the variance if this is enabled, otherwise # use the robust timerit mean / min times chunk_iter = ub.chunks(ti.times, ti.bestof) times = list(map(min, chunk_iter)) # TODO: timerit method for this for time in times: row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) else: row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) time_key = 'time' if RECORD_ALL else 'min' # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values(time_key) if RECORD_ALL: # Show the min / mean if we record all min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1) mean_times = data.groupby('key')[['time' ]].mean().rename({'time': 'mean'}, axis=1) stats_data = pd.concat([min_times, mean_times], axis=1) stats_data = stats_data.sort_values('min') else: stats_data = data USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} other_keys = sorted( set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'}) for params, variants in stats_data.groupby(other_keys): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) mean_speedup = variants['mean'].max() / variants['mean'] stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup min_speedup = variants['min'].max() / variants['min'] stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) print('Statistics:') print(stats_data) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('Aggregated Rankings =\n{}'.format(skill_agg)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw) ax.set_title('Benchmark Name') ax.set_xlabel('Size (todo: A better x-variable description)') ax.set_ylabel('Time (todo: A better y-variable description)') # ax.set_xscale('log') # ax.set_yscale('log') try: __IPYTHON__ except NameError: plt.show()
def benchmark_pathlib_vs_fspath(): import ubelt as ub import pathlib import pandas as pd import random import timerit import os def method_pathlib(inputs): p = pathlib.Path(*inputs) def method_ospath(inputs): p = os.path.join(*inputs) method_lut = locals() # can populate this some other way ti = timerit.Timerit(10000, bestof=10, verbose=2) basis = { 'method': ['method_pathlib', 'method_ospath'], 'num_parts': [2, 4, 8, 12, 16], } xlabel = 'num_parts' kw_labels = [] group_labels = { 'style': [], 'size': [], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) n = params['num_parts'] inputs = [chr(random.randint(97, 120)) for _ in range(n)] kwargs['inputs'] = inputs method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(**kwargs) row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('min') print(data) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('Time') ax.set_ylabel('Number of parts')
def benchmark_repeat_vs_reduce_mul(): import ubelt as ub import pandas as pd import timerit def reduce_daq_rec(func, arrs): if len(arrs) == 1: return arrs[0] if len(arrs) == 2: return func(arrs[0], arrs[1]) elif len(arrs) == 3: return func(func(arrs[0], arrs[1]), arrs[3]) else: arrs1 = arrs[0::2] arrs2 = arrs[1::2] res1 = reduce_daq_rec(func, arrs1) res2 = reduce_daq_rec(func, arrs2) res = func(res1, res2) return res def reduce_daq_iter(func, arrs): """ https://www.baeldung.com/cs/convert-recursion-to-iteration https://stackoverflow.com/questions/159590/way-to-go-from-recursion-to-iteration arrs = [2, 3, 5, 7, 11, 13, 17, 21] """ raise NotImplementedError # TODO: make the iterative version from collections import deque empty_result = None stack = deque([(arrs, empty_result)]) idx = 0 while stack: print('----') print('stack = {}'.format(ub.repr2(list(stack), nl=1))) arrs0, result = stack.pop() if len(arrs0) == 0: raise Exception if result is not None: # raise Exception results = [result] while stack: next_arrs0, next_result = stack.pop() if next_result is None: break else: results.append(next_result) if results: if len(results) == 1: stack.append((results, results[0])) else: stack.append((results, None)) if next_result is None: stack.append((next_arrs0, None)) elif result is None: if len(arrs0) == 1: result = arrs0[0] stack.append((arrs0, result)) # return arrs0[0] if len(arrs0) == 2: result = func(arrs0[0], arrs0[1]) stack.append((arrs0, result)) elif len(arrs0) == 3: result = func(func(arrs0[0], arrs0[1]), arrs0[3]) stack.append((arrs0, result)) else: arrs01 = arrs0[0::2] arrs02 = arrs0[1::2] stack.append((arrs0, empty_result)) stack.append((arrs01, empty_result)) stack.append((arrs02, empty_result)) # res1 = reduce_daq_rec(func, arrs01) # res2 = reduce_daq_rec(func, arrs2) # res = func(res1, res2) idx += 1 if idx > 10: raise Exception return res def method_daq_rec(arrs): return reduce_daq_rec(np.multiply, arrs) def method_repeat(arrs): """ helper code: arr_names = ['a{:02d}'.format(idx) for idx in range(1, 32 + 1)] lhs = ', '.join(arr_names) rhs = ' * '.join(arr_names) print(f'{lhs} = arrs') print(f'ret = {rhs}') """ # Hard coded pure python syntax for multiplying if len(arrs) == 4: a01, a02, a03, a04 = arrs ret = a01 * a02 * a03 * a04 elif len(arrs) == 8: a01, a02, a03, a04, a05, a06, a07, a08 = arrs ret = a01 * a02 * a03 * a04 * a05 * a06 * a07 * a08 elif len(arrs) == 32: a01, a02, a03, a04, a05, a06, a07, a08, a09, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28, a29, a30, a31, a32 = arrs ret = a01 * a02 * a03 * a04 * a05 * a06 * a07 * a08 * a09 * a10 * a11 * a12 * a13 * a14 * a15 * a16 * a17 * a18 * a19 * a20 * a21 * a22 * a23 * a24 * a25 * a26 * a27 * a28 * a29 * a30 * a31 * a32 return ret def method_reduce(arrs): ret = np.multiply.reduce(arrs) return ret def method_stack(arrs): stacked = np.stack(arrs) ret = stacked.prod(axis=0) return ret method_lut = locals() # can populate this some other way ti = timerit.Timerit(10000, bestof=10, verbose=2) basis = { 'method': ['method_repeat', 'method_reduce', 'method_stack', 'method_daq_rec'], 'arr_size': [10, 100, 1000, 10000], 'num_arrs': [4, 8, 32], } xlabel = 'arr_size' kw_labels = [] group_labels = { 'style': ['num_arrs'], 'size': [], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) arr_size = params['arr_size'] num_arrs = params['num_arrs'] arrs = [] for _ in range(num_arrs): arr = np.random.rand(arr_size) arrs.append(arr) kwargs['arrs'] = arrs method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(**kwargs) row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('min') print(data) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('Array Size') ax.set_ylabel('Time')
def __init__(api): api.base = 'https://pogoapi.net/api/v1/' api.routes = { 'pokemon_stats': api.base + 'pokemon_stats.json', 'current_pokemon_moves': api.base + 'current_pokemon_moves.json', 'pokemon_evolutions': api.base + 'pokemon_evolutions.json', 'cp_multiplier': api.base + 'cp_multiplier.json', } api.data = {} for key, url in api.routes.items(): data_fpath = ub.grabdata(url, verbose=1) with open(data_fpath, 'r') as file: data = json.load(file) api.data[key] = data # Make the API global for now pokemon_stats = api.data['pokemon_stats'] _name_to_stats = ub.group_items( pokemon_stats, lambda item: item['pokemon_name'].lower()) _name_to_stats = dict(_name_to_stats) api.name_to_stats = _name_to_stats _name_to_items = ub.group_items( api.data['current_pokemon_moves'], lambda item: item['pokemon_name'].lower()) _name_to_items.default_factory = None _name_to_items = dict(_name_to_items) # base = 'http://pokeapi.co/api/v2/pokemon/' api.name_to_moves = _name_to_items evolutions = api.data['pokemon_evolutions'] _name_to_evolutions = ub.group_items( evolutions, lambda item: item['pokemon_name'].lower()) _name_to_evolutions = dict(_name_to_evolutions) for key, form_stats in api.name_to_stats.items(): if key not in _name_to_evolutions: noevos = [] for s in form_stats: empty = ub.dict_isect( s, {'form', 'pokemon_name', 'pokemon_id'}) empty['evolutions'] = [] noevos.append(empty) _name_to_evolutions[key] = noevos import networkx as nx evo_graph = nx.DiGraph() for name, form_evo_list in _name_to_evolutions.items(): for form_evo in form_evo_list: u = form_evo['pokemon_name'].lower() evo_graph.add_node(u) for evo in form_evo['evolutions']: v = evo['pokemon_name'].lower() evo_graph.add_edge(u, v) # if 0: # print(forest_str(evo_graph)) api.name_to_family = {} api.name_to_base = {} evo_graph.remove_edges_from(nx.selfloop_edges(evo_graph)) api.evo_graph = evo_graph for cc in list(nx.connected_components(api.evo_graph.to_undirected())): bases = [n for n in cc if len(evo_graph.pred[n]) == 0] base = bases[0] for n in cc: api.name_to_family[n] = cc api.name_to_base[n] = base # base_pokmeon = [n for n in evo_graph.nodes if len(evo_graph.pred[n]) == 0] api.name_to_evolutions = _name_to_evolutions # api.name_to_family = {} # for base in base_pokmeon: # family = list(nx.dfs_postorder_nodes(evo_graph, base)) # for name in family: # api.name_to_family[name] = family # evos = api.name_to_evolutions[name] # for evo in evos: # evo['base'] = base # for evo in evos['evolutions']: # evo['base'] api.learnable = { 'stunfisk_galarian': { 'fast': [ 'MUD_SHOT', 'METAL_CLAW', ], 'charge': [ 'EARTHQUAKE', 'FLASH_CANNON', 'MUDDY_WATER', 'ROCK_SLIDE', ] } }
def benchmark_nested_break(): """ There are several ways to do a nested break, but which one is best? https://twitter.com/nedbat/status/1515345787563220996 """ import ubelt as ub import pandas as pd import timerit import itertools as it def method1_itertools(iter1, iter2): for i, j in it.product(iter1, iter2): if i == 20 and j == 20: break def method2_except(iter1, iter2): class Found(Exception): pass try: for i in iter1: for j in iter2: if i == 20 and j == 20: raise Found except Found: pass class FoundPredef(Exception): pass def method2_5_except_predef(iter1, iter2): try: for i in iter1: for j in iter2: if i == 20 and j == 20: raise FoundPredef except FoundPredef: pass def method3_gendef(iter1, iter2): def genfunc(): for i in iter1: for j in iter2: yield i, j for i, j in genfunc(): if i == 20 and j == 20: break def method4_genexp(iter1, iter2): genexpr = ((i, j) for i in iter1 for j in iter2) for i, j in genexpr: if i == 20 and j == 20: break method_lut = locals() # can populate this some other way # Change params here to modify number of trials ti = timerit.Timerit(1000, bestof=10, verbose=1) # if True, record every trail run and show variance in seaborn # if False, use the standard timerit min/mean measures RECORD_ALL = True # These are the parameters that we benchmark over import numpy as np basis = { 'method': ['method1_itertools', 'method2_except', 'method2_5_except_predef', 'method3_gendef', 'method4_genexp'], # 'n1': np.logspace(1, np.log2(100), 30, base=2).astype(int), # 'n2': np.logspace(1, np.log2(100), 30, base=2).astype(int), 'size': np.logspace(1, np.log2(10000), 30, base=2).astype(int), 'input_style': ['range', 'list', 'customized_iter'], # 'param_name': [param values], } xlabel = 'size' xinput_labels = ['n1', 'n2', 'size'] # Set these to param labels that directly transfer to method kwargs kw_labels = [] # Set these to empty lists if they are not used group_labels = { 'style': ['input_style'], 'size': [], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel} - xinput_labels) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) def make_input(params): # Given the parameterization make the benchmark function input # n1 = params['n1'] # n2 = params['n2'] size = params['size'] n1 = int(np.sqrt(size)) n2 = int(np.sqrt(size)) if params['input_style'] == 'list': iter1 = list(range(n1)) iter2 = list(range(n1)) elif params['input_style'] == 'range': iter1 = range(n1) iter2 = range(n2) elif params['input_style'] == 'customized_iter': import random def rando1(): rng1 = random.Random(0) for _ in range(n1): yield rng1.randint(0, n2) def rando2(): rng2 = random.Random(1) for _ in range(n1): yield rng2.randint(0, n2) iter1 = rando1() iter2 = rando2() else: raise KeyError return {'iter1': iter1, 'iter2': iter2} # For each variation of your experiment, create a row. rows = [] for params in grid_iter: # size = params['n1'] * params['n2'] # params['size'] = size group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) # Make any modifications you need to compute input kwargs for each # method here. kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... kwargs.update(make_input(params)) with timer: # Put the logic you want to time here method(**kwargs) if RECORD_ALL: # Seaborn will show the variance if this is enabled, otherwise # use the robust timerit mean / min times # chunk_iter = ub.chunks(ti.times, ti.bestof) # times = list(map(min, chunk_iter)) # TODO: timerit method for this times = ti.robust_times() for time in times: row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) else: row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) time_key = 'time' if RECORD_ALL else 'min' # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values(time_key) if RECORD_ALL: # Show the min / mean if we record all min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1) mean_times = data.groupby('key')[['time']].mean().rename({'time': 'mean'}, axis=1) stats_data = pd.concat([min_times, mean_times], axis=1) stats_data = stats_data.sort_values('min') else: stats_data = data USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} other_keys = sorted(set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'}) for params, variants in stats_data.groupby(other_keys): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) mean_speedup = variants['mean'].max() / variants['mean'] stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup min_speedup = variants['min'].max() / variants['min'] stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) print('Statistics:') print(stats_data) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('method_ratings = {}'.format(ub.repr2(method_ratings, nl=1))) print('Aggregated Rankings =\n{}'.format(skill_agg)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw) ax.set_title(f'Benchmark Nested Breaks: #Trials {ti.num}, bestof {ti.bestof}') ax.set_xlabel(f'{xlabel}') ax.set_ylabel('Time') ax.set_xscale('log') ax.set_yscale('log') try: __IPYTHON__ except NameError: plt.show()
def benchamrk_det_nms(): """ Benchmarks different implementations of non-max-supression on the CPU, GPU, and using cython / numpy / torch. CommandLine: xdoctest -m ~/code/kwimage/dev/bench_nms.py benchamrk_det_nms --show SeeAlso: PJR Darknet NonMax supression https://github.com/pjreddie/darknet/blob/master/src/box.c Lightnet NMS https://gitlab.com/EAVISE/lightnet/blob/master/lightnet/data/transform/_postprocess.py#L116 """ # N = 200 # bestof = 50 N = 1 bestof = 1 # xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500, 2000] # max number of boxes yolo will spit out at a time max_boxes = 19 * 19 * 5 xdata = [ 10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500, max_boxes ] # xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500] # Demo values xdata = [0, 1, 2, 3, 10, 100, 200, 300, 500] if ub.argflag('--small'): xdata = [10, 100, 500, 1000, 1500, 2000, 5000, 10000] if ub.argflag('--medium'): xdata = [ 1000, 5000, 10000, 20000, 50000, ] if ub.argflag('--large'): xdata = [ 1000, 5000, 10000, 20000, 50000, 100000, ] if ub.argflag('--extra-large'): xdata = [ 1000, 2000, 10000, 20000, 40000, 100000, 200000, ] title_parts = [] SMALL_BOXES = ub.argflag('--small-boxes') if SMALL_BOXES: title_parts.append('small boxes') else: title_parts.append('large boxes') # NOTE: for large images we may have up to 21,850,753 detections! thresh = float(ub.argval('--thresh', default=0.4)) title_parts.append('thresh={:.2f}'.format(thresh)) from kwimage.algo.algo_nms import available_nms_impls valid_impls = available_nms_impls() print('valid_impls = {!r}'.format(valid_impls)) basis = { 'type': ['ndarray', 'tensor', 'tensor0'], # 'daq': [True, False], # 'daq': [False], # 'device': [None], # 'impl': valid_impls, 'impl': valid_impls + ['auto'], } if ub.argflag('--daq'): basis['daq'] = [True, False] # if torch.cuda.is_available(): # basis['device'].append(0) combos = [ ub.dzip(basis.keys(), vals) for vals in it.product(*basis.values()) ] def is_valid_combo(combo): # if combo['impl'] in {'py', 'cython_cpu'} and combo['device'] is not None: # return False # if combo['type'] == 'ndarray' and combo['impl'] == 'cython_gpu': # if combo['device'] is None: # return False # if combo['type'] == 'ndarray' and combo['impl'] != 'cython_gpu': # if combo['device'] is not None: # return False # if combo['type'].endswith('0'): # if combo['impl'] in {'numpy', 'cython_gpu', 'cython_cpu'}: # return False # if combo['type'] == 'ndarray': # if combo['impl'] in {'torch'}: # return False REMOVE_SLOW = True if REMOVE_SLOW: known_bad = [ { 'impl': 'torch', 'type': 'tensor' }, { 'impl': 'numpy', 'type': 'tensor' }, # {'impl': 'cython_gpu', 'type': 'tensor'}, { 'impl': 'cython_cpu', 'type': 'tensor' }, # {'impl': 'torch', 'type': 'tensor0'}, { 'impl': 'numpy', 'type': 'tensor0' }, # {'impl': 'cython_gpu', 'type': 'tensor0'}, # {'impl': 'cython_cpu', 'type': 'tensor0'}, { 'impl': 'torchvision', 'type': 'ndarray' }, ] for known in known_bad: if all(combo[key] == val for key, val in known.items()): return False return True combos = list(filter(is_valid_combo, combos)) times = ub.ddict(list) for num in xdata: if num > 10000: N = 1 bestof = 1 if num > 1000: N = 3 bestof = 1 if num > 100: N = 10 bestof = 3 elif num > 10: N = 100 bestof = 10 else: N = 1000 bestof = 10 print('\n\n---- number of boxes = {} ----\n'.format(num)) outputs = {} ti = ub.Timerit(N, bestof=bestof, verbose=1) # Build random test boxes and scores np_dets1 = kwimage.Detections.random(num // 2, scale=1000.0, rng=0) np_dets1.data['boxes'] = np_dets1.boxes.to_xywh() if SMALL_BOXES: max_dim = 100 np_dets1.boxes.data[..., 2] = np.minimum(np_dets1.boxes.width, max_dim).ravel() np_dets1.boxes.data[..., 3] = np.minimum(np_dets1.boxes.height, max_dim).ravel() np_dets2 = copy.deepcopy(np_dets1) np_dets2.boxes.translate(10, inplace=True) # add boxes that will definately be removed np_dets = kwimage.Detections.concatenate([np_dets1, np_dets2]) # make all scores unique to ensure comparability np_dets.scores[:] = np.linspace(0, 1, np_dets.num_boxes()) np_dets.data['scores'] = np_dets.scores.astype(np.float32) np_dets.boxes.data = np_dets.boxes.data.astype(np.float32) typed_data = {} # ---------------------------------- import netharn as nh for combo in combos: print('combo = {}'.format(ub.repr2(combo, nl=0))) label = nh.util.make_idstr(combo) mode = combo.copy() # if mode['impl'] == 'cython_gpu': # mode['device_id'] = mode['device'] mode_type = mode.pop('type') if mode_type in typed_data: dets = typed_data[mode_type] else: if mode_type == 'ndarray': dets = np_dets.numpy() elif mode_type == 'tensor': dets = np_dets.tensor(None) elif mode_type == 'tensor0': dets = np_dets.tensor(0) else: raise KeyError typed_data[mode_type] = dets for timer in ti.reset(label): with timer: keep = dets.non_max_supression(thresh=thresh, **mode) torch.cuda.synchronize() times[ti.label].append(ti.min()) outputs[ti.label] = ensure_numpy_indices(keep) # ---------------------------------- # Check that all kept boxes do not have more than `threshold` ious if 0: for key, keep_idxs in outputs.items(): kept = np_dets.take(keep_idxs).boxes ious = kept.ious(kept) max_iou = (np.tril(ious) - np.eye(len(ious))).max() if max_iou > thresh: print('{} produced a bad result with max_iou={}'.format( key, max_iou)) # Check result consistency: print('\nResult stats:') for key in sorted(outputs.keys()): print(' * {:<20}: num={}'.format(key, len(outputs[key]))) print('\nResult overlaps (method1, method2: jaccard):') datas = [] for k1, k2 in it.combinations(sorted(outputs.keys()), 2): idxs1 = set(outputs[k1]) idxs2 = set(outputs[k2]) jaccard = len(idxs1 & idxs2) / max(len(idxs1 | idxs2), 1) datas.append((k1, k2, jaccard)) datas = sorted(datas, key=lambda x: -x[2]) for k1, k2, jaccard in datas: print(' * {:<20}, {:<20}: {:0.4f}'.format(k1, k2, jaccard)) if True: ydata = {key: 1.0 / np.array(vals) for key, vals in times.items()} ylabel = 'Hz' reverse = True yscale = 'symlog' else: ydata = {key: np.array(vals) for key, vals in times.items()} ylabel = 'seconds' reverse = False yscale = 'linear' scores = {key: vals[-1] for key, vals in ydata.items()} ydata = ub.dict_subset(ydata, ub.argsort(scores, reverse=reverse)) ### times_of_interest = [0, 10, 100, 200, 1000] times_of_interest = xdata lines = [] record = lines.append record('### times_of_interest = {!r}'.format(times_of_interest)) for x in times_of_interest: if times_of_interest[-1] == x: record('else:') elif times_of_interest[0] == x: record('if num <= {}:'.format(x)) else: record('elif num <= {}:'.format(x)) if x in xdata: pos = xdata.index(x) score_wrt_x = {} for key, vals in ydata.items(): score_wrt_x[key] = vals[pos] typekeys = ['tensor0', 'tensor', 'ndarray'] type_groups = dict([(b, ub.group_items(score_wrt_x, lambda y: y.endswith(b))[True]) for b in typekeys]) # print('\n=========') # print('x = {!r}'.format(x)) record(' if code not in {!r}:'.format(set(typekeys))) record(' raise KeyError(code)') for typekey, group in type_groups.items(): # print('-------') record(' if code == {!r}:'.format(typekey)) # print('typekey = {!r}'.format(typekey)) # print('group = {!r}'.format(group)) group_x = ub.dict_isect(score_wrt_x, group) valid_keys = ub.argsort(group_x, reverse=True) valid_x = ub.dict_subset(group_x, valid_keys) # parts = [','.split(k) for k in valid_keys] ordered_impls = [] ordered_impls2 = ub.odict() for k in valid_keys: vals = valid_x[k] p = k.split(',') d = dict(i.split('=') for i in p) ordered_impls2[d['impl']] = vals ordered_impls.append(d['impl']) ordered_impls = list(ub.oset(ordered_impls) - {'auto'}) ordered_impls2.pop('auto') record(' # {}'.format( ub.repr2(ordered_impls2, precision=1, nl=0, explicit=True))) record(' preference = {}'.format( ub.repr2(ordered_impls, nl=0))) record('### end times of interest ') print(ub.indent('\n'.join(lines), ' ' * 8)) ### markers = { key: 'o' if 'auto' in key else '' for key, score in scores.items() } if ub.argflag('--daq'): markers = { key: '+' if 'daq=True' in key else '' for key, score in scores.items() } labels = { key: '{:.2f} {} - {}'.format(score, ylabel[0:3], key) for key, score in scores.items() } title = 'NSM-impl speed: ' + ', '.join(title_parts) import kwplot kwplot.autompl() kwplot.multi_plot( xdata, ydata, xlabel='num boxes', ylabel=ylabel, label=labels, yscale=yscale, title=title, marker=markers, # xscale='symlog', ) kwplot.show_if_requested()
'size': ['subsize'], 'style': ['subsize'], } hue_labels = ub.oset(basis) - {xlabel} if group_labels: hue_labels = hue_labels - set.union(*map(set, group_labels.values())) group_labels['hue'] = list(hue_labels) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) data_kwargs = ub.dict_isect(params.copy(), data_kwkeys) func_kwargs = generate_data(**data_kwargs) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): with timer: # MAIN LOGIC method(**func_kwargs) for time in ti.times: row = { # 'mean': ti.mean(),
group_labels = { 'size': ['subsize'], 'style': ['subsize'], } hue_labels = ub.oset(basis) - {xlabel} if group_labels: hue_labels = hue_labels - set.union(*map(set, group_labels.values())) group_labels['hue'] = list(hue_labels) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) data_kwargs = ub.dict_isect(params.copy(), data_kwkeys) func_kwargs = generate_data(**data_kwargs) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): with timer: # MAIN LOGIC method(**func_kwargs) for time in ti.times:
def ford_circles(): """ Draw Ford Circles This is a Ford Circle diagram of the Rationals and Float32 numbers. Only 163 of the 32608 rationals I generated can be exactly represented by a float32. [MF 14] [MF 95] [MF 14] https://www.youtube.com/watch?v=83ZjYvkdzYI&list=PL5A714C94D40392AB&index=14 [MF 95] https://www.youtube.com/watch?v=gATEJ3f3FBM&list=PL5A714C94D40392AB&index=95 Examples: import kwplot kwplot.autompl() """ import kwplot import ubelt as ub import matplotlib as mpl plt = kwplot.autoplt() sns = kwplot.autosns() # NOQA limit = 256 * 256 print('limit = {!r}'.format(limit)) rats_to_plot = set() maxx = 1 _iter = Rational.members(limit=limit) _genrat = set(ub.ProgIter(_iter, total=limit, desc='gen rats')) rats_to_plot |= _genrat rats_to_plot2 = sorted({Rational(r % maxx) for r in rats_to_plot} | {maxx}) floats = sorted( ub.unique(map(float, rats_to_plot2), key=lambda f: f.as_integer_ratio())) print(f'{len(rats_to_plot) = }') print(f'{len(rats_to_plot2) = }') print(f'{len(floats) = }') import numpy as np ax = kwplot.figure(fnum=1, doclf=True).gca() prog = ub.ProgIter(sorted(rats_to_plot2), verbose=1) dtype = np.float32 patches = ub.ddict(list) errors = [] for rat in prog: denominator = rat.denominator radius = 1 / (2 * (denominator * denominator)) point = (rat, radius) flt = dtype(rat) a, b = flt.as_integer_ratio() flt_as_rat = Rational(a, b) error = abs(rat - flt_as_rat) if error == 0: new_circle = plt.Circle(point, radius, facecolor='dodgerblue', edgecolor='none', linewidth=0, alpha=0.5) patches['good'].append(new_circle) else: errors.append(error) # Plot a line for error new_circle = plt.Circle(point, radius, facecolor='orangered', edgecolor='none', linewidth=0, alpha=0.5) patches['bad'].append(new_circle) ax.plot((rat - error, rat + error), (radius, radius), 'x-', color='darkgray') print(ub.map_vals(len, patches)) total = float(sum(errors)) print('total = {!r}'.format(total)) print(max(errors)) print(min(errors)) for v in patches.values(): first = ub.peek(v) prop = ub.dict_isect(first.properties(), ['facecolor', 'linewidth', 'alpha', 'edgecolor']) col = mpl.collections.PatchCollection(v, **prop) ax.add_collection(col) # Lets look for the holes in IEEE float # for flt in ub.ProgIter(sorted(floats), verbose=1): kwplot.phantom_legend({ f'rationals without a {dtype}': 'orangered', f'rationals with a {dtype}': 'dodgerblue', f'x-x indicates {dtype} approximation error': 'darkgray', }) ax.set_title('Holes in IEEE 754 Float64') ax.set_xlabel('A rational number') ax.set_ylabel('The squared rational denominator') # import numpy as np # points = np.array([c.center for c in _circles]) # maxx, maxy = points.max(axis=0) # print('maxx = {!r}'.format(maxx)) # print('maxy = {!r}'.format(maxy)) # maxx, maxy = maxx // 2, maxy // 2 # ax.set_xlim(0, np.sqrt(int(maxx))) # ax.set_ylim(0, np.sqrt(int(maxy))) # ax.set_aspect('equal') # ax.set_xlim(0.2, 0.22) ax.set_xlim(0, 1) ax.set_ylim(0, 0.1)
def benchmark_unpack(): """ What is faster unpacking items with slice syntax or tuple-unpacking Slice unpacking seems to be a tad faster. """ import ubelt as ub import random import pandas as pd import timerit import string def tuple_unpack(items): *prefix, key = items return prefix, key def slice_unpack(items): prefix, key = items[:-1], items[-1] return prefix, key method_lut = locals() # can populate this some other way ti = timerit.Timerit(5000, bestof=3, verbose=2) basis = { 'method': ['tuple_unpack', 'slice_unpack'], 'size': list(range(1, 64 + 1)), 'type': ['string', 'float'], } xlabel = 'size' kw_labels = [] group_labels = { 'style': ['type'], 'size': [], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) size = params['size'] method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): if type == 'string': items = [ ''.join(random.choices(string.printable, k=5)) for _ in range(size) ] elif type == 'float': items = [random.random() for _ in range(size)] with timer: method(items) for time in ti.times: row = { 'time': time, 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('time') summary_rows = [] for method, group in data.groupby('method'): row = {} row['method'] = method row['mean'] = group['time'].mean() row['std'] = group['time'].std() row['min'] = group['time'].min() row['max'] = group['time'].max() summary_rows.append(row) print(pd.DataFrame(summary_rows).sort_values('mean')) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='time', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('Execution time') ax.set_ylabel('Size of slices')