def difference(self, other): """ Set difference Example: >>> self = ChannelSpec('rgb|disparity,flowx|flowy') >>> other = ChannelSpec('rgb') >>> self.difference(other) >>> other = ChannelSpec('flowx') >>> self.difference(other) """ assert len(list(other.keys())) == 1, 'can take diff with one stream' other_norm = ub.oset(ub.peek(other.normalize().values())) self_norm = self.normalize() new_streams = [] for key, parts in self_norm.items(): new_parts = ub.oset(parts) - ub.oset(other_norm) # shrink the representation of a complex r|g|b to an alias if # possible. # TODO: make this more efficient for alias, alias_spec in self._known.items(): alias_parts = ub.oset(alias_spec.split('|')) index = subsequence_index(new_parts, alias_parts) if index is not None: oset_delitem(new_parts, index) oset_insert(new_parts, index.start, alias) new_stream = '|'.join(new_parts) new_streams.append(new_stream) new_spec = ','.join(new_streams) new = self.__class__(new_spec) return new
def test_equality(): def check(a, b): # Self checks assert a == a assert a >= a assert a <= a assert not a < a assert not a > a assert not a != a # Lesser checks assert a < b assert a <= b assert a != b assert not a == b # Greater checks assert b > a assert b >= a assert b != a assert not b == a a = ub.oset([]) b = ub.oset([1]) c = ub.oset([1, 2]) d = ub.oset([1, 2, 3]) check(a, b) check(b, c) check(c, d) check(a, d) check(a, d)
def _epochs_to_remove(existing_epochs, num_keep): """ doctest: >>> import netharn as nh >>> harn = FitHarn({}) >>> rng = np.random.RandomState(0) >>> harn.monitor = nh.Monitor(minimize=['loss'], maximize=['miou']) >>> for epoch in range(200): >>> harn.monitor.update(epoch, {'loss': rng.rand(), >>> 'miou': rng.rand()}) >>> existing_epochs = list(range(0, 200, 4)) >>> num_keep = 10 """ keep = set() recent = existing_epochs[-num_keep:] keep.update(recent) if harn.monitor: for best_epochs in harn.monitor.best_epochs().values(): best = ub.oset(best_epochs).intersection(existing_epochs) keep.update(best[:num_keep]) to_remove = set(existing_epochs) - keep return to_remove
def _epochs_to_remove(epochs): """ Doctest: >>> harn = FitHarness() >>> rng = np.random.RandomState(0) >>> for epoch in range(200): >>> harn.monitor.update(epoch, {'loss': rng.rand(), >>> 'miou': rng.rand()}) >>> epochs = list(range(0, 200, 4)) """ num_keep_recent = 10 num_keep_best = 10 keep = set() recent = epochs[-num_keep_recent:] keep.update(recent) if harn.monitor: best_epochs = harn.monitor.best_epochs() best = ub.oset(best_epochs).intersection(epochs) keep.update(best[-num_keep_best:]) to_remove = set(epochs) - keep return to_remove
def _epochs_to_remove(harn, existing_epochs, num_keep_recent, num_keep_best, keep_freq): """ Unit testable helper for `cleanup_snapshots`. Determines which epochs to remove given which epoches exist. Keeps `keep_freq` most recent, `num_keep_best` best, and one every `keep_freq` epochs. Doctest: >>> import netharn as nh >>> harn = FitHarn({}) >>> rng = np.random.RandomState(0) >>> harn.monitor = nh.Monitor(minimize=['loss'], maximize=['miou']) >>> for epoch in range(200): >>> harn.monitor.update(epoch, {'loss': rng.rand(), >>> 'miou': rng.rand()}) >>> existing_epochs = list(range(0, 200, 4)) >>> num_keep_best = 10 >>> num_keep_recent = 10 >>> keep_freq = 10 >>> to_remove = harn._epochs_to_remove(existing_epochs, >>> num_keep_recent, num_keep_best, >>> keep_freq) >>> assert len(existing_epochs) - len(to_remove) < 40 """ keep = set() recent = existing_epochs[-num_keep_recent:] keep.update(recent) # TODO: add a config for always keeping specific iterations in # multiples of X. if harn.monitor: for best_epochs in harn.monitor.best_epochs().values(): best = ub.oset(best_epochs).intersection(existing_epochs) keep.update(best[:num_keep_best]) # Keep a strided sampling of epochs epoch_arr = np.array(existing_epochs) flags = ((epoch_arr % keep_freq) == 0) sampled = epoch_arr[flags] keep.update(sampled) to_remove = set(existing_epochs) - keep return to_remove
def argparse(self, parser=None, special_options=False): """ construct or update an argparse.ArgumentParser CLI parser Args: parser (None | argparse.ArgumentParser): if specified this parser is updated with options from this config. special_options (bool, default=False): adds special scriptconfig options, namely: --config, --dumps, and --dump. Returns: argparse.ArgumentParser : a new or updated argument parser CommandLine: xdoctest -m scriptconfig.config Config.argparse:0 xdoctest -m scriptconfig.config Config.argparse:1 TODO: A good CLI spec for lists might be # In the case where ``key`` ends with and ``=``, assume the list is # given as a comma separated string with optional square brakets at # each end. --key=[f] # In the case where ``key`` does not end with equals and we know # the value is supposd to be a list, then we consume arguments # until we hit the next one that starts with '--' (which means # that list items cannot start with -- but they can contains # commas) FIXME: * In the case where we have an nargs='+' action, and we specify the option with an `=`, and then we give position args after it there is no way to modify behavior of the action to just look at the data in the string without modifying the ArgumentParser itself. The action object has no control over it. For example `--foo=bar baz biz` will parse as `[baz, biz]` which is really not what we want. We may be able to overload ArgumentParser to fix this. Example: >>> # You can now make instances of this class >>> import scriptconfig >>> self = scriptconfig.Config.demo() >>> parser = self.argparse() >>> parser.print_help() >>> # xdoctest: +REQUIRES(PY3) >>> # Python2 argparse does a hard sys.exit instead of raise >>> ns, extra = parser.parse_known_args() Example: >>> # You can now make instances of this class >>> import scriptconfig as scfg >>> class MyConfig(scfg.Config): >>> description = 'my CLI description' >>> default = { >>> 'path1': scfg.Value(None, position=1, alias='src'), >>> 'path2': scfg.Value(None, position=2, alias='dst'), >>> 'dry': scfg.Value(False, isflag=True), >>> 'approx': scfg.Value(False, isflag=False, alias=['a1', 'a2']), >>> } >>> self = MyConfig() >>> special_options = True >>> parser = None >>> parser = self.argparse(special_options=special_options) >>> parser.print_help() >>> self._read_argv(argv=['objection', '42', '--path1=overruled!']) >>> print('self = {!r}'.format(self)) Ignore: >>> self._read_argv(argv=['hi','--path1=foobar']) >>> self._read_argv(argv=['hi', 'hello', '--path1=foobar']) >>> self._read_argv(argv=['hi', 'hello', '--path1=foobar', '--help']) >>> self._read_argv(argv=['--path1=foobar', '--path1=baz']) >>> print('self = {!r}'.format(self)) """ import argparse if parser is None: parserkw = self._parserkw() parser = argparse.ArgumentParser(**parserkw) # Use custom action used to mark which values were explicitly set on # the commandline parser._explicitly_given = set() parent = self class ParseAction(argparse.Action): def __init__(self, *args, **kwargs): super(ParseAction, self).__init__(*args, **kwargs) # with script config nothing should be required by default all # positional arguments should have keyword arg variants Setting # required=False here will prevent positional args from # erroring if they are not specified. I dont think there are # other side effects, but we should make sure that is actually # the case. self.required = False if self.type is None: # Is this the right place to put this? def _mytype(value): key = self.dest template = parent.default[key] if not isinstance(template, Value): # smartcast non-valued params from commandline value = smartcast.smartcast(value) else: value = template.cast(value) return value self.type = _mytype # print('self.type = {!r}'.format(self.type)) def __call__(action, parser, namespace, values, option_string=None): # print('CALL action = {!r}'.format(action)) # print('option_string = {!r}'.format(option_string)) # print('values = {!r}'.format(values)) if isinstance(values, list) and len(values): # We got a list of lists, which we hack into a flat list if isinstance(values[0], list): import itertools as it values = list(it.chain(*values)) setattr(namespace, action.dest, values) parser._explicitly_given.add(action.dest) # IRC: this ensures each key has a real Value class _metadata = { key: self._data[key] for key, value in self._default.items() if isinstance(self._data[key], Value) } # :type: Dict[str, Value] _positions = {k: v.position for k, v in _metadata.items() if v.position is not None} if _positions: if ub.find_duplicates(_positions.values()): raise Exception('two values have the same position') _keyorder = ub.oset(ub.argsort(_positions)) _keyorder |= (ub.oset(self._default) - _keyorder) else: _keyorder = list(self._default.keys()) def _add_arg(parser, name, key, argkw, positional, isflag, isalias): _argkw = argkw.copy() if isalias: _argkw['help'] = 'alias of {}'.format(key) _argkw.pop('default', None) # flags cannot have flag aliases isflag = False elif positional: parser.add_argument(name, **_argkw) if isflag: # Can we support both flag and setitem methods of cli # parsing? if not isinstance(_argkw.get('default', None), bool): raise ValueError('can only use isflag with bools') _argkw.pop('type', None) _argkw.pop('choices', None) _argkw.pop('action', None) _argkw.pop('nargs', None) _argkw['dest'] = key _argkw_true = _argkw.copy() _argkw_true['action'] = 'store_true' _argkw_false = _argkw.copy() _argkw_false['action'] = 'store_false' _argkw_false.pop('help', None) parser.add_argument('--' + name, **_argkw_true) parser.add_argument('--no-' + name, **_argkw_false) else: parser.add_argument('--' + name, **_argkw) mode = 1 alias_registry = [] for key, value in self._data.items(): # key: str # value: Any | Value argkw = {} argkw['help'] = '' positional = None isflag = False if key in _metadata: # Use the metadata in the Value class to enhance argparse _value = _metadata[key] argkw.update(_value.parsekw) value = _value.value isflag = _value.isflag positional = _value.position else: _value = value if isinstance(value, Value) else None if not argkw['help']: argkw['help'] = '<undocumented>' argkw['default'] = value argkw['action'] = ParseAction name = key _add_arg(parser, name, key, argkw, positional, isflag, isalias=False) if _value is not None: if _value.alias: alts = _value.alias alts = alts if ub.iterable(alts) else [alts] for alias in alts: tup = (alias, key, argkw) alias_registry.append(tup) if mode == 0: name = alias _add_arg(parser, name, key, argkw, positional, isflag, isalias=True) if mode == 1: for tup in alias_registry: (alias, key, argkw) = tup name = alias dest = key _add_arg(parser, name, dest, argkw, positional, isflag, isalias=True) if special_options: parser.add_argument('--config', default=None, help=ub.codeblock( ''' special scriptconfig option that accepts the path to a on-disk configuration file, and loads that into this {!r} object. ''').format(self.__class__.__name__)) parser.add_argument('--dump', default=None, help=ub.codeblock( ''' If specified, dump this config to disk. ''').format(self.__class__.__name__)) parser.add_argument('--dumps', action='store_true', help=ub.codeblock( ''' If specified, dump this config stdout ''').format(self.__class__.__name__)) return parser
def benchmark_nested_break(): """ There are several ways to do a nested break, but which one is best? https://twitter.com/nedbat/status/1515345787563220996 """ import ubelt as ub import pandas as pd import timerit import itertools as it def method1_itertools(iter1, iter2): for i, j in it.product(iter1, iter2): if i == 20 and j == 20: break def method2_except(iter1, iter2): class Found(Exception): pass try: for i in iter1: for j in iter2: if i == 20 and j == 20: raise Found except Found: pass class FoundPredef(Exception): pass def method2_5_except_predef(iter1, iter2): try: for i in iter1: for j in iter2: if i == 20 and j == 20: raise FoundPredef except FoundPredef: pass def method3_gendef(iter1, iter2): def genfunc(): for i in iter1: for j in iter2: yield i, j for i, j in genfunc(): if i == 20 and j == 20: break def method4_genexp(iter1, iter2): genexpr = ((i, j) for i in iter1 for j in iter2) for i, j in genexpr: if i == 20 and j == 20: break method_lut = locals() # can populate this some other way # Change params here to modify number of trials ti = timerit.Timerit(1000, bestof=10, verbose=1) # if True, record every trail run and show variance in seaborn # if False, use the standard timerit min/mean measures RECORD_ALL = True # These are the parameters that we benchmark over import numpy as np basis = { 'method': ['method1_itertools', 'method2_except', 'method2_5_except_predef', 'method3_gendef', 'method4_genexp'], # 'n1': np.logspace(1, np.log2(100), 30, base=2).astype(int), # 'n2': np.logspace(1, np.log2(100), 30, base=2).astype(int), 'size': np.logspace(1, np.log2(10000), 30, base=2).astype(int), 'input_style': ['range', 'list', 'customized_iter'], # 'param_name': [param values], } xlabel = 'size' xinput_labels = ['n1', 'n2', 'size'] # Set these to param labels that directly transfer to method kwargs kw_labels = [] # Set these to empty lists if they are not used group_labels = { 'style': ['input_style'], 'size': [], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel} - xinput_labels) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) def make_input(params): # Given the parameterization make the benchmark function input # n1 = params['n1'] # n2 = params['n2'] size = params['size'] n1 = int(np.sqrt(size)) n2 = int(np.sqrt(size)) if params['input_style'] == 'list': iter1 = list(range(n1)) iter2 = list(range(n1)) elif params['input_style'] == 'range': iter1 = range(n1) iter2 = range(n2) elif params['input_style'] == 'customized_iter': import random def rando1(): rng1 = random.Random(0) for _ in range(n1): yield rng1.randint(0, n2) def rando2(): rng2 = random.Random(1) for _ in range(n1): yield rng2.randint(0, n2) iter1 = rando1() iter2 = rando2() else: raise KeyError return {'iter1': iter1, 'iter2': iter2} # For each variation of your experiment, create a row. rows = [] for params in grid_iter: # size = params['n1'] * params['n2'] # params['size'] = size group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) # Make any modifications you need to compute input kwargs for each # method here. kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... kwargs.update(make_input(params)) with timer: # Put the logic you want to time here method(**kwargs) if RECORD_ALL: # Seaborn will show the variance if this is enabled, otherwise # use the robust timerit mean / min times # chunk_iter = ub.chunks(ti.times, ti.bestof) # times = list(map(min, chunk_iter)) # TODO: timerit method for this times = ti.robust_times() for time in times: row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) else: row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) time_key = 'time' if RECORD_ALL else 'min' # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values(time_key) if RECORD_ALL: # Show the min / mean if we record all min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1) mean_times = data.groupby('key')[['time']].mean().rename({'time': 'mean'}, axis=1) stats_data = pd.concat([min_times, mean_times], axis=1) stats_data = stats_data.sort_values('min') else: stats_data = data USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} other_keys = sorted(set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'}) for params, variants in stats_data.groupby(other_keys): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) mean_speedup = variants['mean'].max() / variants['mean'] stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup min_speedup = variants['min'].max() / variants['min'] stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) print('Statistics:') print(stats_data) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('method_ratings = {}'.format(ub.repr2(method_ratings, nl=1))) print('Aggregated Rankings =\n{}'.format(skill_agg)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw) ax.set_title(f'Benchmark Nested Breaks: #Trials {ti.num}, bestof {ti.bestof}') ax.set_xlabel(f'{xlabel}') ax.set_ylabel('Time') ax.set_xscale('log') ax.set_yscale('log') try: __IPYTHON__ except NameError: plt.show()
def run_pvpoke_ultra_experiment(): """ https://pvpoke.com/battle/matrix/ !pip install selenium """ """ Relevant page items: <button class="add-poke-btn button">+ Add Pokemon</button> '//*[@id="main"]/div[3]/div[3]/div/div[1]/button[1]' '/html/body/div[1]/div/div[3]/div[3]/div/div[1]/button[1]' <input class="poke-search" type="text" placeholder="Search name"> /html/body/div[5]/div/div[3]/div[1]/input /html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1] Level Cap /html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5] # IV GROUP ivs-group save-poke import sys, ubelt sys.path.append(ubelt.expandpath('~/code/pypogo')) from pypogo.pvpoke_experiment import * # NOQA from pypogo.pvpoke_experiment import _oldstuff """ from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import Select import ubelt as ub import os import pathlib import time import pandas as pd import pypogo # Requires the driver be in the PATH fpath = ensure_selenium_chromedriver() os.environ['PATH'] = os.pathsep.join( ub.oset(os.environ['PATH'].split(os.pathsep)) | ub.oset([str(fpath.parent)])) url = 'https://pvpoke.com/battle/matrix/' # chrome_exe = ub.find_exe("google-chrome") driver = webdriver.Chrome() driver.get(url) league = 'Great' # league = 'Master40' if league == 'Great': league_box_target = 'Great League (CP 1500)' have_ivs = list( ub.oset([ tuple([int(x) for x in p.strip().split(',') if x]) for p in ub.codeblock(''' 10, 10, 12, 10, 12, 14, 10, 12, 14, 10, 13, 10, 10, 13, 12, 10, 14, 14, 11, 12, 14, 11, 14, 12, 11, 14, 15, 11, 15, 11, 11, 15, 11, 11, 15, 12, 11, 15, 12, 12, 10, 12, 12, 11, 12, 12, 12, 15, 12, 14, 11, 12, 14, 15, 12, 15, 11, 12, 15, 12 12, 15, 12, 13, 11, 13 13, 12, 10 13, 12, 13, 13, 13, 10, 13, 13, 11, 13, 15, 10, 13, 15, 11, 13, 15, 11, 14, 10, 12, 14, 11, 10, 14, 11, 10, 14, 13, 11 14, 13, 14, 15, 10, 12 15, 11, 10, 15, 11, 11, 15, 12, 11 ''').split('\n') ])) to_check_mons = [ pypogo.Pokemon('Deoxys', form='defense', ivs=ivs, moves=['Counter', 'Rock Slide', 'Psycho Boost']).maximize(1500) for ivs in have_ivs ] meta_text = 'Great League Meta' elif league == 'Master40': league_box_target = 'Master League (Level 40)' meta_text = 'Master League Meta' # Test the effect of best buddies vs the master league to_check_mons = [ pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40), pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41), pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=40), pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=41), pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=40), pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=41), pypogo.Pokemon('Giratina', form='origin', ivs=[15, 14, 15], level=40), pypogo.Pokemon('Giratina', form='origin', ivs=[15, 14, 15], level=41), pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=40), pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=41), pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=40), pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=41), pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=40), pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=41), ] for mon in to_check_mons: mon.populate_all() else: pass leage_select = driver.find_elements_by_class_name('league-select')[0] leage_select.click() leage_select.send_keys(league_box_target) leage_select.click() leage_select.text.split('\n') leage_select.send_keys('\n') leage_select.send_keys('\n') def add_pokemon(mon): add_poke1_button = driver.find_elements_by_class_name( 'add-poke-btn')[0] add_poke1_button.click() select_drop = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/select') if 1: import xdev all_names = select_drop.text.split('\n') distances = xdev.edit_distance(mon.display_name(), all_names) chosen_name = all_names[ub.argmin(distances)] else: chosen_name = mon.name search_box = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/input') search_box.send_keys(chosen_name) advanced_ivs_arrow = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1]') advanced_ivs_arrow.click() level40_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[2]' ) level41_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[3]' ) level50_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[4]' ) level51_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5]' ) if mon.level >= 51: level51_cap.click() elif mon.level >= 50: level50_cap.click() elif mon.level >= 41: level41_cap.click() elif mon.level >= 40: level40_cap.click() level_box = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/input' ) level_box.click() level_box.clear() level_box.clear() level_box.send_keys(str(mon.level)) iv_a = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[1]' ) iv_d = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[2]' ) iv_s = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[3]' ) # TODO # driver.find_elements_by_class_name('move-select') iv_a.clear() iv_a.send_keys(str(mon.ivs[0])) iv_d.clear() iv_d.send_keys(str(mon.ivs[1])) iv_s.clear() iv_s.send_keys(str(mon.ivs[2])) # USE_MOVES = 1 if mon.moves is not None: # mon.populate_all() fast_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[1]') fast_select.click() fast_select.send_keys(mon.pvp_fast_move['name']) fast_select.send_keys(Keys.ENTER) charge1_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[2]') charge1_select.click() charge1_select.send_keys(mon.pvp_charge_moves[0]['name']) charge1_select.send_keys(Keys.ENTER) charge2_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[3]') charge2_select.click() charge2_select.send_keys(mon.pvp_charge_moves[1]['name']) charge2_select.send_keys(Keys.ENTER) save_button = driver.find_elements_by_class_name('save-poke')[0] save_button.click() quickfills = driver.find_elements_by_class_name('quick-fill-select') quickfill = quickfills[1] quickfill.text.split('\n') quickfill.click() quickfill.send_keys(meta_text) quickfill.click() import pypogo # mon1 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40) # mon2 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41) if 1: for mon in to_check_mons: pass add_pokemon(mon) shield_selectors = driver.find_elements_by_class_name('shield-select') shield_selectors[2].click() shield_selectors[2].send_keys('No shields') shield_selectors[2].send_keys(Keys.ENTER) shield_selectors[3].click() shield_selectors[3].send_keys('No shields') shield_selectors[3].send_keys(Keys.ENTER) shield_selectors[0].click() battle_btn = driver.find_elements_by_class_name('battle-btn')[0] battle_btn.click() # Clear previous downloaded files dlfolder = pathlib.Path(ub.expandpath('$HOME/Downloads')) for old_fpath in list(dlfolder.glob('_vs*.csv')): old_fpath.unlink() time.sleep(2.0) # Download new data dl_btn = driver.find_element_by_xpath( '//*[@id="main"]/div[4]/div[9]/div/a') dl_btn.click() while len(list(dlfolder.glob('_vs*.csv'))) < 1: pass new_fpaths = list(dlfolder.glob('_vs*.csv')) assert len(new_fpaths) == 1 fpath = new_fpaths[0] data = pd.read_csv(fpath, header=0, index_col=0) if 1: # GROUP ANALYSIS data.sum(axis=1).sort_values() (data > 500).sum(axis=1).sort_values() flipped = [] for key, col in data.T.iterrows(): if not ub.allsame(col > 500): flipped.append(key) flip_df = data.loc[:, flipped] def color(x): if x > 500: return ub.color_text(str(x), 'green') else: return ub.color_text(str(x), 'red') print(flip_df.applymap(color)) print(flip_df.columns.tolist()) (data > 500) else: # PAIR ANALYSIS pairs = list(ub.iter_window(range(len(data)), step=2)) for i, j in pairs: print('-----') matchup0 = data.iloc[i] matchup1 = data.iloc[j] delta = matchup1 - matchup0 print(delta[delta != 0]) wins0 = matchup0 > 500 wins1 = matchup1 > 500 flips = (wins0 != wins1) flipped_vs = matchup0.index[flips] num_flips = sum(flips) print('flipped_vs = {!r}'.format(flipped_vs)) print('num_flips = {!r}'.format(num_flips)) print(matchup0.mean()) print(matchup1.mean()) print(matchup1.mean() / matchup0.mean())
def benchmark_pathlib_vs_fspath(): import ubelt as ub import pathlib import pandas as pd import random import timerit import os def method_pathlib(inputs): p = pathlib.Path(*inputs) def method_ospath(inputs): p = os.path.join(*inputs) method_lut = locals() # can populate this some other way ti = timerit.Timerit(10000, bestof=10, verbose=2) basis = { 'method': ['method_pathlib', 'method_ospath'], 'num_parts': [2, 4, 8, 12, 16], } xlabel = 'num_parts' kw_labels = [] group_labels = { 'style': [], 'size': [], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) n = params['num_parts'] inputs = [chr(random.randint(97, 120)) for _ in range(n)] kwargs['inputs'] = inputs method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(**kwargs) row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('min') print(data) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('Time') ax.set_ylabel('Number of parts')
def test_extend(): self = ub.oset() self.extend([3, 1, 2, 3]) assert self == [3, 1, 2]
def benchmark_repeat_vs_reduce_mul(): import ubelt as ub import pandas as pd import timerit def reduce_daq_rec(func, arrs): if len(arrs) == 1: return arrs[0] if len(arrs) == 2: return func(arrs[0], arrs[1]) elif len(arrs) == 3: return func(func(arrs[0], arrs[1]), arrs[3]) else: arrs1 = arrs[0::2] arrs2 = arrs[1::2] res1 = reduce_daq_rec(func, arrs1) res2 = reduce_daq_rec(func, arrs2) res = func(res1, res2) return res def reduce_daq_iter(func, arrs): """ https://www.baeldung.com/cs/convert-recursion-to-iteration https://stackoverflow.com/questions/159590/way-to-go-from-recursion-to-iteration arrs = [2, 3, 5, 7, 11, 13, 17, 21] """ raise NotImplementedError # TODO: make the iterative version from collections import deque empty_result = None stack = deque([(arrs, empty_result)]) idx = 0 while stack: print('----') print('stack = {}'.format(ub.repr2(list(stack), nl=1))) arrs0, result = stack.pop() if len(arrs0) == 0: raise Exception if result is not None: # raise Exception results = [result] while stack: next_arrs0, next_result = stack.pop() if next_result is None: break else: results.append(next_result) if results: if len(results) == 1: stack.append((results, results[0])) else: stack.append((results, None)) if next_result is None: stack.append((next_arrs0, None)) elif result is None: if len(arrs0) == 1: result = arrs0[0] stack.append((arrs0, result)) # return arrs0[0] if len(arrs0) == 2: result = func(arrs0[0], arrs0[1]) stack.append((arrs0, result)) elif len(arrs0) == 3: result = func(func(arrs0[0], arrs0[1]), arrs0[3]) stack.append((arrs0, result)) else: arrs01 = arrs0[0::2] arrs02 = arrs0[1::2] stack.append((arrs0, empty_result)) stack.append((arrs01, empty_result)) stack.append((arrs02, empty_result)) # res1 = reduce_daq_rec(func, arrs01) # res2 = reduce_daq_rec(func, arrs2) # res = func(res1, res2) idx += 1 if idx > 10: raise Exception return res def method_daq_rec(arrs): return reduce_daq_rec(np.multiply, arrs) def method_repeat(arrs): """ helper code: arr_names = ['a{:02d}'.format(idx) for idx in range(1, 32 + 1)] lhs = ', '.join(arr_names) rhs = ' * '.join(arr_names) print(f'{lhs} = arrs') print(f'ret = {rhs}') """ # Hard coded pure python syntax for multiplying if len(arrs) == 4: a01, a02, a03, a04 = arrs ret = a01 * a02 * a03 * a04 elif len(arrs) == 8: a01, a02, a03, a04, a05, a06, a07, a08 = arrs ret = a01 * a02 * a03 * a04 * a05 * a06 * a07 * a08 elif len(arrs) == 32: a01, a02, a03, a04, a05, a06, a07, a08, a09, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28, a29, a30, a31, a32 = arrs ret = a01 * a02 * a03 * a04 * a05 * a06 * a07 * a08 * a09 * a10 * a11 * a12 * a13 * a14 * a15 * a16 * a17 * a18 * a19 * a20 * a21 * a22 * a23 * a24 * a25 * a26 * a27 * a28 * a29 * a30 * a31 * a32 return ret def method_reduce(arrs): ret = np.multiply.reduce(arrs) return ret def method_stack(arrs): stacked = np.stack(arrs) ret = stacked.prod(axis=0) return ret method_lut = locals() # can populate this some other way ti = timerit.Timerit(10000, bestof=10, verbose=2) basis = { 'method': ['method_repeat', 'method_reduce', 'method_stack', 'method_daq_rec'], 'arr_size': [10, 100, 1000, 10000], 'num_arrs': [4, 8, 32], } xlabel = 'arr_size' kw_labels = [] group_labels = { 'style': ['num_arrs'], 'size': [], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) arr_size = params['arr_size'] num_arrs = params['num_arrs'] arrs = [] for _ in range(num_arrs): arr = np.random.rand(arr_size) arrs.append(arr) kwargs['arrs'] = arrs method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(**kwargs) row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('min') print(data) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('Array Size') ax.set_ylabel('Time')
def test_frames_are_in_order(): import kwcoco import ubelt as ub import random def is_sorted(x): return x == sorted(x) total_frames = 30 total_videos = 4 max_frames_per_video = 20 # Seed rng for reproducibility rng = random.Random(926960862) # Initialize empty dataset dset = kwcoco.CocoDataset() # Add some number of videos vidid_pool = [ dset.add_video('vid_{:03d}'.format(vididx)) for vididx in range(total_videos) ] vidid_to_frame_pool = { vidid: ub.oset(range(max_frames_per_video)) for vidid in vidid_pool } # Add some number of frames to the videos in a random order for imgidx in range(total_frames): vidid = rng.choice(vidid_pool) frame_pool = vidid_to_frame_pool[vidid] assert frame_pool, 'ran out of frames' frame_index = rng.choice(frame_pool) frame_pool.remove(frame_index) name = 'img_{:03d}'.format(imgidx) dset.add_image(video_id=vidid, frame_index=frame_index, name=name) # Test that our image ids are always ordered by frame ids vidid_to_gids = dset.index.vidid_to_gids gids_were_in_order = [] for vidid, gids in vidid_to_gids.items(): gids_were_in_order.append(is_sorted(gids)) frame_idxs = [dset.imgs[gid]['frame_index'] for gid in gids] # Note: this check is always valid assert is_sorted(frame_idxs), ( 'images in vidid_to_gids must be sorted by frame_index') # Note: this check has a chance of failing for other params / seeds assert not all(gids_were_in_order), ( 'the probability we randomly have ordered image ids is low, ' 'and 0 when we seed the rng') try: import sqlalchemy # NOQA except Exception: pass else: # Test that the sql view works too sql_dset = dset.view_sql(memory=True) vidid_to_gids = dict(sql_dset.index.vidid_to_gids) gids_were_in_order = [] for vidid, gids in vidid_to_gids.items(): gids_were_in_order.append(is_sorted(gids)) frame_idxs = [dset.imgs[gid]['frame_index'] for gid in gids] # Note: this check is always valid assert is_sorted(frame_idxs), ( 'images in vidid_to_gids must be sorted by frame_index') # Note: this check has a chance of failing for other params / seeds assert not all(gids_were_in_order), ( 'the probability we randomly have ordered image ids is low, ' 'and 0 when we seed the rng')
def auto_argparse(func): """ Transform a function with a Google Style Docstring into an `argparse.ArgumentParser`. TODO: - [ ] Handle booleans consistently, allow --flag=True and --flag=False Args: func (callable): function with kwargs """ from xdoctest.docstr import docscrape_google as scrape import ast import argparse import ubelt as ub import inspect spec = inspect.getargspec(func) # Parse default values from the function dynamically try: import xinspect kwdefaults = xinspect.get_func_kwargs(func) except Exception as ex: raise kwdefaults = dict(zip(spec.args[-len(spec.defaults):], spec.defaults)) # Parse help and description information from a google-style docstring docstr = func.__doc__ description = scrape.split_google_docblocks(docstr)[0][1][0].strip() # TODO: allow scraping from the kwargs block as well google_args = { argdict['name']: argdict for argdict in scrape.parse_google_args(docstr) } argnames = ub.oset(spec.args) | ub.oset(kwdefaults) argnames = (ub.oset(google_args) & argnames) | argnames # DEBUG = 1 # if DEBUG: # print(ub.repr2(google_args)) # Create the argument parser and register each argument parser = argparse.ArgumentParser( description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter) for arg in argnames: argkw = {} if arg in kwdefaults: argkw['default'] = kwdefaults[arg] if arg in google_args: garg = google_args[arg] argkw['help'] = garg['desc'] # print('-----') # print('argkw = {}, {}'.format(arg, ub.repr2(argkw))) try: if garg['type'] == 'PathLike': argkw['type'] = str elif garg['type'] == 'bool': def _parse_bool(s): return bool(ast.literal_eval(s)) argkw['type'] = _parse_bool else: argkw['type'] = eval(garg['type'], {}) # literal_eval doesnt handle types # argkw['type'] = ast.literal_eval(garg['type']) except Exception as ex: # print('{}, ex = {!r}'.format(arg, ex)) pass # print('-----') # print('argkw = {}, {!r}'.format(arg, argkw)) parser.add_argument('--' + arg, **argkw) return parser
def make_optimizer(hyper, named_parameters): """ Instantiate the optimizer defined by the hyperparams Contains special logic to create param groups Example: >>> import netharn as nh >>> config = {'optimizer': 'sgd', 'params': [ >>> {'lr': 3e-3, 'params': '.*\\.bias'}, >>> {'lr': 1e-3, 'params': '.*\\.weight'}, >>> #{'lr': 100, 'params': '.*\\.doesnotmatch'}, >>> ]} >>> optim_ = nh.api.Optimizer.coerce(config) >>> hyper = nh.HyperParams(optimizer=optim_) >>> model = nh.models.ToyNet1d() >>> named_parameters = list(model.named_parameters()) >>> optimizer = hyper.make_optimizer(named_parameters) >>> print('optimizer = {!r}'.format(optimizer)) """ if hyper._optimizer_info['instance'] is not None: return hyper._optimizer_info['instance'] # What happens if we want to group parameters optim_kw = hyper.optimizer_params.copy() params = optim_kw.pop('params', None) if params is None: param_groups = [p for (name, p) in named_parameters] else: import re named_parameters = list(named_parameters) name_to_param = dict(named_parameters) param_groups = [] if isinstance(params, dict): # remember the group key groups = [{'key': k, **g} for k, g in params.items()] if isinstance(params, list): groups = params PREVENT_DUPLICATES = 1 seen_ = set() for group in groups: # Transform param grouping specifications into real params group = group.copy() spec = group.pop('params') if isinstance(spec, list): if len(spec): first = ub.peek(spec) if isinstance(first, str): real_params = [name_to_param[k] for k in spec] elif isinstance(first, torch.nn.Parameter): real_params = spec else: raise TypeError(type(first)) else: real_params = [] # Python 3.6 doesn't have re.Pattern elif isinstance(spec, str) or hasattr(spec, 'match'): if hasattr(spec, 'match'): pat = spec else: pat = re.compile(spec) real_params = [ p for name, p in name_to_param.items() if pat.match(name) ] else: raise TypeError(type(spec)) if PREVENT_DUPLICATES: # give priority to earlier params # This is Python 3.6+ only real_params = list(ub.oset(real_params) - seen_) seen_.update(real_params) group['params'] = real_params param_groups.append(group) CHECK = 1 if CHECK: # Determine if we are using the same param more than once # or if we are not using a param at all. # NOTE: torch does do a duplicate check. param_group_ids = [] for group in param_groups: ids = list(map(id, group['params'])) param_group_ids.append(ids) all_param_ids = [id(p) for n, p in named_parameters] flat_ids = list(ub.flatten(param_group_ids)) freq = ub.dict_hist(flat_ids, labels=all_param_ids) num_unused = any(v == 0 for v in freq.values()) num_dups = any(v > 1 for v in freq.values()) if num_unused: warnings.warn( 'There are {} unused params'.format(num_unused)) if num_dups: warnings.warn( 'There are {} duplicate params'.format(num_dups)) optimizer = hyper.optimizer_cls(param_groups, **optim_kw) return optimizer
def benchmark_unpack(): """ What is faster unpacking items with slice syntax or tuple-unpacking Slice unpacking seems to be a tad faster. """ import ubelt as ub import random import pandas as pd import timerit import string def tuple_unpack(items): *prefix, key = items return prefix, key def slice_unpack(items): prefix, key = items[:-1], items[-1] return prefix, key method_lut = locals() # can populate this some other way ti = timerit.Timerit(5000, bestof=3, verbose=2) basis = { 'method': ['tuple_unpack', 'slice_unpack'], 'size': list(range(1, 64 + 1)), 'type': ['string', 'float'], } xlabel = 'size' kw_labels = [] group_labels = { 'style': ['type'], 'size': [], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) size = params['size'] method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): if type == 'string': items = [ ''.join(random.choices(string.printable, k=5)) for _ in range(size) ] elif type == 'float': items = [random.random() for _ in range(size)] with timer: method(items) for time in ti.times: row = { 'time': time, 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('time') summary_rows = [] for method, group in data.groupby('method'): row = {} row['method'] = method row['mean'] = group['time'].mean() row['std'] = group['time'].std() row['min'] = group['time'].min() row['max'] = group['time'].max() summary_rows.append(row) print(pd.DataFrame(summary_rows).sort_values('mean')) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='time', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('Execution time') ax.set_ylabel('Size of slices')
def load_partial_state(model, model_state_dict, leftover=None, ignore_unset=False, verbose=2, mangle=True, association=None, initializer=None): """ CommandLine: python -m netharn.initializers.nninit_base load_partial_state Args: model (torch.nn.Module): module to initialize model_state_dict (dict): state dict we wish to transfer leftover (callable): fallback method for initializing incompatible areas, if none then those areas are left as-is. association (str): controls how we search for the association between the two model states. Can be strict, module-hack, prefix-hack, or embedding. Default is: prefix-hack. mangle (bool, default=True): If True, mangles tensors that have the same key, but different shapes forcing them to fit. This might destroy information when forcing a a larger tensor into a smaller tensor, or leave extra uninitialized room when a small tensor is placed in a larger one. Note be careful when mangling a classification layer if class indexes are not aligned. verbose (int): verbosity level Returns: Dict: info - summary of actions taken TODO: - [ ] Allow user to specify how incompatible layers are handled. Notes: Have you ever had the scenario where Has anyone ever had a problem where you had a torch model with a state dict with keys that looked like: `mymodel.detector.layer1.conv.weight`, but you had a pretrained weight file with keys that looked like: `module.layer1.conv.weight`? The latest version of `netharn.initializers.functional.load_patial_state` can handle this by solving a maximum-common-subtree-isomorphism problem. This computes the largest possible mapping between the two state dictionaries that share consistent suffixes. >>> # This means you can load an off-the-shelf unmodified pretrained resnet50 >>> # where the keys might look something like this: >>> resnet_keys = { >>> 'conv1.weight', >>> 'layer1.0.conv1.weight', >>> 'layer1.0.conv2.weight', >>> 'layer1.0.conv3.weight', >>> 'layer1.0.downsample.0.weight', >>> 'layer2.0.conv1.weight', >>> 'layer2.0.conv2.weight', >>> 'layer2.0.conv3.weight', >>> 'layer3.0.conv1.weight', >>> 'layer4.0.conv1.weight', >>> 'fc.weight', >>> 'fc.bias', >>> } >>> # >>> # And perhaps you have a model that has a state dict where keys >>> # look like this: >>> model_keys = { >>> 'preproc.conv1.weight' >>> 'backbone.layer1.0.conv1.weight', >>> 'backbone.layer1.0.conv2.weight', >>> 'backbone.layer1.0.conv3.weight', >>> 'backbone.layer1.0.downsample.0.weight', >>> 'backbone.layer2.0.conv1.weight', >>> 'backbone.layer2.0.conv2.weight', >>> 'backbone.layer2.0.conv3.weight', >>> 'backbone.layer3.0.conv1.weight', >>> 'backbone.layer4.0.conv1.weight', >>> 'head.conv1' >>> 'head.conv2' >>> 'head.fc.weight' >>> 'head.fc.bias' >>> } >>> # >>> # We can compute a partial mapping between them >>> subpaths1, subpaths2 = maximum_common_ordered_subpaths(resnet_keys, model_keys) >>> print(ub.repr2(ub.dzip(subpaths1, subpaths2))) { 'layer1.0.conv2.weight': 'backbone.layer1.0.conv2.weight', 'layer1.0.conv3.weight': 'backbone.layer1.0.conv3.weight', 'layer1.0.downsample.0.weight': 'backbone.layer1.0.downsample.0.weight', 'layer2.0.conv1.weight': 'backbone.layer2.0.conv1.weight', 'layer2.0.conv2.weight': 'backbone.layer2.0.conv2.weight', 'layer2.0.conv3.weight': 'backbone.layer2.0.conv3.weight', 'layer3.0.conv1.weight': 'backbone.layer3.0.conv1.weight', 'layer4.0.conv1.weight': 'backbone.layer4.0.conv1.weight', } Also, if the sizes of the tensor don't quite fit, they will be mangled, i.e. "shoved-in" as best as possible. Example: >>> import netharn as nh >>> self1 = nh.models.ToyNet2d(input_channels=1, num_classes=10) >>> self2 = nh.models.ToyNet2d(input_channels=3, num_classes=2) >>> self1.hack_param1 = torch.nn.Parameter(torch.rand(1)) >>> self2.hack_param1 = torch.nn.Parameter(torch.rand(3)) >>> self2.hack_param2 = torch.nn.Parameter(torch.rand(3)) >>> model_state_dict = self1.state_dict() >>> load_partial_state(self2, model_state_dict) >>> load_partial_state(self2, model_state_dict, leftover=torch.nn.init.kaiming_normal_) Example: >>> import netharn as nh >>> xpu = nh.XPU(None) >>> self1 = nh.models.ToyNet2d() >>> self2 = xpu.mount(self1) >>> load_partial_state(self2, self1.state_dict()) >>> load_partial_state(self1, self2.state_dict()) >>> # Add extra nonsense to state-dict >>> extra_state_dict = {'extra.' + k: v for k, v in self1.state_dict().items()} >>> extra_state_dict['stats'] = ub.peek(extra_state_dict.values()).clone() >>> model = self2 >>> model_state_dict = extra_state_dict >>> load_partial_state(self2, extra_state_dict) Example: >>> # xdoctest: +REQUIRES(--slow) >>> from netharn.initializers.functional import * # NOQA >>> import torchvision >>> import torch >>> resnet50 = torchvision.models.resnet50() >>> class CustomModel(torch.nn.Module): >>> def __init__(self): >>> super().__init__() >>> self.module = resnet50 >>> self.extra = torch.nn.Linear(1, 1) >>> model = CustomModel() >>> model_state_dict = resnet50.state_dict() >>> model_state_dict2 = {'prefix.' + k: v for k, v in model_state_dict.items()} >>> import ubelt as ub >>> with ub.Timer(verbose=2, label='strict'): >>> load_partial_state(model, model_state_dict, association='strict', verbose=0) >>> with ub.Timer(verbose=2, label='prefix-hack'): >>> load_partial_state(model, model_state_dict, association='prefix-hack', verbose=0) >>> with ub.Timer(verbose=2, label='module-hack'): >>> load_partial_state(model, model_state_dict, association='module-hack', verbose=0) >>> with ub.Timer(verbose=2, label='embedding'): >>> load_partial_state(model, model_state_dict, association='embedding', verbose=0) >>> load_partial_state(model, model_state_dict, association='prefix-hack', verbose=1) >>> load_partial_state(model, model_state_dict, association='module-hack', verbose=1) CommandLine: xdoctest -m /home/joncrall/code/netharn/netharn/initializers/functional.py load_partial_state:2 --slow """ if association is None: association = 'module-hack' # old default # association = 'prefix-hack' # new default if initializer is not None: import warnings warnings.warn('initializer is deprecated use leftover') leftover = initializer self_state = model.state_dict() def _fix_keys(model_state_dict): """ Hack around DataParallel wrapper. If there is nothing in common between the two models check to see if prepending 'module.' to other keys fixes it. """ other_keys = set(model_state_dict) self_keys = set(self_state) common_keys = other_keys.intersection(self_keys) if not common_keys: if association == 'strict': pass elif association == 'module-hack': # If there are no common keys try a hack prefix = 'module.' def smap(f, ss): return set(map(f, ss)) def fix1(k): return prefix + k def fix2(k): if k.startswith(prefix): return k[len(prefix):] if smap(fix1, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix1, model_state_dict) elif smap(fix2, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix2, model_state_dict) elif association == 'prefix-hack': import functools def add_prefix(k, prefix): return prefix + k def remove_prefix(k, prefix): if k.startswith(prefix): return k[len(prefix):] # set1 = other_keys # target_set2 = self_keys found = _best_prefix_transform(other_keys, self_keys) if found is not None: for action, prefix in found['transform']: if action == 'add': func = functools.partial(add_prefix, prefix=prefix) elif action == 'remove': func = functools.partial(remove_prefix, prefix=prefix) else: raise AssertionError model_state_dict = ub.map_keys(func, model_state_dict) elif association == 'embedding': if verbose > 1: print( 'Using subpath embedding assocation, may take some time' ) # I believe this is the correct way to solve the problem paths1 = sorted(other_keys) paths2 = sorted(self_state) subpaths1, subpaths2 = maximum_common_ordered_subpaths( paths1, paths2) mapping = ub.dzip(subpaths1, subpaths2) if verbose > 1: print('mapping = {}'.format(ub.repr2(mapping, nl=1))) model_state_dict = ub.map_keys(lambda k: mapping.get(k, k), model_state_dict) else: raise KeyError(association) return model_state_dict other_state = _fix_keys(model_state_dict) self_unset_keys = set( self_state.keys()) # will end up as keys in our that were not set other_unused_keys = set(other_state.keys( )) # will end up as keys in the other model that were not used seen_keys = ub.ddict(set) for key, other_value in other_state.items(): if key not in self_state: if verbose > 0: print('Skipping {} because it does not exist'.format(key)) seen_keys['skipped'].add(key) else: self_value = self_state[key] if other_value.size() == self_value.size(): self_state[key] = other_value self_unset_keys.remove(key) other_unused_keys.remove(key) seen_keys['full_add'].add(key) elif len(other_value.size()) == len(self_value.size()): if not mangle: if verbose > 0: print( 'Skipping {} due to incompatable size and mangle=False' .format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) elif key.endswith('bias'): if verbose > 0: print( 'Skipping {} due to incompatable size'.format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) else: if leftover is None: if verbose > 0: print( 'Skipping {} due to incompatable size and no default initializer' .format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) else: if verbose > 0: print('Partially add {} with incompatable size'. format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) # Initialize all weights in case any are unspecified if leftover is None: try: leftover(self_state[key]) except Exception: if verbose > 0: print('Unable to init {} with {}'.format( key, leftover)) # Transfer as much as possible min_size = np.minimum(self_state[key].shape, other_value.shape) sl = tuple([slice(0, s) for s in min_size]) self_state[key][sl] = other_value[sl] # if shock_partial: # # Shock weights because we are doing something weird # # might help the network recover in case this is # # not a good idea # shock(self_state[key], func=leftover) self_unset_keys.remove(key) other_unused_keys.remove(key) if self_state[key].numel() < other_value.numel(): seen_keys['partial_add_some'].add(key) else: seen_keys['partial_add_all'].add(key) else: if verbose > 0: print('Skipping {} due to incompatable size'.format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) if ignore_unset is True: self_unset_keys = [] elif ignore_unset: self_unset_keys = list(ub.oset(self_unset_keys) - set(ignore_unset)) if (self_unset_keys or other_unused_keys or seen_keys['partial_add_some'] or seen_keys['partial_add_all']): if verbose > 0: if seen_keys: print('Pretrained weights are a partial fit') else: print('Pretrained weights do not fit!') if verbose > 1: print('Seen Keys: {}'.format(ub.repr2(seen_keys, nl=2))) print('Self Unset Keys: {}'.format(ub.repr2(self_unset_keys, nl=1))) print('Other Unused keys: {}'.format( ub.repr2(other_unused_keys, nl=1))) print('summary:') seen_sum = ub.map_vals(len, seen_keys) print('Seen Num: {}'.format(ub.repr2(seen_sum, nl=2))) print('Self Unset Num: {}'.format( ub.repr2(len(self_unset_keys), nl=1))) print('Other Unused Num: {}'.format( ub.repr2(len(other_unused_keys), nl=1))) if leftover: if verbose > 0: print('Initializing unused keys using {}'.format(leftover)) for key in self_unset_keys: if key.endswith('.num_batches_tracked'): pass # ignore num_batches_tracked elif key.endswith('.bias'): self_state[key].fill_(0) else: try: leftover(self_state[key]) except Exception: if verbose > 0: print('Unable to init {} with {}'.format( key, leftover)) else: if verbose > 0: print('Pretrained weights are a perfect fit') model.load_state_dict(self_state) info = { 'seen': seen_keys, 'self_unset': self_unset_keys, 'other_unused': other_unused_keys } return info
def benchamrk_det_nms(): """ Benchmarks different implementations of non-max-supression on the CPU, GPU, and using cython / numpy / torch. CommandLine: xdoctest -m ~/code/kwimage/dev/bench_nms.py benchamrk_det_nms --show SeeAlso: PJR Darknet NonMax supression https://github.com/pjreddie/darknet/blob/master/src/box.c Lightnet NMS https://gitlab.com/EAVISE/lightnet/blob/master/lightnet/data/transform/_postprocess.py#L116 """ # N = 200 # bestof = 50 N = 1 bestof = 1 # xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500, 2000] # max number of boxes yolo will spit out at a time max_boxes = 19 * 19 * 5 xdata = [ 10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500, max_boxes ] # xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500] # Demo values xdata = [0, 1, 2, 3, 10, 100, 200, 300, 500] if ub.argflag('--small'): xdata = [10, 100, 500, 1000, 1500, 2000, 5000, 10000] if ub.argflag('--medium'): xdata = [ 1000, 5000, 10000, 20000, 50000, ] if ub.argflag('--large'): xdata = [ 1000, 5000, 10000, 20000, 50000, 100000, ] if ub.argflag('--extra-large'): xdata = [ 1000, 2000, 10000, 20000, 40000, 100000, 200000, ] title_parts = [] SMALL_BOXES = ub.argflag('--small-boxes') if SMALL_BOXES: title_parts.append('small boxes') else: title_parts.append('large boxes') # NOTE: for large images we may have up to 21,850,753 detections! thresh = float(ub.argval('--thresh', default=0.4)) title_parts.append('thresh={:.2f}'.format(thresh)) from kwimage.algo.algo_nms import available_nms_impls valid_impls = available_nms_impls() print('valid_impls = {!r}'.format(valid_impls)) basis = { 'type': ['ndarray', 'tensor', 'tensor0'], # 'daq': [True, False], # 'daq': [False], # 'device': [None], # 'impl': valid_impls, 'impl': valid_impls + ['auto'], } if ub.argflag('--daq'): basis['daq'] = [True, False] # if torch.cuda.is_available(): # basis['device'].append(0) combos = [ ub.dzip(basis.keys(), vals) for vals in it.product(*basis.values()) ] def is_valid_combo(combo): # if combo['impl'] in {'py', 'cython_cpu'} and combo['device'] is not None: # return False # if combo['type'] == 'ndarray' and combo['impl'] == 'cython_gpu': # if combo['device'] is None: # return False # if combo['type'] == 'ndarray' and combo['impl'] != 'cython_gpu': # if combo['device'] is not None: # return False # if combo['type'].endswith('0'): # if combo['impl'] in {'numpy', 'cython_gpu', 'cython_cpu'}: # return False # if combo['type'] == 'ndarray': # if combo['impl'] in {'torch'}: # return False REMOVE_SLOW = True if REMOVE_SLOW: known_bad = [ { 'impl': 'torch', 'type': 'tensor' }, { 'impl': 'numpy', 'type': 'tensor' }, # {'impl': 'cython_gpu', 'type': 'tensor'}, { 'impl': 'cython_cpu', 'type': 'tensor' }, # {'impl': 'torch', 'type': 'tensor0'}, { 'impl': 'numpy', 'type': 'tensor0' }, # {'impl': 'cython_gpu', 'type': 'tensor0'}, # {'impl': 'cython_cpu', 'type': 'tensor0'}, { 'impl': 'torchvision', 'type': 'ndarray' }, ] for known in known_bad: if all(combo[key] == val for key, val in known.items()): return False return True combos = list(filter(is_valid_combo, combos)) times = ub.ddict(list) for num in xdata: if num > 10000: N = 1 bestof = 1 if num > 1000: N = 3 bestof = 1 if num > 100: N = 10 bestof = 3 elif num > 10: N = 100 bestof = 10 else: N = 1000 bestof = 10 print('\n\n---- number of boxes = {} ----\n'.format(num)) outputs = {} ti = ub.Timerit(N, bestof=bestof, verbose=1) # Build random test boxes and scores np_dets1 = kwimage.Detections.random(num // 2, scale=1000.0, rng=0) np_dets1.data['boxes'] = np_dets1.boxes.to_xywh() if SMALL_BOXES: max_dim = 100 np_dets1.boxes.data[..., 2] = np.minimum(np_dets1.boxes.width, max_dim).ravel() np_dets1.boxes.data[..., 3] = np.minimum(np_dets1.boxes.height, max_dim).ravel() np_dets2 = copy.deepcopy(np_dets1) np_dets2.boxes.translate(10, inplace=True) # add boxes that will definately be removed np_dets = kwimage.Detections.concatenate([np_dets1, np_dets2]) # make all scores unique to ensure comparability np_dets.scores[:] = np.linspace(0, 1, np_dets.num_boxes()) np_dets.data['scores'] = np_dets.scores.astype(np.float32) np_dets.boxes.data = np_dets.boxes.data.astype(np.float32) typed_data = {} # ---------------------------------- import netharn as nh for combo in combos: print('combo = {}'.format(ub.repr2(combo, nl=0))) label = nh.util.make_idstr(combo) mode = combo.copy() # if mode['impl'] == 'cython_gpu': # mode['device_id'] = mode['device'] mode_type = mode.pop('type') if mode_type in typed_data: dets = typed_data[mode_type] else: if mode_type == 'ndarray': dets = np_dets.numpy() elif mode_type == 'tensor': dets = np_dets.tensor(None) elif mode_type == 'tensor0': dets = np_dets.tensor(0) else: raise KeyError typed_data[mode_type] = dets for timer in ti.reset(label): with timer: keep = dets.non_max_supression(thresh=thresh, **mode) torch.cuda.synchronize() times[ti.label].append(ti.min()) outputs[ti.label] = ensure_numpy_indices(keep) # ---------------------------------- # Check that all kept boxes do not have more than `threshold` ious if 0: for key, keep_idxs in outputs.items(): kept = np_dets.take(keep_idxs).boxes ious = kept.ious(kept) max_iou = (np.tril(ious) - np.eye(len(ious))).max() if max_iou > thresh: print('{} produced a bad result with max_iou={}'.format( key, max_iou)) # Check result consistency: print('\nResult stats:') for key in sorted(outputs.keys()): print(' * {:<20}: num={}'.format(key, len(outputs[key]))) print('\nResult overlaps (method1, method2: jaccard):') datas = [] for k1, k2 in it.combinations(sorted(outputs.keys()), 2): idxs1 = set(outputs[k1]) idxs2 = set(outputs[k2]) jaccard = len(idxs1 & idxs2) / max(len(idxs1 | idxs2), 1) datas.append((k1, k2, jaccard)) datas = sorted(datas, key=lambda x: -x[2]) for k1, k2, jaccard in datas: print(' * {:<20}, {:<20}: {:0.4f}'.format(k1, k2, jaccard)) if True: ydata = {key: 1.0 / np.array(vals) for key, vals in times.items()} ylabel = 'Hz' reverse = True yscale = 'symlog' else: ydata = {key: np.array(vals) for key, vals in times.items()} ylabel = 'seconds' reverse = False yscale = 'linear' scores = {key: vals[-1] for key, vals in ydata.items()} ydata = ub.dict_subset(ydata, ub.argsort(scores, reverse=reverse)) ### times_of_interest = [0, 10, 100, 200, 1000] times_of_interest = xdata lines = [] record = lines.append record('### times_of_interest = {!r}'.format(times_of_interest)) for x in times_of_interest: if times_of_interest[-1] == x: record('else:') elif times_of_interest[0] == x: record('if num <= {}:'.format(x)) else: record('elif num <= {}:'.format(x)) if x in xdata: pos = xdata.index(x) score_wrt_x = {} for key, vals in ydata.items(): score_wrt_x[key] = vals[pos] typekeys = ['tensor0', 'tensor', 'ndarray'] type_groups = dict([(b, ub.group_items(score_wrt_x, lambda y: y.endswith(b))[True]) for b in typekeys]) # print('\n=========') # print('x = {!r}'.format(x)) record(' if code not in {!r}:'.format(set(typekeys))) record(' raise KeyError(code)') for typekey, group in type_groups.items(): # print('-------') record(' if code == {!r}:'.format(typekey)) # print('typekey = {!r}'.format(typekey)) # print('group = {!r}'.format(group)) group_x = ub.dict_isect(score_wrt_x, group) valid_keys = ub.argsort(group_x, reverse=True) valid_x = ub.dict_subset(group_x, valid_keys) # parts = [','.split(k) for k in valid_keys] ordered_impls = [] ordered_impls2 = ub.odict() for k in valid_keys: vals = valid_x[k] p = k.split(',') d = dict(i.split('=') for i in p) ordered_impls2[d['impl']] = vals ordered_impls.append(d['impl']) ordered_impls = list(ub.oset(ordered_impls) - {'auto'}) ordered_impls2.pop('auto') record(' # {}'.format( ub.repr2(ordered_impls2, precision=1, nl=0, explicit=True))) record(' preference = {}'.format( ub.repr2(ordered_impls, nl=0))) record('### end times of interest ') print(ub.indent('\n'.join(lines), ' ' * 8)) ### markers = { key: 'o' if 'auto' in key else '' for key, score in scores.items() } if ub.argflag('--daq'): markers = { key: '+' if 'daq=True' in key else '' for key, score in scores.items() } labels = { key: '{:.2f} {} - {}'.format(score, ylabel[0:3], key) for key, score in scores.items() } title = 'NSM-impl speed: ' + ', '.join(title_parts) import kwplot kwplot.autompl() kwplot.multi_plot( xdata, ydata, xlabel='num boxes', ylabel=ylabel, label=labels, yscale=yscale, title=title, marker=markers, # xscale='symlog', ) kwplot.show_if_requested()
def ensure_selenium_chromedriver(): """ os.environ['webdriver.chrome.driver'] = ensure_selenium_chromedriver() """ import requests import zipfile timeout = 5.0 def latest_version(): rsp = requests.get( 'http://chromedriver.storage.googleapis.com/LATEST_RELEASE', timeout=timeout) if rsp.status_code != 200: raise Exception version = rsp.text.strip() return version # version = latest_version() # version = '91.0.4472.19' # version = '90.0.4430.24' version = '92.0.4515.107' known_hashs = { '91.0.4472.19': '49622b740b1c7e66b87179a2642f6c57f21a97fc844c84b30a48', '90.0.4430.24': 'b85313de6abc1b44f26a0e12e20cb66657b840417f5ac6018946', '92.0.4515.107': '844c0e04bbbfd286617af2d7facd3d6cf7d3491b1e78120f8e0', } url = 'http://chromedriver.storage.googleapis.com/{}/chromedriver_linux64.zip'.format( version) bin_dpath = pathlib.Path(ub.expandpath('~/.local/bin')) download_dpath = bin_dpath / f'chromedriver_{version}' download_dpath.mkdir(exist_ok=True, parents=True) zip_fpath = ub.grabdata( url, hash_prefix=known_hashs.get(version, 'unknown-version'), dpath=download_dpath, ) zip_fpath = pathlib.Path(zip_fpath) # dpath = zip_fpath.parent # TODO: version the binary chromedriver_fpath_real = download_dpath / 'chromedriver' chromedriver_fpath_link = bin_dpath / 'chromedriver' if not chromedriver_fpath_real.exists( ) or not chromedriver_fpath_link.exists(): # Also check hash? zfile = zipfile.ZipFile(str(zip_fpath)) try: fpath = zfile.extract('chromedriver', path=chromedriver_fpath_real.parent) finally: zfile.close() chromedriver_fpath_real_ = pathlib.Path(fpath) assert chromedriver_fpath_real_.exists() ub.symlink(chromedriver_fpath_real_, chromedriver_fpath_link, overwrite=True) if not ub.WIN32: print('add permission chromedriver_fpath_real_ = {!r}'.format( chromedriver_fpath_real_)) st = os.stat(chromedriver_fpath_real_) os.chmod(chromedriver_fpath_real_, st.st_mode | stat.S_IEXEC) os.environ['PATH'] = os.pathsep.join( ub.oset(os.environ['PATH'].split(os.pathsep)) | ub.oset([str(chromedriver_fpath_link.parent)])) return chromedriver_fpath_link
def _heuristic_auto_nms_impl(code, num, valid=None): """ Defined with help from ``~/code/kwimage/dev/bench_nms.py`` Args: code (str): text that indicates which type of data you have tensor0 is a tensor on a cuda device, tensor is on the cpu, and numpy is a ndarray. num (int): number of boxes you have to supress. valid (List[str]): the list of valid implementations, an error will be raised if heuristic preferences do not intersect with this list. Ignore: _impls._funcs valid_pref = ub.oset(preference) & set(_impls._funcs.keys()) python ~/code/kwimage/dev/bench_nms.py --show --small-boxes --thresh=0.6 """ if code not in {'tensor0', 'tensor', 'ndarray'}: raise KeyError(code) if num <= 10: if code == 'tensor0': # dict(cython_cpu=4118.4, torchvision=3042.5, cython_gpu=2244.4, torch=841.9) preference = ['cython_cpu', 'torchvision', 'cython_gpu', 'torch'] if code == 'tensor': # dict(torchvision=5857.1, cython_gpu=3058.1) preference = ['torchvision', 'cython_gpu', 'torch', 'numpy'] if code == 'ndarray': # dict(cython_cpu=12226.1, numpy=7759.1, cython_gpu=3679.0, torch=1786.2) preference = ['cython_cpu', 'numpy', 'cython_gpu', 'torch'] elif num <= 100: if code == 'tensor0': # dict(cython_cpu=4160.7, torchvision=3089.9, cython_gpu=2261.8, torch=846.8) preference = [ 'cython_cpu', 'torchvision', 'cython_gpu', 'torch', 'numpy' ] if code == 'tensor': # dict(torchvision=5875.3, cython_gpu=3076.9) preference = ['torchvision', 'cython_gpu', 'torch', 'numpy'] if code == 'ndarray': # dict(cython_cpu=12256.7, cython_gpu=3702.9, numpy=2311.3, torch=1738.0) preference = ['cython_cpu', 'cython_gpu', 'numpy', 'torch'] elif num <= 200: if code == 'tensor0': # dict(cython_cpu=3460.8, torchvision=2912.9, cython_gpu=2125.2, torch=782.4) preference = ['cython_cpu', 'torchvision', 'cython_gpu', 'torch'] if code == 'tensor': # dict(torchvision=3394.6, cython_gpu=2641.2) preference = ['torchvision', 'cython_gpu', 'torch', 'numpy'] if code == 'ndarray': # dict(cython_cpu=8220.6, cython_gpu=3114.5, torch=1240.7, numpy=309.5) preference = ['cython_cpu', 'cython_gpu', 'torch', 'numpy'] elif num <= 300: if code == 'tensor0': # dict(torchvision=2647.1, cython_cpu=2264.9, cython_gpu=1915.5, torch=672.0) preference = ['torchvision', 'cython_cpu', 'cython_gpu', 'torch'] if code == 'tensor': # dict(cython_gpu=2496.9, torchvision=1781.1) preference = ['cython_gpu', 'torchvision', 'torch', 'numpy'] if code == 'ndarray': # dict(cython_cpu=4085.6, cython_gpu=2944.4, torch=799.8, numpy=173.0) preference = ['cython_cpu', 'cython_gpu', 'torch', 'numpy'] else: if code == 'tensor0': # dict(torchvision=2585.5, cython_gpu=1868.7, cython_cpu=1650.6, torch=623.1) preference = ['torchvision', 'cython_gpu', 'cython_cpu', 'torch'] if code == 'tensor': # dict(cython_gpu=2463.1, torchvision=1126.2) preference = ['cython_gpu', 'torchvision', 'torch', 'numpy'] if code == 'ndarray': # dict(cython_gpu=2880.2, cython_cpu=2432.5, torch=511.9, numpy=114.0) preference = ['cython_gpu', 'cython_cpu', 'torch', 'numpy'] if valid: valid_pref = ub.oset(preference) & valid else: valid_pref = preference if not valid_pref: raise Exception( 'no valid nms algo: code={}, num={}, valid={}, preference={}, valid_pref={}' .format(code, num, valid, preference, valid_pref)) impl = valid_pref[0] return impl
def benchmark_dict_diff_impl(): import ubelt as ub import pandas as pd import timerit import random def method_diffkeys(*args): first_dict = args[0] keys = set(first_dict) keys.difference_update(*map(set, args[1:])) new0 = dict((k, first_dict[k]) for k in keys) return new0 def method_diffkeys_list(*args): first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) keep_keys = [k for k in first_dict.keys() if k not in remove_keys] new = dict((k, first_dict[k]) for k in keep_keys) return new def method_diffkeys_oset(*args): first_dict = args[0] keys = ub.oset(first_dict) keys.difference_update(*map(set, args[1:])) new0 = dict((k, first_dict[k]) for k in keys) return new0 def method_ifkeys_setcomp(*args): first_dict = args[0] remove_keys = {k for ks in args[1:] for k in ks} new1 = dict((k, v) for k, v in first_dict.items() if k not in remove_keys) return new1 def method_ifkeys_setunion(*args): first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new2 = dict((k, v) for k, v in first_dict.items() if k not in remove_keys) return new2 def method_ifkeys_getitem(*args): first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new3 = dict((k, first_dict[k]) for k in first_dict.keys() if k not in remove_keys) return new3 def method_ifkeys_dictcomp(*args): # Cannot use until 3.6 is dropped (it is faster) first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new4 = {k: v for k, v in first_dict.items() if k not in remove_keys} return new4 def method_ifkeys_dictcomp_getitem(*args): # Cannot use until 3.6 is dropped (it is faster) first_dict = args[0] remove_keys = set.union(*map(set, args[1:])) new4 = {k: first_dict[k] for k in first_dict.keys() if k not in remove_keys} return new4 method_lut = locals() # can populate this some other way def make_data(num_items, num_other, remove_fraction, keytype): if keytype == 'str': keytype = str if keytype == 'int': keytype = int first_keys = [random.randint(0, 1000) for _ in range(num_items)] k = int(remove_fraction * len(first_keys)) remove_sets = [list(ub.unique(random.choices(first_keys, k=k) + [random.randint(0, 1000) for _ in range(num_items)])) for _ in range(num_other)] first_dict = {keytype(k): k for k in first_keys} args = [first_dict] + [{keytype(k): k for k in ks} for ks in remove_sets] return args ti = timerit.Timerit(200, bestof=1, verbose=2) basis = { 'method': [ # Cant use because unordered # 'method_diffkeys', # Cant use because python 3.6 'method_ifkeys_dictcomp', 'method_ifkeys_dictcomp_getitem', 'method_ifkeys_setunion', 'method_ifkeys_getitem', 'method_diffkeys_list', # Probably not good # 'method_ifkeys_setcomp', # 'method_diffkeys_oset', ], 'num_items': [10, 100, 1000], 'num_other': [1, 3, 5], # 'num_other': [1], 'remove_fraction': [0, 0.2, 0.5, 0.7, 1.0], # 'remove_fraction': [0.2, 0.8], 'keytype': ['str', 'int'], # 'keytype': ['str'], # 'param_name': [param values], } xlabel = 'num_items' kw_labels = ['num_items', 'num_other', 'remove_fraction', 'keytype'] group_labels = { 'style': ['num_other', 'keytype'], 'size': ['remove_fraction'], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) args = make_data(**kwargs) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(*args) row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values('min') print(data) # for each parameter setting, group all methods with that used those exact # comparable params. Then rank how good each method did. That will be a # preference profile. We will give that preference profile a weight (e.g. # based on the fastest method in the bunch) and then aggregate them with # some voting method. USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} weighted_rankings = ub.ddict(lambda: ub.ddict(float)) for params, variants in data.groupby(['num_other', 'keytype', 'remove_fraction', 'num_items']): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) # Choose a ranking weight scheme weight = variants['mean'].min() # weight = 1 for rank, method in enumerate(ranking): weighted_rankings[method][rank] += weight weighted_rankings[method]['total'] += weight # Probably a more robust voting method to do this weight_rank_rows = [] for method_name, ranks in weighted_rankings.items(): weights = ub.dict_diff(ranks, ['total']) p_rank = ub.map_vals(lambda w: w / ranks['total'], weights) for rank, w in p_rank.items(): weight_rank_rows.append({'rank': rank, 'weight': w, 'name': method_name}) weight_rank_df = pd.DataFrame(weight_rank_rows) piv = weight_rank_df.pivot(['name'], ['rank'], ['weight']) print(piv) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('skill_agg =\n{}'.format(skill_agg)) aggregated = (piv * piv.columns.levels[1].values).sum(axis=1).sort_values() print('weight aggregated =\n{}'.format(aggregated)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='min', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('A better x-variable description') ax.set_ylabel('A better y-variable description')
def benchmark_template(): import ubelt as ub import pandas as pd import timerit def method1(x, y, z): ret = [] for i in range((x + y) * z): ret.append(i) return ret def method2(x, y, z): ret = [i for i in range((x + y) * z)] return ret method_lut = locals() # can populate this some other way # Change params here to modify number of trials ti = timerit.Timerit(100, bestof=10, verbose=1) # if True, record every trail run and show variance in seaborn # if False, use the standard timerit min/mean measures RECORD_ALL = True # These are the parameters that we benchmark over basis = { 'method': ['method1', 'method2'], 'x': list(range(7)), 'y': [0, 100], 'z': [2, 3] # 'param_name': [param values], } xlabel = 'x' # Set these to param labels that directly transfer to method kwargs kw_labels = ['x', 'y', 'z'] # Set these to empty lists if they are not used group_labels = { 'style': ['y'], 'size': ['z'], } group_labels['hue'] = list((ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) # Make any modifications you need to compute input kwargs for each # method here. kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... with timer: # Put the logic you want to time here method(**kwargs) if RECORD_ALL: # Seaborn will show the variance if this is enabled, otherwise # use the robust timerit mean / min times chunk_iter = ub.chunks(ti.times, ti.bestof) times = list(map(min, chunk_iter)) # TODO: timerit method for this for time in times: row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) else: row = { 'mean': ti.mean(), 'min': ti.min(), 'key': key, **group_keys, **params, } rows.append(row) time_key = 'time' if RECORD_ALL else 'min' # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) data = data.sort_values(time_key) if RECORD_ALL: # Show the min / mean if we record all min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1) mean_times = data.groupby('key')[['time' ]].mean().rename({'time': 'mean'}, axis=1) stats_data = pd.concat([min_times, mean_times], axis=1) stats_data = stats_data.sort_values('min') else: stats_data = data USE_OPENSKILL = 1 if USE_OPENSKILL: # Lets try a real ranking method # https://github.com/OpenDebates/openskill.py import openskill method_ratings = {m: openskill.Rating() for m in basis['method']} other_keys = sorted( set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'}) for params, variants in stats_data.groupby(other_keys): variants = variants.sort_values('mean') ranking = variants['method'].reset_index(drop=True) mean_speedup = variants['mean'].max() / variants['mean'] stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup min_speedup = variants['min'].max() / variants['min'] stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup if USE_OPENSKILL: # The idea is that each setting of parameters is a game, and each # "method" is a player. We rank the players by which is fastest, # and update their ranking according to the Weng-Lin Bayes ranking # model. This does not take the fact that some "games" (i.e. # parameter settings) are more important than others, but it should # be fairly robust on average. old_ratings = [[r] for r in ub.take(method_ratings, ranking)] new_values = openskill.rate(old_ratings) # Not inplace new_ratings = [openskill.Rating(*new[0]) for new in new_values] method_ratings.update(ub.dzip(ranking, new_ratings)) print('Statistics:') print(stats_data) if USE_OPENSKILL: from openskill import predict_win win_prob = predict_win([[r] for r in method_ratings.values()]) skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False) print('Aggregated Rankings =\n{}'.format(skill_agg)) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw) ax.set_title('Benchmark Name') ax.set_xlabel('Size (todo: A better x-variable description)') ax.set_ylabel('Time (todo: A better y-variable description)') # ax.set_xscale('log') # ax.set_yscale('log') try: __IPYTHON__ except NameError: plt.show()
def benchmark_mul_vs_pow(): import ubelt as ub import pandas as pd import timerit from functools import reduce import operator as op import itertools as it def method_pow_via_mul_raw(n): """ Construct a function that does multiplication of a value n times """ return eval('lambda v: ' + ' * '.join(['v'] * n)) def method_pow_via_mul_for(v, n): ret = v for _ in range(1, n): ret = ret * v return ret def method_pow_via_mul_reduce(v, n): """ Alternative way to multiply a value n times """ return reduce(op.mul, it.repeat(v, n)) def method_pow_via_pow(v, n): return v ** n method_lut = locals() # can populate this some other way ti = timerit.Timerit(500000, bestof=1000, verbose=2) basis = { 'method': ['method_pow_via_mul_raw', 'method_pow_via_pow'], 'n': list(range(1, 20)), 'v': ['random-int', 'random-float'], # 'param_name': [param values], } xlabel = 'n' kw_labels = ['v', 'n'] group_labels = { 'style': ['v'], 'size': [], } group_labels['hue'] = list( (ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values()))) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2( ub.dict_isect(params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1) kwargs = ub.dict_isect(params.copy(), kw_labels) method = method_lut[params['method']] # Timerit will run some user-specified number of loops. # and compute time stats with similar methodology to timeit if params['method'] == 'method_pow_via_mul_raw': method = method(kwargs.pop('n')) for timer in ti.reset(key): # Put any setup logic you dont want to time here. # ... import random if kwargs['v'] == 'random': kwargs['v'] = random.randint(1, 31000) if random.random() > 0.5 else random.random() elif kwargs['v'] == 'random-int': kwargs['v'] = random.randint(1, 31000) elif kwargs['v'] == 'random-float': kwargs['v'] = random.random() with timer: # Put the logic you want to time here method(**kwargs) for time in map(min, ub.chunks(ti.times, ti.bestof)): row = { # 'mean': ti.mean(), 'time': time, 'key': key, **group_keys, **params, } rows.append(row) # The rows define a long-form pandas data array. # Data in long-form makes it very easy to use seaborn. data = pd.DataFrame(rows) # data = data.sort_values('time') print(data) plot = True if plot: # import seaborn as sns # kwplot autosns works well for IPython and script execution. # not sure about notebooks. import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() plotkw = {} for gname, labels in group_labels.items(): if labels: plotkw[gname] = gname + '_key' # Your variables may change ax = kwplot.figure(fnum=1, doclf=True).gca() sns.lineplot(data=data, x=xlabel, y='time', marker='o', ax=ax, **plotkw) ax.set_title('Benchmark') ax.set_xlabel('N') ax.set_ylabel('Time') ax.set_yscale('log') plt.show()
def load_partial_state(model, model_state_dict, leftover=None, ignore_unset=False, verbose=2, mangle=True, association=None, initializer=None): """ CommandLine: python -m netharn.initializers.nninit_base load_partial_state Args: model (torch.nn.Module): module to initialize model_state_dict (dict): state dict we wish to transfer leftover (callable): fallback method for initializing incompatible areas, if none then those areas are left as-is. association (str): controls how we search for the association between the two model states. Can be strict, module-hack, prefix-hack, or embedding. Default is: prefix-hack. mangle (bool, default=True): If True, mangles tensors that have the same key, but different shapes forcing them to fit. This might destroy information when forcing a a larger tensor into a smaller tensor, or leave extra uninitialized room when a small tensor is placed in a larger one. Note be careful when mangling a classification layer if class indexes are not aligned. verbose (int): verbosity level Returns: Dict: info - summary of actions taken TODO: - [ ] Allow user to specify how incompatible layers are handled. Notes: Have you ever had the scenario where Has anyone ever had a problem where you had a torch model with a state dict with keys that looked like: `mymodel.detector.layer1.conv.weight`, but you had a pretrained weight file with keys that looked like: `module.layer1.conv.weight`? The latest version of `netharn.initializers.functional.load_patial_state` can handle this by solving a maximum-common-subtree-isomorphism problem. This computes the largest possible mapping between the two state dictionaries that share consistent suffixes. >>> # This means you can load an off-the-shelf unmodified pretrained resnet50 >>> # where the keys might look something like this: >>> resnet_keys = { >>> 'conv1.weight', >>> 'layer1.0.conv1.weight', >>> 'layer1.0.conv2.weight', >>> 'layer1.0.conv3.weight', >>> 'layer1.0.downsample.0.weight', >>> 'layer2.0.conv1.weight', >>> 'layer2.0.conv2.weight', >>> 'layer2.0.conv3.weight', >>> 'layer3.0.conv1.weight', >>> 'layer4.0.conv1.weight', >>> 'fc.weight', >>> 'fc.bias', >>> } >>> # >>> # And perhaps you have a model that has a state dict where keys >>> # look like this: >>> model_keys = { >>> 'preproc.conv1.weight' >>> 'backbone.layer1.0.conv1.weight', >>> 'backbone.layer1.0.conv2.weight', >>> 'backbone.layer1.0.conv3.weight', >>> 'backbone.layer1.0.downsample.0.weight', >>> 'backbone.layer2.0.conv1.weight', >>> 'backbone.layer2.0.conv2.weight', >>> 'backbone.layer2.0.conv3.weight', >>> 'backbone.layer3.0.conv1.weight', >>> 'backbone.layer4.0.conv1.weight', >>> 'head.conv1' >>> 'head.conv2' >>> 'head.fc.weight' >>> 'head.fc.bias' >>> } >>> # >>> # We can compute a partial mapping between them >>> subpaths1, subpaths2 = maximum_common_ordered_subpaths(resnet_keys, model_keys) >>> print(ub.repr2(ub.dzip(subpaths1, subpaths2))) { 'layer1.0.conv2.weight': 'backbone.layer1.0.conv2.weight', 'layer1.0.conv3.weight': 'backbone.layer1.0.conv3.weight', 'layer1.0.downsample.0.weight': 'backbone.layer1.0.downsample.0.weight', 'layer2.0.conv1.weight': 'backbone.layer2.0.conv1.weight', 'layer2.0.conv2.weight': 'backbone.layer2.0.conv2.weight', 'layer2.0.conv3.weight': 'backbone.layer2.0.conv3.weight', 'layer3.0.conv1.weight': 'backbone.layer3.0.conv1.weight', 'layer4.0.conv1.weight': 'backbone.layer4.0.conv1.weight', } Also, if the sizes of the tensor don't quite fit, they will be mangled, i.e. "shoved-in" as best as possible. Example: >>> import netharn as nh >>> # --- >>> model_other = nh.models.ToyNet2d(input_channels=1, num_classes=10) >>> model_other.hack_param1 = torch.nn.Parameter(torch.rand(1)) >>> model_other.hack_param3 = torch.nn.Parameter(torch.rand(3)) >>> model_other.hack_param5 = torch.nn.Parameter(torch.rand(3)) >>> # --- >>> model_self = nh.models.ToyNet2d(input_channels=3, num_classes=2) >>> model_self.hack_param1 = torch.nn.Parameter(torch.rand(3)) >>> model_self.hack_param2 = torch.nn.Parameter(torch.rand(3)) >>> model_self.hack_param4 = torch.nn.Parameter(torch.rand(3)) >>> # --- >>> model_state_dict = model_other.state_dict() >>> load_partial_state(model_self, model_state_dict) >>> load_partial_state(model_self, model_state_dict, leftover=torch.nn.init.kaiming_normal_) >>> _ = load_partial_state(model_self, model_state_dict, leftover=torch.nn.init.kaiming_normal_, association='embedding') Example: >>> from netharn.initializers.functional import * # NOQA >>> import netharn as nh >>> xpu = nh.XPU(None) >>> self1 = nh.models.ToyNet2d() >>> self2 = xpu.mount(self1) >>> load_partial_state(self2, self1.state_dict()) >>> load_partial_state(self1, self2.state_dict()) >>> # Add extra nonsense to state-dict >>> extra_state_dict = {'extra.' + k: v for k, v in self1.state_dict().items()} >>> extra_state_dict['stats'] = ub.peek(extra_state_dict.values()).clone() >>> model = self2 >>> model_state_dict = extra_state_dict >>> load_partial_state(self2, extra_state_dict, association='embedding') Example: >>> # xdoctest: +REQUIRES(--slow) >>> from netharn.initializers.functional import * # NOQA >>> import torchvision >>> import torch >>> resnet50 = torchvision.models.resnet50() >>> class CustomModel(torch.nn.Module): >>> def __init__(self): >>> super().__init__() >>> self.module = resnet50 >>> self.extra = torch.nn.Linear(1, 1) >>> model = CustomModel() >>> model_state_dict = resnet50.state_dict() >>> model_state_dict2 = {'prefix.' + k: v for k, v in model_state_dict.items()} >>> import ubelt as ub >>> with ub.Timer(verbose=2, label='strict'): >>> load_partial_state(model, model_state_dict, association='strict', verbose=0) >>> with ub.Timer(verbose=2, label='prefix-hack'): >>> load_partial_state(model, model_state_dict, association='prefix-hack', verbose=0) >>> with ub.Timer(verbose=2, label='module-hack'): >>> load_partial_state(model, model_state_dict, association='module-hack', verbose=0) >>> with ub.Timer(verbose=2, label='embedding'): >>> load_partial_state(model, model_state_dict, association='embedding', verbose=0) >>> load_partial_state(model, model_state_dict, association='prefix-hack', verbose=1) >>> load_partial_state(model, model_state_dict, association='module-hack', verbose=1) Ignore: >>> from bioharn.models.new_models_v1 import * # NOQA >>> channels = ChannelSpec.coerce('rgb') >>> input_stats = None >>> self = MM_HRNetV2_w18_MaskRCNN(classes=3, channels=channels) >>> filename = self.pretrained_url >>> self._init_backbone_from_pretrained(self.pretrained_url) >>> from bioharn.models.mm_models import _load_mmcv_weights >>> model_state = _load_mmcv_weights(filename) >>> self.detector.backbone.chan_backbones.rgb >>> model = self >>> model_state_dict = model_state from netharn.initializers.functional import * # NOQA import xdev globals().update(**xdev.get_func_kwargs(load_partial_state)) CommandLine: xdoctest -m /home/joncrall/code/netharn/netharn/initializers/functional.py load_partial_state:2 --slow """ if association is None: association = 'module-hack' # old default # association = 'prefix-hack' # new default if initializer is not None: warnings.warn('initializer is deprecated use leftover') leftover = initializer self_state = model.state_dict() def _fix_keys(model_state_dict): """ Hack around DataParallel wrapper. If there is nothing in common between the two models check to see if prepending 'module.' to other keys fixes it. """ other_keys = set(model_state_dict) self_keys = set(self_state) if 0: # Automatic way to reduce nodes in the trees? # If node b always follows node a, can we contract it? nodes1 = [n for p in other_keys for n in p.split('.')] nodes2 = [n for p in self_keys for n in p.split('.')] tups1 = list(tup for key in other_keys for tup in ub.iter_window(key.split('.'), 2)) tups2 = list(tup for key in self_keys for tup in ub.iter_window(key.split('.'), 2)) x = ub.ddict(list) for a, b in tups1: x[a].append(b) for a, b in tups2: x[a].append(b) nodehist = ub.dict_hist(nodes1 + nodes2) for k, v in x.items(): print('----') print(k) print(nodehist[k]) follow_hist = ub.dict_hist(v) print(follow_hist) total = sum(follow_hist.values()) if ub.allsame(follow_hist.values()) and total == nodehist[k]: print('CONTRACT') # pair_freq = ub.dict_hist(ub.flatten([tups1, tups2])) # print(forest_str(paths_to_otree(other_keys, '.'))) # common_keys = other_keys.intersection(self_keys) # if not common_keys: if not other_keys.issubset(self_keys): if association == 'strict': pass elif association == 'module-hack': # If there are no common keys try a hack prefix = 'module.' def smap(f, ss): return set(map(f, ss)) def fix1(k): return prefix + k def fix2(k): if k.startswith(prefix): return k[len(prefix):] if smap(fix1, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix1, model_state_dict) elif smap(fix2, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix2, model_state_dict) elif association == 'prefix-hack': import functools def add_prefix(k, prefix): return prefix + k def remove_prefix(k, prefix): if k.startswith(prefix): return k[len(prefix):] # set1 = other_keys # target_set2 = self_keys found = _best_prefix_transform(other_keys, self_keys) if found is not None: for action, prefix in found['transform']: if action == 'add': func = functools.partial(add_prefix, prefix=prefix) elif action == 'remove': func = functools.partial(remove_prefix, prefix=prefix) else: raise AssertionError model_state_dict = ub.map_keys(func, model_state_dict) elif association in {'embedding', 'isomorphism'}: if verbose > 1: print('Using subpath {} association, may take some time'. format(association)) # I believe this is the correct way to solve the problem paths1 = sorted(other_keys) paths2 = sorted(self_state) if 1: # hack to filter to reduce tree size in embedding problem def shrink_paths(paths): new_paths = [] for p in paths: p = p.replace('.0', ':0') p = p.replace('.1', ':1') p = p.replace('.2', ':2') p = p.replace('.3', ':3') p = p.replace('.4', ':4') p = p.replace('.5', ':5') p = p.replace('.6', ':6') p = p.replace('.7', ':7') p = p.replace('.8', ':8') p = p.replace('.9', ':9') p = p.replace('.weight', ':weight') p = p.replace('.bias', ':bias') p = p.replace('.num_batches_tracked', ':num_batches_tracked') p = p.replace('.running_mean', ':running_mean') p = p.replace('.running_var', ':running_var') # p = p.replace('.conv1', ':conv1') # p = p.replace('.conv2', ':conv2') # p = p.replace('.conv3', ':conv3') # p = p.replace('.bn1', ':bn1') # p = p.replace('.bn2', ':bn2') # p = p.replace('.bn3', ':bn3') new_paths.append(p) return new_paths # Reducing the depth saves a lot of time paths1_ = shrink_paths(paths1) paths2_ = shrink_paths(paths2) subpaths1, subpaths2 = maximum_common_ordered_subpaths( paths1_, paths2_, sep='.', mode=association) subpaths1 = [p.replace(':', '.') for p in subpaths1] subpaths2 = [p.replace(':', '.') for p in subpaths2] mapping = ub.dzip(subpaths1, subpaths2) if verbose > 1: other_unmapped = sorted(other_keys - set(mapping.keys())) self_unmapped = sorted(self_keys - set(mapping.values())) print('-- embed association (other -> self) --') print('mapping = {}'.format(ub.repr2(mapping, nl=1))) print('self_unmapped = {}'.format( ub.repr2(self_unmapped, nl=1))) print('other_unmapped = {}'.format( ub.repr2(other_unmapped, nl=1))) print('len(mapping) = {}'.format( ub.repr2(len(mapping), nl=1))) print('len(self_unmapped) = {}'.format( ub.repr2(len(self_unmapped), nl=1))) print('len(other_unmapped) = {}'.format( ub.repr2(len(other_unmapped), nl=1))) print('-- end embed association --') # HACK: something might be wrong, there was an instance with # HRNet_w32 where multiple keys mapped to the same key # bad keys were incre_modules.3.0.conv1.weight and conv1.weight # # This will not error, but may produce bad output try: model_state_dict = ub.map_keys(lambda k: mapping.get(k, k), model_state_dict) except Exception as ex: HACK = 1 if HACK: new_state_dict_ = {} for k, v in model_state_dict.items(): new_state_dict_[mapping.get(k, k)] = v model_state_dict = new_state_dict_ warnings.warn('ex = {!r}'.format(ex)) else: raise else: raise KeyError(association) return model_state_dict other_state = _fix_keys(model_state_dict) self_unset_keys = set( self_state.keys()) # will end up as keys in our that were not set other_unused_keys = set(other_state.keys( )) # will end up as keys in the other model that were not used seen_keys = ub.ddict(set) for key, other_value in other_state.items(): if key not in self_state: if verbose > 0: print('Skipping {} because it does not exist'.format(key)) seen_keys['skipped'].add(key) else: self_value = self_state[key] if other_value.size() == self_value.size(): self_state[key] = other_value self_unset_keys.remove(key) other_unused_keys.remove(key) seen_keys['full_add'].add(key) elif len(other_value.size()) == len(self_value.size()): if not mangle: if verbose > 0: print( 'Skipping {} due to incompatable size and mangle=False' .format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) elif key.endswith('bias'): if verbose > 0: print( 'Skipping {} due to incompatable size'.format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) else: if leftover is None: if verbose > 0: print( 'Skipping {} due to incompatable size and no default initializer' .format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) else: if verbose > 0: print('Partially add {} with incompatable size'. format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) # Initialize all weights in case any are unspecified if leftover is None: try: leftover(self_state[key]) except Exception: if verbose > 0: print('Unable to init {} with {}'.format( key, leftover)) # Transfer as much as possible min_size = np.minimum(self_state[key].shape, other_value.shape) sl = tuple([slice(0, s) for s in min_size]) self_state[key][sl] = other_value[sl] # if shock_partial: # # Shock weights because we are doing something weird # # might help the network recover in case this is # # not a good idea # shock(self_state[key], func=leftover) self_unset_keys.remove(key) other_unused_keys.remove(key) if self_state[key].numel() < other_value.numel(): seen_keys['partial_add_some'].add(key) else: seen_keys['partial_add_all'].add(key) else: if verbose > 0: print('Skipping {} due to incompatable size'.format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) if ignore_unset is True: self_unset_keys = [] elif ignore_unset: self_unset_keys = list(ub.oset(self_unset_keys) - set(ignore_unset)) if (self_unset_keys or other_unused_keys or seen_keys['partial_add_some'] or seen_keys['partial_add_all']): if verbose > 0: if seen_keys: print('Pretrained weights are a partial fit') else: print('Pretrained weights do not fit!') if verbose > 1: print('Seen Keys: {}'.format(ub.repr2(seen_keys, nl=2))) print('Self Unset Keys: {}'.format(ub.repr2(self_unset_keys, nl=1))) print('Other Unused keys: {}'.format( ub.repr2(other_unused_keys, nl=1))) print('summary:') seen_sum = ub.map_vals(len, seen_keys) print('Seen Num: {}'.format(ub.repr2(seen_sum, nl=2))) print('Self Unset Num: {}'.format( ub.repr2(len(self_unset_keys), nl=1))) print('Other Unused Num: {}'.format( ub.repr2(len(other_unused_keys), nl=1))) if leftover: if verbose > 0: print('Initializing unused keys using {}'.format(leftover)) for key in self_unset_keys: if key.endswith('.num_batches_tracked'): pass # ignore num_batches_tracked elif key.endswith('.bias'): self_state[key].fill_(0) else: try: leftover(self_state[key]) except Exception: if verbose > 0: print('Unable to init {} with {}'.format( key, leftover)) else: if verbose > 0: print('Pretrained weights are a perfect fit') model.load_state_dict(self_state) info = { 'seen': seen_keys, 'self_unset': self_unset_keys, 'other_unused': other_unused_keys } return info
def method_diffkeys_oset(*args): first_dict = args[0] keys = ub.oset(first_dict) keys.difference_update(*map(set, args[1:])) new0 = dict((k, first_dict[k]) for k in keys) return new0
np.logspace(0, np.log(10000) / np.log(base), num=10, base=base).round().astype(int), 'subsize': [2, 8, 32, 64], } data_kwkeys = ub.compatible(basis, generate_data) func_kwkeys = ub.compatible(basis, method_lut[basis['method'][0]]) # These variable influence what is plotted on the x-asis y-axis and # with different line types xlabel = 'size' ylabel = 'time' group_labels = { 'size': ['subsize'], 'style': ['subsize'], } hue_labels = ub.oset(basis) - {xlabel} if group_labels: hue_labels = hue_labels - set.union(*map(set, group_labels.values())) group_labels['hue'] = list(hue_labels) grid_iter = list(ub.named_product(basis)) # For each variation of your experiment, create a row. rows = [] for params in grid_iter: group_keys = {} for gname, labels in group_labels.items(): group_keys[gname + '_key'] = ub.repr2(ub.dict_isect( params, labels), compact=1, si=1) key = ub.repr2(params, compact=1, si=1)
def load_partial_state(model, model_state_dict, initializer=None, ignore_unset=False, verbose=2): """ CommandLine: python -m netharn.initializers.nninit_base load_partial_state Example: >>> import netharn as nh >>> self1 = nh.models.ToyNet2d(input_channels=1, num_classes=10) >>> self2 = nh.models.ToyNet2d(input_channels=3, num_classes=2) >>> model_state_dict = self1.state_dict() >>> load_partial_state(self2, model_state_dict) Example: >>> import netharn as nh >>> xpu = nh.XPU(None) >>> self1 = nh.models.ToyNet2d() >>> self2 = xpu.mount(self1) >>> load_partial_state(self2, self1.state_dict()) >>> load_partial_state(self1, self2.state_dict()) """ self_state = model.state_dict() def _fix_keys(model_state_dict): """ Hack around DataParallel wrapper. If there is nothing in common between the two models check to see if prepending 'module.' to other keys fixes it. """ other_keys = set(model_state_dict) self_keys = set(self_state) if not other_keys.intersection(self_keys): prefix = 'module.' def smap(f, ss): return set(map(f, ss)) def fix1(k): return prefix + k def fix2(k): if k.startswith(prefix): return k[len(prefix):] if smap(fix1, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix1, model_state_dict) elif smap(fix2, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix2, model_state_dict) return model_state_dict other_state = _fix_keys(model_state_dict) self_unset_keys = set( self_state.keys()) # will end up as keys in our that were not set other_unused_keys = set(other_state.keys( )) # will end up as keys in the other model that were not used seen_keys = ub.ddict(set) for key, other_value in other_state.items(): if key not in self_state: if verbose > 0: print('Skipping {} because it does not exist'.format(key)) seen_keys['skipped'].add(key) else: self_value = self_state[key] if other_value.size() == self_value.size(): self_state[key] = other_value self_unset_keys.remove(key) other_unused_keys.remove(key) seen_keys['full_add'].add(key) elif len(other_value.size()) == len(self_value.size()): if key.endswith('bias'): if verbose > 0: print( 'Skipping {} due to incompatable size'.format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) else: if initializer is None: if verbose > 0: print( 'Skipping {} due to incompatable size and no default initializer' .format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) else: if verbose > 0: print('Partially add {} with incompatable size'. format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) # Initialize all weights in case any are unspecified if initializer is not None: initializer(self_state[key]) # Transfer as much as possible min_size = np.minimum(self_state[key].shape, other_value.shape) sl = tuple([slice(0, s) for s in min_size]) self_state[key][sl] = other_value[sl] # if shock_partial: # # Shock weights because we are doing something weird # # might help the network recover in case this is # # not a good idea # shock(self_state[key], func=initializer) self_unset_keys.remove(key) other_unused_keys.remove(key) seen_keys['partial_add'].add(key) else: if verbose > 0: print('Skipping {} due to incompatable size'.format(key)) print(' * self = {!r}'.format(self_value.size())) print(' * other = {!r}'.format(other_value.size())) seen_keys['skipped'].add(key) if ignore_unset is True: self_unset_keys = [] elif ignore_unset: self_unset_keys = list(ub.oset(self_unset_keys) - set(ignore_unset)) if self_unset_keys or other_unused_keys: if verbose > 0: if seen_keys: print('Pretrained weights are a partial fit') else: print('Pretrained weights do not fit!') if verbose > 1: print('Seen Keys: {}'.format(ub.repr2(seen_keys, nl=2))) print('Self Unset Keys: {}'.format(ub.repr2(self_unset_keys, nl=1))) print('Other Unused keys: {}'.format( ub.repr2(other_unused_keys, nl=1))) if initializer: if verbose > 0: print('Initializing unused keys using {}'.format(initializer)) for key in self_unset_keys: if key.endswith('.bias'): self_state[key].fill_(0) else: initializer(self_state[key]) else: if verbose > 0: print('Pretrained weights are a perfect fit') model.load_state_dict(self_state)