def make_pipeline_text(self): blocks = [] blocks.append(ub.codeblock( ''' # ---------------------- # nodes # ''')) for proc in self.procs.values(): node_text = proc.make_node_text() if node_text: blocks.append(node_text) blocks.append(ub.codeblock( ''' # ---------------------- # connections # ''')) for proc in self.procs.values(): edge_text = proc.make_edge_text() if edge_text: blocks.append(edge_text) blocks.append(ub.codeblock( ''' # ---------------------- # global pipeline config # ''')) blocks.append(self.make_global_text()) text = '\n\n'.join(blocks) return text
def make_initd(fpath): name = splitext(basename(fpath))[0] text = INITD_TEMPLATE.format(name=name, fpath=fpath) startup_hook = expanduser(join('~/local/ubuntu/startup/init.d', name)) ub.writeto(startup_hook, text) print(ub.codeblock( ''' RUN: sudo cp {startup_hook} /etc/init.d/{name} sudo chmod +x /etc/init.d/{name} # sudo update-rc.d /etc/init.d/{name} defaults sudo update-rc.d {name} defaults service {name} start ''').format(**locals()))
def make_node_text(self): """ Creates a text based definition of this node for a .pipe file """ fmtstr = ub.codeblock( ''' process {name} :: {type} ''') parts = [fmtstr.format(name=self.name, type=self.type)] if self.config: if isinstance(self.config, six.string_types): parts.extend(self.config.splitlines()) else: for key, val in self.config.items(): parts.append(' :{key} {val}'.format(key=key, val=val)) text = '\n'.join(parts) return text
def make_edge_text(self): """ Creates a text based definition of all incoming conections to this node for a .pipe file """ fmtstr = ub.codeblock( ''' connect from {oport_abs_name} to {iport_abs_name} ''') parts = [] for iport in self.iports.values(): for oport in iport.connections: if oport is not None: part = fmtstr.format( oport_abs_name=oport.absname(), iport_abs_name=iport.absname(), ) parts.append(part) text = '\n'.join(parts) return text
def argparse(self, parser=None, special_options=False): """ construct or update an argparse.ArgumentParser CLI parser Args: parser (None | argparse.ArgumentParser): if specified this parser is updated with options from this config. special_options (bool, default=False): adds special scriptconfig options, namely: --config, --dumps, and --dump. Returns: argparse.ArgumentParser : a new or updated argument parser CommandLine: xdoctest -m scriptconfig.config Config.argparse:0 xdoctest -m scriptconfig.config Config.argparse:1 TODO: A good CLI spec for lists might be # In the case where ``key`` ends with and ``=``, assume the list is # given as a comma separated string with optional square brakets at # each end. --key=[f] # In the case where ``key`` does not end with equals and we know # the value is supposd to be a list, then we consume arguments # until we hit the next one that starts with '--' (which means # that list items cannot start with -- but they can contains # commas) FIXME: * In the case where we have an nargs='+' action, and we specify the option with an `=`, and then we give position args after it there is no way to modify behavior of the action to just look at the data in the string without modifying the ArgumentParser itself. The action object has no control over it. For example `--foo=bar baz biz` will parse as `[baz, biz]` which is really not what we want. We may be able to overload ArgumentParser to fix this. Example: >>> # You can now make instances of this class >>> import scriptconfig >>> self = scriptconfig.Config.demo() >>> parser = self.argparse() >>> parser.print_help() >>> # xdoctest: +REQUIRES(PY3) >>> # Python2 argparse does a hard sys.exit instead of raise >>> ns, extra = parser.parse_known_args() Example: >>> # You can now make instances of this class >>> import scriptconfig as scfg >>> class MyConfig(scfg.Config): >>> description = 'my CLI description' >>> default = { >>> 'path1': scfg.Value(None, position=1, alias='src'), >>> 'path2': scfg.Value(None, position=2, alias='dst'), >>> 'dry': scfg.Value(False, isflag=True), >>> 'approx': scfg.Value(False, isflag=False, alias=['a1', 'a2']), >>> } >>> self = MyConfig() >>> special_options = True >>> parser = None >>> parser = self.argparse(special_options=special_options) >>> parser.print_help() >>> self._read_argv(argv=['objection', '42', '--path1=overruled!']) >>> print('self = {!r}'.format(self)) Ignore: >>> self._read_argv(argv=['hi','--path1=foobar']) >>> self._read_argv(argv=['hi', 'hello', '--path1=foobar']) >>> self._read_argv(argv=['hi', 'hello', '--path1=foobar', '--help']) >>> self._read_argv(argv=['--path1=foobar', '--path1=baz']) >>> print('self = {!r}'.format(self)) """ import argparse if parser is None: parserkw = self._parserkw() parser = argparse.ArgumentParser(**parserkw) # Use custom action used to mark which values were explicitly set on # the commandline parser._explicitly_given = set() parent = self class ParseAction(argparse.Action): def __init__(self, *args, **kwargs): super(ParseAction, self).__init__(*args, **kwargs) # with script config nothing should be required by default all # positional arguments should have keyword arg variants Setting # required=False here will prevent positional args from # erroring if they are not specified. I dont think there are # other side effects, but we should make sure that is actually # the case. self.required = False if self.type is None: # Is this the right place to put this? def _mytype(value): key = self.dest template = parent.default[key] if not isinstance(template, Value): # smartcast non-valued params from commandline value = smartcast.smartcast(value) else: value = template.cast(value) return value self.type = _mytype # print('self.type = {!r}'.format(self.type)) def __call__(action, parser, namespace, values, option_string=None): # print('CALL action = {!r}'.format(action)) # print('option_string = {!r}'.format(option_string)) # print('values = {!r}'.format(values)) if isinstance(values, list) and len(values): # We got a list of lists, which we hack into a flat list if isinstance(values[0], list): import itertools as it values = list(it.chain(*values)) setattr(namespace, action.dest, values) parser._explicitly_given.add(action.dest) # IRC: this ensures each key has a real Value class _metadata = { key: self._data[key] for key, value in self._default.items() if isinstance(self._data[key], Value) } # :type: Dict[str, Value] _positions = {k: v.position for k, v in _metadata.items() if v.position is not None} if _positions: if ub.find_duplicates(_positions.values()): raise Exception('two values have the same position') _keyorder = ub.oset(ub.argsort(_positions)) _keyorder |= (ub.oset(self._default) - _keyorder) else: _keyorder = list(self._default.keys()) def _add_arg(parser, name, key, argkw, positional, isflag, isalias): _argkw = argkw.copy() if isalias: _argkw['help'] = 'alias of {}'.format(key) _argkw.pop('default', None) # flags cannot have flag aliases isflag = False elif positional: parser.add_argument(name, **_argkw) if isflag: # Can we support both flag and setitem methods of cli # parsing? if not isinstance(_argkw.get('default', None), bool): raise ValueError('can only use isflag with bools') _argkw.pop('type', None) _argkw.pop('choices', None) _argkw.pop('action', None) _argkw.pop('nargs', None) _argkw['dest'] = key _argkw_true = _argkw.copy() _argkw_true['action'] = 'store_true' _argkw_false = _argkw.copy() _argkw_false['action'] = 'store_false' _argkw_false.pop('help', None) parser.add_argument('--' + name, **_argkw_true) parser.add_argument('--no-' + name, **_argkw_false) else: parser.add_argument('--' + name, **_argkw) mode = 1 alias_registry = [] for key, value in self._data.items(): # key: str # value: Any | Value argkw = {} argkw['help'] = '' positional = None isflag = False if key in _metadata: # Use the metadata in the Value class to enhance argparse _value = _metadata[key] argkw.update(_value.parsekw) value = _value.value isflag = _value.isflag positional = _value.position else: _value = value if isinstance(value, Value) else None if not argkw['help']: argkw['help'] = '<undocumented>' argkw['default'] = value argkw['action'] = ParseAction name = key _add_arg(parser, name, key, argkw, positional, isflag, isalias=False) if _value is not None: if _value.alias: alts = _value.alias alts = alts if ub.iterable(alts) else [alts] for alias in alts: tup = (alias, key, argkw) alias_registry.append(tup) if mode == 0: name = alias _add_arg(parser, name, key, argkw, positional, isflag, isalias=True) if mode == 1: for tup in alias_registry: (alias, key, argkw) = tup name = alias dest = key _add_arg(parser, name, dest, argkw, positional, isflag, isalias=True) if special_options: parser.add_argument('--config', default=None, help=ub.codeblock( ''' special scriptconfig option that accepts the path to a on-disk configuration file, and loads that into this {!r} object. ''').format(self.__class__.__name__)) parser.add_argument('--dump', default=None, help=ub.codeblock( ''' If specified, dump this config to disk. ''').format(self.__class__.__name__)) parser.add_argument('--dumps', action='store_true', help=ub.codeblock( ''' If specified, dump this config stdout ''').format(self.__class__.__name__)) return parser
def main(): candidates = None mode = 'ultra' if mode == 'great': candidate_csv_text = ub.codeblock( ''' registeel,LOCK_ON,FLASH_CANNON,FOCUS_BLAST,22,10,14,15 stunfisk_galarian,MUD_SHOT,ROCK_SLIDE,EARTHQUAKE,25,11,14,14 # altaria,DRAGON_BREATH,SKY_ATTACK,DRAGON_PULSE,26.5,14,12,13 skarmory,AIR_SLASH,SKY_ATTACK,FLASH_CANNON,26,11,13,10 azumarill,BUBBLE,ICE_BEAM,HYDRO_PUMP,38,12,15,13 dewgong,ICE_SHARD,ICY_WIND,WATER_PULSE,26.5,15,08,15 # umbreon,SNARL,FOUL_PLAY,LAST_RESORT,24.5,15,10,15 # farfetchd_galarian,FURY_CUTTER,LEAF_BLADE,BRAVE_BIRD,33.5,12,15,15 hypno,CONFUSION,SHADOW_BALL,THUNDER_PUNCH,25.5,13,15,14 # hypno,CONFUSION,SHADOW_BALL,FOCUS_BLAST,25.5,13,15,14 # machamp-shadow,COUNTER,ROCK_SLIDE,CROSS_CHOP,18,5,11,10 victreebel_shadow-shadow,RAZOR_LEAF,LEAF_BLADE,FRUSTRATION,22.5,4,14,14 ''') candidate_explicit = [ Pokemon('medicham', ivs=[7, 15, 14], level=41.5), Pokemon('medicham', ivs=[7, 15, 14], level=43.0), Pokemon('medicham', ivs=[7, 15, 14]).maximize(1500), Pokemon('machamp', [1, 15, 6], cp=1493), Pokemon('altaria', [1, 11, 8], cp=1496), Pokemon('skarmory', [0, 15, 13], cp=1495), Pokemon('umbreon', [1, 8, 8], cp=1495), Pokemon('registeel', [10, 14, 15], cp=1487), Pokemon('stunfisk', [11, 14, 14], form='Galarian', cp=1498), Pokemon('cresselia', [7, 14, 8], cp=1493), Pokemon('vigoroth', [0, 10, 9], cp=1495), Pokemon('drifblim', [4, 14, 13], cp=1498), Pokemon('haunter', [6, 13, 15], cp=1498), Pokemon('mantine', [6, 13, 14], cp=1497), Pokemon('politoed', [3, 5, 13], cp=1493), Pokemon('charizard', [3, 15, 14], cp=1485), Pokemon('gengar', [5, 11, 14], cp=1483), Pokemon('mew', [15, 12, 11], cp=1470), Pokemon('dewgong', [15, 8, 15]).maximize(1500), Pokemon('azumarill', [12, 15, 13]).maximize(1500), Pokemon('hypno', [13, 15, 14]).maximize(1500), ] for cand in candidate_explicit: cand.populate_cp() stat_products = [cand.stat_product for cand in candidate_explicit] sortx = ub.argsort(stat_products) candidate_explicit = list(ub.take(candidate_explicit, sortx)) stat_products = list(ub.take(stat_products, sortx)) print('stat_products = {}'.format(ub.repr2(stat_products, nl=1))) print('candidate_explicit = {}'.format(ub.repr2(candidate_explicit, nl=1))) for cand in candidate_explicit: print('cand.adjusted = {}, {:.2f}, {}'.format(ub.repr2(cand.adjusted, nl=0, precision=2), cand.stat_product, cand)) if mode == 'ultra': candidate_csv_text = ub.codeblock( ''' cresselia,PSYCHO_CUT,MOONBLAST,FUTURE_SIGHT togekiss,CHARM,FLAMETHROWER,ANCIENT_POWER articuno,ICE_SHARD,ICY_WIND,HURRICANE swampert,MUD_SHOT,MUDDY_WATER,EARTHQUAKE venusaur,VINE_WHIP,FRENZY_PLANT,SLUDGE_BOMB ''') candidates = [ Pokemon('Gengar', (7, 14, 14), cp=2500, moves=['SHADOW_CLAW', 'SHADOW_PUNCH', 'SHADOW_BALL']), Pokemon('Togekiss', (15, 15, 14), cp=2469, moves=['CHARM', 'FLAMETHROWER', 'AERIAL_ACE']), Pokemon('Venusaur', (15, 13, 13), cp=2482, moves=['VINE_WHIP', 'FRENZY_PLANT', 'SLUDGE_BOMB']), Pokemon('Muk', (9, 7, 4), cp=2486, form='Alola', moves=['SNARL', 'DARK_PULSE', 'SLUDGE_WAVE']), Pokemon('Swampert', (0, 2, 14), cp=2500, moves=['WATER_GUN', 'HYDRO_CANNON', 'SLUDGE_WAVE']), Pokemon('Empoleon', (0, 10, 14), cp=2495, moves=['WATERFALL', 'HYDRO_CANNON', 'DRILL_PECK']), Pokemon('sirfetch’d', (4, 11, 12), cp=2485, form='Galarian', moves=['COUNTER', 'CLOSE_COMBAT', 'LEAF_BLADE']), ] # else: # raise KeyError(mode) if candidates is None: candidates = [] for line in candidate_csv_text.split('\n'): line = line.strip() if line.startswith('#'): continue if line: row = line.split(',') cand = Pokemon.from_pvpoke_row(row) candidates.append(cand) print(ub.repr2(api.learnable)) if mode == 'ultra': base = 'https://pvpoke.com/team-builder/all/2500' base = 'https://pvpoke.com/team-builder/premier/2500' elif mode == 'great': base = 'https://pvpoke.com/team-builder/all/1500' sep = '%2C' import itertools as it print('candidates = {!r}'.format(candidates)) for team in it.combinations(candidates, 3): # if not any('registeel' in p.name for p in team): # continue # if not any('victree' in p.name for p in team): # continue # if len(set(p.name for p in team)) != 3: # continue suffix = sep.join([p.to_pvpoke_url() for p in team]) url = base + '/' + suffix print(url)
def main(): mode = 'great' if mode == 'great': candidate_csv_text = ub.codeblock(''' registeel,LOCK_ON,FLASH_CANNON,FOCUS_BLAST,22,10,14,15 stunfisk_galarian,MUD_SHOT,ROCK_SLIDE,EARTHQUAKE,25,11,14,14 # altaria,DRAGON_BREATH,SKY_ATTACK,DRAGON_PULSE,26.5,14,12,13 skarmory,AIR_SLASH,SKY_ATTACK,FLASH_CANNON,26,11,13,10 azumarill,BUBBLE,ICE_BEAM,HYDRO_PUMP,38,12,15,13 dewgong,ICE_SHARD,ICY_WIND,WATER_PULSE,26.5,15,08,15 # umbreon,SNARL,FOUL_PLAY,LAST_RESORT,24.5,15,10,15 # farfetchd_galarian,FURY_CUTTER,LEAF_BLADE,BRAVE_BIRD,33.5,12,15,15 hypno,CONFUSION,SHADOW_BALL,THUNDER_PUNCH,25.5,13,15,14 # hypno,CONFUSION,SHADOW_BALL,FOCUS_BLAST,25.5,13,15,14 # machamp-shadow,COUNTER,ROCK_SLIDE,CROSS_CHOP,18,5,11,10 victreebel_shadow-shadow,RAZOR_LEAF,LEAF_BLADE,FRUSTRATION,22.5,4,14,14 ''') elif mode == 'ultra': candidate_csv_text = ub.codeblock(''' cresselia,PSYCHO_CUT,MOONBLAST,FUTURE_SIGHT togekiss,CHARM,FLAMETHROWER,ANCIENT_POWER articuno,ICE_SHARD,ICY_WIND,HURRICANE swampert,MUD_SHOT,MUDDY_WATER,EARTHQUAKE venusaur,VINE_WHIP,FRENZY_PLANT,SLUDGE_BOMB ''') else: raise KeyError(mode) candidates = [] for line in candidate_csv_text.split('\n'): line = line.strip() if line.startswith('#'): continue if line: row = line.split(',') cand = Pokemon.from_pvpoke_row(row) candidates.append(cand) # for self in candidates: # self.populate_stats() # for self in candidates: # print('self = {!r}'.format(self)) # print(self.calc_cp()) print(ub.repr2(api.learnable)) if mode == 'ultra': base = 'https://pvpoke.com/team-builder/all/2500' elif mode == 'great': base = 'https://pvpoke.com/team-builder/all/1500' sep = '%2C' import itertools as it print('candidates = {!r}'.format(candidates)) for team in it.combinations(candidates, 3): # if not any('registeel' in p.name for p in team): # continue if not any('victree' in p.name for p in team): continue if len(set(p.name for p in team)) != 3: continue suffix = sep.join([p.to_pvpoke_url() for p in team]) url = base + '/' + suffix print(url)
candidates = seagate_msrps for cand in candidates: cand['cost_per_TB'] = cand['cost'] / cand['TB'] import pandas as pd df = pd.DataFrame(candidates) df = df.sort_values('cost_per_TB') print(df) print('candidates = {}'.format(ub.repr2(candidates, nl=1, precision=2, align=':'))) considering = ub.codeblock( ''' https://www.lg.com/us/monitors/lg-27UK650-W-4k-uhd-led-monitor# https://www.lg.com/us/monitors/lg-27GL850-B-gaming-monitor https://www.amazon.com/Dell-U2720QM-UltraSharp-Ultra-Thin-DisplayPort/dp/B08F5J8S6Y?ref_=ast_sto_dp https://www.amazon.com/Dell-U2720QM-UltraSharp-Ultra-Thin-DisplayPort/dp/B08F5J8S6Y?ref_=ast_sto_dp # Has 2560x1440 resolution, which is what I like anyway # https://www.amazon.com/LG-27GL83A-B-Ultragear-Compatible-Monitor/dp/B07YGZL8XF/ref=sr_1_2_sspa?dchild=1&keywords=4k+monitor&qid=1619557709&sr=8-2-spons&psc=1&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUEzNjBXRVNRRkNMWU5QJmVuY3J5cHRlZElkPUEwMDgxNzQ4MUQ3RE9SM0dTTkkyTCZlbmNyeXB0ZWRBZElkPUEwNTM2NTc0MTE5SVJJVzQySjJGQyZ3aWRnZXROYW1lPXNwX2F0ZiZhY3Rpb249Y2xpY2tSZWRpcmVjdCZkb05vdExvZ0NsaWNrPXRydWU= WQHD ''')
def __repr__(self): return ub.codeblock(''' <This repr has newlines, and the first line is long: * line1 * line2> ''')
def main(): import ubelt as ub header = ub.codeblock(''' import ubelt as ub ti = ub.Timerit(100, bestof=10, verbose=2) d = { 'keyboard_debug': False, 'snapshot_after_error': True, # Try to checkpoint before crashing 'show_prog': True, 'use_tqdm': None, 'prog_backend': 'progiter', 'ignore_inf_loss_parts': False, 'use_tensorboard': True, 'export_modules': [], 'large_loss': 1000, 'num_keep': 2, 'keep_freq': 20, } num_inner_loops = 10000 def access_dict_direct(): for i in range(num_inner_loops): if d['ignore_inf_loss_parts']: pass for timer in ti.reset('access_dict_direct'): with timer: access_dict_direct() ''') parts = [header] for n in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100, 1000]: func_text, funcname = _gen_cluttered_func(n=n) time_text = ub.codeblock(''' {func_text} for timer in ti.reset('{funcname}'): with timer: {funcname}() ''').format(func_text=func_text, funcname=funcname) parts.append(time_text) block = '\n'.join(parts) prog_text = ub.codeblock(''' import ubelt as ub def main(): {block} if __name__ == '__main__': main() ''').format(block=ub.indent(block)) # prog_text = 'def main():\n' + ub.indent(block) + 'if __name__ == "__main__":\n main()' fpath = 'bench_local_clutter.py' with open(fpath, 'w') as file: file.write(prog_text) ub.cmd('python ' + fpath, verbose=3)
def make_dummy_package(dpath, pkgname='mkinit_dummy_module'): """ Creates a dummy package structure with or without __init__ files """ root = ub.ensuredir(join(dpath, pkgname)) ub.delete(root) ub.ensuredir(root) paths = { 'root': root, 'submod1': ub.touch(join(root, 'submod1.py')), 'submod2': ub.touch(join(root, 'submod2.py')), 'subdir1': ub.ensuredir(join(root, 'subdir1')), 'subdir2': ub.ensuredir(join(root, 'subdir2')), } paths['subdir1_init'] = ub.touch(join(paths['subdir1'], '__init__.py')) paths['subdir2_init'] = ub.touch(join(paths['subdir2'], '__init__.py')) paths['root_init'] = ub.touch(join(paths['root'], '__init__.py')) ub.writeto(paths['subdir1_init'], ub.codeblock( ''' simple_subattr1 = "hello world" simple_subattr2 = "hello world" _private_attr = "hello world" ''')) ub.writeto(paths['subdir2_init'], ub.codeblock( ''' __all__ = ['public_attr'] public_attr = "hello world" private_attr = "hello world" ''')) ub.writeto(paths['submod1'], ub.codeblock( ''' import six attr1 = True attr2 = six.moves.zip # ------------------------ if True: good_attr_01 = None if False: bad_attr_false1 = None if None: bad_attr_none1 = None # ------------------------ if True: good_attr_02 = None else: bad_attr_true2 = None if False: bad_attr_false2 = None else: good_attr_03 = None if None: bad_attr_none2 = None else: good_attr_04 = None # ------------------------ if True: good_attr_05 = None elif False: bad_attr3 = None else: bad_attr3 = None if False: bad_attr_elif_True3_0 = None elif True: good_attr_06 = None else: bad_attr_elif_True3_1 = None # ------------------------ import sys if sys.version_info.major == 3: good_attr_07 = 'py3' bad_attr_uncommon4_1 = None else: good_attr_07 = 'py2' bad_attr_uncommon4_0 = None # ------------------------ # This is static, so maybe another_val exists as a global if sys.version_info.major == good_attr_07: good_attr_08 = None bad_attr_uncommon5_1 = None bad_attr_uncommon5_0 = None elif sys: good_attr_08 = None bad_attr_uncommon5_1 = None else: good_attr_08 = None bad_attr_uncommon5_0 = None # ------------------------ flag1 = sys.version_info.major < 10 flag2 = sys.version_info.major > 10 flag3 = sys.version_info.major > 10 if flag1: bad_attr_num6 = 1 elif flag2: bad_attr_num6 = 1 elif flag3: bad_attr_num6 = 1 if flag1: bad_attr_num6_0 = 1 elif 0: bad_attr_num0 = 1 elif 1: bad_attr_09 = 1 else: bad_attr13 = 1 if flag1: good_attr_09 = 1 elif 1: good_attr_09 = 1 bad_attr_09_1 = 1 elif 2 == 3: pass # ------------------------ if 'foobar': good_attr_10 = 1 if False: bad_attr_str7 = 1 elif (1, 2): good_attr_11 = 1 elif True: bad_attr_true8 = 1 # ------------------------ if flag1 != flag2: good_attr_12 = None else: bad_attr_12 = None raise Exception # ------------------------ try: good_attr_13 = None bad_attr_13 = None except Exception: good_attr_13 = None # ------------------------ try: good_attr_14 = None except Exception: bad_attr_14 = None raise # ------------------------ def func1(): pass class class1(): pass if __name__ == '__main__': bad_attr_main = None if __name__ == 'something_else': bad_something_else = None ''')) return paths
def main(): # TODO: find a better place for root ROOT = join(os.getcwd()) # ROOT = '.' os.chdir(ROOT) NAME = 'pyhesaff' VERSION = '0.1.2' DOCKER_TAG = '{}-{}'.format(NAME, VERSION) QUAY_REPO = 'quay.io/erotemic/manylinux-for' DOCKER_URI = '{QUAY_REPO}:{DOCKER_TAG}'.format(**locals()) dockerfile_fpath = join(ROOT, 'Dockerfile') # This docker code is very specific for building linux binaries. # We will need to do a bit of refactoring to handle OSX and windows. # But the goal is to get at least one OS working end-to-end. """ Notes: docker run --rm -it quay.io/pypa/manylinux2010_x86_64 /bin/bash --- ls /opt/python """ BASE_IMAGE = 'quay.io/pypa/manylinux2010_x86_64' docker_code = ub.codeblock(f''' FROM {BASE_IMAGE} RUN yum install lz4-devel -y RUN MB_PYTHON_TAG=cp27-cp27m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp27-cp27mu && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp35-cp35m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp36-cp36m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp37-cp37m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp38-cp38 && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja ''') docker_code2 = '\n\n'.join( [ub.paragraph(p) for p in docker_code.split('\n\n')]) try: print(ub.color_text('\n--- DOCKER CODE ---', 'white')) print(ub.highlight_code(docker_code2, 'docker')) print(ub.color_text('--- END DOCKER CODE ---\n', 'white')) except Exception: pass with open(dockerfile_fpath, 'w') as file: file.write(docker_code2) docker_build_cli = ' '.join([ 'docker', 'build', '--tag {}'.format(DOCKER_TAG), '-f {}'.format(dockerfile_fpath), '.', ]) print('docker_build_cli = {!r}'.format(docker_build_cli)) if ub.argflag('--dry'): print('DRY RUN') print('WOULD RUN') print(docker_build_cli) else: info = ub.cmd(docker_build_cli, verbose=3, shell=True) if info['ret'] != 0: print(ub.color_text('\n--- FAILURE ---', 'red')) print('Failed command:') print(info['command']) print(info['err']) print('NOTE: sometimes reruning the command manually works') raise Exception('Building docker failed with exit code {}'.format( info['ret'])) else: print(ub.color_text('\n--- SUCCESS ---', 'green')) print( ub.highlight_code( ub.codeblock(r''' # Finished creating the docker image. # To test / export / publish you can do something like this: # Test that we can get a bash terminal docker run -it {DOCKER_TAG} /bin/bash # Create a tag for the docker image docker tag {DOCKER_TAG} {DOCKER_URI} # Export your docker image to a file docker save -o ${ROOT}/{DOCKER_TAG}.docker.tar {DOCKER_TAG} # Login to a docker registry (we are using quay) # In some cases this works, docker login # But you may need to specify secret credentials load_secrets echo "QUAY_USERNAME = $QUAY_USERNAME" docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io unload_secrets # Upload the docker image to quay.io docker push {DOCKER_URI} ''').format( NAME=NAME, ROOT=ROOT, DOCKER_TAG=DOCKER_TAG, DOCKER_URI=DOCKER_URI, ), 'bash', )) PUBLISH = 0 if PUBLISH: cmd1 = 'docker tag {DOCKER_TAG} {DOCKER_URI}'.format(**locals()) cmd2 = 'docker push {DOCKER_URI}'.format(**locals()) print('-- <push cmds> ---') print(cmd1) print(cmd2) print('-- </push cmds> ---')
""" Looks at the `git_*.{sh,ph}` scripts and makes corresponding `git-*` scripts """ import glob from os.path import dirname, join, basename, splitext import ubelt as ub SCRIPT_HEADER = ub.codeblock( r''' #!/bin/bash # References: # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" ''') SCRIPT_FOOTER_FMT = '$DIR/../{fname} "$@"' def setup_git_scripts(): dpath = dirname(__file__) git_sh_scripts = list(glob.glob(join(dpath, 'git_*.sh')))
class MathSymbolsUnicode: """ https://en.wikipedia.org/wiki/Mathematical_operators_and_symbols_in_Unicode https://www.quora.com/What-do-mathbb-C-mathbb-F-mathbb-H-mathbb-N-mathbb-Q-mathbb-R-mathbb-S-and-mathbb-Z-mean https://mathworld.wolfram.com/Doublestruck.html https://peterjamesthomas.com/maths-science/a-brief-taxonomy-of-numbers/ http://xahlee.info/comp/unicode_math_operators.html # Example: # >>> """ sym_elementof = 'ϵ' sym_finitefield = '𝔽' sym_natural = 'ℕ' sym_integral = 'ℤ' sym_rational = 'ℚ' sym_complex = 'ℂ' sym_quaternions = 'ℍ' sym_octernion = '𝕆' sym_irrational = 'ℙ' sym_real = 'ℝ' sym_floating = '𝕃' sym_list = '[]' _greek_alphabet = ub.codeblock( """ Α α Alpha a Β β Beta b Γ γ Gamma g Δ δ Delta d Ε ε Epsilon e Ζ ζ Zeta z Η η Eta h Θ θ Theta th Ι ι Iota i Κ κ Kappa k Λ λ Lambda l Μ μ Mu m Ν ν Nu n Ξ ξ Xi x Ο ο Omicron o Π π Pi p Ρ ρ Rho r Σ σ,ς * Sigma s Τ τ Tau t Υ υ Upsilon u Φ φ Phi ph Χ χ Chi ch Ψ ψ Psi ps Ω ω Omega o Superscripts: ⁰ ¹ ² ³ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁺ ⁻ ⁼ ⁽ ⁾ ⁿ ⁱ Subscripts: ₀ ₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₊ ₋ ₌ ₍ ₎ ₐ ₑ ₕ ᵢ ⱼ ₖ ₗ ₘ ₙ ₒ ₚ ᵣ ₛ ₜ ᵤ ᵥ ₓ """) _notes = ub.codeblock( """ # Variables alpha = α beta = β delta = δ epsilon = ε theta = θ lambda = λ mu = μ phi = φ psi = ψ omega = Ω alpha = α beta = β gamma = γ delta = δ epsilon = ε zeta = ζ eta = η theta = θ iota = ι kappa = κ lambda = λ nu = ν mu = μ xi = ξ omicron = ο pi = π rho = ρ sigma = σ tau = τ upsilon = υ phi = φ chi = χ psi = ψ omega = ω # Special meta-numeric symbols pi = π # 3.14... ratio of circle circumference to the diameter tau = 𝜏 # 6.28... ratio of circle circumference to its radius... it makes a tad more sense infinity = ∞ # Existential quantifiers forall = ∀ exists = ∃ forall = ∀ not_exists = ∄ delta_upper = ∆ # Calculus partial = ∂ integral = ∫ # Relational eq = = ne = ≠ le = ≦ ge = ≥ lt = < gt = > approx_eq = ≈ approx_ne = ≇ strict_approx_ne = ≆ propor = ∝ equiv = ≡ not_equiv = ≢ # n-ary operations product = ∏ sumation = ∑ big_isect = ⋂ big_union = ⋃ union = ∪ # Root operations square_root = √ cube_root = ∛ quad_root = ∜ # Logical and = ∧ or = ∨ not = ¬ # Set operations emptyset = ∅ isect = ∩ # Membership elementof = ϵ not_elementof = ∉ subset_lt = ⊂ # strict subset of subset_gt = ⊃ # strict superset of subset_le = ⊆ # subset subset_ge = ⊇ # superset subset_not_ge = ⊉ subset_not_gt = ⊅ subset_not_lt = ⊄ subset_not_le = ⊈ """)
CORE_LAYERS = ub.codeblock( ''' layer {{ bottom: "data" top: "conv1_1" name: "conv1_1" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 64 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv1_1" top: "conv1_1" name: "conv1_1_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv1_1" top: "conv1_1" name: "relu1_1" type: "ReLU" }} layer {{ bottom: "conv1_1" top: "conv1_2" name: "conv1_2" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 64 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv1_2" top: "conv1_2" name: "conv1_2_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv1_2" top: "conv1_2" name: "relu1_2" type: "ReLU" }} layer {{ bottom: "conv1_2" top: "pool1" top: "pool1_mask" name: "pool1" type: "Pooling" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ bottom: "pool1" top: "conv2_1" name: "conv2_1" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 128 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv2_1" top: "conv2_1" name: "conv2_1_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv2_1" top: "conv2_1" name: "relu2_1" type: "ReLU" }} layer {{ bottom: "conv2_1" top: "conv2_2" name: "conv2_2" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 128 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv2_2" top: "conv2_2" name: "conv2_2_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv2_2" top: "conv2_2" name: "relu2_2" type: "ReLU" }} layer {{ bottom: "conv2_2" top: "pool2" top: "pool2_mask" name: "pool2" type: "Pooling" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ bottom: "pool2" top: "conv3_1" name: "conv3_1" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 256 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv3_1" top: "conv3_1" name: "conv3_1_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv3_1" top: "conv3_1" name: "relu3_1" type: "ReLU" }} layer {{ bottom: "conv3_1" top: "conv3_2" name: "conv3_2" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 256 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv3_2" top: "conv3_2" name: "conv3_2_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv3_2" top: "conv3_2" name: "relu3_2" type: "ReLU" }} layer {{ bottom: "conv3_2" top: "conv3_3" name: "conv3_3" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 256 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv3_3" top: "conv3_3" name: "conv3_3_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv3_3" top: "conv3_3" name: "relu3_3" type: "ReLU" }} layer {{ bottom: "conv3_3" top: "pool3" top: "pool3_mask" name: "pool3" type: "Pooling" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ bottom: "pool3" top: "conv4_1" name: "conv4_1" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv4_1" top: "conv4_1" name: "conv4_1_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv4_1" top: "conv4_1" name: "relu4_1" type: "ReLU" }} layer {{ bottom: "conv4_1" top: "conv4_2" name: "conv4_2" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv4_2" top: "conv4_2" name: "conv4_2_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv4_2" top: "conv4_2" name: "relu4_2" type: "ReLU" }} layer {{ bottom: "conv4_2" top: "conv4_3" name: "conv4_3" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv4_3" top: "conv4_3" name: "conv4_3_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv4_3" top: "conv4_3" name: "relu4_3" type: "ReLU" }} layer {{ bottom: "conv4_3" top: "pool4" top: "pool4_mask" name: "pool4" type: "Pooling" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ bottom: "pool4" top: "conv5_1" name: "conv5_1" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv5_1" top: "conv5_1" name: "conv5_1_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv5_1" top: "conv5_1" name: "relu5_1" type: "ReLU" }} layer {{ bottom: "conv5_1" top: "conv5_2" name: "conv5_2" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv5_2" top: "conv5_2" name: "conv5_2_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv5_2" top: "conv5_2" name: "relu5_2" type: "ReLU" }} layer {{ bottom: "conv5_2" top: "conv5_3" name: "conv5_3" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv5_3" top: "conv5_3" name: "conv5_3_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv5_3" top: "conv5_3" name: "relu5_3" type: "ReLU" }} layer {{ bottom: "conv5_3" top: "pool5" top: "pool5_mask" name: "pool5" type: "Pooling" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ name: "upsample5" type: "Upsample" bottom: "pool5" top: "pool5_D" bottom: "pool5_mask" upsample_param {{ scale: 2 upsample_w: 30 upsample_h: 23 }} }} layer {{ bottom: "pool5_D" top: "conv5_3_D" name: "conv5_3_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv5_3_D" top: "conv5_3_D" name: "conv5_3_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv5_3_D" top: "conv5_3_D" name: "relu5_3_D" type: "ReLU" }} layer {{ bottom: "conv5_3_D" top: "conv5_2_D" name: "conv5_2_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv5_2_D" top: "conv5_2_D" name: "conv5_2_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv5_2_D" top: "conv5_2_D" name: "relu5_2_D" type: "ReLU" }} layer {{ bottom: "conv5_2_D" top: "conv5_1_D" name: "conv5_1_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv5_1_D" top: "conv5_1_D" name: "conv5_1_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv5_1_D" top: "conv5_1_D" name: "relu5_1_D" type: "ReLU" }} layer {{ name: "upsample4" type: "Upsample" bottom: "conv5_1_D" top: "pool4_D" bottom: "pool4_mask" upsample_param {{ scale: 2 upsample_w: 60 upsample_h: 45 }} }} layer {{ bottom: "pool4_D" top: "conv4_3_D" name: "conv4_3_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv4_3_D" top: "conv4_3_D" name: "conv4_3_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv4_3_D" top: "conv4_3_D" name: "relu4_3_D" type: "ReLU" }} layer {{ bottom: "conv4_3_D" top: "conv4_2_D" name: "conv4_2_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 512 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv4_2_D" top: "conv4_2_D" name: "conv4_2_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv4_2_D" top: "conv4_2_D" name: "relu4_2_D" type: "ReLU" }} layer {{ bottom: "conv4_2_D" top: "conv4_1_D" name: "conv4_1_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 256 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv4_1_D" top: "conv4_1_D" name: "conv4_1_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv4_1_D" top: "conv4_1_D" name: "relu4_1_D" type: "ReLU" }} layer {{ name: "upsample3" type: "Upsample" bottom: "conv4_1_D" top: "pool3_D" bottom: "pool3_mask" upsample_param {{ scale: 2 }} }} layer {{ bottom: "pool3_D" top: "conv3_3_D" name: "conv3_3_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 256 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv3_3_D" top: "conv3_3_D" name: "conv3_3_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv3_3_D" top: "conv3_3_D" name: "relu3_3_D" type: "ReLU" }} layer {{ bottom: "conv3_3_D" top: "conv3_2_D" name: "conv3_2_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 256 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv3_2_D" top: "conv3_2_D" name: "conv3_2_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv3_2_D" top: "conv3_2_D" name: "relu3_2_D" type: "ReLU" }} layer {{ bottom: "conv3_2_D" top: "conv3_1_D" name: "conv3_1_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 128 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv3_1_D" top: "conv3_1_D" name: "conv3_1_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv3_1_D" top: "conv3_1_D" name: "relu3_1_D" type: "ReLU" }} layer {{ name: "upsample2" type: "Upsample" bottom: "conv3_1_D" top: "pool2_D" bottom: "pool2_mask" upsample_param {{ scale: 2 }} }} layer {{ bottom: "pool2_D" top: "conv2_2_D" name: "conv2_2_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 128 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv2_2_D" top: "conv2_2_D" name: "conv2_2_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv2_2_D" top: "conv2_2_D" name: "relu2_2_D" type: "ReLU" }} layer {{ bottom: "conv2_2_D" top: "conv2_1_D" name: "conv2_1_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 64 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv2_1_D" top: "conv2_1_D" name: "conv2_1_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv2_1_D" top: "conv2_1_D" name: "relu2_1_D" type: "ReLU" }} layer {{ name: "upsample1" type: "Upsample" bottom: "conv2_1_D" top: "pool1_D" bottom: "pool1_mask" upsample_param {{ scale: 2 }} }} layer {{ bottom: "pool1_D" top: "conv1_2_D" name: "conv1_2_D" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: 64 pad: 1 kernel_size: 3 }} }} layer {{ bottom: "conv1_2_D" top: "conv1_2_D" name: "conv1_2_D_bn" type: "BN" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 1 decay_mult: 0 }} bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ bottom: "conv1_2_D" top: "conv1_2_D" name: "relu1_2_D" type: "ReLU" }} layer {{ bottom: "conv1_2_D" top: "conv1_1_D_output{n_classes}" name: "conv1_1_D_output{n_classes}" type: "Convolution" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} num_output: {n_classes} pad: 1 kernel_size: 3 }} }} ''')
def main(): import os ROOT = join(os.getcwd()) ROOT = ub.expandpath('~/code/hesaff') os.chdir(ROOT) VERSION = setup.version PY_VER = sys.version_info.major NAME = 'pyhesaff' tag = '{}-{}-py{}'.format(NAME, VERSION, PY_VER) # context_dpath = ub.ensuredir((ROOT, 'docker/context')) staging_dpath = ub.ensuredir((ROOT, 'docker/staging')) # Prestage the multibuild repo if not exists(join(staging_dpath, 'multibuild')): # FIXME: make robust in the case this fails info = ub.cmd( 'git clone https://github.com/matthew-brett/multibuild.git', cwd=staging_dpath, verbose=3) if not exists(join(staging_dpath, 'opencv')): # FIXME: make robust in the case this fails opencv_version = '4.1.0' fpath = ub.grabdata( 'https://github.com/opencv/opencv/archive/{}.zip'.format( opencv_version), verbose=1) ub.cmd('ln -s {} .'.format(fpath), cwd=staging_dpath, verbose=3) ub.cmd('unzip {}'.format(fpath), cwd=staging_dpath, verbose=3) import shutil shutil.move(join(staging_dpath, 'opencv-' + opencv_version), join(staging_dpath, 'opencv')) stage_self(ROOT, staging_dpath) dockerfile_fpath = join(ROOT, 'Dockerfile') # This docker code is very specific for building linux binaries. # We will need to do a bit of refactoring to handle OSX and windows. # But the goal is to get at least one OS working end-to-end. docker_code = ub.codeblock(''' FROM quay.io/skvark/manylinux1_x86_64 # SETUP ENV ARG MB_PYTHON_VERSION=3.6 ARG ENABLE_CONTRIB=0 ARG ENABLE_HEADLESS=1 ENV PYTHON_VERSION=3.6 ENV PYTHON_ROOT=/opt/python/cp36-cp36m/ ENV PYTHONPATH=/opt/python/cp36-cp36m/lib/python3.6/site-packages/ ENV PATH=/opt/python/cp36-cp36m/bin:$PATH ENV PYTHON_EXE=/opt/python/cp36-cp36m/python ENV MULTIBUILD_DIR=/root/code/multibuild ENV HOME=/root # params to bdist_wheel. used to set osx build target. ENV TEST_DEPENDS="numpy==1.11.1" ENV BDIST_PARAMS="" ENV USE_CCACHE=1 ENV PLAT=x86_64 ENV UNICODE_WIDTH=32 # -e BUILD_COMMANDS="$build_cmds" \ # -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ # -e UNICODE_WIDTH="$UNICODE_WIDTH" \ # -e BUILD_COMMIT="$BUILD_COMMIT" \ # -e CONFIG_PATH="$CONFIG_PATH" \ # -e ENV_VARS_PATH="$ENV_VARS_PATH" \ # -e WHEEL_SDIR="$WHEEL_SDIR" \ # -e MANYLINUX_URL="$MANYLINUX_URL" \ # -e BUILD_DEPENDS="$BUILD_DEPENDS" \ # -e USE_CCACHE="$USE_CCACHE" \ # -e REPO_DIR="$repo_dir" \ # -e PLAT="$PLAT" \ # These are defined in the parent image # ENV JPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include # ENV JPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a RUN mkdir -p /io WORKDIR /root # Setup code / scripts COPY docker/staging/multibuild /root/code/multibuild # Hack to fix issue RUN find $MULTIBUILD_DIR -iname "*.sh" -type f -exec sed -i 's/gh-clone/gh_clone/g' {} + # Copy bash configs (mirrors the environs) COPY docker/config.sh /root/config.sh COPY docker/bashrc.sh /root/.bashrc # Setup a virtualenv RUN source /root/.bashrc && \ $PYTHON_EXE -m pip install --upgrade pip && \ $PYTHON_EXE -m pip install virtualenv && \ $PYTHON_EXE -m virtualenv --python=$PYTHON_EXE $HOME/venv # Install packages in virtual environment RUN source /root/.bashrc && \ pip install cmake ninja scikit-build wheel numpy # This is very different for different operating systems # https://github.com/skvark/opencv-python/blob/master/setup.py COPY docker/staging/opencv /root/code/opencv RUN source /root/.bashrc && \ source code/multibuild/common_utils.sh && \ source code/multibuild/travis_linux_steps.sh && \ mkdir -p /root/code/opencv/build && \ cd /root/code/opencv/build && \ cmake -G "Unix Makefiles" \ -DINSTALL_CREATE_DISTRIB=ON \ -DOPENCV_SKIP_PYTHON_LOADER=ON \ -DBUILD_opencv_apps=OFF \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTS=OFF \ -DBUILD_PERF_TESTS=OFF \ -DBUILD_DOCS=OFF \ -DWITH_QT=OFF \ -DWITH_IPP=OFF \ -DWITH_V4L=ON \ -DBUILD_JPEG=OFF \ -DENABLE_PRECOMPILED_HEADERS=OFF \ -DJPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include \ -DJPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a \ /root/code/opencv # Note: there is no need to compile the above with python # -DPYTHON3_EXECUTABLE=$PYTHON_EXE \ # -DBUILD_opencv_python3=ON \ # -DOPENCV_PYTHON3_INSTALL_PATH=python \ RUN source /root/.bashrc && \ source code/multibuild/common_utils.sh && \ source code/multibuild/travis_linux_steps.sh && \ cd /root/code/opencv/build && \ make -j9 && make install COPY docker/staging/hesaff /root/code/hesaff # # Use skbuild to build hesaff # RUN source /root/.bashrc && \ # cd /root/code/hesaff && \ # CMAKE_FIND_LIBRARY_SUFFIXES=".a;.so" python setup.py build_ext --inplace # export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' # Use cmake to build hesaff 9maybe not needed?) # RUN source /root/.bashrc && \ # mkdir -p /root/code/hesaff/build && \ # cd /root/code/hesaff/build && \ # CXXFLAGS="-std=c++11 $CXXFLAGS" cmake -G "Unix Makefiles" /root/code/hesaff && \ # make # Use skbuild to build hesaff RUN source /root/.bashrc && \ cd /root/code/hesaff && \ python setup.py build && \ python setup.py bdist_wheel # RUN source /root/.bashrc && \ # pip install xdoctest ''') try: print(ub.color_text('\n--- DOCKER CODE ---', 'white')) print(ub.highlight_code(docker_code, 'docker')) print(ub.color_text('--- END DOCKER CODE ---\n', 'white')) except Exception: pass with open(dockerfile_fpath, 'w') as file: file.write(docker_code) docker_build_cli = ' '.join([ 'docker', 'build', # '--build-arg PY_VER={}'.format(PY_VER), '--tag {}'.format(tag), '-f {}'.format(dockerfile_fpath), '.' ]) print('docker_build_cli = {!r}'.format(docker_build_cli)) info = ub.cmd(docker_build_cli, verbose=3, shell=True) if info['ret'] != 0: print(ub.color_text('\n--- FAILURE ---', 'red')) print('Failed command:') print(info['command']) print(info['err']) print('NOTE: sometimes reruning the command manually works') raise Exception('Building docker failed with exit code {}'.format( info['ret'])) else: print(ub.color_text('\n--- SUCCESS ---', 'green')) DEPLOY = True if DEPLOY: VMNT_DIR = '{ROOT}/{NAME}-docker/vmnt'.format(NAME=NAME, ROOT=ROOT) print('VMNT_DIR = {!r}'.format(VMNT_DIR)) ub.ensuredir(VMNT_DIR) # TODO: Correctly mangle the ffmpeg libs as done via # ls -a ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/ # ls ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/.libs # cp code/hesaff/build/libhesaff.so /root/vmnt # cp /root/ffmpeg_build/lib/libavcodec.so.58 /root/vmnt # cp /root/ffmpeg_build/lib/libavformat.so.58 /root/vmnt # cp /root/ffmpeg_build/lib/libavutil.so.56 /root/vmnt # cp /root/ffmpeg_build/lib/libswscale.so.5 /root/vmnt inside_cmds = ' && '.join( ub.codeblock(''' cp code/hesaff/dist/pyhesaff*.whl /root/vmnt ''').split('\n')) docker_run_cli = ' '.join([ 'docker', 'run', '-v {}:/root/vmnt/'.format(VMNT_DIR), '-it', tag, 'bash -c "{}"'.format(inside_cmds) ]) print(docker_run_cli) info = ub.cmd(docker_run_cli, verbose=3) assert info['ret'] == 0 # import shutil # PKG_DIR = join(ROOT, 'pyhesaff') # shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libhesaff-manylinux1_x86_64.so')) # TODO: do this correctly # shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libavcodec.so.58')) # shutil.copy(join(VMNT_DIR, 'libavformat.so.58'), join(PKG_DIR, 'libavformat.so.58')) # shutil.copy(join(VMNT_DIR, 'libavutil.so.56'), join(PKG_DIR, 'libavutil.so.56')) # shutil.copy(join(VMNT_DIR, 'libswscale.so.5'), join(PKG_DIR, 'libswscale.so.5')) # print(ub.highlight_code(ub.codeblock( print( ub.highlight_code( ub.codeblock(r''' # Finished creating the docker image. # To test / export you can do something like this: VMNT_DIR={ROOT}/{NAME}-docker/vmnt mkdir -p VMNT_DIR TAG={tag} # Test that we can get a bash terminal docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash # Move deployment to the vmnt directory docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 -m xdoctest pyhesaff' # Run system tests docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 run_doctests.sh' # Inside bash test that we can fit a new model python -m pyhessaff demo mkdir -p ${ROOT}/{NAME}-docker/dist docker save -o ${ROOT}/{NAME}-docker/dist/{tag}.docker.tar {tag} ''').format(NAME=NAME, ROOT=ROOT, tag=tag), 'bash'))
def run_ibeis(): r""" CommandLine: python -m ibeis python -m ibeis find_installed_tomcat python -m ibeis get_annot_groundtruth:1 """ import ibeis # NOQA #ut.set_process_title('IBEIS_main') #main_locals = ibeis.main() #ibeis.main_loop(main_locals) #ut.set_process_title('IBEIS_main') cmdline_varags = ut.get_cmdline_varargs() if len(cmdline_varags) > 0 and cmdline_varags[0] == 'rsync': from ibeis.scripts import rsync_ibeisdb rsync_ibeisdb.rsync_ibsdb_main() sys.exit(0) if ub.argflag('--devcmd'): # Hack to let devs mess around when using an installer version # TODO: add more hacks #import utool.tests.run_tests #utool.tests.run_tests.run_tests() ut.embed() # Run the tests of other modules elif ub.argflag('--run-utool-tests'): raise Exception('Deprecated functionality') elif ub.argflag('--run-vtool_ibeis-tests'): raise Exception('Deprecated functionality') elif ub.argflag(('--run-ibeis-tests', '--run-tests')): raise Exception('Deprecated functionality') if ub.argflag('-e'): """ ibeis -e print -a default -t default """ # Run dev script if -e given import ibeis.dev # NOQA ibeis.dev.devmain() print('... exiting') sys.exit(0) # Attempt to run a test using the funciton name alone # with the --tf flag # if False: # import ibeis.tests.run_tests # import ibeis.tests.reset_testdbs # import ibeis.scripts.thesis # ignore_prefix = [ # #'ibeis.tests', # 'ibeis.control.__SQLITE3__', # '_autogen_explicit_controller'] # ignore_suffix = ['_grave'] # func_to_module_dict = { # 'demo_bayesnet': 'ibeis.unstable.demobayes', # } # ut.main_function_tester('ibeis', ignore_prefix, ignore_suffix, # func_to_module_dict=func_to_module_dict) #if ub.argflag('-e'): # import ibeis # expt_kw = ut.get_arg_dict(ut.get_func_kwargs(ibeis.run_experiment), # prefix_list=['--', '-']) # ibeis.run_experiment(**expt_kw) # sys.exit(0) doctest_modname = ut.get_argval( ('--doctest-module', '--tmod', '-tm', '--testmod'), type_=str, default=None, help_='specify a module to doctest') if doctest_modname is not None: """ Allow any doctest to be run the main ibeis script python -m ibeis --tmod utool.util_str --test-align:0 python -m ibeis --tmod ibeis.algo.hots.pipeline --test-request_ibeis_query_L0:0 --show python -m ibeis --tf request_ibeis_query_L0:0 --show ./dist/ibeis/IBEISApp --tmod ibeis.algo.hots.pipeline --test-request_ibeis_query_L0:0 --show # NOQA ./dist/ibeis/IBEISApp --tmod utool.util_str --test-align:0 ./dist/IBEIS.app/Contents/MacOS/IBEISApp --tmod utool.util_str --test-align:0 ./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-utool-tests ./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-vtool_ibeis-tests """ print('[ibeis] Testing module') mod_alias_list = { 'exptdraw': 'ibeis.expt.experiment_drawing' } doctest_modname = mod_alias_list.get(doctest_modname, doctest_modname) module = ut.import_modname(doctest_modname) (nPass, nTotal, failed_list, error_report_list) = ut.doctest_funcs(module=module) retcode = 1 - (len(failed_list) == 0) #print(module) sys.exit(retcode) import ibeis main_locals = ibeis.main() execstr = ibeis.main_loop(main_locals) # <DEBUG CODE> if 'back' in main_locals and CMD: back = main_locals['back'] front = getattr(back, 'front', None) # NOQA #front = back.front #ui = front.ui ibs = main_locals['ibs'] # NOQA print('-- EXECSTR --') print(ub.codeblock(execstr)) print('-- /EXECSTR --') exec(execstr)
def write_default_ipython_profile(): """ CommandLine: python ~/local/init/init_ipython_config.py python -c "import xdev, ubelt; xdev.startfile(ubelt.truepath('~/.ipython/profile_default'))" python -c "import xdev, ubelt; xdev.editfile(ubelt.truepath('~/.ipython/profile_default/ipython_config.py'))" References: http://2sn.org/python/ipython_config.py """ dpath = ub.expandpath('~/.ipython/profile_default') ub.ensuredir(dpath) ipy_config_fpath = join(dpath, 'ipython_config.py') ipy_config_text = ub.codeblock( r''' # STARTBLOCK import six c = get_config() # NOQA c.InteractiveShellApp.exec_lines = [] if six.PY2: future_line = ( 'from __future__ import absolute_import, division, print_function, with_statement, unicode_literals') c.InteractiveShellApp.exec_lines.append(future_line) # Fix sip versions try: import sip # http://stackoverflow.com/questions/21217399/pyqt4-qtcore-qvariant-object-instead-of-a-string sip.setapi('QVariant', 2) sip.setapi('QString', 2) sip.setapi('QTextStream', 2) sip.setapi('QTime', 2) sip.setapi('QUrl', 2) sip.setapi('QDate', 2) sip.setapi('QDateTime', 2) if hasattr(sip, 'setdestroyonexit'): sip.setdestroyonexit(False) # This prevents a crash on windows except ImportError as ex: pass except ValueError as ex: print('Warning: Value Error: %s' % str(ex)) pass c.InteractiveShellApp.exec_lines.append('%load_ext autoreload') c.InteractiveShellApp.exec_lines.append('%autoreload 2') #c.InteractiveShellApp.exec_lines.append('%pylab qt4') c.InteractiveShellApp.exec_lines.append('import numpy as np') c.InteractiveShellApp.exec_lines.append('import ubelt as ub') c.InteractiveShellApp.exec_lines.append('import xdev') c.InteractiveShellApp.exec_lines.append('import pandas as pd') c.InteractiveShellApp.exec_lines.append('pd.options.display.max_columns = 40') c.InteractiveShellApp.exec_lines.append('pd.options.display.width = 160') c.InteractiveShellApp.exec_lines.append('pd.options.display.max_rows = 20') c.InteractiveShellApp.exec_lines.append('pd.options.display.float_format = lambda x: \'%.4f\' % (x,)') c.InteractiveShellApp.exec_lines.append('import networkx as nx') c.InteractiveShellApp.exec_lines.append('from os.path import *') c.InteractiveShellApp.exec_lines.append('from six.moves import cPickle as pickle') #c.InteractiveShellApp.exec_lines.append('if \'verbose\' not in vars():\\n verbose = True') import ubelt as ub c.InteractiveShellApp.exec_lines.append(ub.codeblock( """ class classproperty(property): def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)() class vim(object): @classproperty def focus(cls): from vimtk.cplat_ctrl import Window Window.find('GVIM').focus() @classproperty def copy(cls): import time from vimtk.cplat_ctrl import Window gvim_window = Window.find('GVIM') gvim_window.focus() import vimtk import IPython ipy = IPython.get_ipython() lastline = ipy.history_manager.input_hist_parsed[-2] vimtk.cplat.copy_text_to_clipboard(lastline) from vimtk import xctrl xctrl.XCtrl.do( ('focus', 'GVIM'), ('key', 'ctrl+v'), ('focus', 'x-terminal-emulator.X-terminal-emulator') ) """ )) #c.InteractiveShell.autoindent = True #c.InteractiveShell.colors = 'LightBG' #c.InteractiveShell.confirm_exit = False #c.InteractiveShell.deep_reload = True c.InteractiveShell.editor = 'gvim' #c.InteractiveShell.xmode = 'Context' # ENDBOCK ''' ) with open(ipy_config_fpath, 'w') as file: file.write(ipy_config_text + '\n')
def compare_results(): print('Comparing results') import pandas as pd from tabulate import tabulate # Read in output of demo script measure_fpath = 'measurements_haul83.csv' py_df = pd.DataFrame.from_csv(measure_fpath, index_col=None) # Convert python length output from mm into cm for consistency py_df['fishlen'] = py_df['fishlen'] / 10 py_df['current_frame'] = py_df['current_frame'].astype(np.int) # janky CSV parsing py_df['box_pts1'] = py_df['box_pts1'].map(lambda p: eval(p.replace(';', ','), np.__dict__)) py_df['box_pts2'] = py_df['box_pts2'].map(lambda p: eval(p.replace(';', ','), np.__dict__)) py_df['obox1'] = [ctalgo.OrientedBBox(*cv2.minAreaRect(pts[:, None, :].astype(np.int))) for pts in py_df['box_pts1']] py_df['obox2'] = [ctalgo.OrientedBBox(*cv2.minAreaRect(pts[:, None, :].astype(np.int))) for pts in py_df['box_pts2']] py_df.drop(['box_pts1', 'box_pts2'], axis=1, inplace=True) # Remap to matlab names py_df = py_df.rename(columns={ 'error': 'Err', 'fishlen': 'fishLength', 'range': 'fishRange', }) # Load matlab results mat_df = _read_kresimir_results() FORCE_COMPARABLE_RANGE = True # FORCE_COMPARABLE_RANGE = False if FORCE_COMPARABLE_RANGE: # Be absolutely certain we are in comparable regions (may slightly bias # results, against python and in favor of matlab) min_frame = max(mat_df.current_frame.min(), py_df.current_frame.min()) max_frame = min(mat_df.current_frame.max(), py_df.current_frame.max()) print('min_frame = {!r}'.format(min_frame)) print('max_frame = {!r}'.format(max_frame)) mat_df = mat_df[(mat_df.current_frame >= min_frame) & (mat_df.current_frame <= max_frame)] py_df = py_df[(py_df.current_frame >= min_frame) & (py_df.current_frame <= max_frame)] intersect_frames = np.intersect1d(mat_df.current_frame, py_df.current_frame) print('intersecting frames = {} / {} (matlab)'.format( len(intersect_frames), len(set(mat_df.current_frame)))) print('intersecting frames = {} / {} (python)'.format( len(intersect_frames), len(set(py_df.current_frame)))) # Reuse the hungarian algorithm implementation from ctalgo min_assign = ctalgo.FishStereoMeasurments.minimum_weight_assignment correspond = [] for f in intersect_frames: pidxs = np.where(py_df.current_frame == f)[0] midxs = np.where(mat_df.current_frame == f)[0] pdf = py_df.iloc[pidxs] mdf = mat_df.iloc[midxs] ppts1 = np.array([o.center for o in pdf['obox1']]) mpts1 = np.array([o.center for o in mdf['obox1']]) ppts2 = np.array([o.center for o in pdf['obox2']]) mpts2 = np.array([o.center for o in mdf['obox2']]) dists1 = sklearn.metrics.pairwise.pairwise_distances(ppts1, mpts1) dists2 = sklearn.metrics.pairwise.pairwise_distances(ppts2, mpts2) # arbitrarilly chosen threshold thresh = 100 for i, j in min_assign(dists1): d1 = dists1[i, j] d2 = dists2[i, j] if d1 < thresh and d2 < thresh and abs(d1 - d2) < thresh / 4: correspond.append((pidxs[i], midxs[j])) correspond = np.array(correspond) # pflags = np.array(ub.boolmask(correspond.T[0], len(py_df))) mflags = np.array(ub.boolmask(correspond.T[1], len(mat_df))) # print('there are {} detections that seem to be in common'.format(len(correspond))) # print('The QC flags of the common detections are: {}'.format( # ub.dict_hist(mat_df[mflags]['QC'].values))) # print('The QC flags of the other matlab detections are: {}'.format( # ub.dict_hist(mat_df[~mflags]['QC'].values))) print('\n\n----\n## All stats\n') print(ub.codeblock( ''' Overall, the matlab script made {nmat} length measurements and the python script made {npy} length measurements. Here is a table summarizing the average lengths / ranges / errors of each script: ''').format(npy=len(py_df), nmat=len(mat_df))) stats = pd.DataFrame(columns=['python', 'matlab']) for key in ['fishLength', 'fishRange', 'Err']: stats.loc[key, 'python'] = '{:6.2f} ± {:6.2f}'.format(py_df[key].mean(), py_df[key].std()) stats.loc[key, 'matlab'] = '{:6.2f} ± {:6.2f}'.format(mat_df[key].mean(), mat_df[key].std()) stats.loc['nTotal', 'python'] = '{}'.format(len(py_df)) stats.loc['nTotal', 'matlab'] = '{}'.format(len(mat_df)) print(tabulate(stats, headers='keys', tablefmt='psql', stralign='right')) print('\n\n----\n## Only COMMON detections\n') py_df_c = py_df.iloc[correspond.T[0]] mat_df_c = mat_df.iloc[correspond.T[1]] stats = pd.DataFrame(columns=['python', 'matlab']) for key in ['fishLength', 'fishRange', 'Err']: stats.loc[key, 'python'] = '{:6.2f} ± {:6.2f}'.format(py_df_c[key].mean(), py_df_c[key].std()) stats.loc[key, 'matlab'] = '{:6.2f} ± {:6.2f}'.format(mat_df_c[key].mean(), mat_df_c[key].std()) stats.loc['nTotal', 'python'] = '{}'.format(len(py_df_c)) stats.loc['nTotal', 'matlab'] = '{}'.format(len(mat_df_c)) print(ub.codeblock( ''' Now, we investigate how many dections matlab and python made in common. (Note, choosing which dections in one version correspond to which in another is done using a heuristic based on distances between bbox centers and a thresholded minimum assignment problem). Python made {npy_c}/{nmat} = {percent:.2f}% of the detections matlab made ''').format(npy_c=len(py_df_c), nmat=len(mat_df), percent=100 * len(py_df_c) / len(mat_df))) print(tabulate(stats, headers='keys', tablefmt='psql', stralign='right')) print('\n\n----\n## Evaulation using the QC code\n') hist_hit = ub.dict_hist(mat_df[mflags]['QC'].values) hist_miss = ub.dict_hist(mat_df[~mflags]['QC'].values) print(ub.codeblock( ''' However, not all of those matlab detections were good. Because we have detections in corrsepondences with each other we can assign the python detections QC codes. Here is a histogram of the QC codes for these python detections: {} (Note: read histogram as <QC-code>: <frequency>) Here is a histogram of the other matlab detections that python did not find: {} To summarize: python correctly rejected {:.2f}% of the matlab QC=0 detections python correctly accepted {:.2f}% of the matlab QC=1 detections python correctly accepted {:.2f}% of the matlab QC=2 detections Note, that because python made detections that matlab did not make, the remaining {} detections may be right or wrong, but there is no way to tell from this analysis. Lastly, here are the statistics for the common detections that had a non-zero QC code. ''').format( ub.repr2(hist_hit, nl=1), ub.repr2(hist_miss, nl=1), 100 * hist_miss[0] / (hist_hit[0] + hist_miss[0]), 100 * hist_hit[1] / (hist_hit[1] + hist_miss[1]), 100 * hist_hit[2] / (hist_hit[2] + hist_miss[2]), len(py_df) - len(py_df_c) ) ) is_qc = (mat_df_c['QC'] > 0).values mat_df_c = mat_df_c[is_qc] py_df_c = py_df_c[is_qc] stats = pd.DataFrame(columns=['python', 'matlab']) for key in ['fishLength', 'fishRange', 'Err']: stats.loc[key, 'python'] = '{:6.2f} ± {:6.2f}'.format(py_df_c[key].mean(), py_df_c[key].std()) stats.loc[key, 'matlab'] = '{:6.2f} ± {:6.2f}'.format(mat_df_c[key].mean(), mat_df_c[key].std()) stats.loc['nTotal', 'python'] = '{}'.format(len(py_df_c)) stats.loc['nTotal', 'matlab'] = '{}'.format(len(mat_df_c)) print(tabulate(stats, headers='keys', tablefmt='psql', stralign='right'))
CORE_LAYERS = ub.codeblock(""" layer {{ name: "norm" type: "LRN" bottom: "data" top: "norm" lrn_param {{ local_size: 5 alpha: 0.0001 beta: 0.75 }} }} layer {{ name: "conv1" type: "Convolution" bottom: "norm" top: "conv1" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv1" top: "conv1" name: "conv1_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "relu1" type: "ReLU" bottom: "conv1" top: "conv1" }} layer {{ name: "pool1" type: "Pooling" bottom: "conv1" top: "pool1" top: "pool1_mask" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ name: "conv2" type: "Convolution" bottom: "pool1" top: "conv2" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv2" top: "conv2" name: "conv2_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "relu2" type: "ReLU" bottom: "conv2" top: "conv2" }} layer {{ name: "pool2" type: "Pooling" bottom: "conv2" top: "pool2" top: "pool2_mask" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ name: "conv3" type: "Convolution" bottom: "pool2" top: "conv3" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv3" top: "conv3" name: "conv3_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "relu3" type: "ReLU" bottom: "conv3" top: "conv3" }} layer {{ name: "pool3" type: "Pooling" bottom: "conv3" top: "pool3" top: "pool3_mask" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ name: "conv4" type: "Convolution" bottom: "pool3" top: "conv4" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv4" top: "conv4" name: "conv4_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "relu4" type: "ReLU" bottom: "conv4" top: "conv4" }} layer {{ name: "pool4" type: "Pooling" bottom: "conv4" top: "pool4" top: "pool4_mask" pooling_param {{ pool: MAX kernel_size: 2 stride: 2 }} }} layer {{ name: "upsample4" type: "Upsample" bottom: "pool4" bottom: "pool4_mask" top: "upsample4" upsample_param {{ scale: 2 pad_out_h: true }} }} layer {{ name: "conv_decode4" type: "Convolution" bottom: "upsample4" top: "conv_decode4" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv_decode4" top: "conv_decode4" name: "conv_decode4_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "upsample3" type: "Upsample" bottom: "conv_decode4" bottom: "pool3_mask" top: "upsample3" upsample_param {{ scale: 2 }} }} layer {{ name: "conv_decode3" type: "Convolution" bottom: "upsample3" top: "conv_decode3" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv_decode3" top: "conv_decode3" name: "conv_decode3_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "upsample2" type: "Upsample" bottom: "conv_decode3" bottom: "pool2_mask" top: "upsample2" upsample_param {{ scale: 2 }} }} layer {{ name: "conv_decode2" type: "Convolution" bottom: "upsample2" top: "conv_decode2" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv_decode2" top: "conv_decode2" name: "conv_decode2_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "upsample1" type: "Upsample" bottom: "conv_decode2" bottom: "pool1_mask" top: "upsample1" upsample_param {{ scale: 2 }} }} layer {{ name: "conv_decode1" type: "Convolution" bottom: "upsample1" top: "conv_decode1" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: 64 kernel_size: 7 pad: 3 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} layer {{ bottom: "conv_decode1" top: "conv_decode1" name: "conv_decode1_bn" type: "BN" bn_param {{ bn_mode: INFERENCE scale_filler {{ type: "constant" value: 1 }} shift_filler {{ type: "constant" value: 0.001 }} }} }} layer {{ name: "conv_classifier_output{n_classes}" type: "Convolution" bottom: "conv_decode1" top: "conv_classifier_output{n_classes}" param {{ lr_mult: 1 decay_mult: 1 }} param {{ lr_mult: 2 decay_mult: 0 }} convolution_param {{ num_output: {n_classes} kernel_size: 1 weight_filler {{ type: "msra" }} bias_filler {{ type: "constant" }} }} }} """)
core_layer_auto, num_context_lines=10))) return core_layer_auto FIT_FOOTER = ub.codeblock(''' layer {{ name: "loss" type: "SoftmaxWithLoss" bottom: "conv1_1_D_output{n_classes}" bottom: "label" top: "loss" softmax_param {{engine: CAFFE}} loss_param: {{ weight_by_label_freqs: true {class_weights_text} }} }} layer {{ name: "accuracy" type: "Accuracy" bottom: "conv1_1_D_output{n_classes}" bottom: "label" top: "accuracy" top: "per_class_accuracy" }} ''') PREDICT_FOOTER = ub.codeblock(''' layer {{ name: "prob" type: "Softmax"
def make_prototext(image_list_fpath, arch, mode='fit', batch_size=1, n_classes=None, class_weights=None, ignore_label=None, shuffle=None, params=None): assert mode in {'fit', 'predict'} mod = model_modules[arch] if shuffle is None: shuffle = (mode == 'fit') if n_classes is None: n_classes = len(class_weights) elif ignore_label is not None: # this is really weird # with 12 classes we need to make the number of outputs be 11 because # we are ignoring the last label. However, when class_weights are # passed in we only send it the used weights, so that's already the # right number. Not sure what happend when ignore_label=0 and not 11 n_classes -= 1 fmtdict = { 'shuffle': str(shuffle).lower(), 'batch_size': batch_size, 'image_list_fpath': image_list_fpath, 'n_classes': n_classes, 'arch_name': arch, } if image_list_fpath is None: # Input layer when we use blobs # maybe use this def instead? # layer { # name: "input" # type: "Input" # top: "data" # input_param { # shape { # dim: 1 # dim: 3 # dim: 360 # dim: 480 # } # } # } input_layer_fmt = ub.codeblock( ''' input: "data" input_dim: {batch_size} input_dim: 3 input_dim: 360 input_dim: 480 ''') else: # Layer when input is specified in a txt input_layer_fmt = ub.codeblock( ''' name: "{arch_name}" layer {{ name: "data" type: "DenseImageData" top: "data" top: "label" dense_image_data_param {{ source: "{image_list_fpath}" batch_size: {batch_size} shuffle: {shuffle} }} }} ''' ) input_layer = input_layer_fmt.format(**fmtdict) if hasattr(mod, 'make_core_layers'): if params is not None: freeze_before = params['freeze_before'] finetune_decay = params['finetune_decay'] else: freeze_before = 0 finetune_decay = 1 core = mod.make_core_layers(n_classes, freeze_before, finetune_decay) else: core = mod.CORE_LAYERS.format(**fmtdict) if mode == 'fit': # remove batch-norm inference when fitting core = re.sub('^\s*bn_mode:\s*INFERENCE$', '', core, flags=re.M) class_weights_line = ['class_weighting: {}'.format(w) for w in class_weights] class_weights_line += ['ignore_label: {}'.format(ignore_label)] class_weights_text = ub.indent('\n'.join(class_weights_line), ' ' * 4).lstrip() fmtdict['class_weights_text'] = class_weights_text footer_fmt = mod.FIT_FOOTER else: footer_fmt = mod.PREDICT_FOOTER footer = footer_fmt.format(**fmtdict) text = '\n'.join([input_layer, core, footer]) return text
def main(): from os.path import join import six import pytest import ubelt as ub import os from xdoctest import static_analysis as static dpath = ub.ensure_app_cache_dir('import_abi_test') text = ub.codeblock(''' a = 42 ''') os.chdir(dpath) modpath = join(dpath, 'foo.py') ub.writeto(modpath, text) import foo assert foo.a == 42 tag = static._extension_module_tags()[0] # Q: Can augment .py files with an ABI tag # A: No modpath = join(dpath, 'bar.' + tag + '.py') ub.writeto(modpath, text) with pytest.raises(ImportError, match=".*No module named .?bar.*"): import bar # NOQA # Q: Can we augment .so files with an ABI tag # A: Yes (note its not a valid so so it fails) modpath = join(dpath, 'baz.' + tag + '.so') ub.writeto(modpath, text) with pytest.raises(ImportError, match=".*file too short"): import baz # NOQA # Q: Can we augment .so files with an **arbitrary** tag? # A: No modpath = join(dpath, 'buz.' + 'junktag' + '.so') ub.writeto(modpath, text) with pytest.raises(ImportError, match="No module named .*"): import buz # NOQA # Q: Can we augment .so files with an "abi3" tag? # A: Yes, but not in python2 modpath = join(dpath, 'biz.' + 'abi3' + '.so') ub.writeto(modpath, text) if six.PY3: with pytest.raises(ImportError, match=".*file too short"): import biz # NOQA else: with pytest.raises(ImportError, match="No module named .*"): import biz # NOQA # Q: Can we augment .so files with a semi-nice tag? # A: No modpath = join(dpath, 'spam.' + 'linux' + '.so') ub.writeto(modpath, text) with pytest.raises(ImportError, match="No module named .*"): import spam # NOQA modpath = join(dpath, 'eggs.' + 'cp3-abi3-linux_x86_64' + '.so') ub.writeto(modpath, text) with pytest.raises(ImportError, match="No module named .*"): import eggs # NOQA modpath = join(dpath, 'ham.' + 'py3-none-any' + '.so') ub.writeto(modpath, text) with pytest.raises(ImportError, match="No module named .*"): import ham # NOQA
def test_list_of_numpy(): try: import numpy as np except ImportError: import pytest pytest.skip('numpy is optional') import ubelt as ub data = [ np.zeros((3, 3), dtype=np.int32), np.zeros((3, 10), dtype=np.int32), np.zeros((3, 20), dtype=np.int32), np.zeros((3, 30), dtype=np.int32), ] text = ub.repr2(data, nl=2) print(text) assert repr(data) == repr(eval(text)), 'should produce eval-able code' assert text == ub.codeblock(''' [ np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), ] ''') text = ub.repr2(data, max_line_width=10000, nl=2) print(text) assert text == ub.codeblock(''' [ np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), ] ''') text = ub.repr2(data, nl=1) print(text) assert text == ub.codeblock(''' [ np.array([[0, 0, 0],[0, 0, 0],[0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), ] ''') text = ub.repr2(data, nl=0) print(text) assert text == ub.codeblock(''' [np.array([[0, 0, 0],[0, 0, 0],[0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32)] ''')
def main(): def argval(clikey, envkey=None, default=ub.NoParam): if envkey is not None: envval = os.environ.get(envkey) if envval: default = envval return ub.argval(clikey, default=default) DEFAULT_PY_VER = '{}.{}'.format(sys.version_info.major, sys.version_info.minor) PY_VER = argval('--pyver', 'MB_PYTHON_VERSION', default=DEFAULT_PY_VER) dpath = argval('--dpath', None, default=os.getcwd()) PLAT = argval('--plat', 'PLAT', default='x86_64') UNICODE_WIDTH = argval('--unicode_width', 'UNICODE_WIDTH', '32') import multiprocessing MAKE_CPUS = argval('--make_cpus', 'MAKE_CPUS', multiprocessing.cpu_count() + 1) OPENCV_VERSION = '4.1.0' os.chdir(dpath) BASE = 'manylinux1_{}'.format(PLAT) BASE_REPO = 'quay.io/skvark' PY_TAG = 'cp{ver}-cp{ver}m'.format(ver=PY_VER.replace('.', '')) # do we need the unicode width in this tag? DOCKER_TAG = '{}-opencv{}-py{}'.format(BASE, OPENCV_VERSION, PY_VER) if not exists(join(dpath, 'opencv-' + OPENCV_VERSION)): # FIXME: make robust in the case this fails fpath = ub.grabdata('https://github.com/opencv/opencv/archive/{}.zip'.format(OPENCV_VERSION), dpath=dpath, verbose=1) ub.cmd('ln -s {} .'.format(fpath), cwd=dpath, verbose=3) ub.cmd('unzip {}'.format(fpath), cwd=dpath, verbose=3) dockerfile_fpath = join(dpath, 'Dockerfile_' + DOCKER_TAG) # This docker code is very specific for building linux binaries. # We will need to do a bit of refactoring to handle OSX and windows. # But the goal is to get at least one OS working end-to-end. docker_code = ub.codeblock( ''' FROM {BASE_REPO}/{BASE} # SETUP ENV ARG MB_PYTHON_VERSION={PY_VER} ENV PYTHON_VERSION={PY_VER} ENV PYTHON_ROOT=/opt/python/{PY_TAG}/ ENV PYTHONPATH=/opt/python/{PY_TAG}/lib/python{PY_VER}/site-packages/ ENV PATH=/opt/python/{PY_TAG}/bin:$PATH ENV PYTHON_EXE=/opt/python/{PY_TAG}/bin/python ENV HOME=/root ENV PLAT={PLAT} ENV UNICODE_WIDTH={UNICODE_WIDTH} # Update python environment RUN echo "$PYTHON_EXE" RUN $PYTHON_EXE -m pip install --upgrade pip && \ $PYTHON_EXE -m pip install cmake ninja scikit-build wheel numpy # This is very different for different operating systems # https://github.com/skvark/opencv-python/blob/master/setup.py COPY opencv-{OPENCV_VERSION} /root/code/opencv RUN mkdir -p /root/code/opencv/build && \ cd /root/code/opencv/build && \ cmake -G "Unix Makefiles" \ -DINSTALL_CREATE_DISTRIB=ON \ -DOPENCV_SKIP_PYTHON_LOADER=ON \ -DBUILD_opencv_apps=OFF \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTS=OFF \ -DBUILD_PERF_TESTS=OFF \ -DBUILD_DOCS=OFF \ -DWITH_QT=OFF \ -DWITH_IPP=OFF \ -DWITH_V4L=ON \ -DBUILD_JPEG=OFF \ -DENABLE_PRECOMPILED_HEADERS=OFF \ /root/code/opencv # Note: there is no need to compile the above with python # -DPYTHON3_EXECUTABLE=$PYTHON_EXE \ # -DBUILD_opencv_python3=ON \ # -DOPENCV_PYTHON3_INSTALL_PATH=python \ RUN cd /root/code/opencv/build && make -j{MAKE_CPUS} && make install '''.format(**locals())) try: print(ub.color_text('\n--- DOCKER CODE ---', 'white')) print(ub.highlight_code(docker_code, 'docker')) print(ub.color_text('--- END DOCKER CODE ---\n', 'white')) except Exception: pass with open(dockerfile_fpath, 'w') as file: file.write(docker_code) docker_build_cli = ' '.join([ 'docker', 'build', # '--build-arg PY_VER={}'.format(PY_VER), '--tag {}'.format(DOCKER_TAG), '-f {}'.format(dockerfile_fpath), '.' ]) print('docker_build_cli = {!r}'.format(docker_build_cli)) info = ub.cmd(docker_build_cli, verbose=3, shell=True) if info['ret'] != 0: print(ub.color_text('\n--- FAILURE ---', 'red')) print('Failed command:') print(info['command']) print(info['err']) print('NOTE: sometimes reruning the command manually works') raise Exception('Building docker failed with exit code {}'.format(info['ret'])) else: # write out what the tag is with open(join(dpath, 'opencv-docker-tag.txt'), 'w') as file: file.write(DOCKER_TAG) print(ub.color_text('\n--- SUCCESS ---', 'green'))
def test_list_of_numpy(): import numpy as np import ubelt as ub data = [ np.zeros((3, 3), dtype=np.int32), np.zeros((3, 10), dtype=np.int32), np.zeros((3, 20), dtype=np.int32), np.zeros((3, 30), dtype=np.int32), ] string = ub.repr2(data, nl=2) print(string) assert repr(data) == repr(eval(string)), 'should produce eval-able code' assert string == ub.codeblock(''' [ np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), ] ''') string = ub.repr2(data, max_line_width=10000, nl=2) print(string) assert string == ub.codeblock(''' [ np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), ] ''') string = ub.repr2(data, nl=1) print(string) assert string == ub.codeblock(''' [ np.array([[0, 0, 0],[0, 0, 0],[0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), ] ''') string = ub.repr2(data, nl=0) print(string) assert string == ub.codeblock(''' [np.array([[0, 0, 0],[0, 0, 0],[0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32), np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0],[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int32)] ''')
def export_model_code(dpath, model, initkw=None, export_modules=[]): """ Exports the class used to define a pytorch model as a new python module. Exports the minimum amount of code needed to make a self-contained Python module defining the pytorch model class. This exports the actual source code. The advantage of using this over pickle is that the original code can change arbitrarilly because all dependencies on the original code are removed in the exported code. Args: dpath (str): directory to dump the model model (tuple or type or object): class or class instance (e.g. torch.nn.Module) name (str): name to use for the file (defaults to the classname) initkw (dict): if specified, creates the function `make`, which initializes the network with the specific arguments. export_modules (List[str]): A list of modules that the exported code should not depend on. Any code referenced from these modules will be statically extracted and copied into the model definition. Note that this feature is experimental. Returns: str: static_modpath: path to the saved model file. While you could put the output path in your PYTHONPATH, it is best to use `ub.import_module_from_path` to "load" the model instead. Example: >>> from torch_liberator.exporter import export_model_code >>> from torchvision.models import densenet >>> import torchvision >>> from os.path import basename >>> initkw = {'growth_rate': 16} >>> model = densenet.DenseNet(**initkw) >>> dpath = ub.ensure_app_cache_dir('torch_liberator/tests') >>> static_modpath = export_model_code(dpath, model, initkw) >>> print('static_modpath = {!r}'.format(static_modpath)) ... >>> mod_fname = (basename(static_modpath)) >>> print('mod_fname = {!r}'.format(mod_fname)) >>> if torchvision.__version__ == '0.2.2': >>> if six.PY2: >>> assert mod_fname == 'DenseNet_b7ec43.py', 'got={}'.format(mod_fname) >>> else: >>> assert mod_fname == 'DenseNet_256629.py', 'got={}'.format(mod_fname) >>> # now the module can be loaded >>> module = ub.import_module_from_path(static_modpath) >>> loaded = module.make() >>> assert model.features.denseblock1.denselayer1.conv2.out_channels == 16 >>> assert loaded.features.denseblock1.denselayer1.conv2.out_channels == 16 >>> assert model is not loaded """ if isinstance(model, type): model_class = model else: model_class = model.__class__ classname = model_class.__name__ if initkw is None: raise NotImplementedError( 'ERROR: The params passed to the model __init__ must be available') footer = '' else: # First see if we can get away with a simple encoding of initkw try: # Do not use repr. The text produced is non-deterministic for # dictionaries. Instead, use ub.repr2, which is deterministic. init_text = ub.repr2(initkw, nl=1) eval(init_text, {}) init_code = ub.codeblock('initkw = {}').format(init_text) except Exception: # fallback to pickle warnings.warn('Initialization params might not be serialized ' 'deterministically') init_bytes = repr(pickle.dumps(initkw, protocol=0)) init_code = ub.codeblock(''' import pickle initkw = pickle.loads({}) ''').format(init_bytes) init_code = ub.indent(init_code).lstrip() # create a function to instanciate the class footer = '\n\n' + ub.codeblock(''' __pt_export_version__ = '{__pt_export_version__}' def get_initkw(): """ creates an instance of the model """ {init_code} return initkw def get_model_cls(): model_cls = {classname} return model_cls def make(): """ creates an instance of the model """ initkw = get_initkw() model_cls = get_model_cls() model = model_cls(**initkw) return model ''').format(classname=classname, init_code=init_code, __pt_export_version__=__pt_export_version__) # TODO: assert that the name "make" is not used in the model body body = closer.source_closure(model_class, expand_names=export_modules) body_footer = body + footer + '\n' # dont need to hash the header, because comments are removed anyway # with open('debug-closer.py', 'w') as file: # file.write(body_footer) hashid = hash_code(body_footer) header = ub.codeblock(''' """ This module was autogenerated by torch_liberator/export/exporter.py original_module={} classname={} timestamp={} hashid={} """ ''').format(model_class.__module__, classname, ub.timestamp(), hashid) sourcecode = header + '\n' + body_footer static_modname = classname + '_' + hashid[0:6] static_modpath = join(dpath, static_modname + '.py') with open(static_modpath, 'w') as file: file.write(sourcecode) return static_modpath
INITD_TEMPLATE = ub.codeblock( ''' #!/bin/sh ### BEGIN INIT INFO # Provides: {name} # Required-Start: $local_fs $network # Required-Stop: $local_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: custom script # Description: custom script ### END INIT INFO PIDFILE="/var/run/{name}.pid" NAME="{name}" DAEMON="{fpath}" # Return 0, process already started. # Return 1, start cpu_temp do_start() {{ if [ -f $PIDFILE ]; then return 0 fi $DAEMON & echo "$!" > $PIDFILE return 1 }} # Return 0, process not started. # Return 1, kill process do_stop() {{ if [ ! -f $PIDFILE ]; then return 0 fi kill -9 `cat $PIDFILE` rm $PIDFILE return 1 }} case "$1" in start) do_start case "$?" in 0) echo "$NAME already started." ;; 1) echo "Started $NAME." ;; esac ;; stop) do_stop case "$?" in 0) echo "$NAME has not started." ;; 1) echo "Killed $NAME." ;; esac ;; status) if [ ! -r "$PIDFILE" ]; then echo "$NAME is not running." exit 3 fi if read pid < "$PIDFILE" && ps -p "$pid" > /dev/null 2>&1; then echo "$NAME is running." exit 0 else echo "$NAME is not running but $PIDFILE exists." exit 1 fi ;; *) N=/etc/init.d/$NAME echo "Usage: $N {{start|stop|status}}" >&2 exit 1 ;; esac exit 0 ''')
def run_pvpoke_ultra_experiment(): """ https://pvpoke.com/battle/matrix/ !pip install selenium """ """ Relevant page items: <button class="add-poke-btn button">+ Add Pokemon</button> '//*[@id="main"]/div[3]/div[3]/div/div[1]/button[1]' '/html/body/div[1]/div/div[3]/div[3]/div/div[1]/button[1]' <input class="poke-search" type="text" placeholder="Search name"> /html/body/div[5]/div/div[3]/div[1]/input /html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1] Level Cap /html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5] # IV GROUP ivs-group save-poke import sys, ubelt sys.path.append(ubelt.expandpath('~/code/pypogo')) from pypogo.pvpoke_experiment import * # NOQA from pypogo.pvpoke_experiment import _oldstuff """ from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import Select import ubelt as ub import os import pathlib import time import pandas as pd import pypogo # Requires the driver be in the PATH fpath = ensure_selenium_chromedriver() os.environ['PATH'] = os.pathsep.join( ub.oset(os.environ['PATH'].split(os.pathsep)) | ub.oset([str(fpath.parent)])) url = 'https://pvpoke.com/battle/matrix/' # chrome_exe = ub.find_exe("google-chrome") driver = webdriver.Chrome() driver.get(url) league = 'Great' # league = 'Master40' if league == 'Great': league_box_target = 'Great League (CP 1500)' have_ivs = list( ub.oset([ tuple([int(x) for x in p.strip().split(',') if x]) for p in ub.codeblock(''' 10, 10, 12, 10, 12, 14, 10, 12, 14, 10, 13, 10, 10, 13, 12, 10, 14, 14, 11, 12, 14, 11, 14, 12, 11, 14, 15, 11, 15, 11, 11, 15, 11, 11, 15, 12, 11, 15, 12, 12, 10, 12, 12, 11, 12, 12, 12, 15, 12, 14, 11, 12, 14, 15, 12, 15, 11, 12, 15, 12 12, 15, 12, 13, 11, 13 13, 12, 10 13, 12, 13, 13, 13, 10, 13, 13, 11, 13, 15, 10, 13, 15, 11, 13, 15, 11, 14, 10, 12, 14, 11, 10, 14, 11, 10, 14, 13, 11 14, 13, 14, 15, 10, 12 15, 11, 10, 15, 11, 11, 15, 12, 11 ''').split('\n') ])) to_check_mons = [ pypogo.Pokemon('Deoxys', form='defense', ivs=ivs, moves=['Counter', 'Rock Slide', 'Psycho Boost']).maximize(1500) for ivs in have_ivs ] meta_text = 'Great League Meta' elif league == 'Master40': league_box_target = 'Master League (Level 40)' meta_text = 'Master League Meta' # Test the effect of best buddies vs the master league to_check_mons = [ pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40), pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41), pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=40), pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=41), pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=40), pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=41), pypogo.Pokemon('Giratina', form='origin', ivs=[15, 14, 15], level=40), pypogo.Pokemon('Giratina', form='origin', ivs=[15, 14, 15], level=41), pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=40), pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=41), pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=40), pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=41), pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=40), pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=41), ] for mon in to_check_mons: mon.populate_all() else: pass leage_select = driver.find_elements_by_class_name('league-select')[0] leage_select.click() leage_select.send_keys(league_box_target) leage_select.click() leage_select.text.split('\n') leage_select.send_keys('\n') leage_select.send_keys('\n') def add_pokemon(mon): add_poke1_button = driver.find_elements_by_class_name( 'add-poke-btn')[0] add_poke1_button.click() select_drop = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/select') if 1: import xdev all_names = select_drop.text.split('\n') distances = xdev.edit_distance(mon.display_name(), all_names) chosen_name = all_names[ub.argmin(distances)] else: chosen_name = mon.name search_box = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/input') search_box.send_keys(chosen_name) advanced_ivs_arrow = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1]') advanced_ivs_arrow.click() level40_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[2]' ) level41_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[3]' ) level50_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[4]' ) level51_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5]' ) if mon.level >= 51: level51_cap.click() elif mon.level >= 50: level50_cap.click() elif mon.level >= 41: level41_cap.click() elif mon.level >= 40: level40_cap.click() level_box = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/input' ) level_box.click() level_box.clear() level_box.clear() level_box.send_keys(str(mon.level)) iv_a = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[1]' ) iv_d = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[2]' ) iv_s = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[3]' ) # TODO # driver.find_elements_by_class_name('move-select') iv_a.clear() iv_a.send_keys(str(mon.ivs[0])) iv_d.clear() iv_d.send_keys(str(mon.ivs[1])) iv_s.clear() iv_s.send_keys(str(mon.ivs[2])) # USE_MOVES = 1 if mon.moves is not None: # mon.populate_all() fast_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[1]') fast_select.click() fast_select.send_keys(mon.pvp_fast_move['name']) fast_select.send_keys(Keys.ENTER) charge1_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[2]') charge1_select.click() charge1_select.send_keys(mon.pvp_charge_moves[0]['name']) charge1_select.send_keys(Keys.ENTER) charge2_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[3]') charge2_select.click() charge2_select.send_keys(mon.pvp_charge_moves[1]['name']) charge2_select.send_keys(Keys.ENTER) save_button = driver.find_elements_by_class_name('save-poke')[0] save_button.click() quickfills = driver.find_elements_by_class_name('quick-fill-select') quickfill = quickfills[1] quickfill.text.split('\n') quickfill.click() quickfill.send_keys(meta_text) quickfill.click() import pypogo # mon1 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40) # mon2 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41) if 1: for mon in to_check_mons: pass add_pokemon(mon) shield_selectors = driver.find_elements_by_class_name('shield-select') shield_selectors[2].click() shield_selectors[2].send_keys('No shields') shield_selectors[2].send_keys(Keys.ENTER) shield_selectors[3].click() shield_selectors[3].send_keys('No shields') shield_selectors[3].send_keys(Keys.ENTER) shield_selectors[0].click() battle_btn = driver.find_elements_by_class_name('battle-btn')[0] battle_btn.click() # Clear previous downloaded files dlfolder = pathlib.Path(ub.expandpath('$HOME/Downloads')) for old_fpath in list(dlfolder.glob('_vs*.csv')): old_fpath.unlink() time.sleep(2.0) # Download new data dl_btn = driver.find_element_by_xpath( '//*[@id="main"]/div[4]/div[9]/div/a') dl_btn.click() while len(list(dlfolder.glob('_vs*.csv'))) < 1: pass new_fpaths = list(dlfolder.glob('_vs*.csv')) assert len(new_fpaths) == 1 fpath = new_fpaths[0] data = pd.read_csv(fpath, header=0, index_col=0) if 1: # GROUP ANALYSIS data.sum(axis=1).sort_values() (data > 500).sum(axis=1).sort_values() flipped = [] for key, col in data.T.iterrows(): if not ub.allsame(col > 500): flipped.append(key) flip_df = data.loc[:, flipped] def color(x): if x > 500: return ub.color_text(str(x), 'green') else: return ub.color_text(str(x), 'red') print(flip_df.applymap(color)) print(flip_df.columns.tolist()) (data > 500) else: # PAIR ANALYSIS pairs = list(ub.iter_window(range(len(data)), step=2)) for i, j in pairs: print('-----') matchup0 = data.iloc[i] matchup1 = data.iloc[j] delta = matchup1 - matchup0 print(delta[delta != 0]) wins0 = matchup0 > 500 wins1 = matchup1 > 500 flips = (wins0 != wins1) flipped_vs = matchup0.index[flips] num_flips = sum(flips) print('flipped_vs = {!r}'.format(flipped_vs)) print('num_flips = {!r}'.format(num_flips)) print(matchup0.mean()) print(matchup1.mean()) print(matchup1.mean() / matchup0.mean())
def build_pyproject(self): # data = toml.loads((self.template_dpath / 'pyproject.toml').read_text()) # print('data = {}'.format(ub.repr2(data, nl=5))) pyproj_config = ub.AutoDict() # {'tool': {}} if 'binpy' in self.config['tags']: pyproj_config['build-system']['requires'] = [ "setuptools>=41.0.1", # setuptools_scm[toml] "wheel", "scikit-build>=0.9.0", "numpy", "ninja" ] pyproj_config['tool']['cibuildwheel'].update({ 'build': "cp37-* cp38-* cp39-* cp310-*", 'build-frontend': "build", 'skip': "pp* cp27-* cp34-* cp35-* cp36-* *-musllinux_*", 'build-verbosity': 1, 'test-requires': ["-r requirements/tests.txt"], 'test-command': "python {project}/run_tests.py" }) if True: cibw = pyproj_config['tool']['cibuildwheel'] req_commands = { 'linux': [ 'yum install epel-release lz4 lz4-devel -y', ], 'windows': [ 'choco install lz4 -y', ], 'macos': [ 'brew install lz4', ] } for plat in req_commands.keys(): cmd = ' && '.join(req_commands[plat]) cibw[plat]['before-all'] = cmd WITH_PYTEST_INI = 1 if WITH_PYTEST_INI: pytest_ini_opts = pyproj_config['tool']['pytest']['ini_options'] pytest_ini_opts[ 'addopts'] = "-p no:doctest --xdoctest --xdoctest-style=google --ignore-glob=setup.py" pytest_ini_opts[ 'norecursedirs'] = ".git ignore build __pycache__ dev _skbuild" pytest_ini_opts['filterwarnings'] = [ "default", "ignore:.*No cfgstr given in Cacher constructor or call.*:Warning", "ignore:.*Define the __nice__ method for.*:Warning", "ignore:.*private pytest class or function.*:Warning", ] WITH_COVERAGE = 1 if WITH_COVERAGE: pyproj_config['tool']['coverage'].update( toml.loads( ub.codeblock(''' [run] branch = true [report] exclude_lines =[ "pragma: no cover", ".* # pragma: no cover", ".* # nocover", "def __repr__", "raise AssertionError", "raise NotImplementedError", "if 0:", "if trace is not None", "verbose = .*", "^ *raise", "^ *pass *$", "if _debug:", "if __name__ == .__main__.:", ".*if six.PY2:" ] omit=[ "{REPO_NAME}/__main__.py", "*/setup.py" ] ''').format(REPO_NAME=self.repo_name))) text = toml.dumps(pyproj_config) return text
def get_func_sourcecode(func, strip_def=False, strip_ret=False, strip_docstr=False, strip_comments=False, remove_linenums=None, strip_decor=False): """ wrapper around inspect.getsource but takes into account utool decorators strip flags are very hacky as of now Args: func (function): strip_def (bool): strip_ret (bool): (default = False) strip_docstr (bool): (default = False) strip_comments (bool): (default = False) remove_linenums (None): (default = None) Example: >>> # build test data >>> func = get_func_sourcecode >>> strip_def = True >>> strip_ret = True >>> sourcecode = get_func_sourcecode(func, strip_def) >>> print('sourcecode = {}'.format(sourcecode)) """ inspect.linecache.clearcache() # HACK: fix inspect bug sourcefile = inspect.getsourcefile(func) if hasattr(func, '_utinfo'): # DEPRICATE func2 = func._utinfo['orig_func'] sourcecode = get_func_sourcecode(func2) elif sourcefile is not None and (sourcefile != '<string>'): try_limit = 2 for num_tries in range(try_limit): try: #print(func) sourcecode = inspect.getsource(func) if not isinstance(sourcecode, six.text_type): sourcecode = sourcecode.decode('utf-8') #print(sourcecode) except (IndexError, OSError, SyntaxError): print('WARNING: Error getting source') inspect.linecache.clearcache() if num_tries + 1 != try_limit: tries_left = try_limit - num_tries - 1 print('Attempting %d more time(s)' % (tries_left)) else: raise else: sourcecode = None if strip_def: # hacky # TODO: use redbaron or something like that for a more robust appraoch sourcecode = textwrap.dedent(sourcecode) regex_decor = '^@.' + REGEX_NONGREEDY regex_defline = '^def [^:]*\\):\n' patern = '(' + regex_decor + ')?' + regex_defline RE_FLAGS = re.MULTILINE | re.DOTALL RE_KWARGS = {'flags': RE_FLAGS} nodef_source = re.sub(patern, '', sourcecode, **RE_KWARGS) sourcecode = textwrap.dedent(nodef_source) #print(sourcecode) pass if strip_ret: r""" \s is a whitespace char """ return_ = named_field('return', 'return .*$') prereturn = named_field('prereturn', r'^\s*') return_bref = bref_field('return') prereturn_bref = bref_field('prereturn') regex = prereturn + return_ repl = prereturn_bref + 'pass # ' + return_bref sourcecode_ = re.sub(regex, repl, sourcecode, flags=re.MULTILINE) sourcecode = sourcecode_ pass if strip_docstr or strip_comments: # pip install pyminifier # References: http://code.activestate.com/recipes/576704/ #from pyminifier import minification, token_utils def remove_docstrings_or_comments(source): """ TODO: commit clean version to pyminifier """ import tokenize from six.moves import StringIO io_obj = StringIO(source) out = '' prev_toktype = tokenize.INDENT last_lineno = -1 last_col = 0 for tok in tokenize.generate_tokens(io_obj.readline): token_type = tok[0] token_string = tok[1] start_line, start_col = tok[2] end_line, end_col = tok[3] if start_line > last_lineno: last_col = 0 if start_col > last_col: out += (' ' * (start_col - last_col)) # Remove comments: if strip_comments and token_type == tokenize.COMMENT: pass elif strip_docstr and token_type == tokenize.STRING: if prev_toktype != tokenize.INDENT: # This is likely a docstring; double-check we're not inside an operator: if prev_toktype != tokenize.NEWLINE: if start_col > 0: out += token_string else: out += token_string prev_toktype = token_type last_col = end_col last_lineno = end_line return out sourcecode = remove_docstrings_or_comments(sourcecode) #sourcecode = minification.remove_comments_and_docstrings(sourcecode) #tokens = token_utils.listified_tokenizer(sourcecode) #minification.remove_comments(tokens) #minification.remove_docstrings(tokens) #token_utils.untokenize(tokens) if strip_decor: try: import redbaron red = redbaron.RedBaron(ub.codeblock(sourcecode)) except Exception: hack_text = ub.ensure_unicode(ub.codeblock(sourcecode)).encode( 'ascii', 'replace') red = redbaron.RedBaron(hack_text) pass if len(red) == 1: redfunc = red[0] if redfunc.type == 'def': # Remove decorators del redfunc.decorators[:] sourcecode = redfunc.dumps() if remove_linenums is not None: source_lines = sourcecode.strip('\n').split('\n') delete_items_by_index(source_lines, remove_linenums) sourcecode = '\n'.join(source_lines) return sourcecode
def main(cls, cmdline=0, **kwargs): """ Ignore: repodir = ub.Path('~/code/pyflann_ibeis').expand() kwargs = { 'repodir': repodir, 'tags': ['binpy', 'erotemic', 'github'], } cmdline = 0 Example: repodir = ub.Path.appdir('pypkg/demo/my_new_repo') import sys, ubelt sys.path.append(ubelt.expandpath('~/misc/templates/PYPKG')) from apply_template import * # NOQA kwargs = { 'repodir': repodir, } cmdline = 0 """ import ubelt as ub config = TemplateConfig(cmdline=cmdline, data=kwargs) repo_dpath = ub.Path(config['repodir']) repo_dpath.ensuredir() IS_NEW_REPO = 0 create_new_repo_info = ub.codeblock(''' # TODO: # At least instructions on how to create a new repo, or maybe an # API call https://github.com/new git init ''') print(create_new_repo_info) if IS_NEW_REPO: # TODO: git init # TODO: github or gitlab register pass self = TemplateApplier(config) self.setup().gather_tasks() self.setup().apply() if config['setup_secrets']: setup_secrets_fpath = self.repo_dpath / 'dev/setup_secrets.sh' if 'erotemic' in self.config['tags']: environ_export = 'setup_package_environs_github_erotemic' upload_secret_cmd = 'upload_github_secrets' elif 'pyutils' in self.config['tags']: environ_export = 'setup_package_environs_github_pyutils' upload_secret_cmd = 'upload_github_secrets' elif 'kitware' in self.config['tags']: environ_export = 'setup_package_environs_gitlab_kitware' upload_secret_cmd = 'upload_gitlab_repo_secrets' else: raise Exception import cmd_queue script = cmd_queue.Queue.create() script.submit( ub.codeblock(f''' cd {self.repo_dpath} source {setup_secrets_fpath} {environ_export} load_secrets export_encrypted_code_signing_keys git commit -am "Updated secrets" {upload_secret_cmd} ''')) script.rprint()
#!/usr/bin/env python """ Looks at the `git_*.{sh,ph}` scripts and makes corresponding `git-*` scripts """ import glob from os.path import dirname, join, basename, splitext import ubelt as ub SCRIPT_HEADER = ub.codeblock(r''' #!/bin/bash # References: # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" ''') SCRIPT_FOOTER_FMT = '$DIR/../{fname} "$@"' def setup_git_scripts(): dpath = dirname(__file__) git_sh_scripts = list(glob.glob(join(dpath, 'git_*.sh'))) git_py_scripts = list(glob.glob(join(dpath, 'git_*.py')))
def main(cls, cmdline=False, **kwargs): config = cls(cmdline=cmdline, data=kwargs) print(ub.codeblock(config['text']))
def make_baseline_truthfiles(): work_dir = ub.truepath('~/work') data_dir = ub.truepath('~/data') challenge_data_dir = join(data_dir, 'viame-challenge-2018') challenge_work_dir = join(work_dir, 'viame-challenge-2018') ub.ensuredir(challenge_work_dir) img_root = join(challenge_data_dir, 'phase0-imagery') annot_dir = join(challenge_data_dir, 'phase0-annotations') fpaths = list(glob.glob(join(annot_dir, '*.json'))) # ignore the non-bounding box nwfsc and afsc datasets for now # exclude = ('nwfsc', 'afsc', 'mouss', 'habcam') # exclude = ('nwfsc', 'afsc', 'mouss',) # fpaths = [p for p in fpaths if not basename(p).startswith(exclude)] import json dsets = ub.odict() for fpath in fpaths: key = basename(fpath).split('.')[0] dsets[key] = json.load(open(fpath, 'r')) print('Merging') merged = coco_union(dsets) merged_fpath = join(challenge_work_dir, 'phase0-merged.mscoco.json') with open(merged_fpath, 'w') as fp: json.dump(merged, fp, indent=4) import copy self = CocoDataset(copy.deepcopy(merged), img_root=img_root, autobuild=False) self._build_index() self.run_fixes() if True: # remove all point annotations print('Remove point annotations') to_remove = [] for ann in self.dataset['annotations']: if ann['roi_shape'] == 'point': to_remove.append(ann) for ann in to_remove: self.dataset['annotations'].remove(ann) self._build_index() # remove empty images print('Remove empty images') to_remove = [] for gid in self.imgs.keys(): aids = self.gid_to_aids.get(gid, []) if not aids: to_remove.append(self.imgs[gid]) for img in to_remove: self.dataset['images'].remove(img) self._build_index() print('# self.anns = {!r}'.format(len(self.anns))) print('# self.imgs = {!r}'.format(len(self.imgs))) print('# self.cats = {!r}'.format(len(self.cats))) catname_to_nannots = ub.map_keys(lambda x: self.cats[x]['name'], ub.map_vals(len, self.cid_to_aids)) catname_to_nannots = ub.odict(sorted(catname_to_nannots.items(), key=lambda kv: kv[1])) print(ub.repr2(catname_to_nannots)) if False: # aid = list(self.anns.values())[0]['id'] # self.show_annotation(aid) gids = sorted([gid for gid, aids in self.gid_to_aids.items() if aids]) # import utool as ut # for gid in ut.InteractiveIter(gids): for gid in gids: from matplotlib import pyplot as plt fig = plt.figure(1) fig.clf() self.show_annotation(gid=gid) fig.canvas.draw() for ann in self.anns.values(): primary_aid = ann['id'] print('primary_aid = {!r}'.format(primary_aid)) print(len(self.gid_to_aids[ann['image_id']])) if 'roi_shape' not in ann: ann['roi_shape'] = 'bounding_box' if ann['roi_shape'] == 'boundingBox': pass if ann['roi_shape'] == 'point': primary_aid = ann['id'] print('primary_aid = {!r}'.format(primary_aid)) print(len(self.gid_to_aids[ann['image_id']])) break # Split into train / test set print('Splitting') skf = StratifiedGroupKFold(n_splits=2) groups = [ann['image_id'] for ann in self.anns.values()] y = [ann['category_id'] for ann in self.anns.values()] X = [ann['id'] for ann in self.anns.values()] split = list(skf.split(X=X, y=y, groups=groups))[0] train_idx, test_idx = split print('Taking subsets') aid_to_gid = {aid: ann['image_id'] for aid, ann in self.anns.items()} train_aids = list(ub.take(X, train_idx)) test_aids = list(ub.take(X, test_idx)) train_gids = sorted(set(ub.take(aid_to_gid, train_aids))) test_gids = sorted(set(ub.take(aid_to_gid, test_aids))) train_dset = self.subset(train_gids) test_dset = self.subset(test_gids) print('---------') print('# train_dset.anns = {!r}'.format(len(train_dset.anns))) print('# train_dset.imgs = {!r}'.format(len(train_dset.imgs))) print('# train_dset.cats = {!r}'.format(len(train_dset.cats))) print('---------') print('# test_dset.anns = {!r}'.format(len(test_dset.anns))) print('# test_dset.imgs = {!r}'.format(len(test_dset.imgs))) print('# test_dset.cats = {!r}'.format(len(test_dset.cats))) train_dset._ensure_imgsize() test_dset._ensure_imgsize() print('Writing') with open(join(challenge_work_dir, 'phase0-merged-train.mscoco.json'), 'w') as fp: json.dump(train_dset.dataset, fp, indent=4) with open(join(challenge_work_dir, 'phase0-merged-test.mscoco.json'), 'w') as fp: json.dump(test_dset.dataset, fp, indent=4) # Make a detectron yaml file config_text = ub.codeblock( """ MODEL: TYPE: generalized_rcnn CONV_BODY: ResNet.add_ResNet50_conv4_body NUM_CLASSES: {num_classes} FASTER_RCNN: True NUM_GPUS: 1 SOLVER: WEIGHT_DECAY: 0.0001 LR_POLICY: steps_with_decay BASE_LR: 0.01 GAMMA: 0.1 # 1x schedule (note TRAIN.IMS_PER_BATCH: 1) MAX_ITER: 180000 STEPS: [0, 120000, 160000] RPN: SIZES: (32, 64, 128, 256, 512) FAST_RCNN: ROI_BOX_HEAD: ResNet.add_ResNet_roi_conv5_head ROI_XFORM_METHOD: RoIAlign TRAIN: WEIGHTS: https://s3-us-west-2.amazonaws.com/detectron/ImageNetPretrained/MSRA/R-50.pkl DATASETS: ('/work/viame-challenge-2018/phase0-merged-train.mscoco.json',) IM_DIR: '/data/viame-challenge-2018/phase0-imagery' SCALES: (800,) MAX_SIZE: 1333 IMS_PER_BATCH: 1 BATCH_SIZE_PER_IM: 512 TEST: DATASETS: ('/work/viame-challenge-2018/phase0-merged-test.mscoco.json',) IM_DIR: '/data/viame-challenge-2018/phase0-imagery' SCALES: (800,) MAX_SIZE: 1333 NMS: 0.5 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 1000 OUTPUT_DIR: /work/viame-challenge-2018/output """) config_text = config_text.format( num_classes=len(self.cats), ) ub.writeto(join(challenge_work_dir, 'phase0-faster-rcnn.yaml'), config_text) docker_cmd = ('nvidia-docker run ' '-v {work_dir}:/work -v {data_dir}:/data ' '-it detectron:c2-cuda9-cudnn7 bash').format( work_dir=work_dir, data_dir=data_dir) train_cmd = ('python2 tools/train_net.py ' '--cfg /work/viame-challenge-2018/phase0-faster-rcnn.yaml ' 'OUTPUT_DIR /work/viame-challenge-2018/output') hacks = ub.codeblock( """ git remote add Erotemic https://github.com/Erotemic/Detectron.git git fetch --all git checkout general_dataset # curl https://github.com/Erotemic/Detectron/blob/42d44b2d155c775dc509b6a44518d0c582f8cdf5/tools/train_net.py # wget https://github.com/Erotemic/Detectron/blob/42d44b2d155c775dc509b6a44518d0c582f8cdf5/lib/core/config.py """) print(docker_cmd) print(train_cmd)
def compare_results(): print('Comparing results') import pandas as pd from tabulate import tabulate # Read in output of demo script measure_fpath = 'measurements_haul83.csv' py_df = pd.DataFrame.from_csv(measure_fpath, index_col=None) # Convert python length output from mm into cm for consistency py_df['fishlen'] = py_df['fishlen'] / 10 py_df['current_frame'] = py_df['current_frame'].astype(np.int) # janky CSV parsing py_df['box_pts1'] = py_df['box_pts1'].map( lambda p: eval(p.replace(';', ','), np.__dict__)) py_df['box_pts2'] = py_df['box_pts2'].map( lambda p: eval(p.replace(';', ','), np.__dict__)) py_df['obox1'] = [ ctalgo.OrientedBBox(*cv2.minAreaRect(pts[:, None, :].astype(np.int))) for pts in py_df['box_pts1'] ] py_df['obox2'] = [ ctalgo.OrientedBBox(*cv2.minAreaRect(pts[:, None, :].astype(np.int))) for pts in py_df['box_pts2'] ] py_df.drop(['box_pts1', 'box_pts2'], axis=1, inplace=True) # Remap to matlab names py_df = py_df.rename(columns={ 'error': 'Err', 'fishlen': 'fishLength', 'range': 'fishRange', }) # Load matlab results mat_df = _read_kresimir_results() FORCE_COMPARABLE_RANGE = True # FORCE_COMPARABLE_RANGE = False if FORCE_COMPARABLE_RANGE: # Be absolutely certain we are in comparable regions (may slightly bias # results, against python and in favor of matlab) min_frame = max(mat_df.current_frame.min(), py_df.current_frame.min()) max_frame = min(mat_df.current_frame.max(), py_df.current_frame.max()) print('min_frame = {!r}'.format(min_frame)) print('max_frame = {!r}'.format(max_frame)) mat_df = mat_df[(mat_df.current_frame >= min_frame) & (mat_df.current_frame <= max_frame)] py_df = py_df[(py_df.current_frame >= min_frame) & (py_df.current_frame <= max_frame)] intersect_frames = np.intersect1d(mat_df.current_frame, py_df.current_frame) print('intersecting frames = {} / {} (matlab)'.format( len(intersect_frames), len(set(mat_df.current_frame)))) print('intersecting frames = {} / {} (python)'.format( len(intersect_frames), len(set(py_df.current_frame)))) # Reuse the hungarian algorithm implementation from ctalgo min_assign = ctalgo.FishStereoMeasurments.minimum_weight_assignment correspond = [] for f in intersect_frames: pidxs = np.where(py_df.current_frame == f)[0] midxs = np.where(mat_df.current_frame == f)[0] pdf = py_df.iloc[pidxs] mdf = mat_df.iloc[midxs] ppts1 = np.array([o.center for o in pdf['obox1']]) mpts1 = np.array([o.center for o in mdf['obox1']]) ppts2 = np.array([o.center for o in pdf['obox2']]) mpts2 = np.array([o.center for o in mdf['obox2']]) dists1 = sklearn.metrics.pairwise.pairwise_distances(ppts1, mpts1) dists2 = sklearn.metrics.pairwise.pairwise_distances(ppts2, mpts2) # arbitrarilly chosen threshold thresh = 100 for i, j in min_assign(dists1): d1 = dists1[i, j] d2 = dists2[i, j] if d1 < thresh and d2 < thresh and abs(d1 - d2) < thresh / 4: correspond.append((pidxs[i], midxs[j])) correspond = np.array(correspond) # pflags = np.array(ub.boolmask(correspond.T[0], len(py_df))) mflags = np.array(ub.boolmask(correspond.T[1], len(mat_df))) # print('there are {} detections that seem to be in common'.format(len(correspond))) # print('The QC flags of the common detections are: {}'.format( # ub.dict_hist(mat_df[mflags]['QC'].values))) # print('The QC flags of the other matlab detections are: {}'.format( # ub.dict_hist(mat_df[~mflags]['QC'].values))) print('\n\n----\n## All stats\n') print( ub.codeblock(''' Overall, the matlab script made {nmat} length measurements and the python script made {npy} length measurements. Here is a table summarizing the average lengths / ranges / errors of each script: ''').format(npy=len(py_df), nmat=len(mat_df))) stats = pd.DataFrame(columns=['python', 'matlab']) for key in ['fishLength', 'fishRange', 'Err']: stats.loc[key, 'python'] = '{:6.2f} ± {:6.2f}'.format( py_df[key].mean(), py_df[key].std()) stats.loc[key, 'matlab'] = '{:6.2f} ± {:6.2f}'.format( mat_df[key].mean(), mat_df[key].std()) stats.loc['nTotal', 'python'] = '{}'.format(len(py_df)) stats.loc['nTotal', 'matlab'] = '{}'.format(len(mat_df)) print(tabulate(stats, headers='keys', tablefmt='psql', stralign='right')) print('\n\n----\n## Only COMMON detections\n') py_df_c = py_df.iloc[correspond.T[0]] mat_df_c = mat_df.iloc[correspond.T[1]] stats = pd.DataFrame(columns=['python', 'matlab']) for key in ['fishLength', 'fishRange', 'Err']: stats.loc[key, 'python'] = '{:6.2f} ± {:6.2f}'.format( py_df_c[key].mean(), py_df_c[key].std()) stats.loc[key, 'matlab'] = '{:6.2f} ± {:6.2f}'.format( mat_df_c[key].mean(), mat_df_c[key].std()) stats.loc['nTotal', 'python'] = '{}'.format(len(py_df_c)) stats.loc['nTotal', 'matlab'] = '{}'.format(len(mat_df_c)) print( ub.codeblock(''' Now, we investigate how many dections matlab and python made in common. (Note, choosing which dections in one version correspond to which in another is done using a heuristic based on distances between bbox centers and a thresholded minimum assignment problem). Python made {npy_c}/{nmat} = {percent:.2f}% of the detections matlab made ''').format(npy_c=len(py_df_c), nmat=len(mat_df), percent=100 * len(py_df_c) / len(mat_df))) print(tabulate(stats, headers='keys', tablefmt='psql', stralign='right')) print('\n\n----\n## Evaulation using the QC code\n') hist_hit = ub.dict_hist(mat_df[mflags]['QC'].values) hist_miss = ub.dict_hist(mat_df[~mflags]['QC'].values) print( ub.codeblock(''' However, not all of those matlab detections were good. Because we have detections in corrsepondences with each other we can assign the python detections QC codes. Here is a histogram of the QC codes for these python detections: {} (Note: read histogram as <QC-code>: <frequency>) Here is a histogram of the other matlab detections that python did not find: {} To summarize: python correctly rejected {:.2f}% of the matlab QC=0 detections python correctly accepted {:.2f}% of the matlab QC=1 detections python correctly accepted {:.2f}% of the matlab QC=2 detections Note, that because python made detections that matlab did not make, the remaining {} detections may be right or wrong, but there is no way to tell from this analysis. Lastly, here are the statistics for the common detections that had a non-zero QC code. ''').format(ub.repr2(hist_hit, nl=1), ub.repr2(hist_miss, nl=1), 100 * hist_miss[0] / (hist_hit[0] + hist_miss[0]), 100 * hist_hit[1] / (hist_hit[1] + hist_miss[1]), 100 * hist_hit[2] / (hist_hit[2] + hist_miss[2]), len(py_df) - len(py_df_c))) is_qc = (mat_df_c['QC'] > 0).values mat_df_c = mat_df_c[is_qc] py_df_c = py_df_c[is_qc] stats = pd.DataFrame(columns=['python', 'matlab']) for key in ['fishLength', 'fishRange', 'Err']: stats.loc[key, 'python'] = '{:6.2f} ± {:6.2f}'.format( py_df_c[key].mean(), py_df_c[key].std()) stats.loc[key, 'matlab'] = '{:6.2f} ± {:6.2f}'.format( mat_df_c[key].mean(), mat_df_c[key].std()) stats.loc['nTotal', 'python'] = '{}'.format(len(py_df_c)) stats.loc['nTotal', 'matlab'] = '{}'.format(len(mat_df_c)) print(tabulate(stats, headers='keys', tablefmt='psql', stralign='right'))
def _complete_source(line, state_indent, line_iter): """ helper remove lines from the iterator if they are needed to complete source This uses :func:`static.is_balanced_statement` to do the heavy lifting Example: >>> from xdoctest.parser import * # NOQA >>> from xdoctest.parser import _complete_source >>> state_indent = 0 >>> line = '>>> x = { # The line is not finished' >>> remain_lines = ['>>> 1:2,', '>>> 3:4,', '>>> 5:6}', '>>> y = 7'] >>> line_iter = enumerate(remain_lines, start=1) >>> finished = list(_complete_source(line, state_indent, line_iter)) >>> final = chr(10).join([t[1] for t in finished]) >>> print(final) """ norm_line = line[state_indent:] # Normalize line indentation prefix = norm_line[:4] suffix = norm_line[4:] assert prefix.strip() in {'>>>', '...' }, ('unexpected prefix: {!r}'.format(prefix)) yield line, norm_line source_parts = [suffix] # These hacks actually modify the input doctest slighly HACK_TRIPLE_QUOTE_FIX = True try: while not static.is_balanced_statement(source_parts, only_tokens=True): line_idx, next_line = next(line_iter) norm_line = next_line[state_indent:] prefix = norm_line[:4] suffix = norm_line[4:] if prefix.strip() not in {'>>>', '...', ''}: # nocover error = True if HACK_TRIPLE_QUOTE_FIX: # TODO: make a more robust patch if any("'''" in s or '"""' in s for s in source_parts): # print('HACK FIXING TRIPLE QUOTE') next_line = next_line[:state_indent] + '... ' + norm_line norm_line = '... ' + norm_line prefix = '' suffix = norm_line error = False if error: if DEBUG: print(' * !!!ERROR!!!') print(' * source_parts = {!r}'.format(source_parts)) print(' * prefix = {!r}'.format(prefix)) print(' * norm_line = {!r}'.format(norm_line)) print(' * !!!!!!!!!!!!!') raise SyntaxError( 'Bad indentation in doctest on line {}: {!r}'.format( line_idx, next_line)) source_parts.append(suffix) yield next_line, norm_line except StopIteration: if DEBUG: import ubelt as ub print('<FAIL DID NOT COMPLETE SOURCE>') import traceback tb_text = traceback.format_exc() tb_text = ub.highlight_code(tb_text) tb_text = ub.indent(tb_text) print(tb_text) # print(' * line_iter = {!r}'.format(line_iter)) print(' * state_indent = {!r}'.format(state_indent)) print(' * line = {!r}'.format(line)) # print('source =\n{}'.format('\n'.join(source_parts))) print('# Ensure that the following line should actually fail') print('source_parts = {}'.format(ub.repr2(source_parts, nl=2))) print( ub.codeblock(r''' from xdoctest import static_analysis as static static.is_balanced_statement(source_parts, only_tokens=False) static.is_balanced_statement(source_parts, only_tokens=True) text = '\n'.join(source_parts) print(text) static.six_axt_parse(text) ''')) print('</FAIL DID NOT COMPLETE SOURCE>') # sys.exit(1) # TODO: use AST to reparse all doctest parts to discover where the # syntax error in the doctest is and then raise it. raise exceptions.IncompleteParseError( 'ill-formed doctest: all parts have been processed ' 'but the doctest source is not balanced') else: if DEBUG > 1: import ubelt as ub print('<SUCCESS COMPLETED SOURCE>') # print(' * line_iter = {!r}'.format(line_iter)) print('source_parts = {}'.format(ub.repr2(source_parts, nl=2))) print('</SUCCESS COMPLETED SOURCE>')
def find_chain(head, authors=None, preserve_tags=True, oldest_commit=None): """ Find a chain of commits starting at the HEAD. If `authors` is specified the commits must be from one of these authors. The term chain is used in the graph-theory sense. It is a list of commits where all non-endpoint commits have exactly one parent and one child. TODO: - [ ] allow a chain to include branches if all messages on all branches conform to the chain pattern (e.g. wip) def search(node, current_path): if current_path: pass child_paths = [] for parent in node.parents: path = search(parent, current_path) child_paths.append(path) if len(child_paths) == 0: pass if len(child_paths) == 1: # normal one parent case pass else: pass # Branching case # ACCEPT THE BRANCHING PATHS IF: # * PARENT OF ALL PATHS HAVE A COMMON ENDPOINT # * HANDLE CASE WHERE PATHS OVERLAPS Args: head (git.Commit): starting point authors (set): valid authors preserve_tags (bool, default=True): if True the chain is not allowed to extend past any tags. If a set, then we will not procede past any tag with a name in the set. Example: >>> # assuming you are in a git repo >>> chain = find_chain(git.Repo().head.commit) """ chain = [] commit = head if preserve_tags: tags = head.repo.tags if isinstance(preserve_tags, (set, list, tuple)): tags = {tag for tag in tags if tag.name in preserve_tags} tagged_commits = {tag.commit for tag in tags} while len(commit.parents) <= 1: if authors is not None and commit.author.name not in authors: break if len(commit.parents) == 0: # Hmm it seems that including the initial commit in a chain causes # problems, issue a warning warnings.warn(ub.codeblock( ''' This script contains a known issue, where the initial commit is not included when it "should" be part of a streak. To squash the entire branch, use the following workaround: git branch -m master old_master git checkout --orphan master git commit -am "initial commit" git branch -D old_master ''')) break if oldest_commit is not None: if commit.hexsha.startswith(oldest_commit): break if oldest_commit in tagged_commits: break if preserve_tags: # If we are preserving tags, break the chain once we find one if commit in tagged_commits: break chain.append(commit) if len(commit.parents) > 0: commit = commit.parents[0] else: break return chain