def test_global_color_disable(): """ CommandLine: xdoctest -m /home/joncrall/code/ubelt/tests/test_color.py test_global_color_disable """ import ubelt as ub text = 'text = "hi"' has_color = ub.color_text(text, 'red') != ub.color_text(text, None) text1a = ub.color_text(text, 'red') text1b = ub.highlight_code(text) if has_color: assert text != text1a assert text != text1b # Force colors to be disabled prev = ub.util_colors.NO_COLOR try: ub.util_colors.NO_COLOR = True text2a = ub.color_text(text, 'red') text2b = ub.highlight_code(text) assert text == text2a assert text == text2b finally: # Re-enable coloration ub.util_colors.NO_COLOR = prev text3a = ub.color_text(text, 'red') text3b = ub.highlight_code(text) if has_color: assert text != text3a assert text != text3b
def short_status(repo): r""" Example: >>> repo = Repo(dpath=ub.truepath('.')) >>> result = repo.short_status() >>> print(result) """ import ubelt as ub prefix = repo.dpath with ChdirContext(repo.dpath, verbose=False): info = ub.cmd('git status', verbose=False) out = info['out'] # parse git status is_clean_msg1 = 'Your branch is up-to-date with' is_clean_msgs = [ 'nothing to commit, working directory clean', 'nothing to commit, working tree clean', ] msg2 = 'nothing added to commit but untracked files present' needs_commit_msgs = [ 'Changes to be committed', 'Changes not staged for commit', 'Your branch is ahead of', ] suffix = '' if is_clean_msg1 in out and any(msg in out for msg in is_clean_msgs): suffix += ub.color_text('is clean', 'blue') if msg2 in out: suffix += ub.color_text('has untracked files', 'yellow') if any(msg in out for msg in needs_commit_msgs): suffix += ub.color_text('has changes', 'red') print(prefix + ' ' + suffix)
def message(monitor, ansi=True): """ A status message with optional ANSI coloration Args: ansi (bool, default=True): if False disables ANSI coloration Returns: str: message for logging Example: >>> from netharn.monitor import * >>> monitor = Monitor() >>> print(monitor.message(ansi=False)) vloss is unevaluated >>> monitor.update(0, {'loss': 1.0}) >>> print(monitor.message(ansi=False)) vloss: 1.0000 (n_bad=00, best=1.0000) >>> monitor.update(0, {'loss': 2.0}) >>> print(monitor.message(ansi=False)) vloss: 1.4000 (n_bad=01, best=1.0000) >>> monitor.update(0, {'loss': 0.1}) >>> print(monitor.message(ansi=False)) vloss: 0.8800 (n_bad=00, best=0.8800) """ if not monitor._epochs: message = 'vloss is unevaluated' if ansi: message = ub.color_text(message, 'blue') else: prev_loss = monitor._smooth_metrics[-1]['loss'] best_loss = monitor._best_smooth_metrics['loss'] message = 'vloss: {:.4f} (n_bad={:02d}, best={:.4f})'.format( prev_loss, monitor._n_bad_epochs, best_loss, ) if monitor.patience is None: patience = monitor.max_epoch else: patience = monitor.patience if ansi: if monitor._n_bad_epochs <= int(patience * .25): message = ub.color_text(message, 'green') elif monitor._n_bad_epochs >= int(patience * .75): message = ub.color_text(message, 'red') else: message = ub.color_text(message, 'yellow') return message
def main(): import sys print(ub.color_text(' === TEST MULTIARG CLASSMETHOD ===', 'blue')) print('sys.version_info = {!r}'.format(sys.version_info)) unbound_methods = { # 'classmethod_self_args': MyType.classmethod_self_args, # 'classmethod_args': MyType.classmethod_args, # 'method_self_args': MyType.method_self_args, 'method_args': MyType.method_args, 'compat_method_args': MyType.compat_method_args, } for key, unbound in unbound_methods.items(): print('---') print('key = {!r}'.format(key)) print('unbound = {!r}'.format(unbound)) try: instance = MyType() print('instance = {!r}'.format(instance)) bound = getattr(instance, key) print('bound = {!r}'.format(bound)) print('Call as classmethod with 2 args') unbound(instance, 42) unbound(32, 42) print('Call as bound method with 2 args') bound(instance, 42) bound(32, 42) print('Call as classmethod with 1 args') unbound(instance) unbound(32) print('Call as bound method with 1 args') bound(instance) bound(32) print('Call as classmethod with 0 args') unbound() print('Call as bound method with 0 args') bound() except Exception as ex: print(ub.color_text('XXX ex = {!r}'.format(ex), 'red')) else: print(ub.color_text('OOO worked', 'green'))
def _ldd_tree(fpath, depth=0): prefix = '│ ' * (depth) + '├──' if fpath in ldd_seen: # if 'python' in fpath or 'py' in basename(fpath): # print(ub.color_text(prefix + fpath + ' [SEEN]', 'red')) # else: # print(prefix + fpath + ' [SEEN]') raise StopIteration() else: if print_tree: if 'python' in fpath or 'py' in basename(fpath): print(ub.color_text(prefix + fpath, 'red')) else: print(prefix + fpath) if not exists(fpath): yield fpath + ' (not found)' # print('ERROR: does not exist fpath = {!r}'.format(fpath)) ldd_seen[fpath] = None else: yield fpath # print('fpath = {!r}'.format(fpath)) if fpath not in ldd_seen: ldd_seen[fpath] = list(ldd(fpath)) for child in ldd_seen[fpath]: for _ in _ldd_tree(child, depth=depth + 1): yield _
def message(early_stop): if early_stop.prev_epoch is None: return 'vloss is unevaluated' # if early_stop.is_improved(): # message = 'vloss: {:.4f} (new_best)'.format(early_stop.best_loss) message = 'vloss: {:.4f} (n_bad_epochs={:2d}, best={:.4f})'.format( early_stop.prev_loss, early_stop.n_bad_epochs, early_stop.best_loss, ) if early_stop.n_bad_epochs <= int(early_stop.patience * .25): message = ub.color_text(message, 'green') elif early_stop.n_bad_epochs >= int(early_stop.patience * .75): message = ub.color_text(message, 'red') else: message = ub.color_text(message, 'yellow') return message
def predict(harn, have_true=True): # Import the right version of caffe print(ub.color_text('[segnet] begin prediction', 'blue')) harn.prepare_test_model(force=False) harn.test.make_dumpsafe_names() net = harn.make_net() assert harn.test_batch_size == 1 # have_true = bool(harn.test.gt_paths) if not have_true: def load_batch_data(bx): """ bx = 0 """ offset = bx * harn.test_batch_size blob_data = net.blobs['data'].data for jx in range(harn.test_batch_size): # push data into the network ix = offset + jx im_hwc = util.imread(harn.test.im_paths[ix]) im_hwc = im_hwc[:, :, ::-1] im_chw = np.transpose(im_hwc, (2, 0, 1)).astype(np.float32) blob_data[jx, :, :, :] = im_chw n_iter = int(harn.test.n_input / harn.test_batch_size) for bx in ub.ProgIter(range(n_iter), label='forward batch', freq=1): if not have_true: load_batch_data(bx) net.forward() blobs = net.blobs harn.dump_predictions(blobs, bx, have_true=have_true)
def test_unable_to_find_color(): import ubelt as ub import pytest if ub.util_colors.NO_COLOR: pytest.skip() with pytest.warns(UserWarning): text = ub.color_text('text', 'wizbang') assert text == 'text', 'bad colors should pass the text back'
def message(monitor): if not monitor.epochs: return ub.color_text('vloss is unevaluated', 'blue') # if monitor.is_improved(): # message = 'vloss: {:.4f} (new_best)'.format(monitor.best_loss) prev_loss = monitor.smooth_metrics[-1]['loss'] best_loss = monitor.best_smooth_metrics['loss'] message = 'vloss: {:.4f} (n_bad_epochs={:2d}, best={:.4f})'.format( prev_loss, monitor.n_bad_epochs, best_loss, ) if monitor.n_bad_epochs <= int(monitor.patience * .25): message = ub.color_text(message, 'green') elif monitor.n_bad_epochs >= int(monitor.patience * .75): message = ub.color_text(message, 'red') else: message = ub.color_text(message, 'yellow') return message
def bench_isinstance_vs_attr(): instances = { 'base1': Base1(), 'base2': Base2(), 'derived2': Derived2(), } import ubelt as ub ti = ub.Timerit(100000, bestof=500, verbose=1, unit='us') # Do this twice, but keep the second measure data = ub.AutoDict() for selfname, self in instances.items(): print(ub.color_text('--- SELF = {} ---'.format(selfname), 'blue')) subdata = data[selfname] = {} for timer in ti.reset('isinstance(self, Base1)'): with timer: isinstance(self, Base1) subdata[ti.label] = ti.min() for timer in ti.reset('isinstance(self, Base2)'): with timer: isinstance(self, Base2) subdata[ti.label] = ti.min() for timer in ti.reset('isinstance(self, Derived2)'): with timer: isinstance(self, Derived2) subdata[ti.label] = ti.min() for timer in ti.reset('getattr(self, "class_attr1", False)'): with timer: getattr(self, 'class_attr1', False) subdata[ti.label] = ti.min() for timer in ti.reset('getattr(self, "attr1", False)'): with timer: getattr(self, 'attr1', False) subdata[ti.label] = ti.min() try: import pandas as pd df = pd.DataFrame(data) * 1e9 try: from kwil.util.util_pandas import _to_string_monkey print(_to_string_monkey(df, key='minima')) except Exception: print(df) except ImportError: print('no pandas') print(ub.repr2(data, nl=2, precision=4))
def humanize_dollars(dollars, named=False, colored=False): if named: text = '${}'.format(named_large_number(dollars)) # return '${:5.02g}'.format(float(dollars)) else: text = '${:5.02g}'.format(float(dollars)) if colored: if dollars > 1e16: # Blue is VERY safe text = ub.color_text(text, 'blue') elif dollars > 1e6: # Green is safe text = ub.color_text(text, 'green') elif dollars > 1e3: # Yellow is short-term safe text = ub.color_text(text, 'yellow') else: # Red is not safe text = ub.color_text(text, 'red') return text
def cprint(text, color=None): """ provides some color to terminal output Args: text (str): color (str): Ignore: assert color in ['', 'yellow', 'blink', 'lightgray', 'underline', 'darkyellow', 'blue', 'darkblue', 'faint', 'fuchsia', 'black', 'white', 'red', 'brown', 'turquoise', 'bold', 'darkred', 'darkgreen', 'reset', 'standout', 'darkteal', 'darkgray', 'overline', 'purple', 'green', 'teal', 'fuscia'] Example0: >>> import pygments.console >>> msg_list = list(pygments.console.codes.keys()) >>> color_list = list(pygments.console.codes.keys()) >>> [cprint(text, color) for text, color in zip(msg_list, color_list)] Example1: >>> import pygments.console >>> print('line1') >>> cprint('line2', 'red') >>> cprint('line3', 'blue') >>> cprint('line4', 'fuchsia') >>> cprint('line5', 'reset') >>> cprint('line5', 'fuchsia') >>> print('line6') """ if False and ub.WIN32: # Ignore colors on windows. Seems to cause a recursion error print(text) else: try: if color is None: print(ub.color_text(text, 'blue')) else: print(ub.color_text(text, color)) except RecursionError: print(text)
def highlight_regex(str_, pat, reflags=0, color='red'): """ FIXME Use pygments instead """ import re matches = list(re.finditer(pat, str_, flags=reflags)) colored = str_ for match in reversed(matches): start = match.start() end = match.end() colored_part = ub.color_text(colored[start:end], color) colored = colored[:start] + colored_part + colored[end:] return colored
def humanize_seconds(seconds, colored=True, precision=4, named=False): minutes = seconds / 60. hours = minutes / 60. days = hours / 24. years = days / 365. if minutes < 1: raw = (seconds, 'seconds') elif hours < 1: raw = (minutes, 'minutes') elif days < 1: raw = (hours, 'hours') elif years < 1: raw = (days, 'days') else: raw = (years, 'years') count, unit = raw count_ = round(float(count), 4) if named: ret = '{} {}'.format(named_large_number(count), unit) else: ret = ('{:.' + str(precision) + 'g} {}').format(count_, unit) if colored: if years > 1e5: # Blue is VERY safe ret = ub.color_text(ret, 'blue') elif years > 80: # Green is safe ret = ub.color_text(ret, 'green') elif years > 10: # Yellow is short-term safe ret = ub.color_text(ret, 'yellow') else: # Red is not safe ret = ub.color_text(ret, 'red') return ret
def prepare_solver(harn, force=False): if force or not harn.solver_fpath: harn.train.prepare_input() harn.train.prepare_gtstats(harn.task) gtstats = harn.train.gtstats class_weights = gtstats.loss_weight.loc[harn.task.classnames] class_weights = class_weights.drop(harn.task.ignore_classnames) print(ub.color_text('[segnet] Preparing solver', 'blue')) harn.train_base = ub.ensuredir((harn.archdir, 'train')) if harn.init_pretrained_fpath is not None: if harn.train_init_id is None: harn.train_init_id = util.hash_file( harn.init_pretrained_fpath)[:harn.abbrev] else: harn.train_init_id = None harn.train_hyper_id = util.hash_data( harn.params.hyper_id())[:harn.abbrev] harn.train_id = '{}_{}_{}_{}'.format(harn.train.input_id, harn.arch, harn.train_init_id, harn.train_hyper_id) print('+=========') print('harn.hyper_strid = {!r}'.format(harn.params.hyper_id())) print('harn.train_init_id = {!r}'.format(harn.train_init_id)) print('harn.arch = {!r}'.format(harn.arch)) print('harn.train_hyper_id = {!r}'.format(harn.train_hyper_id)) print('harn.train_id = {!r}'.format(harn.train_id)) print('+=========') harn.solver_dpath = ub.ensuredir( (harn.train_base, 'input_' + harn.train.input_id, 'solver_{}'.format(harn.train_id))) harn.solver_fpath = models.make_solver_file( harn.train.input_fpath, arch=harn.arch, dpath=harn.solver_dpath, params=harn.params, gpu_num=harn.gpu_num, modelkw={ 'batch_size': harn.train_batch_size, 'class_weights': class_weights, 'n_classes': len(harn.task.classnames), 'ignore_label': harn.task.ignore_label, }, )
def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False): """ issues a command on a repo Example: >>> # DISABLE_DOCTEST >>> repo = dirname(dirname(ub.__file__)) >>> command = 'git status' >>> sudo = False >>> result = repocmd(repo, command, sudo) >>> print(result) """ WIN32 = sys.platform.startswith('win32') if WIN32: assert not sudo, 'cant sudo on windows' if command == 'short_status': return repo.short_status() command_list = [command] cmdstr = '\n '.join([cmd_ for cmd_ in command_list]) if not dry: import ubelt as ub print('+--- *** repocmd(%s) *** ' % (cmdstr,)) print('repo=%s' % ub.color_text(repo.dpath, 'yellow')) verbose = True with repo.chdir_context(): ret = None for count, command in enumerate(command_list): if dry: print(command) continue if not sudo or WIN32: cmdinfo = ub.cmd(command, verbose=1) out, err, ret = ub.take(cmdinfo, ['out', 'err', 'ret']) else: out, err, ret = ub.cmd('sudo ' + command) if verbose > 1: print('ret(%d) = %r' % (count, ret,)) if ret != 0: if error == 'raise': raise Exception('Failed command %r' % (command,)) elif error == 'return': return out else: raise ValueError('unknown flag error=%r' % (error,)) if return_out: return out if not dry: print('L____')
def gitcmd(repo, command): try: import ubelt as ub except ImportError: print() print("************") try: print('repo=%s' % ub.color_text(repo.dpath, 'yellow')) except Exception: print('repo = %r ' % (repo,)) os.chdir(repo) if command.find('git') != 0 and command != 'gcwip': command = 'git ' + command os.system(command) print("************") else: repo_ = Repo(dpath=repo) repo_.issue(command)
def ensure(repo, dry=False): """ Ensure that the repo is checked out on your local machine, that the correct branch is checked out, and the upstreams are targeting the correct remotes. """ if repo.verbose > 0: if dry: repo.debug(ub.color_text('Checking {}'.format(repo), 'blue')) else: repo.debug(ub.color_text('Ensuring {}'.format(repo), 'blue')) if not exists(repo.dpath): repo.debug('NEED TO CLONE {}'.format(repo)) if dry: return repo.ensure_clone() repo._assert_clean() # Ensure all registered remotes exist for remote_name, remote_url in repo.remotes.items(): try: remote = repo.pygit.remotes[remote_name] have_urls = list(remote.urls) if remote_url not in have_urls: print('WARNING: REMOTE NAME EXIST BUT URL IS NOT {}. ' 'INSTEAD GOT: {}'.format(remote_url, have_urls)) except (IndexError): try: print('NEED TO ADD REMOTE {}->{} FOR {}'.format( remote_name, remote_url, repo)) if not dry: repo._cmd('git remote add {} {}'.format( remote_name, remote_url)) except ShellException: if remote_name == repo.remote: # Only error if the main remote is not available raise # Ensure we have the right remote try: remote = repo.pygit.remotes[repo.remote] except IndexError: if not dry: raise AssertionError('Something went wrong') else: remote = None if remote is not None: try: if not remote.exists(): raise IndexError else: repo.debug( 'The requested remote={} name exists'.format(remote)) except IndexError: repo.debug('WARNING: remote={} does not exist'.format(remote)) else: if remote.exists(): repo.debug('Requested remote does exists') remote_branchnames = [ ref.remote_head for ref in remote.refs ] if repo.branch not in remote_branchnames: repo.info( 'Branch name not found in local remote. Attempting to fetch' ) if dry: repo.info('dry run, not fetching') else: repo._cmd('git fetch {}'.format(remote.name)) repo.info('Fetch was successful') else: repo.debug('Requested remote does NOT exist') # Ensure the remote points to the right place if repo.url not in list(remote.urls): repo.debug( 'WARNING: The requested url={} disagrees with remote urls={}' .format(repo.url, list(remote.urls))) if dry: repo.info('Dry run, not updating remote url') else: repo.info('Updating remote url') repo._cmd('git remote set-url {} {}'.format( repo.remote, repo.url)) # Ensure we are on the right branch if repo.branch != repo.pygit.active_branch.name: repo.debug('NEED TO SET BRANCH TO {} for {}'.format( repo.branch, repo)) try: repo._cmd('git checkout {}'.format(repo.branch)) except ShellException: repo.debug( 'Checkout failed. Branch name might be ambiguous. Trying again' ) try: repo._cmd('git checkout -b {} {}/{}'.format( repo.branch, repo.remote, repo.branch)) except ShellException: raise Exception('does the branch exist on the remote?') tracking_branch = repo.pygit.active_branch.tracking_branch() if tracking_branch is None or tracking_branch.remote_name != repo.remote: repo.debug('NEED TO SET UPSTREAM FOR FOR {}'.format(repo)) try: remote = repo.pygit.remotes[repo.remote] if not remote.exists(): raise IndexError except IndexError: repo.debug( 'WARNING: remote={} does not exist'.format(remote)) else: if remote.exists(): remote_branchnames = [ ref.remote_head for ref in remote.refs ] if repo.branch not in remote_branchnames: if dry: repo.info( 'Branch name not found in local remote. Dry run, use ensure to attempt to fetch' ) else: repo.info( 'Branch name not found in local remote. Attempting to fetch' ) repo._cmd('git fetch {}'.format(repo.remote)) remote_branchnames = [ ref.remote_head for ref in remote.refs ] if repo.branch not in remote_branchnames: raise Exception( 'Branch name still does not exist') if not dry: repo._cmd( 'git branch --set-upstream-to={remote}/{branch} {branch}' .format(remote=repo.remote, branch=repo.branch)) else: repo.info('Would attempt to set upstream') # Print some status repo.debug(' * branch = {} -> {}'.format( repo.pygit.active_branch.name, repo.pygit.active_branch.tracking_branch(), ))
def benchmark_attribute_access(): """ How fast are different methods of accessing attributes? Lets find out! """ instances = { 'simple': Simple(), 'complex': Complex(), 'slot_simple': SimpleWithSlots(), 'slot_complex': ComplexWithSlots(), } import ubelt as ub ti = ub.Timerit(100000, bestof=500, verbose=1, unit='us') # Do this twice, but keep the second measure data = ub.AutoDict() for selfname, self in instances.items(): print(ub.color_text('--- SELF = {} ---'.format(selfname), 'blue')) subdata = data[selfname] = {} for timer in ti.reset('self.attr1'): with timer: self.attr1 subdata[ti.label] = ti.min() for timer in ti.reset('getattr(self, attr1)'): with timer: getattr(self, 'attr1') subdata[ti.label] = ti.min() attrs = ['attr1', 'attr2'] for attrname in attrs: for timer in ti.reset('hasattr(self, {})'.format(attrname)): with timer: hasattr(self, attrname) subdata[ti.label] = ti.min() for timer in ti.reset('getattr(self, {}, None)'.format(attrname)): with timer: getattr(self, attrname, None) subdata[ti.label] = ti.min() if 'slot' not in selfname.lower(): for timer in ti.reset( 'self.__dict__.get({}, None)'.format(attrname)): with timer: self.__dict__.get(attrname, None) subdata[ti.label] = ti.min() for timer in ti.reset('try/except: self.attr2'): with timer: try: x = self.attr2 except AttributeError: x = None subdata[ti.label] = ti.min() for timer in ti.reset('try/except: self.attr1'): with timer: try: x = self.attr1 except AttributeError: x = None subdata[ti.label] = ti.min() del x try: import pandas as pd df = pd.DataFrame(data) * 1e9 try: from kwil.util.util_pandas import _to_string_monkey print(_to_string_monkey(df, key='minima')) except Exception: print(df) except ImportError: print('no pandas') print(ub.repr2(data, nl=2, precision=4))
def ensure(repo, dry=False): """ Ensure that the repo is checked out on your local machine, that the correct branch is checked out, and the upstreams are targeting the correct remotes. """ if repo.verbose > 0: if dry: repo.debug(ub.color_text('Checking {}'.format(repo), 'blue')) else: repo.debug(ub.color_text('Ensuring {}'.format(repo), 'blue')) if not exists(repo.dpath): repo.debug('NEED TO CLONE {}: {}'.format(repo, repo.url)) if dry: return repo.ensure_clone() repo._assert_clean() # Ensure we have the right remote remote = repo._registered_remote(dry=dry) if remote is not None: try: if not remote.exists(): raise IndexError else: repo.debug( 'The requested remote={} name exists'.format(remote)) except IndexError: repo.debug('WARNING: remote={} does not exist'.format(remote)) else: if remote.exists(): repo.debug('Requested remote does exists') remote_branchnames = [ ref.remote_head for ref in remote.refs ] if repo.branch not in remote_branchnames: repo.info( 'Branch name not found in local remote. Attempting to fetch' ) if dry: repo.info('dry run, not fetching') else: repo._cmd('git fetch {}'.format(remote.name)) repo.info('Fetch was successful') else: repo.debug('Requested remote does NOT exist') # Ensure the remote points to the right place if repo.url not in list(remote.urls): repo.debug( ub.paragraph(''' 'WARNING: The requested url={} disagrees with remote urls={} ''').format(repo.url, list(remote.urls))) if dry: repo.info('Dry run, not updating remote url') else: repo.info('Updating remote url') repo._cmd('git remote set-url {} {}'.format( repo.remote, repo.url)) # Ensure we are on the right branch try: active_branch_name = repo.pygit.active_branch.name except TypeError: # We may be on a tag, not a branch candidates = [ tag for tag in repo.pygit.tags if tag.name == repo.branch ] if len(candidates) != 1: raise else: # branch is actually a tag assert len(candidates) == 1 want_tag = candidates[0] is_on_correct_commit = (repo.pygit.head.commit.hexsha == want_tag.commit.hexsha) ref_is_tag = True else: ref_is_tag = False tracking_branch = repo.pygit.active_branch.tracking_branch() is_on_correct_commit = repo.branch == active_branch_name if not is_on_correct_commit: repo.debug('NEED TO SET BRANCH TO {} for {}'.format( repo.branch, repo)) if dry: repo.info('Dry run, not setting branch') else: try: repo._cmd('git checkout {}'.format(repo.branch)) except ShellException: repo.debug( 'Checkout failed. Branch name might be ambiguous. Trying again' ) try: repo._cmd('git fetch {}'.format(remote.name)) repo._cmd('git checkout -b {} {}/{}'.format( repo.branch, repo.remote, repo.branch)) except ShellException: raise Exception( 'does the branch exist on the remote?') if not ref_is_tag: if tracking_branch is None or tracking_branch.remote_name != repo.remote: repo.debug('NEED TO SET UPSTREAM FOR FOR {}'.format(repo)) try: remote = repo.pygit.remotes[repo.remote] if not remote.exists(): raise IndexError except IndexError: repo.debug( 'WARNING: remote={} does not exist'.format(remote)) else: if remote.exists(): remote_branchnames = [ ref.remote_head for ref in remote.refs ] if repo.branch not in remote_branchnames: if dry: repo.info( 'Branch name not found in local remote. Dry run, use ensure to attempt to fetch' ) else: repo.info( 'Branch name not found in local remote. Attempting to fetch' ) repo._cmd('git fetch {}'.format( repo.remote)) remote_branchnames = [ ref.remote_head for ref in remote.refs ] if repo.branch not in remote_branchnames: raise Exception( 'Branch name still does not exist') if not dry: repo._cmd( 'git branch --set-upstream-to={remote}/{branch} {branch}' .format(remote=repo.remote, branch=repo.branch)) else: repo.info('Would attempt to set upstream') # Check if the current head is tagged head_tags = [ tag for tag in repo.pygit.tags if tag.commit.hexsha == repo.pygit.head.commit.hexsha ] # Print some status try: repo.debug(' * branch = {} -> {}'.format( repo.pygit.active_branch.name, repo.pygit.active_branch.tracking_branch(), )) except Exception: pass if head_tags: repo.debug(' * head_tags = {}'.format(head_tags))
def run_xval_evaluation(task, arch='segnet_proper', pretrained=None, hyperparams={}, xval_method='predef', test_keys=None, fit=True): """ Writes test/train data files containing the image paths that will be used for testing and training as well as the appropriate solvers and prediction models. Args: fit (bool): if False, we will only evaluate models that have been trained so far (useful for getting results from existing while a model is not done training) CommandLine: export PYTHONPATH=$HOME/code/fletch/build-py3/install/lib/python3.5/site-packages:$PYTHONPATH python -m clab.tasks DivaV1.run_xval_evaluation python -m clab.tasks DivaV1.run_xval_evaluation --batch-size=1 python -m clab.tasks DivaV1.run_xval_evaluation rsync -avpr aretha:sseg/sseg-data/xval-solvers ~/sseg/sseg-data/xval-solvers Script: >>> from clab.tasks import * >>> import clab >>> task = DivaV1(clean=2) >>> arch = 'segnet_proper' >>> pretrained = '/home/local/KHQ/jon.crall/store/segnet-exact/SegNet-Tutorial/Models/Training/segnet_iter_30000.caffemodel' >>> pretrained = 'segnet_proper_camvid.caffemodel' >>> hyperparams = {'freeze_before': -23, 'max_iter': 10000} >>> # Use external dataset to increase the amount of training data >>> tutorial_dir = './SegNet-Tutorial' >>> task.extend_data_from(clab.tasks.CamVid(tutorial_dir)) >>> task.run_xval_evaluation() """ from clab import harness # from clab import models xval_base = abspath(task.exptdir('xval')) xval_results = [] for idx, xval_split in enumerate( task.xval_splits(xval_method, test_keys=test_keys)): print(ub.color_text('XVAL iteration {}'.format(idx), 'blue')) # harn = task.harness_from_xval(idx, arch=arch, # pretrained=pretrained, # hyperparams=hyperparams, # xval_method=xval_method, # test_keys=test_keys) # (train_data, test_data) = xval_split xval_dpath = ub.ensuredir((xval_base, 'split_{:0=2}'.format(idx))) (train, test) = xval_split harn = harness.Harness(workdir=xval_dpath, arch=arch) harn.set_inputs(train, test) # harn.train.im_paths = train_data[0] # harn.train.gt_paths = train_data[1] # harn.test.im_paths = test_data[0] # harn.test.gt_paths = test_data[1] # assert len(harn.train.im_paths) > 0 # assert len(harn.train.gt_paths) > 0 # assert len(harn.test.im_paths) > 0 # assert len(harn.test.gt_paths) > 0 harn.params.update(hyperparams) harn.init_pretrained_fpath = pretrained harn.test.prepare_images() harn.train.prepare_images() harn.gpu_num = gpu_util.find_unused_gpu(min_memory=6000) print('harn.gpu_num = {!r}'.format(harn.gpu_num)) if harn.gpu_num is not None: avail_mb = gpu_util.gpu_info()[harn.gpu_num]['mem_avail'] # Estimate how much we can fit in memory # TODO: estimate this from the model arch instead. # (90 is a mgic num corresponding to segnet_proper) harn.train_batch_size = int( (avail_mb * 90) // np.prod(task.input_shape)) harn.train_batch_size = int(harn.train_batch_size) if harn.train_batch_size == 0: raise MemoryError( 'not enough GPU memory to train the model') else: # not sure what the best to do on CPU is. Probably nothing. harn.train_batch_size = 4 harn.prepare_solver() # Check if we can resume a previous training state print( ub.color_text('Checking for previous snapshot states', 'blue')) previous_states = harn.snapshot_states() print('Found {} previous_states'.format(len(previous_states))) from clab.backend import iface_caffe as iface solver_info = iface.parse_solver_info(harn.solver_fpath) prev_iter = 0 if previous_states: prev_state = previous_states[-1] prev_iter = iface.snapshot_iterno(prev_state) if prev_iter == 0: if fit: print( ub.color_text('Starting a fresh training session', 'blue')) # harn.fit() list(harn.fit2()) else: print( ub.color_text('Would start a fresh training session', 'yellow')) elif prev_iter < solver_info['max_iter']: if fit: # continue from the previous iteration print( ub.color_text( 'Resume training from iter {}'.format(prev_iter), 'blue')) # harn.fit(prevstate_fpath=prev_state) list(harn.fit2(prevstate_fpath=prev_state)) else: print( ub.color_text( 'Would resume training from iter {}'.format( prev_iter), 'yellow')) else: print( ub.color_text('Already finished training this model', 'blue')) for _ in harn.deploy_trained_for_testing(): # hack to evaulate while deploying harn.evaulate_all() xval_results.append(list(harn._test_results_fpaths())) return xval_results
def postprocess_training_weights(harn, train_weights_path=None, force=False): """ Fixes BN params in the weights file for testing Ideally this should be run after dumping a snapshot to disk (because it requires training data to compute the BN params) However, you can run it before testing IF you still know where all the training data is. """ solver_info = iface.parse_solver_info(harn.solver_fpath) train_model_path = solver_info['train_model_path'] if train_weights_path is None: # Get the most recent set of weights snapshot_models = iface.load_snapshot_weight_paths( solver_info['snapshot_prefix']) train_weights_path = snapshot_models[-1] iterno = iface.snapshot_iterno(train_weights_path) test_weight_suffix = ('{}_{:08d}'.format(harn.train_id, iterno)) harn.test_weights_dir = ub.ensuredir(( harn.solver_dpath, 'testable', )) # Names of the files we are about to write test_deploy_fpath = join( harn.test_weights_dir, 'deploy_{}.prototxt'.format(test_weight_suffix)) test_weights_fpath = join( harn.test_weights_dir, 'test_weights_{}.caffemodel'.format(test_weight_suffix)) BN_calc_path = join( harn.test_weights_dir, '__temp_bn_stats_{}.prototext'.format(test_weight_suffix)) if exists(test_weights_fpath) and not force: # lazy cache harn.test_weights_fpath = test_weights_fpath return test_weights_fpath print(ub.color_text('Postprocessing BN stats', 'blue')) print('Building testable weights from train_model_path = {!r}'.format( train_model_path)) print( 'Testable weights will be saved to harn.test_weights_fpath = {!r}'. format(harn.test_weights_fpath)) harn.prepare_workdir() from pysseg.backend import batch_norm_stats from google.protobuf import text_format train_net = batch_norm_stats.make_testable(train_model_path) with open(BN_calc_path, 'w') as f: f.write(text_format.MessageToString(train_net)) # use testable net to calculate BN layer stats # these must be TRAIN datas train_im_paths, train_gt_paths = batch_norm_stats.extract_dataset( train_net) # HACK: remove aug train_gt_paths = [ p for p in train_gt_paths if '_aug' not in basename(p) ] train_im_paths = [ p for p in train_im_paths if '_aug' not in basename(p) ] # HACK: remove part-scale2 train_gt_paths = [ p for p in train_gt_paths if 'part-scale2' not in basename(p) ] train_im_paths = [ p for p in train_im_paths if 'part-scale2' not in basename(p) ] # This does a single forward pass on the testing dataset so we can get # some stats gpu_num = harn.gpu_num test_net, test_msg = batch_norm_stats.make_test_files( BN_calc_path, train_weights_path, train_im_paths, gpu_num=gpu_num) # save deploy prototxt # (note deploy is a prediction model that is meant to be used with blob # inputs) with open(test_deploy_fpath, 'w') as f: f.write(text_format.MessageToString(test_msg)) test_net.save(test_weights_fpath) harn.test_weights_fpath = test_weights_fpath return test_weights_fpath
def color_text(text): ub.color_text(text)
def main(): # TODO: find a better place for root ROOT = join(os.getcwd()) # ROOT = '.' os.chdir(ROOT) NAME = 'pyhesaff' VERSION = '0.1.2' DOCKER_TAG = '{}-{}'.format(NAME, VERSION) QUAY_REPO = 'quay.io/erotemic/manylinux-for' DOCKER_URI = '{QUAY_REPO}:{DOCKER_TAG}'.format(**locals()) dockerfile_fpath = join(ROOT, 'Dockerfile') # This docker code is very specific for building linux binaries. # We will need to do a bit of refactoring to handle OSX and windows. # But the goal is to get at least one OS working end-to-end. """ Notes: docker run --rm -it quay.io/pypa/manylinux2010_x86_64 /bin/bash --- ls /opt/python """ BASE_IMAGE = 'quay.io/pypa/manylinux2010_x86_64' docker_code = ub.codeblock(f''' FROM {BASE_IMAGE} RUN yum install lz4-devel -y RUN MB_PYTHON_TAG=cp27-cp27m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp27-cp27mu && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp35-cp35m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp36-cp36m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp37-cp37m && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja RUN MB_PYTHON_TAG=cp38-cp38 && \ /opt/python/$MB_PYTHON_TAG/bin/python -m pip install setuptools pip virtualenv -U && \ /opt/python/$MB_PYTHON_TAG/bin/python -m virtualenv ./venv-$MB_PYTHON_TAG && \ source ./venv-$MB_PYTHON_TAG/bin/activate && \ pip install scikit-build cmake ninja ''') docker_code2 = '\n\n'.join( [ub.paragraph(p) for p in docker_code.split('\n\n')]) try: print(ub.color_text('\n--- DOCKER CODE ---', 'white')) print(ub.highlight_code(docker_code2, 'docker')) print(ub.color_text('--- END DOCKER CODE ---\n', 'white')) except Exception: pass with open(dockerfile_fpath, 'w') as file: file.write(docker_code2) docker_build_cli = ' '.join([ 'docker', 'build', '--tag {}'.format(DOCKER_TAG), '-f {}'.format(dockerfile_fpath), '.', ]) print('docker_build_cli = {!r}'.format(docker_build_cli)) if ub.argflag('--dry'): print('DRY RUN') print('WOULD RUN') print(docker_build_cli) else: info = ub.cmd(docker_build_cli, verbose=3, shell=True) if info['ret'] != 0: print(ub.color_text('\n--- FAILURE ---', 'red')) print('Failed command:') print(info['command']) print(info['err']) print('NOTE: sometimes reruning the command manually works') raise Exception('Building docker failed with exit code {}'.format( info['ret'])) else: print(ub.color_text('\n--- SUCCESS ---', 'green')) print( ub.highlight_code( ub.codeblock(r''' # Finished creating the docker image. # To test / export / publish you can do something like this: # Test that we can get a bash terminal docker run -it {DOCKER_TAG} /bin/bash # Create a tag for the docker image docker tag {DOCKER_TAG} {DOCKER_URI} # Export your docker image to a file docker save -o ${ROOT}/{DOCKER_TAG}.docker.tar {DOCKER_TAG} # Login to a docker registry (we are using quay) # In some cases this works, docker login # But you may need to specify secret credentials load_secrets echo "QUAY_USERNAME = $QUAY_USERNAME" docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io unload_secrets # Upload the docker image to quay.io docker push {DOCKER_URI} ''').format( NAME=NAME, ROOT=ROOT, DOCKER_TAG=DOCKER_TAG, DOCKER_URI=DOCKER_URI, ), 'bash', )) PUBLISH = 0 if PUBLISH: cmd1 = 'docker tag {DOCKER_TAG} {DOCKER_URI}'.format(**locals()) cmd2 = 'docker push {DOCKER_URI}'.format(**locals()) print('-- <push cmds> ---') print(cmd1) print(cmd2) print('-- </push cmds> ---')
def main(): import os ROOT = join(os.getcwd()) ROOT = ub.expandpath('~/code/hesaff') os.chdir(ROOT) VERSION = setup.version PY_VER = sys.version_info.major NAME = 'pyhesaff' tag = '{}-{}-py{}'.format(NAME, VERSION, PY_VER) # context_dpath = ub.ensuredir((ROOT, 'docker/context')) staging_dpath = ub.ensuredir((ROOT, 'docker/staging')) # Prestage the multibuild repo if not exists(join(staging_dpath, 'multibuild')): # FIXME: make robust in the case this fails info = ub.cmd( 'git clone https://github.com/matthew-brett/multibuild.git', cwd=staging_dpath, verbose=3) if not exists(join(staging_dpath, 'opencv')): # FIXME: make robust in the case this fails opencv_version = '4.1.0' fpath = ub.grabdata( 'https://github.com/opencv/opencv/archive/{}.zip'.format( opencv_version), verbose=1) ub.cmd('ln -s {} .'.format(fpath), cwd=staging_dpath, verbose=3) ub.cmd('unzip {}'.format(fpath), cwd=staging_dpath, verbose=3) import shutil shutil.move(join(staging_dpath, 'opencv-' + opencv_version), join(staging_dpath, 'opencv')) stage_self(ROOT, staging_dpath) dockerfile_fpath = join(ROOT, 'Dockerfile') # This docker code is very specific for building linux binaries. # We will need to do a bit of refactoring to handle OSX and windows. # But the goal is to get at least one OS working end-to-end. docker_code = ub.codeblock(''' FROM quay.io/skvark/manylinux1_x86_64 # SETUP ENV ARG MB_PYTHON_VERSION=3.6 ARG ENABLE_CONTRIB=0 ARG ENABLE_HEADLESS=1 ENV PYTHON_VERSION=3.6 ENV PYTHON_ROOT=/opt/python/cp36-cp36m/ ENV PYTHONPATH=/opt/python/cp36-cp36m/lib/python3.6/site-packages/ ENV PATH=/opt/python/cp36-cp36m/bin:$PATH ENV PYTHON_EXE=/opt/python/cp36-cp36m/python ENV MULTIBUILD_DIR=/root/code/multibuild ENV HOME=/root # params to bdist_wheel. used to set osx build target. ENV TEST_DEPENDS="numpy==1.11.1" ENV BDIST_PARAMS="" ENV USE_CCACHE=1 ENV PLAT=x86_64 ENV UNICODE_WIDTH=32 # -e BUILD_COMMANDS="$build_cmds" \ # -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ # -e UNICODE_WIDTH="$UNICODE_WIDTH" \ # -e BUILD_COMMIT="$BUILD_COMMIT" \ # -e CONFIG_PATH="$CONFIG_PATH" \ # -e ENV_VARS_PATH="$ENV_VARS_PATH" \ # -e WHEEL_SDIR="$WHEEL_SDIR" \ # -e MANYLINUX_URL="$MANYLINUX_URL" \ # -e BUILD_DEPENDS="$BUILD_DEPENDS" \ # -e USE_CCACHE="$USE_CCACHE" \ # -e REPO_DIR="$repo_dir" \ # -e PLAT="$PLAT" \ # These are defined in the parent image # ENV JPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include # ENV JPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a RUN mkdir -p /io WORKDIR /root # Setup code / scripts COPY docker/staging/multibuild /root/code/multibuild # Hack to fix issue RUN find $MULTIBUILD_DIR -iname "*.sh" -type f -exec sed -i 's/gh-clone/gh_clone/g' {} + # Copy bash configs (mirrors the environs) COPY docker/config.sh /root/config.sh COPY docker/bashrc.sh /root/.bashrc # Setup a virtualenv RUN source /root/.bashrc && \ $PYTHON_EXE -m pip install --upgrade pip && \ $PYTHON_EXE -m pip install virtualenv && \ $PYTHON_EXE -m virtualenv --python=$PYTHON_EXE $HOME/venv # Install packages in virtual environment RUN source /root/.bashrc && \ pip install cmake ninja scikit-build wheel numpy # This is very different for different operating systems # https://github.com/skvark/opencv-python/blob/master/setup.py COPY docker/staging/opencv /root/code/opencv RUN source /root/.bashrc && \ source code/multibuild/common_utils.sh && \ source code/multibuild/travis_linux_steps.sh && \ mkdir -p /root/code/opencv/build && \ cd /root/code/opencv/build && \ cmake -G "Unix Makefiles" \ -DINSTALL_CREATE_DISTRIB=ON \ -DOPENCV_SKIP_PYTHON_LOADER=ON \ -DBUILD_opencv_apps=OFF \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTS=OFF \ -DBUILD_PERF_TESTS=OFF \ -DBUILD_DOCS=OFF \ -DWITH_QT=OFF \ -DWITH_IPP=OFF \ -DWITH_V4L=ON \ -DBUILD_JPEG=OFF \ -DENABLE_PRECOMPILED_HEADERS=OFF \ -DJPEG_INCLUDE_DIR=/opt/libjpeg-turbo/include \ -DJPEG_LIBRARY=/opt/libjpeg-turbo/lib64/libjpeg.a \ /root/code/opencv # Note: there is no need to compile the above with python # -DPYTHON3_EXECUTABLE=$PYTHON_EXE \ # -DBUILD_opencv_python3=ON \ # -DOPENCV_PYTHON3_INSTALL_PATH=python \ RUN source /root/.bashrc && \ source code/multibuild/common_utils.sh && \ source code/multibuild/travis_linux_steps.sh && \ cd /root/code/opencv/build && \ make -j9 && make install COPY docker/staging/hesaff /root/code/hesaff # # Use skbuild to build hesaff # RUN source /root/.bashrc && \ # cd /root/code/hesaff && \ # CMAKE_FIND_LIBRARY_SUFFIXES=".a;.so" python setup.py build_ext --inplace # export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' # Use cmake to build hesaff 9maybe not needed?) # RUN source /root/.bashrc && \ # mkdir -p /root/code/hesaff/build && \ # cd /root/code/hesaff/build && \ # CXXFLAGS="-std=c++11 $CXXFLAGS" cmake -G "Unix Makefiles" /root/code/hesaff && \ # make # Use skbuild to build hesaff RUN source /root/.bashrc && \ cd /root/code/hesaff && \ python setup.py build && \ python setup.py bdist_wheel # RUN source /root/.bashrc && \ # pip install xdoctest ''') try: print(ub.color_text('\n--- DOCKER CODE ---', 'white')) print(ub.highlight_code(docker_code, 'docker')) print(ub.color_text('--- END DOCKER CODE ---\n', 'white')) except Exception: pass with open(dockerfile_fpath, 'w') as file: file.write(docker_code) docker_build_cli = ' '.join([ 'docker', 'build', # '--build-arg PY_VER={}'.format(PY_VER), '--tag {}'.format(tag), '-f {}'.format(dockerfile_fpath), '.' ]) print('docker_build_cli = {!r}'.format(docker_build_cli)) info = ub.cmd(docker_build_cli, verbose=3, shell=True) if info['ret'] != 0: print(ub.color_text('\n--- FAILURE ---', 'red')) print('Failed command:') print(info['command']) print(info['err']) print('NOTE: sometimes reruning the command manually works') raise Exception('Building docker failed with exit code {}'.format( info['ret'])) else: print(ub.color_text('\n--- SUCCESS ---', 'green')) DEPLOY = True if DEPLOY: VMNT_DIR = '{ROOT}/{NAME}-docker/vmnt'.format(NAME=NAME, ROOT=ROOT) print('VMNT_DIR = {!r}'.format(VMNT_DIR)) ub.ensuredir(VMNT_DIR) # TODO: Correctly mangle the ffmpeg libs as done via # ls -a ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/ # ls ~/.local/conda/envs/py36/lib/python3.6/site-packages/cv2/.libs # cp code/hesaff/build/libhesaff.so /root/vmnt # cp /root/ffmpeg_build/lib/libavcodec.so.58 /root/vmnt # cp /root/ffmpeg_build/lib/libavformat.so.58 /root/vmnt # cp /root/ffmpeg_build/lib/libavutil.so.56 /root/vmnt # cp /root/ffmpeg_build/lib/libswscale.so.5 /root/vmnt inside_cmds = ' && '.join( ub.codeblock(''' cp code/hesaff/dist/pyhesaff*.whl /root/vmnt ''').split('\n')) docker_run_cli = ' '.join([ 'docker', 'run', '-v {}:/root/vmnt/'.format(VMNT_DIR), '-it', tag, 'bash -c "{}"'.format(inside_cmds) ]) print(docker_run_cli) info = ub.cmd(docker_run_cli, verbose=3) assert info['ret'] == 0 # import shutil # PKG_DIR = join(ROOT, 'pyhesaff') # shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libhesaff-manylinux1_x86_64.so')) # TODO: do this correctly # shutil.copy(join(VMNT_DIR, 'libhesaff.so'), join(PKG_DIR, 'libavcodec.so.58')) # shutil.copy(join(VMNT_DIR, 'libavformat.so.58'), join(PKG_DIR, 'libavformat.so.58')) # shutil.copy(join(VMNT_DIR, 'libavutil.so.56'), join(PKG_DIR, 'libavutil.so.56')) # shutil.copy(join(VMNT_DIR, 'libswscale.so.5'), join(PKG_DIR, 'libswscale.so.5')) # print(ub.highlight_code(ub.codeblock( print( ub.highlight_code( ub.codeblock(r''' # Finished creating the docker image. # To test / export you can do something like this: VMNT_DIR={ROOT}/{NAME}-docker/vmnt mkdir -p VMNT_DIR TAG={tag} # Test that we can get a bash terminal docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash # Move deployment to the vmnt directory docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 -m xdoctest pyhesaff' # Run system tests docker run -v $VMNT_DIR:/root/vmnt -it {tag} bash -c 'cd /root/code/hesaff && python3 run_doctests.sh' # Inside bash test that we can fit a new model python -m pyhessaff demo mkdir -p ${ROOT}/{NAME}-docker/dist docker save -o ${ROOT}/{NAME}-docker/dist/{tag}.docker.tar {tag} ''').format(NAME=NAME, ROOT=ROOT, tag=tag), 'bash'))
def color(x): if x > 500: return ub.color_text(str(x), 'green') else: return ub.color_text(str(x), 'red')
def __repr__(self): return ub.color_text('<AnInstance>', 'yellow')
def _entropy_refine(depth, nodes, jdxs): """ Recursively descend the class tree starting at the coursest level. At each level we decide if the items will take a category at this level of granulatority or try to take a more fine-grained label. Args: depth (int): current depth in the tree nodes (list) : set of sibling nodes at a this level jdxs (ArrayLike): item indices that made it to this level (note idxs are used for class indices) """ if DEBUG: print(ub.color_text('* REFINE nodes={}'.format(nodes), 'blue')) # Look at the probabilities of each node at this level idxs = sorted(self.node_to_idx[node] for node in nodes) if ignore_class_idxs: ignore_nodes = set(ub.take(self.idx_to_node, ignore_class_idxs)) idxs = sorted(set(idxs) - set(ignore_class_idxs)) if len(idxs) == 0: raise ValueError('Cannot ignore all top-level classes') probs = flat_class_probs[jdxs][:, idxs] # Choose a highest probability category to predict at this level pred_conf, pred_cx = impl.max_argmax(probs, axis=1) pred_idxs = np.array(idxs)[impl.numpy(pred_cx)] # Group each example which predicted the same class at this level group_idxs, groupxs = kwarray.group_indices(pred_idxs) if DEBUG: groupxs = list(ub.take(groupxs, group_idxs.argsort())) group_idxs = group_idxs[group_idxs.argsort()] # print('groupxs = {!r}'.format(groupxs)) # print('group_idxs = {!r}'.format(group_idxs)) for idx, groupx in zip(group_idxs, groupxs): # Get the children of this node (idx) node = self.idx_to_node[idx] children = sorted(self.graph.successors(node)) if ignore_class_idxs: children = sorted(set(children) - ignore_nodes) if children: # Check if it would be simple to refine the coarse category # current prediction into one of its finer-grained child # categories. Do this by considering the entropy at this # level if we replace this coarse-node with the child # fine-nodes. Then compare that entropy to what we would # get if we were perfectly uncertain about the child node # prediction (i.e. the worst case). If the entropy we get # is much lower than the worst case, then it is simple to # descend the tree and predict a finer-grained label. # Expand this node into all of its children child_idxs = set(self.node_to_idx[child] for child in children) # Get example indices (jdxs) assigned to category idx groupx.sort() group_jdxs = jdxs[groupx] # Expand this parent node, but keep the parent's siblings ommer_idxs = sorted(set(idxs) - {idx}) # Note: ommer = Aunt/Uncle expanded_idxs = sorted(ommer_idxs) + sorted(child_idxs) expanded_probs = flat_class_probs[group_jdxs][:, expanded_idxs] # Compute the entropy of the expanded distribution h_expanded = _criterion(expanded_probs) # Probability assigned to the parent p_parent = flat_class_probs[group_jdxs][:, idx:idx + 1] # Get the absolute probabilities assigned the parents siblings ommer_probs = flat_class_probs[group_jdxs][:, sorted(ommer_idxs)] # Compute the worst-case entropy after expanding the node # In the worst case the parent probability is distributed # uniformly among all of its children c = len(children) child_probs_worst = impl.tile(p_parent / c, reps=[1, c]) expanded_probs_worst = impl.hstack([ommer_probs, child_probs_worst]) h_expanded_worst = _criterion(expanded_probs_worst) # Normalize the entropy we got by the worst case. # eps = float(np.finfo(np.float32).min) eps = 1e-30 complexity_ratio = h_expanded / (h_expanded_worst + eps) simplicity_ratio = 1 - complexity_ratio # If simplicity ratio is over a threshold refine the parent refine_flags = simplicity_ratio > thresh if always_refine_idxs is not None: if idx in always_refine_idxs: refine_flags[:] = 1 if len(child_idxs) == 1: # hack: always refine when there is one child, in this # case the simplicity measure will always be zero, # which is likely a problem with this criterion. refine_flags[:] = 1 refine_flags = kwarray.ArrayAPI.numpy(refine_flags).astype(np.bool) if DEBUG: print('-----------') print('idx = {!r}'.format(idx)) print('node = {!r}'.format(self.idx_to_node[idx])) print('ommer_idxs = {!r}'.format(ommer_idxs)) print('ommer_nodes = {!r}'.format( list(ub.take(self.idx_to_node, ommer_idxs)))) print('depth = {!r}'.format(depth)) import pandas as pd print('expanded_probs =\n{}'.format( ub.repr2(expanded_probs, precision=2, with_dtype=0, supress_small=True))) df = pd.DataFrame({ 'h': h_expanded, 'h_worst': h_expanded_worst, 'c_ratio': complexity_ratio, 's_ratio': simplicity_ratio, 'flags': refine_flags.astype(np.uint8) }) print(df) print('-----------') if np.any(refine_flags): refine_jdxs = group_jdxs[refine_flags] refine_idxs, refine_conf = _entropy_refine(depth + 1, children, refine_jdxs) # Overwrite course decisions with refined decisions. refine_groupx = groupx[refine_flags] pred_idxs[refine_groupx] = refine_idxs pred_conf[refine_groupx] = refine_conf return pred_idxs, pred_conf
def apply(registery, funcname, num_workers=0, **kwargs): print(ub.color_text('--- APPLY {} ---'.format(funcname), 'white')) print(' * num_workers = {!r}'.format(num_workers)) if num_workers == 0: processed_repos = [] for repo in registery.repos: print(ub.color_text('--- REPO = {} ---'.format(repo), 'blue')) try: getattr(repo, funcname)(**kwargs) except DirtyRepoError: print( ub.color_text('Ignoring dirty repo={}'.format(repo), 'red')) processed_repos.append(repo) else: from concurrent import futures # with futures.ThreadPoolExecutor(max_workers=num_workers) as pool: with futures.ProcessPoolExecutor(max_workers=num_workers) as pool: tasks = [] for i, repo in enumerate(registery.repos): future = pool.submit(worker, repo, funcname, kwargs) future.repo = repo tasks.append(future) processed_repos = [] for future in futures.as_completed(tasks): repo = future.repo print( ub.color_text('--- REPO = {} ---'.format(repo), 'blue')) try: repo = future.result() except DirtyRepoError: print( ub.color_text( 'Ignoring dirty repo={}'.format(repo), 'red')) else: print(repo._getlogs()) processed_repos.append(repo) print( ub.color_text('--- FINISHED APPLY {} ---'.format(funcname), 'white')) SHOW_CMDLOG = 1 if SHOW_CMDLOG: print('LOGGED COMMANDS') import os ORIG_CWD = MY_CWD = os.getcwd() for repo in processed_repos: print('# --- For repo = {!r} --- '.format(repo)) for t in repo._logged_cmds: cmd, cwd = t if cwd is None: cwd = os.get_cwd() if cwd != MY_CWD: print('cd ' + ub.shrinkuser(cwd)) MY_CWD = cwd print(cmd) print('cd ' + ub.shrinkuser(ORIG_CWD))
def submit_query_request( qreq_, use_cache=None, use_bigcache=None, verbose=None, save_qcache=None, use_supercache=None, invalidate_supercache=None, ): """ Called from qreq_.execute Checks a big cache for qaid2_cm. If cache miss, tries to load each cm individually. On an individual cache miss, it preforms the query. CommandLine: python -m wbia.algo.hots.match_chips4 --test-submit_query_request Examples: >>> # SLOW_DOCTEST >>> # xdoctest: +SKIP >>> from wbia.algo.hots.match_chips4 import * # NOQA >>> import wbia >>> qaid_list = [1] >>> daid_list = [1, 2, 3, 4, 5] >>> use_bigcache = True >>> use_cache = True >>> ibs = wbia.opendb(db='testdb1') >>> qreq_ = ibs.new_query_request(qaid_list, daid_list, verbose=True) >>> cm_list = submit_query_request(qreq_=qreq_) """ # Get flag defaults if necessary if verbose is None: verbose = pipeline.VERB_PIPELINE if use_cache is None: use_cache = USE_CACHE if save_qcache is None: save_qcache = SAVE_CACHE if use_bigcache is None: use_bigcache = USE_BIGCACHE if use_supercache is None: use_supercache = USE_SUPERCACHE # Create new query request object to store temporary state if verbose: # logger.info('[mc4] --- Submit QueryRequest_ --- ') logger.info( ub.color_text('[mc4] --- Submit QueryRequest_ --- ', 'yellow')) assert qreq_ is not None, 'query request must be prebuilt' # Check fo empty queries try: assert len(qreq_.daids) > 0, 'there are no database chips' assert len(qreq_.qaids) > 0, 'there are no query chips' except AssertionError as ex: ut.printex( ex, 'Impossible query request', iswarning=True, keys=['qreq_.qaids', 'qreq_.daids'], ) if ut.SUPER_STRICT: raise cm_list = [None for qaid in qreq_.qaids] else: # --- BIG CACHE --- # Do not use bigcache single queries is_big = len(qreq_.qaids) > MIN_BIGCACHE_BUNDLE use_bigcache_ = use_bigcache and use_cache and is_big if use_bigcache_ or save_qcache: cacher = qreq_.get_big_cacher() if use_bigcache_: try: qaid2_cm = cacher.load() cm_list = [qaid2_cm[qaid] for qaid in qreq_.qaids] except (IOError, AttributeError): pass else: return cm_list # ------------ # Execute query request qaid2_cm = execute_query_and_save_L1( qreq_, use_cache, save_qcache, verbose=verbose, use_supercache=use_supercache, invalidate_supercache=invalidate_supercache, ) # ------------ if save_qcache and is_big: cacher.save(qaid2_cm) cm_list = [qaid2_cm[qaid] for qaid in qreq_.qaids] return cm_list
def lr_range_scan(harn, low=1e-6, high=10.0, num=8, niter_train=1, niter_vali=0, scale='log'): """ This takes longer than lr_range_test, but is theoretically more robust. Args: low (float, default=1e-6): minimum lr to scan high (float, default=1.0): maximum lr to scan num (int, default=32): number of lrs to scan niter_train (int, default=10): number of training batches to test for each lr niter_vali (int, default=0): number of validation batches to test for each lr Notes: New better lr test: * For each lr in your scan range * reset model to original state * run a few batches and backprop after each * iterate over those batches again and compute their loss, but dont backprop this time. * associate the average of those losses with the learning rate. Example: >>> from netharn.prefit.lr_tests import * >>> import netharn as nh >>> harn = nh.FitHarn.demo().initialize() >>> result = lr_range_scan(harn) >>> print('result = {!r}'.format(result)) >>> # xdoctest: +REQUIRES(--show) >>> import kwplot >>> result.draw() >>> kwplot.show_if_requested() Ignore: >>> import sys >>> sys.path.append('/home/joncrall/code/netharn/examples') >>> from mnist import setup_harn >>> harn = setup_harn().initialize() >>> harn.preferences['prog_backend'] = 'progiter' >>> result = lr_range_scan(harn, niter_train=100, niter_vali=10, num=32) >>> print('result = {!r}'.format(result)) >>> result.draw() Ignore: >>> from netharn.mixins import * >>> import sys >>> sys.path.append('/home/joncrall/code/netharn/examples') >>> from ggr_matching import setup_harn >>> harn = setup_harn(workers=6, xpu=0).initialize() >>> result = lr_range_scan(harn, niter_train=6 * 2) >>> print('recommended_lr = {!r}'.format(recommended_lr)) >>> draw() TODO: - [ ] ensure that this is a randomized sample of the validation dataset. - [ ] cache the dataset if it fits into memory after we run the first epoch. """ import netharn as nh use_vali = bool(niter_vali) # These are the learning rates we will scan through if scale == 'linear': learning_rates = np.linspace(low, high, num=num, endpoint=True) elif scale == 'log': log_b, base = np.log10, 10 learning_rates = np.logspace(log_b(low), log_b(high), num=num, base=base, endpoint=True) else: raise KeyError(scale) orig_model_state = copy.deepcopy(harn.model.state_dict()) orig_optim_state = copy.deepcopy(harn.optimizer.state_dict()) _orig_on_batch = harn.on_batch _orig_on_epoch = harn.on_epoch try: failed_lr = None best_lr = 0 best_loss = float('inf') records = {} records['train'] = ub.ddict(list) if use_vali: records['vali'] = ub.ddict(list) # TODO: find a way to disable callabacks nicely harn.on_batch = lambda *args, **kw: None harn.on_epoch = lambda *args, **kw: None prog = harn._make_prog(learning_rates, desc='scan learning rates', disable=not harn.preferences['show_prog'], total=len(learning_rates), leave=True, dynamic_ncols=True, position=0) harn.info('Running lr-scan') for lr in prog: # prog.set_description(ub.color_text('scan lr = {:.2g}'.format(lr), 'darkgreen')) if hasattr(prog, 'ensure_newline'): prog.ensure_newline() # Reset model back to its original state harn.optimizer.load_state_dict(orig_optim_state) harn.model.load_state_dict(orig_model_state) # Set the optimizer to the current lr we are scanning harn.optimizer.param_groups[0]['lr'] = lr # Run a partial training and validation epoch try: epoch_metrics = {} epoch_metrics['train'] = harn._demo_epoch('train', learn=True, max_iter=niter_train) curr_loss = epoch_metrics['train']['loss'] if curr_loss > 1000: raise nh.fit_harn.TrainingDiverged if use_vali: epoch_metrics['vali'] = harn._demo_epoch('vali', learn=False, max_iter=niter_vali) curr_loss = epoch_metrics['vali']['loss'] if curr_loss < best_loss: best_lr = lr best_loss = curr_loss curr_text = ub.color_text('curr_lr={:.2g}, best_loss={:.2f}'.format(lr, curr_loss), 'green') else: curr_text = ub.color_text('curr_lr={:.2g}, best_loss={:.2f}'.format(lr, curr_loss), 'red') if hasattr(prog, 'ensure_newline'): prog.set_extra( curr_text + ', ' + ub.color_text('best_lr={:.2g}, best_loss={:.2f}'.format(best_lr, best_loss), 'white') ) prog.update() prog.ensure_newline() except nh.fit_harn.TrainingDiverged: harn.info('Learning is causing divergence, stopping lr-scan') for tag in records.keys(): records[tag]['lr'].append(lr) records[tag]['loss'].append(1000) raise for tag, epoch_metrics in epoch_metrics.items(): records[tag]['lr'].append(lr) for key, value in epoch_metrics.items(): records[tag][key].append(value) harn.info('Finished lr-scan') except nh.fit_harn.TrainingDiverged: failed_lr = lr print('failed_lr = {!r}'.format(failed_lr)) except Exception: failed_lr = lr print('failed_lr = {!r}'.format(failed_lr)) raise finally: # Reset model back to its original state harn.optimizer.load_state_dict(orig_optim_state) harn.model.load_state_dict(orig_model_state) harn.on_batch = _orig_on_batch harn.on_epoch = _orig_on_epoch # Choose an lr to recommend tag = 'vali' if 'vali' in records else 'train' if records.get(tag, []): best_idx = ub.argmin(records[tag]['loss']) best_lr = records[tag]['lr'][best_idx] # Because we aren't doing crazy EWMAs and because we reset weights, we # should be able to simply recommend the lr that minimized the loss. # We may be able to do parabolic extrema finding to get an even better # estimate in most cases. recommended_lr = best_lr else: recommended_lr = None # Give the user a way of visualizing what we did def draw(): import kwplot plt = kwplot.autoplt() plotkw = dict( xlabel='learning-rate', ylabel='loss', xscale=scale, ymin=0, xmin='data', xmax=high, fnum=1, ) R = 2 if 'vali' in records else 1 for i, tag in enumerate(['train', 'vali']): if tag in records: kwplot.multi_plot( xdata=records[tag]['lr'], ydata=records[tag]['loss'], ymax=1.2 * np.percentile(records[tag]['loss'], 60) / .6, pnum=(1, R, i), title=tag + ' lr-scan', **plotkw) plt.plot(best_lr, best_loss, '*') result = TestResult(records, recommended_lr) result.draw = draw return result
def on_epoch(harn): """ Custom code executed at the end of each epoch. This function can optionally return a dictionary containing any scalar quality metrics that you wish to log and monitor. (Note these will be plotted to tensorboard if that is installed). Notes: It is ok to do some medium lifting in this function because it is run relatively few times. Returns: dict: dictionary of scalar metrics for netharn to log CommandLine: xdoctest -m netharn.examples.classification ClfHarn.on_epoch Example: >>> harn = setup_harn().initialize() >>> harn._demo_epoch('vali', max_iter=10) >>> harn.on_epoch() """ from netharn.metrics import clf_report dset = harn.datasets[harn.current_tag] probs = np.vstack(harn._accum_confusion_vectors['probs']) y_true = np.hstack(harn._accum_confusion_vectors['y_true']) y_pred = np.hstack(harn._accum_confusion_vectors['y_pred']) # _pred = probs.argmax(axis=1) # assert np.all(_pred == y_pred) # from netharn.metrics import confusion_vectors # cfsn_vecs = confusion_vectors.ConfusionVectors.from_arrays( # true=y_true, pred=y_pred, probs=probs, classes=dset.classes) # report = cfsn_vecs.classification_report() # combined_report = report['metrics'].loc['combined'].to_dict() # ovr_cfsn = cfsn_vecs.binarize_ovr() # Compute multiclass metrics (new way!) target_names = dset.classes ovr_report = clf_report.ovr_classification_report( y_true, probs, target_names=target_names, metrics=['auc', 'ap', 'mcc', 'brier']) # percent error really isn't a great metric, but its easy and standard. errors = (y_true != y_pred) acc = 1.0 - errors.mean() percent_error = (1.0 - acc) * 100 metrics_dict = ub.odict() metrics_dict['ave_brier'] = ovr_report['ave']['brier'] metrics_dict['ave_mcc'] = ovr_report['ave']['mcc'] metrics_dict['ave_auc'] = ovr_report['ave']['auc'] metrics_dict['ave_ap'] = ovr_report['ave']['ap'] metrics_dict['percent_error'] = percent_error metrics_dict['acc'] = acc harn.info( ub.color_text('ACC FOR {!r}: {!r}'.format(harn.current_tag, acc), 'yellow')) # Clear confusion vectors accumulator for the next epoch harn._accum_confusion_vectors = { 'y_true': [], 'y_pred': [], 'probs': [], } return metrics_dict
def main(): def argval(clikey, envkey=None, default=ub.NoParam): if envkey is not None: envval = os.environ.get(envkey) if envval: default = envval return ub.argval(clikey, default=default) DEFAULT_PY_VER = '{}.{}'.format(sys.version_info.major, sys.version_info.minor) PY_VER = argval('--pyver', 'MB_PYTHON_VERSION', default=DEFAULT_PY_VER) dpath = argval('--dpath', None, default=os.getcwd()) PLAT = argval('--plat', 'PLAT', default='x86_64') UNICODE_WIDTH = argval('--unicode_width', 'UNICODE_WIDTH', '32') import multiprocessing MAKE_CPUS = argval('--make_cpus', 'MAKE_CPUS', multiprocessing.cpu_count() + 1) OPENCV_VERSION = '4.1.0' os.chdir(dpath) BASE = 'manylinux1_{}'.format(PLAT) BASE_REPO = 'quay.io/skvark' PY_TAG = 'cp{ver}-cp{ver}m'.format(ver=PY_VER.replace('.', '')) # do we need the unicode width in this tag? DOCKER_TAG = '{}-opencv{}-py{}'.format(BASE, OPENCV_VERSION, PY_VER) if not exists(join(dpath, 'opencv-' + OPENCV_VERSION)): # FIXME: make robust in the case this fails fpath = ub.grabdata('https://github.com/opencv/opencv/archive/{}.zip'.format(OPENCV_VERSION), dpath=dpath, verbose=1) ub.cmd('ln -s {} .'.format(fpath), cwd=dpath, verbose=3) ub.cmd('unzip {}'.format(fpath), cwd=dpath, verbose=3) dockerfile_fpath = join(dpath, 'Dockerfile_' + DOCKER_TAG) # This docker code is very specific for building linux binaries. # We will need to do a bit of refactoring to handle OSX and windows. # But the goal is to get at least one OS working end-to-end. docker_code = ub.codeblock( ''' FROM {BASE_REPO}/{BASE} # SETUP ENV ARG MB_PYTHON_VERSION={PY_VER} ENV PYTHON_VERSION={PY_VER} ENV PYTHON_ROOT=/opt/python/{PY_TAG}/ ENV PYTHONPATH=/opt/python/{PY_TAG}/lib/python{PY_VER}/site-packages/ ENV PATH=/opt/python/{PY_TAG}/bin:$PATH ENV PYTHON_EXE=/opt/python/{PY_TAG}/bin/python ENV HOME=/root ENV PLAT={PLAT} ENV UNICODE_WIDTH={UNICODE_WIDTH} # Update python environment RUN echo "$PYTHON_EXE" RUN $PYTHON_EXE -m pip install --upgrade pip && \ $PYTHON_EXE -m pip install cmake ninja scikit-build wheel numpy # This is very different for different operating systems # https://github.com/skvark/opencv-python/blob/master/setup.py COPY opencv-{OPENCV_VERSION} /root/code/opencv RUN mkdir -p /root/code/opencv/build && \ cd /root/code/opencv/build && \ cmake -G "Unix Makefiles" \ -DINSTALL_CREATE_DISTRIB=ON \ -DOPENCV_SKIP_PYTHON_LOADER=ON \ -DBUILD_opencv_apps=OFF \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTS=OFF \ -DBUILD_PERF_TESTS=OFF \ -DBUILD_DOCS=OFF \ -DWITH_QT=OFF \ -DWITH_IPP=OFF \ -DWITH_V4L=ON \ -DBUILD_JPEG=OFF \ -DENABLE_PRECOMPILED_HEADERS=OFF \ /root/code/opencv # Note: there is no need to compile the above with python # -DPYTHON3_EXECUTABLE=$PYTHON_EXE \ # -DBUILD_opencv_python3=ON \ # -DOPENCV_PYTHON3_INSTALL_PATH=python \ RUN cd /root/code/opencv/build && make -j{MAKE_CPUS} && make install '''.format(**locals())) try: print(ub.color_text('\n--- DOCKER CODE ---', 'white')) print(ub.highlight_code(docker_code, 'docker')) print(ub.color_text('--- END DOCKER CODE ---\n', 'white')) except Exception: pass with open(dockerfile_fpath, 'w') as file: file.write(docker_code) docker_build_cli = ' '.join([ 'docker', 'build', # '--build-arg PY_VER={}'.format(PY_VER), '--tag {}'.format(DOCKER_TAG), '-f {}'.format(dockerfile_fpath), '.' ]) print('docker_build_cli = {!r}'.format(docker_build_cli)) info = ub.cmd(docker_build_cli, verbose=3, shell=True) if info['ret'] != 0: print(ub.color_text('\n--- FAILURE ---', 'red')) print('Failed command:') print(info['command']) print(info['err']) print('NOTE: sometimes reruning the command manually works') raise Exception('Building docker failed with exit code {}'.format(info['ret'])) else: # write out what the tag is with open(join(dpath, 'opencv-docker-tag.txt'), 'w') as file: file.write(DOCKER_TAG) print(ub.color_text('\n--- SUCCESS ---', 'green'))