def setup_sampler(config): workdir = nh.configure_workdir(config, workdir=join('~/work/siam-ibeis2', config['dbname'])) # TODO: cleanup and hook into ibeis AI if config['dbname'] == 'ggr2': print('Creating torch CocoDataset') root = ub.expandpath('~/data/') print('root = {!r}'.format(root)) train_dset = ndsampler.CocoDataset( data=join(root, 'ggr2-coco/annotations/instances_train2018.json'), img_root=join(root, 'ggr2-coco/images/train2018'), ) train_dset.hashid = 'ggr2-coco-train2018' vali_dset = ndsampler.CocoDataset( data=join(root, 'ggr2-coco/annotations/instances_val2018.json'), img_root=join(root, 'ggr2-coco/images/val2018'), ) vali_dset.hashid = 'ggr2-coco-val2018' print('Creating samplers') samplers = { 'train': ndsampler.CocoSampler(train_dset, workdir=workdir), 'vali': ndsampler.CocoSampler(vali_dset, workdir=workdir), } if config['dbname'] == 'ggr2-revised': print('Creating torch CocoDataset') root = ub.expandpath('~/data/ggr2.coco.revised') print('root = {!r}'.format(root)) train_dset = ndsampler.CocoDataset( data=join(root, 'annotations/instances_train2019.json'), img_root=join(root, 'images/train2019'), ) train_dset.hashid = 'ggr2-coco-revised-train2019' vali_dset = ndsampler.CocoDataset( data=join(root, 'annotations/instances_val2019.json'), img_root=join(root, 'images/val2019'), ) vali_dset.hashid = 'ggr2-coco-revised-val2019' print('Creating samplers') samplers = { 'train': ndsampler.CocoSampler(train_dset, workdir=workdir), 'vali': ndsampler.CocoSampler(vali_dset, workdir=workdir), } else: raise KeyError(config['dbname']) return samplers, workdir
def demo(WindowedSamplerDataset, key='habcam', **kwargs): import ndsampler if key == 'habcam': dset_fpath = ub.expandpath('~/data/noaa/Habcam_2015_g027250_a00102917_c0001_v2_vali.mscoco.json') workdir = ub.expandpath('~/work/bioharn') dset = ndsampler.CocoDataset(dset_fpath) sampler = ndsampler.CocoSampler(dset, workdir=workdir, backend=None) else: sampler = ndsampler.CocoSampler.demo(key) self = WindowedSamplerDataset(sampler, **kwargs) return self
def test_rel_file_link(): dpath = ub.ensure_app_cache_dir('ubelt', 'test_rel_file_link') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) real_fpath = join(ub.ensuredir((dpath, 'dir1')), 'real') link_fpath = join(ub.ensuredir((dpath, 'dir2')), 'link') ub.touch(real_fpath) orig = os.getcwd() try: os.chdir(dpath) real_path = relpath(real_fpath, dpath) link_path = relpath(link_fpath, dpath) link = ub.symlink(real_path, link_path) import sys if sys.platform.startswith('win32') and isfile(link): # Note: if windows hard links the file there is no way we can # tell that it was a symlink. Just verify it exists. from ubelt import _win32_links assert _win32_links._win32_is_hardlinked(real_fpath, link_fpath) else: pointed = ub.util_links._readlink(link) resolved = os.path.realpath( ub.expandpath(join(dirname(link), pointed))) assert os.path.realpath(ub.expandpath(real_fpath)) == resolved except Exception: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) print('TEST FAILED: test_rel_link') print('real_fpath = {!r}'.format(real_fpath)) print('link_fpath = {!r}'.format(link_fpath)) print('real_path = {!r}'.format(real_path)) print('link_path = {!r}'.format(link_path)) try: if 'link' in vars(): print('link = {!r}'.format(link)) if 'pointed' in vars(): print('pointed = {!r}'.format(pointed)) if 'resolved' in vars(): print('resolved = {!r}'.format(resolved)) except Exception: print('...rest of the names are not available') raise finally: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) os.chdir(orig)
def determine_code_dpath(): """ Returns a good place to put the code for the internal dependencies. Returns: PathLike: the directory where you want to store your code In order, the methods used for determing this are: * the `--codedpath` command line flag (may be undocumented in the CLI) * the `--codedir` command line flag (may be undocumented in the CLI) * the CODE_DPATH environment variable * the CODE_DIR environment variable * the directory above this script (e.g. if this is in ~/code/repo/super_setup.py then code dir resolves to ~/code) * the user's ~/code directory. """ import os candidates = [ ub.argval('--codedir', default=''), ub.argval('--codedpath', default=''), os.environ.get('CODE_DPATH', ''), os.environ.get('CODE_DIR', ''), ] valid = [c for c in candidates if c != ''] if len(valid) > 0: code_dpath = valid[0] else: try: # This file should be in the top level of a repo, the directory from # this file should be the code directory. this_fpath = abspath(__file__) code_dpath = abspath(dirname(dirname(this_fpath))) except NameError: code_dpath = ub.expandpath('~/code') if not exists(code_dpath): code_dpath = ub.expandpath(code_dpath) # if CODE_DIR and not exists(CODE_DIR): # import warnings # warnings.warn('environment variable CODE_DIR={!r} was defined, but does not exist'.format(CODE_DIR)) if not exists(code_dpath): raise Exception( ub.codeblock(""" Please specify a correct code_dir using the CLI or ENV. code_dpath={!r} does not exist. """.format(code_dpath))) return code_dpath
def parse_mscoco(): # Test that our implementation can handle the real mscoco data root = ub.expandpath('~/data/standard_datasets/mscoco/') fpath = join(root, 'annotations/instances_val2014.json') img_root = normpath(ub.ensuredir((root, 'images', 'val2014'))) # fpath = join(root, 'annotations/stuff_val2017.json') # img_root = normpath(ub.ensuredir((root, 'images', 'val2017'))) import ujson dataset = ujson.load(open(fpath, 'rb')) import ndsampler dset = ndsampler.CocoDataset(dataset) dset.img_root = img_root gid_iter = iter(dset.imgs.keys()) gid = ub.peek(gid_iter) for gid in ub.ProgIter(gid_iter): img = dset.imgs[gid] ub.grabdata(img['coco_url'], dpath=img_root, verbose=0) anns = [dset.anns[aid] for aid in dset.gid_to_aids[gid]] dset.show_image(gid=gid) ann = anns[0] segmentation = ann['segmentation'] from PIL import Image gpath = join(dset.img_root, img['file_name']) with Image.open(gpath) as pil_img: np_img = np.array(pil_img)
def history(self): """ if available return the history of the model as well """ import netharn as nh if self.info is None: # TODO: check for train_info.json in a few different places fpath = self._rectify_fpath() snap_fpath = ub.expandpath(fpath) candidate_paths = [ join(dirname(snap_fpath), 'train_info.json'), join(dirname(dirname(snap_fpath)), 'train_info.json'), ] info = None for info_fpath in candidate_paths: info_fpath = normpath(info_fpath) try: # Info might be inside of a zipfile info = nh.util.read_json(nh.util.zopen(info_fpath)) break except Exception: pass if info is None: info = '__UNKNOWN__' else: info = self.info return info
def normalize(self): if self['pretrained'] in ['null', 'None']: self['pretrained'] = None if self['datasets'] == 'special:voc': self['train_dataset'] = ub.expandpath('~/data/VOC/voc-trainval.mscoco.json') self['vali_dataset'] = ub.expandpath('~/data/VOC/voc-test-2007.mscoco.json') key = self.get('pretrained', None) or self.get('init', None) if key == 'imagenet': self['pretrained'] = yolo2.initial_imagenet_weights() elif key == 'lightnet': self['pretrained'] = yolo2.demo_voc_weights() if self['pretrained'] is not None: self['init'] = 'pretrained'
def _search_param_in_docstr(docstr, param_str): """ Search `docstr` for type(-s) of `param_str`. >>> _search_param_in_docstr(':type param: int', 'param') ['int'] >>> _search_param_in_docstr('@type param: int', 'param') ['int'] >>> _search_param_in_docstr( ... ':type param: :class:`threading.Thread`', 'param') ['threading.Thread'] >>> bool(_search_param_in_docstr('no document', 'param')) False >>> _search_param_in_docstr(':param int param: some description', 'param') ['int'] """ if DEBUG: open(ub.expandpath('~/jedi-test.txt'), 'a').write('search param\n') # look at #40 to see definitions of those params patterns = [ re.compile(p % re.escape(param_str)) for p in module.DOCSTRING_PARAM_PATTERNS ] for pattern in patterns: match = pattern.search(docstr) if match: return [module._strip_rst_role(match.group(1))] return (module._search_param_in_numpydocstr(docstr, param_str) or list(_search_param_in_googledocstr(docstr, param_str)))
def ensure_voc_coco(dpath=None): """ Download the Pascal VOC data and convert it to coco, if it does exit. Args: dpath (str): download directory. Defaults to "~/data/VOC". Returns: Dict[str, str]: mapping from dataset tags to coco file paths. The original datasets have keys prefixed with underscores. The standard splits keys are train, vali, and test. """ if dpath is None: dpath = ub.expandpath('~/data/VOC') paths = { '_train-2007': join(dpath, 'voc-train-2007.mscoco.json'), '_train-2012': join(dpath, 'voc-train-2007.mscoco.json'), '_val-2007': join(dpath, 'voc-val-2007.mscoco.json'), '_val-2012': join(dpath, 'voc-val-2012.mscoco.json'), 'trainval': join(dpath, 'voc-trainval.mscoco.json'), 'train': join(dpath, 'voc-train.mscoco.json'), 'vali': join(dpath, 'voc-val.mscoco.json'), 'test': join(dpath, 'voc-test-2007.mscoco.json'), } if not all(map(exists, paths.values())): ensure_voc_data(dpath=dpath) convert_voc_to_coco(dpath=dpath) return paths
def expand_module(path): # TODO: use ubelt util_import instead import ubelt as ub _debug = 0 if _debug: import sys print('sys.base_exec_prefix = {!r}'.format(sys.base_exec_prefix)) print('sys.base_prefix = {!r}'.format(sys.base_prefix)) print('sys.exec_prefix = {!r}'.format(sys.exec_prefix)) print('sys.executable = {!r}'.format(sys.executable)) print('sys.implementation = {!r}'.format(sys.implementation)) print('sys.prefix = {!r}'.format(sys.prefix)) print('sys.version = {!r}'.format(sys.version)) print('sys.path = {!r}'.format(sys.path)) import sys extra_path = CONFIG.get('vimtk_sys_path') sys_path = sys.path + [ub.expandpath(p) for p in extra_path] print('expand path = {!r}'.format(path)) modparts = path.split('.') for i in reversed(range(1, len(modparts) + 1)): candidate = '.'.join(modparts[0:i]) print('candidate = {!r}'.format(candidate)) path = ub.modname_to_modpath(candidate, sys_path=sys_path) if path is not None: break print('expanded modname-to-path = {!r}'.format(path)) return path
def _search_return_in_numpydocstr(docstr): """ Search `docstr` (in numpydoc format) for type(-s) of function returns. """ if DEBUG: open(ub.expandpath('~/jedi-test.txt'), 'a').write('search return\n') try: doc = module._get_numpy_doc_string_cls()(docstr) except ImportError: return try: # This is a non-public API. If it ever changes we should be # prepared and return gracefully. returns = doc._parsed_data['Returns'] returns += doc._parsed_data['Yields'] except (KeyError, AttributeError): return for r_name, r_type, r_descr in returns: # Return names are optional and if so the type is in the name if not r_type: r_type = r_name for type_ in module._expand_typestr(r_type): yield type_ # Injected code: # Check for google style return hint for type_ in _search_return_in_googledocstr(docstr): yield type_
def configure_workdir(config={}, **kw): config = _update_defaults(config, kw) if config['workdir'] is None: config['workdir'] = kw['workdir'] workdir = config['workdir'] = ub.expandpath(config['workdir']) ub.ensuredir(workdir) return workdir
def main(): try: script = __file__ except NameError: raise # for Ipython hacking script = ub.expandpath( '~/code/ndsampler/dev/devcheck_python23_isect_index_cache.py') # py2 = ub.find_exe('python2') # py3 = ub.find_exe('python3') # ub.cmd([py2, script, 'load_regions'], shell=True) # ub.cmd([py3, script, 'save_regions'], shell=True) # Register scripts for activating python 2/3 virtual envs that have # ndsampler installed import getpass username = getpass.getuser() if username in ['joncrall', 'jon.crall']: # Hack for Jon's computer activate_cmds = { 'python2': 'we py2.7', 'python3': 'we venv3.6', } else: assert False, 'need to customize activation scripts for your machine' activate_cmds = { 'python2': 'source ~/venv27/bin/activate', 'python3': 'conda activate py36', } def run(py): bash_cmd = ' && '.join([ 'source $HOME/.bashrc', activate_cmds[py], 'python {} access_cache'.format(script), ]) sh_cmd = 'bash -c "{}"'.format(bash_cmd) info = ub.cmd(sh_cmd, shell=True, verbose=3) return info workdir = ub.ensure_app_cache_dir('ndsampler', 'tests', '23_regions') # Save in python3, load in python2 print('\n\n--- SAVE Python3, LOAD Python2 ---') ub.delete(workdir, verbose=1) info = run('python3') # NOQA assert info['ret'] == 0 info = run('python2') # NOQA assert info['ret'] == 0 print('\n\n--- SAVE Python2, LOAD Python3 ---') ub.delete(workdir, verbose=1) # Clear the cache info = run('python2') # NOQA assert info['ret'] == 0 info = run('python3') # NOQA assert info['ret'] == 0
def ensure_voc_data(dpath=None, force=False, years=[2007, 2012]): """ Download the Pascal VOC data if it does not already exist. Example: >>> # xdoctest: +REQUIRES(--download) >>> devkit_dpath = ensure_voc_data() """ if dpath is None: dpath = ub.expandpath('~/data/VOC') devkit_dpath = join(dpath, 'VOCdevkit') # if force or not exists(devkit_dpath): ub.ensuredir(dpath) def extract_tarfile(fpath, dpath='.'): # Old way # ub.cmd('tar xvf "{}" -C "{}"'.format(fpath1, dpath), verbout=1) import tarfile try: tar = tarfile.open(fpath1) tar.extractall(dpath) finally: tar.close() fpath1 = ub.grabdata( 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCdevkit_08-Jun-2007.tar', dpath=dpath) if force or not exists(join(dpath, 'VOCdevkit', 'VOCcode')): extract_tarfile(fpath1, dpath) if 2007 in years: # VOC 2007 train+validation data fpath2 = ub.grabdata( 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar', dpath=dpath) if force or not exists( join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main', 'bird_trainval.txt')): extract_tarfile(fpath2, dpath) # VOC 2007 test data fpath3 = ub.grabdata( 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar', dpath=dpath) if force or not exists( join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main', 'bird_test.txt')): extract_tarfile(fpath3, dpath) if 2012 in years: # VOC 2012 train+validation data fpath4 = ub.grabdata( 'https://pjreddie.com/media/files/VOCtrainval_11-May-2012.tar', dpath=dpath) if force or not exists( join(dpath, 'VOCdevkit', 'VOC2012', 'ImageSets', 'Main', 'bird_trainval.txt')): extract_tarfile(fpath4, dpath) return devkit_dpath
def demo(Repo, ensure=True): repo = Repo( remote='https://github.com/Erotemic/ubelt.git', code_dpath=ub.ensuredir(ub.expandpath('~/tmp/demo-repos')), ) if ensure: repo.ensure() return repo
def cast(self, value=None): if isinstance(value, six.string_types): paths1 = sorted(glob.glob(ub.expandpath(value))) paths2 = smartcast.smartcast(value) if paths1: value = paths1 else: value = paths2 return value
def setup_datasets(workdir=None): if workdir is None: workdir = ub.expandpath('~/data/mnist/') # Define your dataset transform = torchvision.transforms.Compose([ torchvision.transforms.ToTensor(), # torchvision.transforms.Normalize((0.1307,), (0.3081,)) ]) learn_dset = nh.data.MNIST(workdir, transform=transform, train=True, download=True) test_dset = nh.data.MNIST(workdir, transform=transform, train=False, download=True) # split the learning dataset into training and validation # take a subset of data factor = .15 n_vali = int(len(learn_dset) * factor) learn_idx = np.arange(len(learn_dset)) rng = np.random.RandomState(0) rng.shuffle(learn_idx) reduction = int(ub.argval('--reduction', default=1)) vali_idx = torch.LongTensor(learn_idx[:n_vali][::reduction]) train_idx = torch.LongTensor(learn_idx[n_vali:][::reduction]) train_dset = torch.utils.data.Subset(learn_dset, train_idx) vali_dset = torch.utils.data.Subset(learn_dset, vali_idx) datasets = { 'train': train_dset, 'vali': vali_dset, 'test': test_dset, } if not ub.argflag('--test'): del datasets['test'] for tag, dset in datasets.items(): # Construct the PCCs (positive connected components) # These are groups of item indices which are positive matches if isinstance(dset, torch.utils.data.Subset): labels = dset.dataset.train_labels[dset.indices] else: labels = dset.labels unique_labels, groupxs = kwarray.group_indices(labels.numpy()) dset.pccs = [xs.tolist() for xs in groupxs] # Give the training dataset an input_id datasets['train'].input_id = 'mnist_' + ub.hash_data( train_idx.numpy())[0:8] return datasets, workdir
def ensure_voc_data(VOCDataset, dpath=None, force=False, years=[2007, 2012]): """ Download the Pascal VOC 2007 data if it does not already exist. CommandLine: python -m netharn.data.voc VOCDataset.ensure_voc_data Example: >>> # SCRIPT >>> # xdoc: +REQUIRES(--voc) >>> from netharn.data.voc import * # NOQA >>> VOCDataset.ensure_voc_data() """ if dpath is None: dpath = ub.expandpath('~/data/VOC') devkit_dpath = join(dpath, 'VOCdevkit') # if force or not exists(devkit_dpath): ub.ensuredir(dpath) fpath1 = ub.grabdata( 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCdevkit_08-Jun-2007.tar', dpath=dpath) if force or not exists(join(dpath, 'VOCdevkit', 'VOCcode')): ub.cmd('tar xvf "{}" -C "{}"'.format(fpath1, dpath), verbout=1) if 2007 in years: # VOC 2007 train+validation data fpath2 = ub.grabdata( 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar', dpath=dpath) if force or not exists( join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main', 'bird_trainval.txt')): ub.cmd('tar xvf "{}" -C "{}"'.format(fpath2, dpath), verbout=1) # VOC 2007 test data fpath3 = ub.grabdata( 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar', dpath=dpath) if force or not exists( join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main', 'bird_test.txt')): ub.cmd('tar xvf "{}" -C "{}"'.format(fpath3, dpath), verbout=1) if 2012 in years: # VOC 2012 train+validation data fpath4 = ub.grabdata( 'https://pjreddie.com/media/files/VOCtrainval_11-May-2012.tar', dpath=dpath) if force or not exists( join(dpath, 'VOCdevkit', 'VOC2012', 'ImageSets', 'Main', 'bird_trainval.txt')): ub.cmd('tar xvf "{}" -C "{}"'.format(fpath4, dpath), verbout=1) return devkit_dpath
def test_rel_dir_link(): dpath = ub.ensure_app_cache_dir('ubelt', 'test_rel_dir_link') ub.delete(dpath, verbose=2) ub.ensuredir(dpath, verbose=2) real_dpath = join(ub.ensuredir((dpath, 'dir1')), 'real') link_dpath = join(ub.ensuredir((dpath, 'dir2')), 'link') ub.ensuredir(real_dpath) orig = os.getcwd() try: os.chdir(dpath) real_path = relpath(real_dpath, dpath) link_path = relpath(link_dpath, dpath) link = ub.symlink(real_path, link_path) # Note: on windows this is hacked. pointed = ub.util_links._readlink(link) resolved = os.path.realpath(ub.expandpath(join(dirname(link), pointed))) assert os.path.realpath(ub.expandpath(real_dpath)) == resolved except Exception: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) print('TEST FAILED: test_rel_link') print('real_dpath = {!r}'.format(real_dpath)) print('link_dpath = {!r}'.format(link_dpath)) print('real_path = {!r}'.format(real_path)) print('link_path = {!r}'.format(link_path)) try: if 'link' in vars(): print('link = {!r}'.format(link)) if 'pointed' in vars(): print('pointed = {!r}'.format(pointed)) if 'resolved' in vars(): print('resolved = {!r}'.format(resolved)) except Exception: print('...rest of the names are not available') raise finally: util_links._dirstats(dpath) util_links._dirstats(join(dpath, 'dir1')) util_links._dirstats(join(dpath, 'dir2')) os.chdir(orig)
def setup_datasets(workdir=None): if workdir is None: workdir = ub.expandpath('~/data/mnist/') # Define your dataset transform = torchvision.transforms.Compose([ torchvision.transforms.ToTensor(), torchvision.transforms.Normalize((0.1307, ), (0.3081, )) ]) learn_dset = torchvision.datasets.MNIST(workdir, transform=transform, train=True, download=True) test_dset = torchvision.datasets.MNIST(workdir, transform=transform, train=False, download=True) # split the learning dataset into training and validation # take a subset of data factor = .15 n_vali = int(len(learn_dset) * factor) learn_idx = np.arange(len(learn_dset)) rng = np.random.RandomState(0) rng.shuffle(learn_idx) reduction = int(ub.argval('--reduction', default=1)) vali_idx = torch.LongTensor(learn_idx[:n_vali][::reduction]) train_idx = torch.LongTensor(learn_idx[n_vali:][::reduction]) train_dset = torch.utils.data.Subset(learn_dset, train_idx) vali_dset = torch.utils.data.Subset(learn_dset, vali_idx) classes = [ 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine' ] datasets = { 'train': train_dset, 'vali': vali_dset, 'test': test_dset, } for tag, dset in datasets.items(): dset.classes = classes dset.num_classes = len(classes) # Give the training dataset an input_id datasets['train'].input_id = 'mnist_' + ub.hash_data( train_idx.numpy())[0:8] return datasets, workdir
def count_ubelt_usage(): import ubelt as ub import glob from os.path import join names = [ 'xdoctest', 'netharn', 'xdev', 'xinspect', 'ndsampler', 'kwil', 'kwarray', 'kwimage', 'kwplot', 'scriptconfig', ] all_fpaths = [] for name in names: repo_fpath = ub.expandpath(join('~/code', name)) fpaths = glob.glob(join(repo_fpath, '**', '*.py'), recursive=True) for fpath in fpaths: all_fpaths.append((name, fpath)) import re pat = re.compile(r'\bub\.(?P<attr>[a-zA-Z_][A-Za-z_0-9]*)\b') import ubelt as ub pkg_to_hist = ub.ddict(lambda: ub.ddict(int)) for name, fpath in ub.ProgIter(all_fpaths): text = open(fpath, 'r').read() for match in pat.finditer(text): attr = match.groupdict()['attr'] if attr in ub.__all__: pkg_to_hist[name][attr] += 1 hist_iter = iter(pkg_to_hist.values()) usage = next(hist_iter).copy() for other in hist_iter: for k, v in other.items(): usage[k] += v for attr in ub.__all__: usage[attr] += 0 for name in pkg_to_hist.keys(): pkg_to_hist[name] = ub.odict( sorted(pkg_to_hist[name].items(), key=lambda t: t[1])[::-1]) usage = ub.odict(sorted(usage.items(), key=lambda t: t[1])[::-1]) print(ub.repr2(pkg_to_hist, nl=2)) print(ub.repr2(usage, nl=1))
def main(): import argparse parser = argparse.ArgumentParser(prog='manage_runs', description=ub.codeblock(''' Summarize, cleanup, and mange runs in a workdir produced by netharn ''')) parser.add_argument(*('-w', '--workdir'), type=str, help='specify the workdir for your project', default=None) parser.add_argument(*('-f', '--force'), help='dry run', action='store_false', dest='dry') # parser.add_argument(*('-n', '--dry'), help='dry run', action='store_true') parser.add_argument(*('--recent', ), help='num recent to keep', type=int, default=100) parser.add_argument(*('--factor', ), help='keep one every <factor> epochs', type=int, default=1) valid_modes = [ 'clean_runs', 'clean_checkpoints', 'clean_monitor', 'summarize', ] parser.add_argument('--mode', help='valid modes are {}'.format( ', '.join(valid_modes)), default='summarize') args, unknown = parser.parse_known_args() ns = args.__dict__.copy() print('ns = {!r}'.format(ns)) mode = ns.pop('mode') ns['workdir'] = ub.expandpath(ns['workdir']) if mode == 'clean_runs': _devcheck_remove_dead_runs(workdir=ns['workdir'], dry=ns['dry']) elif mode == 'clean_checkpoints': _devcheck_manage_checkpoints(**ns) elif mode == 'clean_monitor': _devcheck_manage_monitor(workdir=ns['workdir'], dry=ns['dry']) elif mode == 'summarize': _summarize_workdir(workdir=ns['workdir']) else: raise KeyError(mode)
def test_pathlib(): try: import pathlib base = pathlib.Path(ub.ensure_app_cache_dir('ubelt')) dpath = base.joinpath('test_pathlib_mkdir') # ensuredir ub.delete(dpath) assert not dpath.exists() got = ub.ensuredir(dpath) assert got.exists() # compressuser assert ub.compressuser(base) == '~/.cache/ubelt' assert ub.augpath(base, prefix='foo') == '/home/joncrall/.cache/fooubelt' ub.expandpath(base) except Exception: import pytest pytest.skip('pathlib is not installed')
def autogen_mkint_utils(): import ubelt as ub # Uses netharn closer until it is ported to a standalone module from liberator import closer closer = closer.Closer() from ubelt import util_import closer.add_dynamic(util_import.split_modpath) closer.add_dynamic(util_import.modpath_to_modname) closer.add_dynamic(util_import.modname_to_modpath) closer.expand(["ubelt"]) text = closer.current_sourcecode() print(text) import redbaron new_baron = redbaron.RedBaron(text) new_names = [n.name for n in new_baron.node_list if n.type in ["class", "def"]] import mkinit from mkinit import util from mkinit.util import util_import # NOQA old_baron = redbaron.RedBaron(open(mkinit.util.util_import.__file__, "r").read()) old_names = [n.name for n in old_baron.node_list if n.type in ["class", "def"]] set(old_names) - set(new_names) set(new_names) - set(old_names) prefix = ub.codeblock( ''' # -*- coding: utf-8 -*- """ This file was autogenerated based on code in ubelt """ from __future__ import print_function, division, absolute_import, unicode_literals ''' ) code = prefix + "\n" + text + "\n" print(code) fpath = ub.expandpath("~/code/mkinit/mkinit/util/util_import.py") open(fpath, "w").write(code)
def demo(): """ CommandLine: # Correctly reports skipped (although an only skipped test report # should probably be yellow) xdoctest -m dev/demo_issues.py demo_requires_skips_all_v1 # Incorrectly reports success xdoctest -m dev/demo_issues.py demo_requires_skips_all_v2 # Correctly reports success xdoctest -m dev/demo_issues.py demo_requires_skips_all_v2 --cliflag # Correctly reports success xdoctest -m dev/demo_issues.py demo_requires_skips_all_v1 --cliflag """ # Programatic reproduction (notice the first one also reports itself in # pytest mode which is also wrong) import xdoctest xdoctest.doctest_callable(demo_requires_skips_all_v1) xdoctest.doctest_callable(demo_requires_skips_all_v2) import sys, ubelt sys.path.append(ubelt.expandpath('~/code/xdoctest/dev')) import demo_issues # Correctly reports skipped xdoctest.doctest_module(demo_issues, command='demo_requires_skips_all_v1', argv=[]) # Incorrectly reports passed xdoctest.doctest_module(demo_issues, command='demo_requires_skips_all_v2', argv=[]) # argv not respected? xdoctest.doctest_module(demo_issues, command='demo_requires_skips_all_v1', argv=['--cliflag']) # argv not respected? xdoctest.doctest_module(demo_issues, command='demo_requires_skips_all_v2', argv=['--cliflag'])
def expand_module_prefix(path): # TODO: we could parse the AST to figure out if the prefix is an alias # for a known module. # Check if the path certainly looks like it could be a chain of python # attribute accessors. if re.match(r'^[\w\d_.]*$', path): extra_path = CONFIG.get('vimtk_sys_path') sys_path = sys.path + [ub.expandpath(p) for p in extra_path] parts = path.split('.') for i in reversed(range(len(parts))): prefix = '.'.join(parts[:i]) path = ub.modname_to_modpath(prefix, sys_path=sys_path) if path is not None: print('expanded prefix = {!r}'.format(path)) return path print('expanded prefix = {!r}'.format(None)) return None
def __init__(repo, **kwargs): repo.name = kwargs.pop('name', None) repo.dpath = kwargs.pop('dpath', None) repo.code_dpath = kwargs.pop('code_dpath', None) repo.remotes = kwargs.pop('remotes', None) repo.remote = kwargs.pop('remote', None) repo.branch = kwargs.pop('branch', 'master') repo._logged_lines = [] repo._logged_cmds = [] if repo.remote is None: if repo.remotes is None: raise ValueError('must specify some remote') else: if len(repo.remotes) > 1: raise ValueError('remotes are ambiguous, specify one') else: repo.remote = ub.peek(repo.remotes) else: if repo.remotes is None: _default_remote = 'origin' repo.remotes = {_default_remote: repo.remote} repo.remote = _default_remote repo.url = repo.remotes[repo.remote] if repo.name is None: suffix = repo.url.split('/')[-1] repo.name = suffix.split('.git')[0] if repo.dpath is None: repo.dpath = join(repo.code_dpath, repo.name) repo.pkg_dpath = join(repo.dpath, repo.name) for path_attr in ['dpath', 'code_dpath']: path = getattr(repo, path_attr) if path is not None: setattr(repo, path_attr, ub.expandpath(path)) repo.verbose = kwargs.pop('verbose', 3) if kwargs: raise ValueError('unknown kwargs = {}'.format(kwargs.keys())) repo._pygit = None
def cmake_clean(dpath='.'): """ """ dpath = ub.expandpath(dpath) cmake_cache_fpath = join(dpath, 'CMakeCache.txt') if not exists(cmake_cache_fpath): raise Exception('This does not look like a cmake build directory. ' 'No CMakeCache.txt exists') fpath_set = set(glob.glob(join(dpath, '*'))) - {cmake_cache_fpath} for fpath in list(fpath_set): if basename(fpath).startswith('_cmake_build_backup_'): fpath_set.remove(fpath) backup_dpath = ub.ensuredir( join(dpath, '_cmake_build_backup_' + ub.timestamp())) for fpath in ub.ProgIter(fpath_set, 'moving files'): shutil.move(fpath, backup_dpath)
def _autogen_xdoctest_utils(): import ubelt as ub # Uses netharn closer until it is ported to a standalone module import netharn as nh closer = nh.export.closer.Closer() from ubelt import util_import closer.add_dynamic(util_import.split_modpath) closer.add_dynamic(util_import.modpath_to_modname) closer.add_dynamic(util_import.modname_to_modpath) closer.add_dynamic(util_import.import_module_from_name) closer.add_dynamic(util_import.import_module_from_path) closer.add_dynamic(util_import._pkgutil_modname_to_modpath) closer.add_dynamic(util_import._importlib_import_modpath) closer.add_dynamic(util_import.is_modname_importable) closer.expand(['ubelt']) text = closer.current_sourcecode() print(text) import redbaron new_baron = redbaron.RedBaron(text) new_names = [n.name for n in new_baron.node_list if n.type in ['class', 'def']] import xdoctest old_baron = redbaron.RedBaron(open(xdoctest.utils.util_import.__file__, 'r').read()) old_names = [n.name for n in old_baron.node_list if n.type in ['class', 'def']] set(old_names) - set(new_names) set(new_names) - set(old_names) prefix = ub.codeblock( ''' # -*- coding: utf-8 -*- """ This file was autogenerated based on code in ubelt """ from __future__ import print_function, division, absolute_import, unicode_literals ''') fpath = ub.expandpath('~/code/xdoctest/xdoctest/utils/util_import.py') open(fpath, 'w').write(prefix + '\n' + text + '\n')
def main(): flake8_errors = [ 'E126', # continuation line hanging-indent 'E127', # continuation line over-indented for visual indent 'E201', # whitespace after '(' 'E202', # whitespace before ']' 'E203', # whitespace before ', ' 'E221', # multiple spaces before operator (TODO: I wish I could make an exception for the equals operator. Is there a way to do this?) 'E222', # multiple spaces after operator 'E241', # multiple spaces after , 'E265', # block comment should start with "# " 'E271', # multiple spaces after keyword 'E272', # multiple spaces before keyword 'E301', # expected 1 blank line, found 0 'E305', # expected 1 blank line after class / func 'E306', # expected 1 blank line before func #'E402', # moduel import not at top 'E501', # line length > 79 'W602', # Old reraise syntax 'E266', # too many leading '#' for block comment 'N801', # function name should be lowercase [N806] 'N802', # function name should be lowercase [N806] 'N803', # argument should be lowercase [N806] 'N805', # first argument of a method should be named 'self' 'N806', # variable in function should be lowercase [N806] 'N811', # constant name imported as non constant 'N813', # camel case 'W504', # line break after binary operator ] flake8_args_list = [ '--max-line-length 79', #'--max-line-length 100', '--ignore=' + ','.join(flake8_errors) ] flake8_args = ' '.join(flake8_args_list) import ubelt as ub import sys loc = ub.expandpath('~/code/xdoctest/xdoctest') command = 'flake8 ' + flake8_args + ' ' + loc print('command = {!r}'.format(command)) info = ub.cmd(command, verbose=3) sys.exit(info['ret'])
def write_default_ipython_profile(): """ CommandLine: python ~/local/init/init_ipython_config.py python -c "import xdev, ubelt; xdev.startfile(ubelt.truepath('~/.ipython/profile_default'))" python -c "import xdev, ubelt; xdev.editfile(ubelt.truepath('~/.ipython/profile_default/ipython_config.py'))" References: http://2sn.org/python/ipython_config.py """ dpath = ub.expandpath('~/.ipython/profile_default') ub.ensuredir(dpath) ipy_config_fpath = join(dpath, 'ipython_config.py') ipy_config_text = ub.codeblock( r''' # STARTBLOCK import six c = get_config() # NOQA c.InteractiveShellApp.exec_lines = [] if six.PY2: future_line = ( 'from __future__ import absolute_import, division, print_function, with_statement, unicode_literals') c.InteractiveShellApp.exec_lines.append(future_line) # Fix sip versions try: import sip # http://stackoverflow.com/questions/21217399/pyqt4-qtcore-qvariant-object-instead-of-a-string sip.setapi('QVariant', 2) sip.setapi('QString', 2) sip.setapi('QTextStream', 2) sip.setapi('QTime', 2) sip.setapi('QUrl', 2) sip.setapi('QDate', 2) sip.setapi('QDateTime', 2) if hasattr(sip, 'setdestroyonexit'): sip.setdestroyonexit(False) # This prevents a crash on windows except ImportError as ex: pass except ValueError as ex: print('Warning: Value Error: %s' % str(ex)) pass c.InteractiveShellApp.exec_lines.append('%load_ext autoreload') c.InteractiveShellApp.exec_lines.append('%autoreload 2') #c.InteractiveShellApp.exec_lines.append('%pylab qt4') c.InteractiveShellApp.exec_lines.append('import numpy as np') c.InteractiveShellApp.exec_lines.append('import ubelt as ub') c.InteractiveShellApp.exec_lines.append('import xdev') c.InteractiveShellApp.exec_lines.append('import pandas as pd') c.InteractiveShellApp.exec_lines.append('pd.options.display.max_columns = 40') c.InteractiveShellApp.exec_lines.append('pd.options.display.width = 160') c.InteractiveShellApp.exec_lines.append('pd.options.display.max_rows = 20') c.InteractiveShellApp.exec_lines.append('pd.options.display.float_format = lambda x: \'%.4f\' % (x,)') c.InteractiveShellApp.exec_lines.append('import networkx as nx') c.InteractiveShellApp.exec_lines.append('from os.path import *') c.InteractiveShellApp.exec_lines.append('from six.moves import cPickle as pickle') #c.InteractiveShellApp.exec_lines.append('if \'verbose\' not in vars():\\n verbose = True') import ubelt as ub c.InteractiveShellApp.exec_lines.append(ub.codeblock( """ class classproperty(property): def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)() class vim(object): @classproperty def focus(cls): from vimtk.cplat_ctrl import Window Window.find('GVIM').focus() @classproperty def copy(cls): import time from vimtk.cplat_ctrl import Window gvim_window = Window.find('GVIM') gvim_window.focus() import vimtk import IPython ipy = IPython.get_ipython() lastline = ipy.history_manager.input_hist_parsed[-2] vimtk.cplat.copy_text_to_clipboard(lastline) from vimtk import xctrl xctrl.XCtrl.do( ('focus', 'GVIM'), ('key', 'ctrl+v'), ('focus', 'x-terminal-emulator.X-terminal-emulator') ) """ )) #c.InteractiveShell.autoindent = True #c.InteractiveShell.colors = 'LightBG' #c.InteractiveShell.confirm_exit = False #c.InteractiveShell.deep_reload = True c.InteractiveShell.editor = 'gvim' #c.InteractiveShell.xmode = 'Context' # ENDBOCK ''' ) with open(ipy_config_fpath, 'w') as file: file.write(ipy_config_text + '\n')