def test_load(): fpath = 'foo.pt' cpu = XPU(None) gpu = XPU(0) gpu_data = gpu.move(torch.FloatTensor([10])) cpu_data = cpu.move(torch.FloatTensor([10])) torch.save(gpu_data, ub.augpath(fpath, 'gpu')) torch.save(cpu_data, ub.augpath(fpath, 'cpu')) gpu_data2 = gpu.load(ub.augpath(fpath, 'cpu')) cpu_data2 = cpu.load(ub.augpath(fpath, 'gpu')) assert not gpu_data2.is_gpu() assert cpu_data2.is_gpu()
def gt_info_from_path(pred_fpath): # Hack to read UrbanMapper specific data gtl_fname = ub.augpath(basename(pred_fpath), suffix='_GTL', ext='.tif') gti_fname = ub.augpath(basename(pred_fpath), suffix='_GTI', ext='.tif') dsm_fname = ub.augpath(basename(pred_fpath), suffix='_DSM', ext='.tif') gtl_fpath = join(train_data_path, gtl_fname) gti_fpath = join(train_data_path, gti_fname) dsm_fpath = join(train_data_path, dsm_fname) gti = util.imread(gti_fpath) gtl = util.imread(gtl_fpath) dsm = util.imread(dsm_fpath) uncertain = (gtl == 65) return gti, uncertain, dsm
def initial_weights(): """ Weights pretrained trained ImageNet by yolo9000-pytorch """ import os url = 'https://data.kitware.com/api/v1/file/5ab513438d777f068578f1d0/download' dpath = ub.ensure_app_cache_dir('clab/yolo_v2') fname = 'darknet19.weights.npz' dest = os.path.join(dpath, fname) if not os.path.exists(dest): command = 'curl -X GET {} > {}'.format(url, dest) ub.cmd(command, verbout=1, shell=True) # url = 'http://acidalia.kitware.com:8000/weights/darknet19.weights.npz' # npz_fpath = ub.grabdata(url, dpath=ub.ensure_app_cache_dir('clab')) # convert to torch weights npz_fpath = dest torch_fpath = ub.augpath(npz_fpath, ext='.pt') if not os.path.exists(torch_fpath): # hack to transform initial state model = Darknet19(num_classes=20) model.load_from_npz(npz_fpath, num_conv=18) torch.save(model.state_dict(), torch_fpath) # from clab import xpu_device # xpu = xpu_device.XPU('gpu') # xpu.load(torch_fpath) # torch.load(torch_fpath) return torch_fpath
def main(): import xdoctest import ubelt as ub import sys modpath = ub.modname_to_modpath('netharn.examples') name_to_path = {} submods = list(xdoctest.static_analysis.package_modpaths(modpath)) for submod in submods: modname = ub.augpath(submod, dpath='', ext='') if not modname.startswith('_'): name_to_path[modname] = submod print('name_to_path = {}'.format(ub.repr2(name_to_path, nl=1))) chosen = None for arg in sys.argv[1:2]: print('arg = {!r}'.format(arg)) if arg in name_to_path: chosen = name_to_path[arg] break print('chosen = {!r}'.format(chosen)) assert chosen is not None module = ub.import_module_from_path(chosen) print('module = {!r}'.format(module)) module.main()
def _autojit_cython(pyx_fpath, verbose=1): """ This idea is that given a pyx file, we try to compile it. We write a stamp file so subsequent calls should be very fast as long as the source pyx has not changed. Parameters ---------- pyx_fpath : str path to the pyx file verbose : int higher is more verbose. """ import shutil # TODO: move necessary ubelt utilities to nx.utils? # Separate this into its own util? if shutil.which("cythonize"): pyx_dpath = dirname(pyx_fpath) # Check if the compiled library exists pyx_base = splitext(basename(pyx_fpath))[0] SO_EXTS = _platform_pylib_exts() so_fname = False for fname in os.listdir(pyx_dpath): if fname.startswith(pyx_base) and fname.endswith(SO_EXTS): so_fname = fname break try: # Currently this functionality depends on ubelt. # We could replace ub.cmd with subprocess.check_call and ub.augpath # with os.path operations, but hash_file and CacheStamp are harder # to replace. We can use "liberator" to statically extract these # and add them to nx.utils though. import ubelt as ub except Exception: return False else: if so_fname is False: # We can compute what the so_fname will be if it doesnt exist so_fname = pyx_base + SO_EXTS[0] so_fpath = join(pyx_dpath, so_fname) depends = [ub.hash_file(pyx_fpath, hasher="sha1")] stamp_fname = ub.augpath(so_fname, ext=".jit.stamp") stamp = ub.CacheStamp( stamp_fname, dpath=pyx_dpath, product=so_fpath, depends=depends, verbose=verbose, ) if stamp.expired(): ub.cmd("cythonize -i {}".format(pyx_fpath), verbose=verbose, check=True) stamp.renew() return True
def 路径_名字处理(路径, 末尾='', 前缀='', 扩展名=None, 名称=None, dpath=None, relative=None, 不包含点=False): return ub.augpath(路径, 末尾, 前缀, 扩展名, 名称, dpath, relative, 不包含点)
def test_pathlib(): try: import pathlib base = pathlib.Path(ub.ensure_app_cache_dir('ubelt')) dpath = base.joinpath('test_pathlib_mkdir') # ensuredir ub.delete(dpath) assert not dpath.exists() got = ub.ensuredir(dpath) assert got.exists() # compressuser assert ub.compressuser(base) == '~/.cache/ubelt' assert ub.augpath(base, prefix='foo') == '/home/joncrall/.cache/fooubelt' ub.expandpath(base) except Exception: import pytest pytest.skip('pathlib is not installed')
def ensure_sql_coco_view(dset): """ Create an SQL view of the COCO dataset """ db_fpath = ub.augpath(dset.fpath, prefix='.', ext='.sqlite') db_uri = 'sqlite:///' + db_fpath # dpath = dirname(dset.fpath) self = CocoSqlDatabase(db_uri) import os needs_rewrite = True if exists(db_fpath): needs_rewrite = (os.stat(dset.fpath).st_mtime > os.stat(db_fpath).st_mtime) if needs_rewrite: # Write to the SQL instance self.delete() self.connect() # Convert a coco file to an sql database self.populate_from(dset) else: self.connect() return self
def detect_cli(config={}): """ CommandLine: python -m bioharn.detect_predict --help CommandLine: python -m bioharn.detect_predict \ --dataset=~/data/noaa/Habcam_2015_g027250_a00102917_c0001_v2_test.mscoco.json \ --deployed=/home/joncrall/work/bioharn/fit/runs/bioharn-det-v11-test-cascade/myovdqvi/deploy_MM_CascadeRCNN_myovdqvi_035_MVKVVR.zip \ --out_dpath=~/work/bioharn/habcam_test_out \ --draw=100 \ --input_dims=512,512 \ --xpu=0 --batch_size=1 Ignore: >>> config = {} >>> config['dataset'] = '~/data/noaa/Habcam_2015_g027250_a00102917_c0001_v2_vali.mscoco.json' >>> config['deployed'] = '/home/joncrall/work/bioharn/fit/runs/bioharn-det-v11-test-cascade/myovdqvi/deploy_MM_CascadeRCNN_myovdqvi_035_MVKVVR.zip' >>> config['out_dpath'] = 'out' """ import kwarray import ndsampler from os.path import basename, join, exists, isfile, isdir # NOQA config = DetectPredictCLIConfig(config, cmdline=True) print('config = {}'.format(ub.repr2(config.asdict()))) out_dpath = ub.expandpath(config.get('out_dpath')) import six if isinstance(config['dataset'], six.string_types): if config['dataset'].endswith('.json'): dataset_fpath = ub.expandpath(config['dataset']) coco_dset = ndsampler.CocoDataset(dataset_fpath) # Running prediction is much faster if you can build a sampler. sampler_backend = { 'type': 'cog', 'config': { 'compress': 'JPEG', }, '_hack_old_names': False, # flip to true to use legacy caches } sampler_backend = None print('coco hashid = {}'.format(coco_dset._build_hashid())) else: sampler_backend = None if exists(config['dataset']) and isfile(config['dataset']): # Single image case image_fpath = ub.expandpath(config['dataset']) coco_dset = ndsampler.CocoDataset() coco_dset.add_image(image_fpath) elif isinstance(config['dataset'], list): # Multiple image case gpaths = config['dataset'] gpaths = [ub.expandpath(g) for g in gpaths] coco_dset = ndsampler.CocoDataset() for gpath in gpaths: coco_dset.add_image(gpath) else: raise TypeError(config['dataset']) draw = config.get('draw') workdir = ub.expandpath(config.get('workdir')) det_outdir = ub.ensuredir((out_dpath, 'pred')) pred_config = ub.dict_subset(config, DetectPredictConfig.default) print('Create sampler') sampler = ndsampler.CocoSampler(coco_dset, workdir=workdir, backend=sampler_backend) print('prepare frames') sampler.frames.prepare(workers=config['workers']) print('Create predictor') predictor = DetectPredictor(pred_config) print('Ensure model') predictor._ensure_model() pred_dataset = coco_dset.dataset.copy() pred_dataset['annotations'] = [] pred_dset = ndsampler.CocoDataset(pred_dataset) # self = predictor predictor.config['verbose'] = 1 pred_gen = predictor.predict_sampler(sampler) buffered_gen = AsyncBufferedGenerator(pred_gen, size=coco_dset.n_images) gid_to_pred = {} prog = ub.ProgIter(buffered_gen, total=coco_dset.n_images, desc='buffered detect') for img_idx, (gid, dets) in enumerate(prog): gid_to_pred[gid] = dets for ann in dets.to_coco(): ann['image_id'] = gid try: catname = ann['category_name'] ann['category_id'] = pred_dset._resolve_to_cid(catname) except KeyError: if 'category_id' not in ann: cid = pred_dset.add_category(catname) ann['category_id'] = cid pred_dset.add_annotation(**ann) single_img_coco = pred_dset.subset([gid]) single_pred_dpath = ub.ensuredir((det_outdir, 'single_image')) single_pred_fpath = join(single_pred_dpath, 'detections_gid_{:08d}.mscoco.json'.format(gid)) single_img_coco.dump(single_pred_fpath, newlines=True) if draw is True or (draw and img_idx < draw): draw_outdir = ub.ensuredir((out_dpath, 'draw')) img_fpath = coco_dset.load_image_fpath(gid) gname = basename(img_fpath) viz_fname = ub.augpath(gname, prefix='detect_', ext='.jpg') viz_fpath = join(draw_outdir, viz_fname) image = kwimage.imread(img_fpath) flags = dets.scores > .2 flags[kwarray.argmaxima(dets.scores, num=10)] = True top_dets = dets.compress(flags) toshow = top_dets.draw_on(image, alpha=None) # kwplot.imshow(toshow) kwimage.imwrite(viz_fpath, toshow, space='rgb') pred_fpath = join(det_outdir, 'detections.mscoco.json') print('Dump detections to pred_fpath = {!r}'.format(pred_fpath)) pred_dset.dump(pred_fpath, newlines=True)
def copy_over_stuff_from_nx_pr(): """ Copy from networkx dev/ordered_subtree_isomorphism b08106baae7987af1dc755a6308fcd11bc21cbc8 """ dst = ub.expandpath('~/code/netharn/netharn/initializers/_nx_ext_v2') from os.path import join nx_repo = dirname(dirname(nx.__file__)) to_copy = [ join(nx_repo, 'networkx/algorithms/string/_autojit.py'), join(nx_repo, 'networkx/algorithms/string/balanced_embedding.py'), join(nx_repo, 'networkx/algorithms/string/balanced_embedding_cython.pyx'), join(nx_repo, 'networkx/algorithms/string/balanced_isomorphism.py'), join(nx_repo, 'networkx/algorithms/string/balanced_isomorphism_cython.pyx'), join(nx_repo, 'networkx/algorithms/string/balanced_sequence.py'), join(nx_repo, 'networkx/algorithms/minors/tree_embedding.py'), join(nx_repo, 'networkx/algorithms/minors/tree_isomorphism.py'), ] import shutil fpath_list = [] for fpath in to_copy: fpath2 = ub.augpath(fpath, dpath=dst) fpath_list.append(fpath2) shutil.copy2(fpath, fpath2) util_fpath = join(dst, 'utils.py') closer = liberator.Closer() closer.add_dynamic(nx.forest_str) closer.add_dynamic(nx.random_ordered_tree) closer.add_dynamic(nx.random_tree) with open(util_fpath, 'w') as file: file.write(closer.current_sourcecode()) from rob import rob_nav force = True # force = 0 rob_nav._ut_sed(r'networkx\.algorithms\.string', 'netharn.initializers._nx_ext_v2', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'networkx/networkx/algorithms/string', 'netharn/netharn/initializers/_nx_ext_v2', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'networkx/algorithms/string', 'netharn/initializers/_nx_ext_v2', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'networkx\.algorithms\.minors', 'netharn.initializers._nx_ext_v2', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'networkx/networkx/algorithms/minors', 'netharn/netharn/initializers/_nx_ext_v2', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'networkx/algorithms/minors', 'netharn/initializers/_nx_ext_v2', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'networkx\.generators\.random_graphs', 'netharn.initializers._nx_ext_v2.utils', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'networkx\.readwrite\.text', 'netharn.initializers._nx_ext_v2.utils', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'nx.random_tree', 'random_tree', fpath_list=[join(dst, 'utils.py')], force=force) rob_nav._ut_sed(r'nx.forest_str', 'forest_str', fpath_list=[join(dst, 'utils.py')], force=force) rob_nav._ut_sed(r'nx.random_ordered_tree', 'random_ordered_tree', fpath_list=[join(dst, 'utils.py')], force=force) # force = 0 rob_nav._ut_sed(r'nx.forest_str', 'forest_str', fpath_list=fpath_list, force=force) rob_nav._ut_sed(r'nx.random_ordered_tree', 'random_ordered_tree', fpath_list=fpath_list, force=force) with open(join(dst, 'tree_embedding.py'), 'a') as file: file.write('\n') file.write( 'from netharn.initializers._nx_ext_v2.utils import forest_str # NOQA\n' ) file.write( 'from netharn.initializers._nx_ext_v2.utils import random_ordered_tree # NOQA\n' ) # Enable default autojit rob_nav._ut_sed(r'# NETWORKX_AUTOJIT', 'NETWORKX_AUTOJIT', fpath_list=fpath_list, force=force) """
def test_augpath_dpath(): assert ub.augpath('foo', dpath='bar') == join('bar', 'foo') assert ub.augpath('foo/bar', dpath='baz') == join('baz', 'bar') assert ub.augpath('', dpath='bar').startswith('bar')
def test_augpath_identity(): assert ub.augpath('foo') == 'foo' assert ub.augpath('foo/bar') == join('foo', 'bar') assert ub.augpath('') == ''
def save_parts(fig, fpath, grouped_axes=None, dpi=None): """ FIXME: this works in mpl 2.0.0, but not 2.0.2 Args: fig (?): fpath (str): file path string dpi (None): (default = None) Returns: list: subpaths CommandLine: python -m draw_func2 save_parts Ignore: >>> # DISABLE_DOCTEST >>> import kwplot >>> kwplot.autompl() >>> import matplotlib as mpl >>> import matplotlib.pyplot as plt >>> def testimg(fname): >>> return plt.imread(mpl.cbook.get_sample_data(fname)) >>> fnames = ['grace_hopper.png', 'ada.png'] * 4 >>> fig = plt.figure(1) >>> for c, fname in enumerate(fnames, start=1): >>> ax = fig.add_subplot(3, 4, c) >>> ax.imshow(testimg(fname)) >>> ax.set_title(fname[0:3] + str(c)) >>> ax.set_xticks([]) >>> ax.set_yticks([]) >>> ax = fig.add_subplot(3, 1, 3) >>> ax.plot(np.sin(np.linspace(0, np.pi * 2))) >>> ax.set_xlabel('xlabel') >>> ax.set_ylabel('ylabel') >>> ax.set_title('title') >>> fpath = 'test_save_parts.png' >>> adjust_subplots(fig=fig, wspace=.3, hspace=.3, top=.9) >>> subpaths = save_parts(fig, fpath, dpi=300) >>> fig.savefig(fpath) >>> ub.startfile(subpaths[0]) >>> ub.startfile(fpath) """ if dpi: # Need to set figure dpi before we draw fig.dpi = dpi # We need to draw the figure before calling get_window_extent # (or we can figure out how to set the renderer object) # if getattr(fig.canvas, 'renderer', None) is None: fig.canvas.draw() # Group axes that belong together if grouped_axes is None: grouped_axes = [] for ax in fig.axes: grouped_axes.append([ax]) subpaths = [] _iter = enumerate(grouped_axes, start=0) _iter = ub.ProgIter(list(_iter), label='save subfig') for count, axs in _iter: subpath = ub.augpath(fpath, suffix=chr(count + 65)) extent = axes_extent(axs).transformed(fig.dpi_scale_trans.inverted()) savekw = {} savekw['transparent'] = ub.argflag('--alpha') if dpi is not None: savekw['dpi'] = dpi savekw['edgecolor'] = 'none' fig.savefig(subpath, bbox_inches=extent, **savekw) subpaths.append(subpath) return subpaths
def make_parts(prep, fullres, scale=1, clear=False): """ Slices the fullres images into smaller parts that fit into the network but are at the original resolution (or higher). >>> from clab.tasks.urban_mapper_3d import * >>> task = UrbanMapper3D(root='~/remote/aretha/data/UrbanMapper3D', workdir='~/data/work/urban_mapper') >>> task.prepare_fullres_inputs() >>> fullres = task.fullres >>> datadir = ub.ensuredir((task.workdir, 'data')) >>> prep = Preprocessor(datadir) >>> scale = 1 >>> clear = False >>> lowres = prep.make_parts(fullres, scale) """ part_config = prep.part_config hashid = hashutil.hash_data(ub.repr2(part_config), hashlen=8) shapestr = '_'.join(list(map(str, prep.input_shape))) mode = 'part-scale{}-{}-{}'.format(scale, shapestr, hashid) parts, flag = prep._mode_new_input(mode, fullres, clear=clear) if flag: return parts input_shape = prep.input_shape overlap = part_config['overlap'] keepbound = part_config['keepbound'] records = list(fullres.iter_records()) for record in ub.ProgIter(records, label='make ' + mode): dump_fname = basename(record['dump_fname']) im_shape = np.array(Image.open(record['im']).size[::-1]) im_shape = tuple(np.floor(im_shape * scale).astype(np.int)) # Consolodate all channels that belong to this record in_paths = record.get('aux').copy() for k in ['im', 'gt']: if k in record: in_paths[k] = record[k] # Read the images for this record and resize if necessary in_images = {k: imutil.imread(v) for k, v in in_paths.items()} # 9% of the time if scale != 1.0: for k in in_images.keys(): interp = cv2.INTER_LANCZOS4 if k == 'im' else cv2.INTER_NEAREST in_images[k] = imutil.imscale(in_images[k], scale, interp)[0] sl_gen = imutil.image_slices(im_shape, input_shape, overlap, keepbound) for idx, rc_slice in enumerate(sl_gen): rsl, csl = rc_slice suffix = '_part{:0=4d}_{:0=3d}_{:0=3d}'.format( idx, rsl.start, csl.start) fname = ub.augpath(dump_fname, suffix=suffix) for k, in_data in in_images.items(): out_data = in_data[rc_slice] out_fpath = join(parts.dirs[k], fname) imutil.imwrite(out_fpath, out_data) # 84% of the time parts.paths[k].append(out_fpath) return parts
def main(bib_fpath=None): r""" intro point to fixbib script CommmandLine: fixbib python -m fixtex bib python -m fixtex bib --dryrun python -m fixtex bib --dryrun --debug """ if bib_fpath is None: bib_fpath = 'My Library.bib' # DEBUG = ub.argflag('--debug') # Read in text and ensure ascii format dirty_text = ut.readfrom(bib_fpath) from fixtex.fix_tex import find_used_citations, testdata_fpaths if exists('custom_extra.bib'): extra_parser = bparser.BibTexParser(ignore_nonstandard_types=False) parser = bparser.BibTexParser() ut.delete_keys(parser.alt_dict, ['url', 'urls']) print('Parsing extra bibtex file') extra_text = ut.readfrom('custom_extra.bib') extra_database = extra_parser.parse(extra_text, partial=False) print('Finished parsing extra') extra_dict = extra_database.get_entry_dict() else: extra_dict = None #udata = dirty_text.decode("utf-8") #dirty_text = udata.encode("ascii", "ignore") #dirty_text = udata # parser = bparser.BibTexParser() # bib_database = parser.parse(dirty_text) # d = bib_database.get_entry_dict() print('BIBTEXPARSER LOAD') parser = bparser.BibTexParser(ignore_nonstandard_types=False, common_strings=True) ut.delete_keys(parser.alt_dict, ['url', 'urls']) print('Parsing bibtex file') bib_database = parser.parse(dirty_text, partial=False) print('Finished parsing') bibtex_dict = bib_database.get_entry_dict() old_keys = list(bibtex_dict.keys()) new_keys = [] for key in ub.ProgIter(old_keys, label='fixing keys'): new_key = key new_key = new_key.replace(':', '') new_key = new_key.replace('-', '_') new_key = re.sub('__*', '_', new_key) new_keys.append(new_key) # assert len(ut.find_duplicate_items(new_keys)) == 0, 'new keys created conflict' assert len(ub.find_duplicates(new_keys)) == 0, 'new keys created conflict' for key, new_key in zip(old_keys, new_keys): if key != new_key: entry = bibtex_dict[key] entry['ID'] = new_key bibtex_dict[new_key] = entry del bibtex_dict[key] # The bibtext is now clean. Print it to stdout #print(clean_text) verbose = None if verbose is None: verbose = 1 # Find citations from the tex documents key_list = None if key_list is None: cacher = ub.Cacher('texcite1', enabled=0) data = cacher.tryload() if data is None: fpaths = testdata_fpaths() key_list, inverse = find_used_citations(fpaths, return_inverse=True) # ignore = ['JP', '?', 'hendrick'] # for item in ignore: # try: # key_list.remove(item) # except ValueError: # pass if verbose: print('Found %d citations used in the document' % (len(key_list), )) data = key_list, inverse cacher.save(data) key_list, inverse = data # else: # key_list = None unknown_pubkeys = [] debug_author = ub.argval('--debug-author', default=None) # ./fix_bib.py --debug_author=Kappes if verbose: print('Fixing %d/%d bibtex entries' % (len(key_list), len(bibtex_dict))) # debug = True debug = False if debug_author is not None: debug = False known_keys = list(bibtex_dict.keys()) missing_keys = set(key_list) - set(known_keys) if extra_dict is not None: missing_keys.difference_update(set(extra_dict.keys())) if missing_keys: print('The library is missing keys found in tex files %s' % (ub.repr2(missing_keys), )) # Search for possible typos: candidate_typos = {} sedlines = [] for key in missing_keys: candidates = ut.closet_words(key, known_keys, num=3, subset=True) if len(candidates) > 1: top = candidates[0] if ut.edit_distance(key, top) == 1: # "sed -i -e 's/{}/{}/g' *.tex".format(key, top) import os replpaths = ' '.join( [relpath(p, os.getcwd()) for p in inverse[key]]) sedlines.append("sed -i -e 's/{}/{}/g' {}".format( key, top, replpaths)) candidate_typos[key] = candidates print('Cannot find key = %r' % (key, )) print('Did you mean? %r' % (candidates, )) print('Quick fixes') print('\n'.join(sedlines)) # group by file just = max([0] + list(map(len, missing_keys))) missing_fpaths = [inverse[key] for key in missing_keys] for fpath in sorted(set(ub.flatten(missing_fpaths))): # ut.fix_embed_globals() subkeys = [k for k in missing_keys if fpath in inverse[k]] print('') ut.cprint('--- Missing Keys ---', 'blue') ut.cprint('fpath = %r' % (fpath, ), 'blue') ut.cprint('{} | {}'.format('Missing'.ljust(just), 'Did you mean?'), 'blue') for key in subkeys: print('{} | {}'.format(ut.highlight_text(key.ljust(just), 'red'), ' '.join(candidate_typos[key]))) # for key in list(bibtex_dict.keys()): if extra_dict is not None: # Extra database takes precidence over regular key_list = list(ut.unique(key_list + list(extra_dict.keys()))) for k, v in extra_dict.items(): bibtex_dict[k] = v full = ub.argflag('--full') for key in key_list: try: entry = bibtex_dict[key] except KeyError: continue self = BibTexCleaner(key, entry, full=full) if debug_author is not None: debug = debug_author in entry.get('author', '') if debug: ut.cprint(' --- ENTRY ---', 'yellow') print(ub.repr2(entry, nl=1)) entry = self.fix() # self.clip_abstract() # self.shorten_keys() # self.fix_authors() # self.fix_year() # old_pubval = self.fix_pubkey() # if old_pubval: # unknown_pubkeys.append(old_pubval) # self.fix_arxiv() # self.fix_general() # self.fix_paper_types() if debug: print(ub.repr2(entry, nl=1)) ut.cprint(' --- END ENTRY ---', 'yellow') bibtex_dict[key] = entry unwanted_keys = set(bibtex_dict.keys()) - set(key_list) if verbose: print('Removing unwanted %d entries' % (len(unwanted_keys))) ut.delete_dict_keys(bibtex_dict, unwanted_keys) if 0: d1 = bibtex_dict.copy() full = True for key, entry in d1.items(): self = BibTexCleaner(key, entry, full=full) pub = self.publication() if pub is None: print(self.entry['ENTRYTYPE']) old = self.fix_pubkey() x1 = self._pubval() x2 = self.standard_pubval(full=full) # if x2 is not None and len(x2) > 5: # print(ub.repr2(self.entry)) if x1 != x2: print('x2 = %r' % (x2, )) print('x1 = %r' % (x1, )) print(ub.repr2(self.entry)) # if 'CVPR' in self.entry.get('booktitle', ''): # if 'CVPR' != self.entry.get('booktitle', ''): # break if old: print('old = %r' % (old, )) d1[key] = self.entry if full: d1 = bibtex_dict.copy() import numpy as np import pandas as pd df = pd.DataFrame.from_dict(d1, orient='index') paged_items = df[~pd.isnull(df['pub_accro'])] has_pages = ~pd.isnull(paged_items['pages']) print('have pages {} / {}'.format(has_pages.sum(), len(has_pages))) print(ub.repr2(paged_items[~has_pages]['title'].values.tolist())) entrytypes = dict(list(df.groupby('pub_type'))) if False: # entrytypes['misc'] g = entrytypes['online'] g = g[g.columns[~np.all(pd.isnull(g), axis=0)]] entrytypes['book'] entrytypes['thesis'] g = entrytypes['article'] g = entrytypes['incollection'] g = entrytypes['conference'] def lookup_pub(e): if e == 'article': return 'journal', 'journal' elif e == 'incollection': return 'booksection', 'booktitle' elif e == 'conference': return 'conference', 'booktitle' return None, None for e, g in entrytypes.items(): print('e = %r' % (e, )) g = g[g.columns[~np.all(pd.isnull(g), axis=0)]] if 'pub_full' in g.columns: place_title = g['pub_full'].tolist() print(ub.repr2(ub.dict_hist(place_title))) else: print('Unknown publications') if 'report' in entrytypes: g = entrytypes['report'] missing = g[pd.isnull(g['title'])] if len(missing): print('Missing Title') print(ub.repr2(missing[['title', 'author']].values.tolist())) if 'journal' in entrytypes: g = entrytypes['journal'] g = g[g.columns[~np.all(pd.isnull(g), axis=0)]] missing = g[pd.isnull(g['journal'])] if len(missing): print('Missing Journal') print(ub.repr2(missing[['title', 'author']].values.tolist())) if 'conference' in entrytypes: g = entrytypes['conference'] g = g[g.columns[~np.all(pd.isnull(g), axis=0)]] missing = g[pd.isnull(g['booktitle'])] if len(missing): print('Missing Booktitle') print(ub.repr2(missing[['title', 'author']].values.tolist())) if 'incollection' in entrytypes: g = entrytypes['incollection'] g = g[g.columns[~np.all(pd.isnull(g), axis=0)]] missing = g[pd.isnull(g['booktitle'])] if len(missing): print('Missing Booktitle') print(ub.repr2(missing[['title', 'author']].values.tolist())) if 'thesis' in entrytypes: g = entrytypes['thesis'] g = g[g.columns[~np.all(pd.isnull(g), axis=0)]] missing = g[pd.isnull(g['institution'])] if len(missing): print('Missing Institution') print(ub.repr2(missing[['title', 'author']].values.tolist())) # import utool # utool.embed() # Overwrite BibDatabase structure bib_database._entries_dict = bibtex_dict bib_database.entries = list(bibtex_dict.values()) #conftitle_to_types_set_hist = {key: set(val) for key, val in conftitle_to_types_hist.items()} #print(ub.repr2(conftitle_to_types_set_hist)) print('Unknown conference keys:') print(ub.repr2(sorted(unknown_pubkeys))) print('len(unknown_pubkeys) = %r' % (len(unknown_pubkeys), )) writer = BibTexWriter() writer.contents = ['comments', 'entries'] writer.indent = ' ' writer.order_entries_by = ('type', 'author', 'year') new_bibtex_str = bibtexparser.dumps(bib_database, writer) # Need to check #jegou_aggregating_2012 # Fix the Journal Abreviations # References: # https://www.ieee.org/documents/trans_journal_names.pdf # Write out clean bibfile in ascii format clean_bib_fpath = ub.augpath(bib_fpath.replace(' ', '_'), suffix='_clean') if not ub.argflag('--dryrun'): ut.writeto(clean_bib_fpath, new_bibtex_str)
def _autojit_cython(pyx_fpath, verbose=1, recompile=False, annotate=False): """ This idea is that given a pyx file, we try to compile it. We write a stamp file so subsequent calls should be very fast as long as the source pyx has not changed. Parameters ---------- pyx_fpath : str path to the pyx file verbose : int higher is more verbose. """ import shutil if verbose > 3: print('_autojit_cython') # TODO: move necessary ubelt utilities to nx.utils? # Separate this into its own util? if shutil.which("cythonize"): pyx_dpath = dirname(pyx_fpath) if verbose > 3: print('pyx_dpath = {!r}'.format(pyx_dpath)) # Check if the compiled library exists pyx_base = splitext(basename(pyx_fpath))[0] SO_EXTS = _platform_pylib_exts() so_fname = False for fname in os.listdir(pyx_dpath): if fname.startswith(pyx_base) and fname.endswith(SO_EXTS): so_fname = fname break if verbose > 3: print('so_fname = {!r}'.format(so_fname)) try: # Currently this functionality depends on ubelt. # We could replace ub.cmd with subprocess.check_call and ub.augpath # with os.path operations, but hash_file and CacheStamp are harder # to replace. We can use "liberator" to statically extract these # and add them to nx.utils though. import ubelt as ub except Exception: if verbose > 3: print('return false, no ubelt') return False else: if so_fname is False: # We can compute what the so_fname will be if it doesnt exist so_fname = pyx_base + SO_EXTS[0] so_fpath = join(pyx_dpath, so_fname) content = ub.readfrom(pyx_fpath) mtime = os.stat(pyx_fpath).st_mtime depends = [ub.hash_data(content, hasher="sha1"), mtime] stamp_fname = ub.augpath(so_fname, ext=".jit.stamp") stamp = ub.CacheStamp( stamp_fname, dpath=pyx_dpath, product=so_fpath, depends=depends, verbose=verbose, ) if verbose > 3: print('stamp = {!r}'.format(stamp)) if recompile or stamp.expired(): # Heuristic to try and grab the numpy include dir or not cythonize_args = ['cythonize'] cythonize_env = os.environ.copy() needs_numpy = 'numpy' in content if needs_numpy: import numpy as np import pathlib numpy_include_dpath = pathlib.Path(np.get_include()) numpy_dpath = (numpy_include_dpath / '../..').resolve() # cythonize_env['CPATH'] = numpy_include_dpath + ':' + cythonize_env.get('CPATH', '') cythonize_env['CFLAGS'] = ' '.join([ '-I{}'.format(numpy_include_dpath), ]) + cythonize_env.get('CFLAGS', '') cythonize_env['LDFLAGS'] = ' '.join([ '-L{} -lnpyrandom'.format(numpy_dpath / 'random/lib'), '-L{} -lnpymath'.format(numpy_dpath / 'core/lib'), ]) + cythonize_env.get('LDFLAGS', '') if annotate: cythonize_args.append('-a') cythonize_args.append('-i {}'.format(pyx_fpath)) cythonize_cmd = ' '.join(cythonize_args) if needs_numpy: print('CFLAGS="{}" '.format(cythonize_env['CFLAGS']) + 'LDFLAGS="{}" '.format(cythonize_env['LDFLAGS']) + cythonize_cmd) ub.cmd(cythonize_cmd, verbose=verbose, check=True, env=cythonize_env) stamp.renew() return True else: if verbose > 2: print('Cythonize not found!')
def check_nitfs(): nitfs = unsafe_grab_nitfs() import xdev import netharn as nh total = 0 for fpath in nitfs: nbytes = nh.util.get_file_info(fpath)['filesize'] print('nbytes = {!r}'.format(xdev.byte_str(nbytes))) total += nbytes print(xdev.byte_str(total)) failed_fpaths = [] passed_fpaths = [] for fpath in nitfs: import kwimage try: kwimage.imread(fpath) passed_fpaths.append(fpath) except Exception: failed_fpaths.append(fpath) print('passed = {}'.format(len(passed_fpaths))) print('failed = {}'.format(len(failed_fpaths))) print('CANT HANDLE') for fpath in failed_fpaths: name = splitext(basename(fpath))[0] desc = NITF_DESC[name] print(desc) for fpath in failed_fpaths: print('\n-----') print('fpath = {!r}'.format(fpath)) try: kwimage.imread(fpath) except Exception: pass print('\n-----') from ndsampler.abstract_frames import _cog_cache_write for gpath in passed_fpaths: cache_gpath = ub.augpath(gpath, ext='.test.api.cog') ub.delete(cache_gpath) # config = {'hack_use_cli': True} config = {'hack_use_cli': False, 'compress': 'LZW'} _cog_cache_write(gpath, cache_gpath, config=config) from ndsampler.abstract_frames import _cog_cache_write for gpath in passed_fpaths: cache_gpath = ub.augpath(gpath, ext='.test.cli.cog') ub.delete(cache_gpath) # config = {'hack_use_cli': True} config = {'hack_use_cli': True, 'compress': 'LZW'} _cog_cache_write(gpath, cache_gpath, config=config) from ndsampler.abstract_frames import _cog_cache_write for gpath in passed_fpaths: cache_gpath = ub.augpath(gpath, ext='.test.cli.cog') ub.delete(cache_gpath) # config = {'hack_use_cli': True} kwimage.imread(gpath) config = {'hack_use_cli': True, 'compress': 'JPEG'} _cog_cache_write(gpath, cache_gpath, config=config)
def script_overlay_aux(): """ """ task = UrbanMapper3D(root='~/remote/aretha/data/UrbanMapper3D', workdir='~/data/work/urban_mapper') fullres = task.load_fullres_inputs(subdir='training') NAN_VAL = -32767 for paths in ub.ProgIter(fullres, adjust=False, freq=1): bgr = util.imread(paths['im']) gtl = util.imread(paths['gt']) gti = util.imread(paths['gt'].replace('GTL', 'GTI')) dsm = util.imread(paths['aux']['dsm']) dsm[(NAN_VAL == dsm)] = np.nan dtm = util.imread(paths['aux']['dtm']) dtm[(NAN_VAL == dtm)] = np.nan diff = dtm - dsm def normalize(chan): min_val = np.nanmax(chan) max_val = np.nanmin(chan) is_nan = np.isnan(chan) norm = chan.copy() norm[~is_nan] = (norm[~is_nan] - min_val) / (max_val - min_val) norm[is_nan] = 0 return norm, min_val, max_val def colorize(chan): norm, min_val, max_val = normalize(chan) domain = np.linspace(min_val, max_val) shape = (norm.shape[0], norm.shape[0] / 25) cb = colorutil.colorbar_image(domain, dpi=200, shape=shape)[:, :, 0: 3] sfy = norm.shape[0] / cb.shape[0] cb = imutil.imscale(cb, scale=(sfy, sfy))[0] color_chan = util.make_heatmask(norm)[:, :, 0:3] color_chan[np.isnan(chan)] = [[0, 0, 1]] blend_chan = util.overlay_colorized(color_chan, bgr, alpha=.2)[:, :, 0:3] blend_gt_chan = draw_instance_contours(blend_chan, gti, gtl, thickness=2, alpha=.3) # Add a colorbar blend_chan = np.hstack([blend_chan, cb]) blend_gt_chan = np.hstack([blend_gt_chan, cb]) return blend_chan, blend_gt_chan blend_dsm, blend_gt_dsm = colorize(dsm) blend_dtm, blend_gt_dtm = colorize(dtm) blend_diff, blend_gt_diff = colorize(diff) base_dpath = ub.ensuredir(join(task.workdir, 'viz')) outputs = { 'blend_diff': blend_diff, 'blend_dsm': blend_dsm, 'blend_dtm': blend_dtm, 'blend_gt_diff': blend_gt_diff, 'blend_gt_dsm': blend_gt_dsm, 'blend_gt_dtm': blend_gt_dtm, } for key, val in outputs.items(): out_dpath = ub.ensuredir((base_dpath, key)) out_fpath = join(out_dpath, ub.augpath(paths['dump_fname'], ext='.png')) util.imwrite(out_fpath, val)