def test_load():
fpath = 'foo.pt'
cpu = XPU(None)
gpu = XPU(0)
gpu_data = gpu.move(torch.FloatTensor([10]))
cpu_data = cpu.move(torch.FloatTensor([10]))
torch.save(gpu_data, ub.augpath(fpath, 'gpu'))
torch.save(cpu_data, ub.augpath(fpath, 'cpu'))
gpu_data2 = gpu.load(ub.augpath(fpath, 'cpu'))
cpu_data2 = cpu.load(ub.augpath(fpath, 'gpu'))
assert not gpu_data2.is_gpu()
assert cpu_data2.is_gpu()
def gt_info_from_path(pred_fpath):
# Hack to read UrbanMapper specific data
gtl_fname = ub.augpath(basename(pred_fpath), suffix='_GTL', ext='.tif')
gti_fname = ub.augpath(basename(pred_fpath), suffix='_GTI', ext='.tif')
dsm_fname = ub.augpath(basename(pred_fpath), suffix='_DSM', ext='.tif')
gtl_fpath = join(train_data_path, gtl_fname)
gti_fpath = join(train_data_path, gti_fname)
dsm_fpath = join(train_data_path, dsm_fname)
gti = util.imread(gti_fpath)
gtl = util.imread(gtl_fpath)
dsm = util.imread(dsm_fpath)
uncertain = (gtl == 65)
return gti, uncertain, dsm
def initial_weights():
"""
Weights pretrained trained ImageNet by yolo9000-pytorch
"""
import os
url = 'https://data.kitware.com/api/v1/file/5ab513438d777f068578f1d0/download'
dpath = ub.ensure_app_cache_dir('clab/yolo_v2')
fname = 'darknet19.weights.npz'
dest = os.path.join(dpath, fname)
if not os.path.exists(dest):
command = 'curl -X GET {} > {}'.format(url, dest)
ub.cmd(command, verbout=1, shell=True)
# url = 'http://acidalia.kitware.com:8000/weights/darknet19.weights.npz'
# npz_fpath = ub.grabdata(url, dpath=ub.ensure_app_cache_dir('clab'))
# convert to torch weights
npz_fpath = dest
torch_fpath = ub.augpath(npz_fpath, ext='.pt')
if not os.path.exists(torch_fpath):
# hack to transform initial state
model = Darknet19(num_classes=20)
model.load_from_npz(npz_fpath, num_conv=18)
torch.save(model.state_dict(), torch_fpath)
# from clab import xpu_device
# xpu = xpu_device.XPU('gpu')
# xpu.load(torch_fpath)
# torch.load(torch_fpath)
return torch_fpath
def main():
import xdoctest
import ubelt as ub
import sys
modpath = ub.modname_to_modpath('netharn.examples')
name_to_path = {}
submods = list(xdoctest.static_analysis.package_modpaths(modpath))
for submod in submods:
modname = ub.augpath(submod, dpath='', ext='')
if not modname.startswith('_'):
name_to_path[modname] = submod
print('name_to_path = {}'.format(ub.repr2(name_to_path, nl=1)))
chosen = None
for arg in sys.argv[1:2]:
print('arg = {!r}'.format(arg))
if arg in name_to_path:
chosen = name_to_path[arg]
break
print('chosen = {!r}'.format(chosen))
assert chosen is not None
module = ub.import_module_from_path(chosen)
print('module = {!r}'.format(module))
module.main()
def _autojit_cython(pyx_fpath, verbose=1):
"""
This idea is that given a pyx file, we try to compile it. We write a stamp
file so subsequent calls should be very fast as long as the source pyx has
not changed.
Parameters
----------
pyx_fpath : str
path to the pyx file
verbose : int
higher is more verbose.
"""
import shutil
# TODO: move necessary ubelt utilities to nx.utils?
# Separate this into its own util?
if shutil.which("cythonize"):
pyx_dpath = dirname(pyx_fpath)
# Check if the compiled library exists
pyx_base = splitext(basename(pyx_fpath))[0]
SO_EXTS = _platform_pylib_exts()
so_fname = False
for fname in os.listdir(pyx_dpath):
if fname.startswith(pyx_base) and fname.endswith(SO_EXTS):
so_fname = fname
break
try:
# Currently this functionality depends on ubelt.
# We could replace ub.cmd with subprocess.check_call and ub.augpath
# with os.path operations, but hash_file and CacheStamp are harder
# to replace. We can use "liberator" to statically extract these
# and add them to nx.utils though.
import ubelt as ub
except Exception:
return False
else:
if so_fname is False:
# We can compute what the so_fname will be if it doesnt exist
so_fname = pyx_base + SO_EXTS[0]
so_fpath = join(pyx_dpath, so_fname)
depends = [ub.hash_file(pyx_fpath, hasher="sha1")]
stamp_fname = ub.augpath(so_fname, ext=".jit.stamp")
stamp = ub.CacheStamp(
stamp_fname,
dpath=pyx_dpath,
product=so_fpath,
depends=depends,
verbose=verbose,
)
if stamp.expired():
ub.cmd("cythonize -i {}".format(pyx_fpath), verbose=verbose, check=True)
stamp.renew()
return True
def 路径_名字处理(路径,
末尾='',
前缀='',
扩展名=None,
名称=None,
dpath=None,
relative=None,
不包含点=False):
return ub.augpath(路径, 末尾, 前缀, 扩展名, 名称, dpath, relative, 不包含点)
def test_pathlib():
try:
import pathlib
base = pathlib.Path(ub.ensure_app_cache_dir('ubelt'))
dpath = base.joinpath('test_pathlib_mkdir')
# ensuredir
ub.delete(dpath)
assert not dpath.exists()
got = ub.ensuredir(dpath)
assert got.exists()
# compressuser
assert ub.compressuser(base) == '~/.cache/ubelt'
assert ub.augpath(base, prefix='foo') == '/home/joncrall/.cache/fooubelt'
ub.expandpath(base)
except Exception:
import pytest
pytest.skip('pathlib is not installed')
def ensure_sql_coco_view(dset):
"""
Create an SQL view of the COCO dataset
"""
db_fpath = ub.augpath(dset.fpath, prefix='.', ext='.sqlite')
db_uri = 'sqlite:///' + db_fpath
# dpath = dirname(dset.fpath)
self = CocoSqlDatabase(db_uri)
import os
needs_rewrite = True
if exists(db_fpath):
needs_rewrite = (os.stat(dset.fpath).st_mtime >
os.stat(db_fpath).st_mtime)
if needs_rewrite:
# Write to the SQL instance
self.delete()
self.connect()
# Convert a coco file to an sql database
self.populate_from(dset)
else:
self.connect()
return self
def detect_cli(config={}):
"""
CommandLine:
python -m bioharn.detect_predict --help
CommandLine:
python -m bioharn.detect_predict \
--dataset=~/data/noaa/Habcam_2015_g027250_a00102917_c0001_v2_test.mscoco.json \
--deployed=/home/joncrall/work/bioharn/fit/runs/bioharn-det-v11-test-cascade/myovdqvi/deploy_MM_CascadeRCNN_myovdqvi_035_MVKVVR.zip \
--out_dpath=~/work/bioharn/habcam_test_out \
--draw=100 \
--input_dims=512,512 \
--xpu=0 --batch_size=1
Ignore:
>>> config = {}
>>> config['dataset'] = '~/data/noaa/Habcam_2015_g027250_a00102917_c0001_v2_vali.mscoco.json'
>>> config['deployed'] = '/home/joncrall/work/bioharn/fit/runs/bioharn-det-v11-test-cascade/myovdqvi/deploy_MM_CascadeRCNN_myovdqvi_035_MVKVVR.zip'
>>> config['out_dpath'] = 'out'
"""
import kwarray
import ndsampler
from os.path import basename, join, exists, isfile, isdir # NOQA
config = DetectPredictCLIConfig(config, cmdline=True)
print('config = {}'.format(ub.repr2(config.asdict())))
out_dpath = ub.expandpath(config.get('out_dpath'))
import six
if isinstance(config['dataset'], six.string_types):
if config['dataset'].endswith('.json'):
dataset_fpath = ub.expandpath(config['dataset'])
coco_dset = ndsampler.CocoDataset(dataset_fpath)
# Running prediction is much faster if you can build a sampler.
sampler_backend = {
'type': 'cog',
'config': {
'compress': 'JPEG',
},
'_hack_old_names': False, # flip to true to use legacy caches
}
sampler_backend = None
print('coco hashid = {}'.format(coco_dset._build_hashid()))
else:
sampler_backend = None
if exists(config['dataset']) and isfile(config['dataset']):
# Single image case
image_fpath = ub.expandpath(config['dataset'])
coco_dset = ndsampler.CocoDataset()
coco_dset.add_image(image_fpath)
elif isinstance(config['dataset'], list):
# Multiple image case
gpaths = config['dataset']
gpaths = [ub.expandpath(g) for g in gpaths]
coco_dset = ndsampler.CocoDataset()
for gpath in gpaths:
coco_dset.add_image(gpath)
else:
raise TypeError(config['dataset'])
draw = config.get('draw')
workdir = ub.expandpath(config.get('workdir'))
det_outdir = ub.ensuredir((out_dpath, 'pred'))
pred_config = ub.dict_subset(config, DetectPredictConfig.default)
print('Create sampler')
sampler = ndsampler.CocoSampler(coco_dset, workdir=workdir,
backend=sampler_backend)
print('prepare frames')
sampler.frames.prepare(workers=config['workers'])
print('Create predictor')
predictor = DetectPredictor(pred_config)
print('Ensure model')
predictor._ensure_model()
pred_dataset = coco_dset.dataset.copy()
pred_dataset['annotations'] = []
pred_dset = ndsampler.CocoDataset(pred_dataset)
# self = predictor
predictor.config['verbose'] = 1
pred_gen = predictor.predict_sampler(sampler)
buffered_gen = AsyncBufferedGenerator(pred_gen, size=coco_dset.n_images)
gid_to_pred = {}
prog = ub.ProgIter(buffered_gen, total=coco_dset.n_images,
desc='buffered detect')
for img_idx, (gid, dets) in enumerate(prog):
gid_to_pred[gid] = dets
for ann in dets.to_coco():
ann['image_id'] = gid
try:
catname = ann['category_name']
ann['category_id'] = pred_dset._resolve_to_cid(catname)
except KeyError:
if 'category_id' not in ann:
cid = pred_dset.add_category(catname)
ann['category_id'] = cid
pred_dset.add_annotation(**ann)
single_img_coco = pred_dset.subset([gid])
single_pred_dpath = ub.ensuredir((det_outdir, 'single_image'))
single_pred_fpath = join(single_pred_dpath, 'detections_gid_{:08d}.mscoco.json'.format(gid))
single_img_coco.dump(single_pred_fpath, newlines=True)
if draw is True or (draw and img_idx < draw):
draw_outdir = ub.ensuredir((out_dpath, 'draw'))
img_fpath = coco_dset.load_image_fpath(gid)
gname = basename(img_fpath)
viz_fname = ub.augpath(gname, prefix='detect_', ext='.jpg')
viz_fpath = join(draw_outdir, viz_fname)
image = kwimage.imread(img_fpath)
flags = dets.scores > .2
flags[kwarray.argmaxima(dets.scores, num=10)] = True
top_dets = dets.compress(flags)
toshow = top_dets.draw_on(image, alpha=None)
# kwplot.imshow(toshow)
kwimage.imwrite(viz_fpath, toshow, space='rgb')
pred_fpath = join(det_outdir, 'detections.mscoco.json')
print('Dump detections to pred_fpath = {!r}'.format(pred_fpath))
pred_dset.dump(pred_fpath, newlines=True)
def copy_over_stuff_from_nx_pr():
"""
Copy from networkx dev/ordered_subtree_isomorphism b08106baae7987af1dc755a6308fcd11bc21cbc8
"""
dst = ub.expandpath('~/code/netharn/netharn/initializers/_nx_ext_v2')
from os.path import join
nx_repo = dirname(dirname(nx.__file__))
to_copy = [
join(nx_repo, 'networkx/algorithms/string/_autojit.py'),
join(nx_repo, 'networkx/algorithms/string/balanced_embedding.py'),
join(nx_repo,
'networkx/algorithms/string/balanced_embedding_cython.pyx'),
join(nx_repo, 'networkx/algorithms/string/balanced_isomorphism.py'),
join(nx_repo,
'networkx/algorithms/string/balanced_isomorphism_cython.pyx'),
join(nx_repo, 'networkx/algorithms/string/balanced_sequence.py'),
join(nx_repo, 'networkx/algorithms/minors/tree_embedding.py'),
join(nx_repo, 'networkx/algorithms/minors/tree_isomorphism.py'),
]
import shutil
fpath_list = []
for fpath in to_copy:
fpath2 = ub.augpath(fpath, dpath=dst)
fpath_list.append(fpath2)
shutil.copy2(fpath, fpath2)
util_fpath = join(dst, 'utils.py')
closer = liberator.Closer()
closer.add_dynamic(nx.forest_str)
closer.add_dynamic(nx.random_ordered_tree)
closer.add_dynamic(nx.random_tree)
with open(util_fpath, 'w') as file:
file.write(closer.current_sourcecode())
from rob import rob_nav
force = True
# force = 0
rob_nav._ut_sed(r'networkx\.algorithms\.string',
'netharn.initializers._nx_ext_v2',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'networkx/networkx/algorithms/string',
'netharn/netharn/initializers/_nx_ext_v2',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'networkx/algorithms/string',
'netharn/initializers/_nx_ext_v2',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'networkx\.algorithms\.minors',
'netharn.initializers._nx_ext_v2',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'networkx/networkx/algorithms/minors',
'netharn/netharn/initializers/_nx_ext_v2',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'networkx/algorithms/minors',
'netharn/initializers/_nx_ext_v2',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'networkx\.generators\.random_graphs',
'netharn.initializers._nx_ext_v2.utils',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'networkx\.readwrite\.text',
'netharn.initializers._nx_ext_v2.utils',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'nx.random_tree',
'random_tree',
fpath_list=[join(dst, 'utils.py')],
force=force)
rob_nav._ut_sed(r'nx.forest_str',
'forest_str',
fpath_list=[join(dst, 'utils.py')],
force=force)
rob_nav._ut_sed(r'nx.random_ordered_tree',
'random_ordered_tree',
fpath_list=[join(dst, 'utils.py')],
force=force)
# force = 0
rob_nav._ut_sed(r'nx.forest_str',
'forest_str',
fpath_list=fpath_list,
force=force)
rob_nav._ut_sed(r'nx.random_ordered_tree',
'random_ordered_tree',
fpath_list=fpath_list,
force=force)
with open(join(dst, 'tree_embedding.py'), 'a') as file:
file.write('\n')
file.write(
'from netharn.initializers._nx_ext_v2.utils import forest_str # NOQA\n'
)
file.write(
'from netharn.initializers._nx_ext_v2.utils import random_ordered_tree # NOQA\n'
)
# Enable default autojit
rob_nav._ut_sed(r'# NETWORKX_AUTOJIT',
'NETWORKX_AUTOJIT',
fpath_list=fpath_list,
force=force)
"""
def test_augpath_dpath():
assert ub.augpath('foo', dpath='bar') == join('bar', 'foo')
assert ub.augpath('foo/bar', dpath='baz') == join('baz', 'bar')
assert ub.augpath('', dpath='bar').startswith('bar')
def test_augpath_identity():
assert ub.augpath('foo') == 'foo'
assert ub.augpath('foo/bar') == join('foo', 'bar')
assert ub.augpath('') == ''
def save_parts(fig, fpath, grouped_axes=None, dpi=None):
"""
FIXME: this works in mpl 2.0.0, but not 2.0.2
Args:
fig (?):
fpath (str): file path string
dpi (None): (default = None)
Returns:
list: subpaths
CommandLine:
python -m draw_func2 save_parts
Ignore:
>>> # DISABLE_DOCTEST
>>> import kwplot
>>> kwplot.autompl()
>>> import matplotlib as mpl
>>> import matplotlib.pyplot as plt
>>> def testimg(fname):
>>> return plt.imread(mpl.cbook.get_sample_data(fname))
>>> fnames = ['grace_hopper.png', 'ada.png'] * 4
>>> fig = plt.figure(1)
>>> for c, fname in enumerate(fnames, start=1):
>>> ax = fig.add_subplot(3, 4, c)
>>> ax.imshow(testimg(fname))
>>> ax.set_title(fname[0:3] + str(c))
>>> ax.set_xticks([])
>>> ax.set_yticks([])
>>> ax = fig.add_subplot(3, 1, 3)
>>> ax.plot(np.sin(np.linspace(0, np.pi * 2)))
>>> ax.set_xlabel('xlabel')
>>> ax.set_ylabel('ylabel')
>>> ax.set_title('title')
>>> fpath = 'test_save_parts.png'
>>> adjust_subplots(fig=fig, wspace=.3, hspace=.3, top=.9)
>>> subpaths = save_parts(fig, fpath, dpi=300)
>>> fig.savefig(fpath)
>>> ub.startfile(subpaths[0])
>>> ub.startfile(fpath)
"""
if dpi:
# Need to set figure dpi before we draw
fig.dpi = dpi
# We need to draw the figure before calling get_window_extent
# (or we can figure out how to set the renderer object)
# if getattr(fig.canvas, 'renderer', None) is None:
fig.canvas.draw()
# Group axes that belong together
if grouped_axes is None:
grouped_axes = []
for ax in fig.axes:
grouped_axes.append([ax])
subpaths = []
_iter = enumerate(grouped_axes, start=0)
_iter = ub.ProgIter(list(_iter), label='save subfig')
for count, axs in _iter:
subpath = ub.augpath(fpath, suffix=chr(count + 65))
extent = axes_extent(axs).transformed(fig.dpi_scale_trans.inverted())
savekw = {}
savekw['transparent'] = ub.argflag('--alpha')
if dpi is not None:
savekw['dpi'] = dpi
savekw['edgecolor'] = 'none'
fig.savefig(subpath, bbox_inches=extent, **savekw)
subpaths.append(subpath)
return subpaths
def make_parts(prep, fullres, scale=1, clear=False):
"""
Slices the fullres images into smaller parts that fit into the network
but are at the original resolution (or higher).
>>> from clab.tasks.urban_mapper_3d import *
>>> task = UrbanMapper3D(root='~/remote/aretha/data/UrbanMapper3D', workdir='~/data/work/urban_mapper')
>>> task.prepare_fullres_inputs()
>>> fullres = task.fullres
>>> datadir = ub.ensuredir((task.workdir, 'data'))
>>> prep = Preprocessor(datadir)
>>> scale = 1
>>> clear = False
>>> lowres = prep.make_parts(fullres, scale)
"""
part_config = prep.part_config
hashid = hashutil.hash_data(ub.repr2(part_config), hashlen=8)
shapestr = '_'.join(list(map(str, prep.input_shape)))
mode = 'part-scale{}-{}-{}'.format(scale, shapestr, hashid)
parts, flag = prep._mode_new_input(mode, fullres, clear=clear)
if flag:
return parts
input_shape = prep.input_shape
overlap = part_config['overlap']
keepbound = part_config['keepbound']
records = list(fullres.iter_records())
for record in ub.ProgIter(records, label='make ' + mode):
dump_fname = basename(record['dump_fname'])
im_shape = np.array(Image.open(record['im']).size[::-1])
im_shape = tuple(np.floor(im_shape * scale).astype(np.int))
# Consolodate all channels that belong to this record
in_paths = record.get('aux').copy()
for k in ['im', 'gt']:
if k in record:
in_paths[k] = record[k]
# Read the images for this record and resize if necessary
in_images = {k: imutil.imread(v)
for k, v in in_paths.items()} # 9% of the time
if scale != 1.0:
for k in in_images.keys():
interp = cv2.INTER_LANCZOS4 if k == 'im' else cv2.INTER_NEAREST
in_images[k] = imutil.imscale(in_images[k], scale,
interp)[0]
sl_gen = imutil.image_slices(im_shape, input_shape, overlap,
keepbound)
for idx, rc_slice in enumerate(sl_gen):
rsl, csl = rc_slice
suffix = '_part{:0=4d}_{:0=3d}_{:0=3d}'.format(
idx, rsl.start, csl.start)
fname = ub.augpath(dump_fname, suffix=suffix)
for k, in_data in in_images.items():
out_data = in_data[rc_slice]
out_fpath = join(parts.dirs[k], fname)
imutil.imwrite(out_fpath, out_data) # 84% of the time
parts.paths[k].append(out_fpath)
return parts
def main(bib_fpath=None):
r"""
intro point to fixbib script
CommmandLine:
fixbib
python -m fixtex bib
python -m fixtex bib --dryrun
python -m fixtex bib --dryrun --debug
"""
if bib_fpath is None:
bib_fpath = 'My Library.bib'
# DEBUG = ub.argflag('--debug')
# Read in text and ensure ascii format
dirty_text = ut.readfrom(bib_fpath)
from fixtex.fix_tex import find_used_citations, testdata_fpaths
if exists('custom_extra.bib'):
extra_parser = bparser.BibTexParser(ignore_nonstandard_types=False)
parser = bparser.BibTexParser()
ut.delete_keys(parser.alt_dict, ['url', 'urls'])
print('Parsing extra bibtex file')
extra_text = ut.readfrom('custom_extra.bib')
extra_database = extra_parser.parse(extra_text, partial=False)
print('Finished parsing extra')
extra_dict = extra_database.get_entry_dict()
else:
extra_dict = None
#udata = dirty_text.decode("utf-8")
#dirty_text = udata.encode("ascii", "ignore")
#dirty_text = udata
# parser = bparser.BibTexParser()
# bib_database = parser.parse(dirty_text)
# d = bib_database.get_entry_dict()
print('BIBTEXPARSER LOAD')
parser = bparser.BibTexParser(ignore_nonstandard_types=False,
common_strings=True)
ut.delete_keys(parser.alt_dict, ['url', 'urls'])
print('Parsing bibtex file')
bib_database = parser.parse(dirty_text, partial=False)
print('Finished parsing')
bibtex_dict = bib_database.get_entry_dict()
old_keys = list(bibtex_dict.keys())
new_keys = []
for key in ub.ProgIter(old_keys, label='fixing keys'):
new_key = key
new_key = new_key.replace(':', '')
new_key = new_key.replace('-', '_')
new_key = re.sub('__*', '_', new_key)
new_keys.append(new_key)
# assert len(ut.find_duplicate_items(new_keys)) == 0, 'new keys created conflict'
assert len(ub.find_duplicates(new_keys)) == 0, 'new keys created conflict'
for key, new_key in zip(old_keys, new_keys):
if key != new_key:
entry = bibtex_dict[key]
entry['ID'] = new_key
bibtex_dict[new_key] = entry
del bibtex_dict[key]
# The bibtext is now clean. Print it to stdout
#print(clean_text)
verbose = None
if verbose is None:
verbose = 1
# Find citations from the tex documents
key_list = None
if key_list is None:
cacher = ub.Cacher('texcite1', enabled=0)
data = cacher.tryload()
if data is None:
fpaths = testdata_fpaths()
key_list, inverse = find_used_citations(fpaths,
return_inverse=True)
# ignore = ['JP', '?', 'hendrick']
# for item in ignore:
# try:
# key_list.remove(item)
# except ValueError:
# pass
if verbose:
print('Found %d citations used in the document' %
(len(key_list), ))
data = key_list, inverse
cacher.save(data)
key_list, inverse = data
# else:
# key_list = None
unknown_pubkeys = []
debug_author = ub.argval('--debug-author', default=None)
# ./fix_bib.py --debug_author=Kappes
if verbose:
print('Fixing %d/%d bibtex entries' %
(len(key_list), len(bibtex_dict)))
# debug = True
debug = False
if debug_author is not None:
debug = False
known_keys = list(bibtex_dict.keys())
missing_keys = set(key_list) - set(known_keys)
if extra_dict is not None:
missing_keys.difference_update(set(extra_dict.keys()))
if missing_keys:
print('The library is missing keys found in tex files %s' %
(ub.repr2(missing_keys), ))
# Search for possible typos:
candidate_typos = {}
sedlines = []
for key in missing_keys:
candidates = ut.closet_words(key, known_keys, num=3, subset=True)
if len(candidates) > 1:
top = candidates[0]
if ut.edit_distance(key, top) == 1:
# "sed -i -e 's/{}/{}/g' *.tex".format(key, top)
import os
replpaths = ' '.join(
[relpath(p, os.getcwd()) for p in inverse[key]])
sedlines.append("sed -i -e 's/{}/{}/g' {}".format(
key, top, replpaths))
candidate_typos[key] = candidates
print('Cannot find key = %r' % (key, ))
print('Did you mean? %r' % (candidates, ))
print('Quick fixes')
print('\n'.join(sedlines))
# group by file
just = max([0] + list(map(len, missing_keys)))
missing_fpaths = [inverse[key] for key in missing_keys]
for fpath in sorted(set(ub.flatten(missing_fpaths))):
# ut.fix_embed_globals()
subkeys = [k for k in missing_keys if fpath in inverse[k]]
print('')
ut.cprint('--- Missing Keys ---', 'blue')
ut.cprint('fpath = %r' % (fpath, ), 'blue')
ut.cprint('{} | {}'.format('Missing'.ljust(just), 'Did you mean?'),
'blue')
for key in subkeys:
print('{} | {}'.format(ut.highlight_text(key.ljust(just), 'red'),
' '.join(candidate_typos[key])))
# for key in list(bibtex_dict.keys()):
if extra_dict is not None:
# Extra database takes precidence over regular
key_list = list(ut.unique(key_list + list(extra_dict.keys())))
for k, v in extra_dict.items():
bibtex_dict[k] = v
full = ub.argflag('--full')
for key in key_list:
try:
entry = bibtex_dict[key]
except KeyError:
continue
self = BibTexCleaner(key, entry, full=full)
if debug_author is not None:
debug = debug_author in entry.get('author', '')
if debug:
ut.cprint(' --- ENTRY ---', 'yellow')
print(ub.repr2(entry, nl=1))
entry = self.fix()
# self.clip_abstract()
# self.shorten_keys()
# self.fix_authors()
# self.fix_year()
# old_pubval = self.fix_pubkey()
# if old_pubval:
# unknown_pubkeys.append(old_pubval)
# self.fix_arxiv()
# self.fix_general()
# self.fix_paper_types()
if debug:
print(ub.repr2(entry, nl=1))
ut.cprint(' --- END ENTRY ---', 'yellow')
bibtex_dict[key] = entry
unwanted_keys = set(bibtex_dict.keys()) - set(key_list)
if verbose:
print('Removing unwanted %d entries' % (len(unwanted_keys)))
ut.delete_dict_keys(bibtex_dict, unwanted_keys)
if 0:
d1 = bibtex_dict.copy()
full = True
for key, entry in d1.items():
self = BibTexCleaner(key, entry, full=full)
pub = self.publication()
if pub is None:
print(self.entry['ENTRYTYPE'])
old = self.fix_pubkey()
x1 = self._pubval()
x2 = self.standard_pubval(full=full)
# if x2 is not None and len(x2) > 5:
# print(ub.repr2(self.entry))
if x1 != x2:
print('x2 = %r' % (x2, ))
print('x1 = %r' % (x1, ))
print(ub.repr2(self.entry))
# if 'CVPR' in self.entry.get('booktitle', ''):
# if 'CVPR' != self.entry.get('booktitle', ''):
# break
if old:
print('old = %r' % (old, ))
d1[key] = self.entry
if full:
d1 = bibtex_dict.copy()
import numpy as np
import pandas as pd
df = pd.DataFrame.from_dict(d1, orient='index')
paged_items = df[~pd.isnull(df['pub_accro'])]
has_pages = ~pd.isnull(paged_items['pages'])
print('have pages {} / {}'.format(has_pages.sum(), len(has_pages)))
print(ub.repr2(paged_items[~has_pages]['title'].values.tolist()))
entrytypes = dict(list(df.groupby('pub_type')))
if False:
# entrytypes['misc']
g = entrytypes['online']
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]
entrytypes['book']
entrytypes['thesis']
g = entrytypes['article']
g = entrytypes['incollection']
g = entrytypes['conference']
def lookup_pub(e):
if e == 'article':
return 'journal', 'journal'
elif e == 'incollection':
return 'booksection', 'booktitle'
elif e == 'conference':
return 'conference', 'booktitle'
return None, None
for e, g in entrytypes.items():
print('e = %r' % (e, ))
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]
if 'pub_full' in g.columns:
place_title = g['pub_full'].tolist()
print(ub.repr2(ub.dict_hist(place_title)))
else:
print('Unknown publications')
if 'report' in entrytypes:
g = entrytypes['report']
missing = g[pd.isnull(g['title'])]
if len(missing):
print('Missing Title')
print(ub.repr2(missing[['title', 'author']].values.tolist()))
if 'journal' in entrytypes:
g = entrytypes['journal']
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]
missing = g[pd.isnull(g['journal'])]
if len(missing):
print('Missing Journal')
print(ub.repr2(missing[['title', 'author']].values.tolist()))
if 'conference' in entrytypes:
g = entrytypes['conference']
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]
missing = g[pd.isnull(g['booktitle'])]
if len(missing):
print('Missing Booktitle')
print(ub.repr2(missing[['title', 'author']].values.tolist()))
if 'incollection' in entrytypes:
g = entrytypes['incollection']
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]
missing = g[pd.isnull(g['booktitle'])]
if len(missing):
print('Missing Booktitle')
print(ub.repr2(missing[['title', 'author']].values.tolist()))
if 'thesis' in entrytypes:
g = entrytypes['thesis']
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]
missing = g[pd.isnull(g['institution'])]
if len(missing):
print('Missing Institution')
print(ub.repr2(missing[['title', 'author']].values.tolist()))
# import utool
# utool.embed()
# Overwrite BibDatabase structure
bib_database._entries_dict = bibtex_dict
bib_database.entries = list(bibtex_dict.values())
#conftitle_to_types_set_hist = {key: set(val) for key, val in conftitle_to_types_hist.items()}
#print(ub.repr2(conftitle_to_types_set_hist))
print('Unknown conference keys:')
print(ub.repr2(sorted(unknown_pubkeys)))
print('len(unknown_pubkeys) = %r' % (len(unknown_pubkeys), ))
writer = BibTexWriter()
writer.contents = ['comments', 'entries']
writer.indent = ' '
writer.order_entries_by = ('type', 'author', 'year')
new_bibtex_str = bibtexparser.dumps(bib_database, writer)
# Need to check
#jegou_aggregating_2012
# Fix the Journal Abreviations
# References:
# https://www.ieee.org/documents/trans_journal_names.pdf
# Write out clean bibfile in ascii format
clean_bib_fpath = ub.augpath(bib_fpath.replace(' ', '_'), suffix='_clean')
if not ub.argflag('--dryrun'):
ut.writeto(clean_bib_fpath, new_bibtex_str)
def _autojit_cython(pyx_fpath, verbose=1, recompile=False, annotate=False):
"""
This idea is that given a pyx file, we try to compile it. We write a stamp
file so subsequent calls should be very fast as long as the source pyx has
not changed.
Parameters
----------
pyx_fpath : str
path to the pyx file
verbose : int
higher is more verbose.
"""
import shutil
if verbose > 3:
print('_autojit_cython')
# TODO: move necessary ubelt utilities to nx.utils?
# Separate this into its own util?
if shutil.which("cythonize"):
pyx_dpath = dirname(pyx_fpath)
if verbose > 3:
print('pyx_dpath = {!r}'.format(pyx_dpath))
# Check if the compiled library exists
pyx_base = splitext(basename(pyx_fpath))[0]
SO_EXTS = _platform_pylib_exts()
so_fname = False
for fname in os.listdir(pyx_dpath):
if fname.startswith(pyx_base) and fname.endswith(SO_EXTS):
so_fname = fname
break
if verbose > 3:
print('so_fname = {!r}'.format(so_fname))
try:
# Currently this functionality depends on ubelt.
# We could replace ub.cmd with subprocess.check_call and ub.augpath
# with os.path operations, but hash_file and CacheStamp are harder
# to replace. We can use "liberator" to statically extract these
# and add them to nx.utils though.
import ubelt as ub
except Exception:
if verbose > 3:
print('return false, no ubelt')
return False
else:
if so_fname is False:
# We can compute what the so_fname will be if it doesnt exist
so_fname = pyx_base + SO_EXTS[0]
so_fpath = join(pyx_dpath, so_fname)
content = ub.readfrom(pyx_fpath)
mtime = os.stat(pyx_fpath).st_mtime
depends = [ub.hash_data(content, hasher="sha1"), mtime]
stamp_fname = ub.augpath(so_fname, ext=".jit.stamp")
stamp = ub.CacheStamp(
stamp_fname,
dpath=pyx_dpath,
product=so_fpath,
depends=depends,
verbose=verbose,
)
if verbose > 3:
print('stamp = {!r}'.format(stamp))
if recompile or stamp.expired():
# Heuristic to try and grab the numpy include dir or not
cythonize_args = ['cythonize']
cythonize_env = os.environ.copy()
needs_numpy = 'numpy' in content
if needs_numpy:
import numpy as np
import pathlib
numpy_include_dpath = pathlib.Path(np.get_include())
numpy_dpath = (numpy_include_dpath / '../..').resolve()
# cythonize_env['CPATH'] = numpy_include_dpath + ':' + cythonize_env.get('CPATH', '')
cythonize_env['CFLAGS'] = ' '.join([
'-I{}'.format(numpy_include_dpath),
]) + cythonize_env.get('CFLAGS', '')
cythonize_env['LDFLAGS'] = ' '.join([
'-L{} -lnpyrandom'.format(numpy_dpath / 'random/lib'),
'-L{} -lnpymath'.format(numpy_dpath / 'core/lib'),
]) + cythonize_env.get('LDFLAGS', '')
if annotate:
cythonize_args.append('-a')
cythonize_args.append('-i {}'.format(pyx_fpath))
cythonize_cmd = ' '.join(cythonize_args)
if needs_numpy:
print('CFLAGS="{}" '.format(cythonize_env['CFLAGS']) +
'LDFLAGS="{}" '.format(cythonize_env['LDFLAGS']) +
cythonize_cmd)
ub.cmd(cythonize_cmd,
verbose=verbose,
check=True,
env=cythonize_env)
stamp.renew()
return True
else:
if verbose > 2:
print('Cythonize not found!')
def check_nitfs():
nitfs = unsafe_grab_nitfs()
import xdev
import netharn as nh
total = 0
for fpath in nitfs:
nbytes = nh.util.get_file_info(fpath)['filesize']
print('nbytes = {!r}'.format(xdev.byte_str(nbytes)))
total += nbytes
print(xdev.byte_str(total))
failed_fpaths = []
passed_fpaths = []
for fpath in nitfs:
import kwimage
try:
kwimage.imread(fpath)
passed_fpaths.append(fpath)
except Exception:
failed_fpaths.append(fpath)
print('passed = {}'.format(len(passed_fpaths)))
print('failed = {}'.format(len(failed_fpaths)))
print('CANT HANDLE')
for fpath in failed_fpaths:
name = splitext(basename(fpath))[0]
desc = NITF_DESC[name]
print(desc)
for fpath in failed_fpaths:
print('\n-----')
print('fpath = {!r}'.format(fpath))
try:
kwimage.imread(fpath)
except Exception:
pass
print('\n-----')
from ndsampler.abstract_frames import _cog_cache_write
for gpath in passed_fpaths:
cache_gpath = ub.augpath(gpath, ext='.test.api.cog')
ub.delete(cache_gpath)
# config = {'hack_use_cli': True}
config = {'hack_use_cli': False, 'compress': 'LZW'}
_cog_cache_write(gpath, cache_gpath, config=config)
from ndsampler.abstract_frames import _cog_cache_write
for gpath in passed_fpaths:
cache_gpath = ub.augpath(gpath, ext='.test.cli.cog')
ub.delete(cache_gpath)
# config = {'hack_use_cli': True}
config = {'hack_use_cli': True, 'compress': 'LZW'}
_cog_cache_write(gpath, cache_gpath, config=config)
from ndsampler.abstract_frames import _cog_cache_write
for gpath in passed_fpaths:
cache_gpath = ub.augpath(gpath, ext='.test.cli.cog')
ub.delete(cache_gpath)
# config = {'hack_use_cli': True}
kwimage.imread(gpath)
config = {'hack_use_cli': True, 'compress': 'JPEG'}
_cog_cache_write(gpath, cache_gpath, config=config)
def script_overlay_aux():
"""
"""
task = UrbanMapper3D(root='~/remote/aretha/data/UrbanMapper3D',
workdir='~/data/work/urban_mapper')
fullres = task.load_fullres_inputs(subdir='training')
NAN_VAL = -32767
for paths in ub.ProgIter(fullres, adjust=False, freq=1):
bgr = util.imread(paths['im'])
gtl = util.imread(paths['gt'])
gti = util.imread(paths['gt'].replace('GTL', 'GTI'))
dsm = util.imread(paths['aux']['dsm'])
dsm[(NAN_VAL == dsm)] = np.nan
dtm = util.imread(paths['aux']['dtm'])
dtm[(NAN_VAL == dtm)] = np.nan
diff = dtm - dsm
def normalize(chan):
min_val = np.nanmax(chan)
max_val = np.nanmin(chan)
is_nan = np.isnan(chan)
norm = chan.copy()
norm[~is_nan] = (norm[~is_nan] - min_val) / (max_val - min_val)
norm[is_nan] = 0
return norm, min_val, max_val
def colorize(chan):
norm, min_val, max_val = normalize(chan)
domain = np.linspace(min_val, max_val)
shape = (norm.shape[0], norm.shape[0] / 25)
cb = colorutil.colorbar_image(domain, dpi=200, shape=shape)[:, :, 0: 3]
sfy = norm.shape[0] / cb.shape[0]
cb = imutil.imscale(cb, scale=(sfy, sfy))[0]
color_chan = util.make_heatmask(norm)[:, :, 0:3]
color_chan[np.isnan(chan)] = [[0, 0, 1]]
blend_chan = util.overlay_colorized(color_chan, bgr, alpha=.2)[:, :, 0:3]
blend_gt_chan = draw_instance_contours(blend_chan, gti, gtl,
thickness=2, alpha=.3)
# Add a colorbar
blend_chan = np.hstack([blend_chan, cb])
blend_gt_chan = np.hstack([blend_gt_chan, cb])
return blend_chan, blend_gt_chan
blend_dsm, blend_gt_dsm = colorize(dsm)
blend_dtm, blend_gt_dtm = colorize(dtm)
blend_diff, blend_gt_diff = colorize(diff)
base_dpath = ub.ensuredir(join(task.workdir, 'viz'))
outputs = {
'blend_diff': blend_diff,
'blend_dsm': blend_dsm,
'blend_dtm': blend_dtm,
'blend_gt_diff': blend_gt_diff,
'blend_gt_dsm': blend_gt_dsm,
'blend_gt_dtm': blend_gt_dtm,
}
for key, val in outputs.items():
out_dpath = ub.ensuredir((base_dpath, key))
out_fpath = join(out_dpath, ub.augpath(paths['dump_fname'], ext='.png'))
util.imwrite(out_fpath, val)
