def test_hash_file():
fpath = join(ub.ensure_app_cache_dir('ubelt'), 'tmp.txt')
ub.writeto(fpath, 'foobar')
hashid1_a = ub.hash_file(fpath,
hasher='sha512',
hashlen=8,
stride=1,
blocksize=1)
hashid2_a = ub.hash_file(fpath,
hasher='sha512',
hashlen=8,
stride=2,
blocksize=1)
hashid1_b = ub.hash_file(fpath,
hasher='sha512',
hashlen=8,
stride=1,
blocksize=10)
hashid2_b = ub.hash_file(fpath,
hasher='sha512',
hashlen=8,
stride=2,
blocksize=10)
assert hashid1_a == hashid1_b
assert hashid2_a != hashid2_b, 'blocksize matters when stride is > 1'
assert hashid1_a != hashid2_a
def test_import_modpath_package():
assert '_tmproot373.sub1.sub2.testmod' not in sys.modules
temp = ub.TempDir().start()
# with ub.TempDir() as temp:
if True:
dpath = temp.dpath
# Create a dummy package heirachy
root = ub.ensuredir((dpath, '_tmproot373'))
sub1 = ub.ensuredir((root, 'sub1'))
sub2 = ub.ensuredir((sub1, 'sub2'))
ub.touch(join(root, '__init__.py'))
ub.touch(join(sub1, '__init__.py'))
ub.touch(join(sub2, '__init__.py'))
modpath = join(sub2, 'testmod.py')
text = ub.codeblock(
'''
a = 'value'
''')
ub.writeto(modpath, text)
assert temp.dpath not in sys.path
module = ub.import_module_from_path(modpath)
assert temp.dpath not in sys.path, 'pythonpath should remain clean'
assert module.a == 'value'
assert module.__file__ == modpath
assert module.__name__ == '_tmproot373.sub1.sub2.testmod'
assert '_tmproot373.sub1.sub2.testmod' in sys.modules
assert '_tmproot373.sub1.sub2' in sys.modules
assert '_tmproot373' in sys.modules
def test_readwrite():
import ubelt as ub
dpath = ub.ensure_app_cache_dir('ubelt')
fpath = dpath + '/' + 'testwrite.txt'
if exists(fpath):
os.remove(fpath)
to_write = 'utf-8 symbols Δ, Й, ק, م, ๗, あ, 叶, 葉, and 말.'
ub.writeto(fpath, to_write, verbose=True)
read_ = ub.readfrom(fpath, verbose=True)
assert read_ == to_write
def make_model_file(input_fpath, arch='segnet_basic', mode='predict',
dpath=None, modelkw={}, params=None):
# assert input_fpath, 'must specify'
model_fname = '{}_{}_model.prototext'.format(arch, mode)
model_fpath = join(dpath, model_fname)
text = make_prototext(input_fpath, arch=arch, mode=mode, params=params,
**modelkw)
ub.writeto(model_fpath, text)
print('made model_fpath = {!r}'.format(ub.compressuser(model_fpath)))
return model_fpath
def write_json(fpath, data):
"""
Write human readable json files
"""
if isinstance(data, pd.DataFrame):
# pretty pandas
json_text = (json.dumps(json.loads(data.to_json()), indent=4))
elif isinstance(data, dict):
json_text = json.dumps(data, cls=LossyJSONEncoder, indent=4)
else:
raise TypeError(type(data))
ub.writeto(fpath, json_text)
def dump_text(self):
print("Dumping Profile Information")
try:
output_text, summary_text = self.get_text()
except AttributeError:
print('profile is not on')
else:
#profile.dump_stats('out.lprof')
import ubelt as ub
print(summary_text)
ub.writeto('profile_output.txt', output_text + '\n' + summary_text)
ub.writeto('profile_output.%s.txt' % (ub.timestamp()),
output_text + '\n' + summary_text)
def deploy_trained_for_testing(harn, stride=1):
"""
Makes the deployable/testable models, and then this function actually
deploys them to the test directory (via a link file).
"""
harn.prepare_test_model(force=False)
for test_weights_fpath in harn.make_testable_weights(stride):
dname = splitext(basename(test_weights_fpath)[5:])[0]
test_weights_dpath = ub.ensuredir((harn.test_dpath, dname))
link_fpath = join(test_weights_dpath,
'test_weights.caffemodel.lnk')
ub.writeto(link_fpath, test_weights_fpath)
yield test_weights_fpath
def make_initd(fpath):
name = splitext(basename(fpath))[0]
text = INITD_TEMPLATE.format(name=name, fpath=fpath)
startup_hook = expanduser(join('~/local/ubuntu/startup/init.d', name))
ub.writeto(startup_hook, text)
print(ub.codeblock(
'''
RUN:
sudo cp {startup_hook} /etc/init.d/{name}
sudo chmod +x /etc/init.d/{name}
# sudo update-rc.d /etc/init.d/{name} defaults
sudo update-rc.d {name} defaults
service {name} start
''').format(**locals()))
def main():
mapper0, mapper2, cats2 = define_fine_challenge_categories()
mapper0, mapper1, cats1 = define_coarse_challenge_categories()
mapper2['shrimp']
mapper1['shrimp']
assert all([k == v for k, v in mapper2.items()])
assert not all([k != v for k, v in mapper1.items()])
raw_to_fine_cat = {k: mapper2[v] for k, v in mapper0.items()}
raw_to_coarse_cat = {k: mapper1[v] for k, v in mapper0.items()}
fine_to_coarse_cat = {}
fine_to_raws = ub.invert_dict(mapper0, 0)
for fine, raws in fine_to_raws.items():
for raw in raws:
coarse = raw_to_coarse_cat[raw]
fine_to_coarse_cat[fine] = coarse
print(ub.repr2(ub.invert_dict(raw_to_fine_cat, False)))
print(ub.repr2(ub.invert_dict(raw_to_coarse_cat, False)))
# Write a python file that contains coarse mappings
text = ub.codeblock(
'''
""" autogenerated file defining the viame challenge 2018 categories """
class FineGrainedChallenge(object):
raw_to_cat = {raw_to_fine_cat}
heirarchy = {cats2}
class CoarseChallenge(object):
raw_to_cat = {raw_to_coarse_cat}
fine_to_cat = {fine_to_coarse_cat}
heirarchy = {cats1}
''').format(
raw_to_fine_cat=ub.repr2(raw_to_fine_cat),
raw_to_coarse_cat=ub.repr2(raw_to_coarse_cat),
fine_to_coarse_cat=ub.repr2(fine_to_coarse_cat),
cats1=ub.repr2(cats1),
cats2=ub.repr2(cats2)
)
import autopep8
pep8_options = {}
new_text = autopep8.fix_code(text, pep8_options)
# print(new_text)
ub.writeto(join(dirname(viame_wrangler.__file__), 'mappings.py'), new_text)
def make_input_file(im_paths, gt_paths=None, ext='.png', dpath=None):
# TODO: remove or refactor (holdover from caffe)
input_fpath = join(dpath, 'inputs.txt')
if gt_paths is not None:
assert fnameutil.check_aligned(
im_paths, gt_paths), ('image names should correspond')
text = '\n'.join([
'{} {}'.format(xpath, ypath)
for xpath, ypath in zip(im_paths, gt_paths)
])
else:
text = '\n'.join(im_paths)
ub.writeto(input_fpath, text)
return input_fpath
def make_solver_file(input_fpath, arch='segnet_basic', dpath=None, modelkw={},
params=None, gpu_num=0):
assert input_fpath, 'must specify'
model_fpath = make_model_file(input_fpath, arch=arch, mode='fit',
dpath=dpath, modelkw=modelkw)
solver_fname = '{}_solver.prototext'.format(arch)
solver_fpath = join(dpath, solver_fname)
snapshot_dpath = ub.ensuredir((dpath, 'snapshots'))
snapshot_prefix = snapshot_dpath + '/'
text = make_solver(model_fpath, snapshot_prefix=snapshot_prefix,
params=params, gpu_num=gpu_num)
ub.writeto(solver_fpath, text)
print('made solver_fpath = {!r}'.format(ub.compressuser(solver_fpath)))
return solver_fpath
def test_import_modpath_basic():
assert 'testmod' not in sys.modules
with ub.TempDir() as temp:
modpath = join(temp.dpath, 'testmod.py')
text = ub.codeblock('''
a = 'value'
''')
ub.writeto(modpath, text)
assert temp.dpath not in sys.path
module = ub.import_module_from_path(modpath)
assert temp.dpath not in sys.path, 'pythonpath should remain clean'
assert module.a == 'value'
assert module.__file__ == modpath
assert module.__name__ == 'testmod'
assert 'testmod' in sys.modules
def dump_text(self):
print("Dumping Profile Information")
try:
output_text, summary_text = self.get_text()
except AttributeError:
print('profile is not on')
else:
#profile.dump_stats('out.lprof')
import ubelt as ub
print(summary_text)
suffix = ub.argval('--profname', default='')
if suffix:
suffix = '_' + suffix
ub.writeto('profile_output{}.txt'.format(suffix), output_text + '\n' + summary_text)
ub.writeto('profile_output{}.{}.txt'.format(suffix, ub.timestamp()),
output_text + '\n' + summary_text)
def generate():
content = ub.readfrom('base_diff.py') + '\n\n'
xdoc_version = content + ub.codeblock('''
if __name__ == '__main__':
import xdoctest
xdoctest.doctest_module(__file__)
''') + '\n'
doc_version = content + ub.codeblock('''
if __name__ == '__main__':
import doctest
doctest.testmod()
''') + '\n'
ub.writeto('_doc_version.py', doc_version)
ub.writeto('_xdoc_version.py', xdoc_version)
def test_cli():
"""
Test command line interaction with kernprof and line_profiler.
References:
https://github.com/pyutils/line_profiler/issues/9
CommandLine:
xdoctest -m ~/code/line_profiler/tests/test_cli.py test_cli
"""
# Create a dummy source file
import ubelt as ub
code = ub.codeblock('''
@profile
def my_inefficient_function():
a = 0
for i in range(10):
a += i
for j in range(10):
a += j
if __name__ == '__main__':
my_inefficient_function()
''')
import tempfile
tmp_dpath = tempfile.mkdtemp()
tmp_src_fpath = join(tmp_dpath, 'foo.py')
ub.writeto(tmp_src_fpath, code)
# Run kernprof on it
info = ub.cmd('kernprof -l {}'.format(tmp_src_fpath),
verbose=3,
cwd=tmp_dpath)
assert info['ret'] == 0
tmp_lprof_fpath = join(tmp_dpath, 'foo.py.lprof')
tmp_lprof_fpath
info = ub.cmd('{} -m line_profiler {}'.format(executable, tmp_lprof_fpath),
cwd=tmp_dpath,
verbose=3)
assert info['ret'] == 0
# Check for some patterns that should be in the output
assert '% Time' in info['out']
assert '7 100' in info['out']
def setup_git_scripts():
dpath = dirname(__file__)
git_sh_scripts = list(glob.glob(join(dpath, 'git_*.sh')))
git_py_scripts = list(glob.glob(join(dpath, 'git_*.py')))
git_scripts = git_py_scripts + git_sh_scripts
for fpath in git_scripts:
fname = basename(fpath)
script_text = (SCRIPT_HEADER + '\n\n' +
SCRIPT_FOOTER_FMT.format(fname=fname) + '\n')
new_fname = splitext(fname)[0].replace('_', '-')
new_fpath = join(dpath, 'scripts', new_fname)
print('writing script {!r}'.format(new_fname))
ub.writeto(new_fpath, script_text)
ub.cmd('chmod +x ' + new_fpath)
ub.cmd('chmod +x ' + fpath)
def prepare_gtstats(self, task, force=False):
"""
Caches stats like class frequency to disk, before we start training
"""
self.prepare_input()
gtstats_fpath = join(self.input_dpath, 'gtstats_v1.json')
print('gtstats_fpath = {!r}'.format(gtstats_fpath))
if not force and exists(gtstats_fpath):
gtstats = pd.read_json(gtstats_fpath)
else:
self.prepare_images()
gt_paths = self.paths['gt']
gtstats = self._compute_gt_info(gt_paths, task)
# pretty writing of json stats
json_text = (json.dumps(json.loads(gtstats.to_json()), indent=4))
ub.writeto(gtstats_fpath, json_text)
self.gtstats = gtstats
return self.gtstats
def setup_git_scripts():
dpath = dirname(__file__)
git_sh_scripts = list(glob.glob(join(dpath, 'git_*.sh')))
git_py_scripts = list(glob.glob(join(dpath, 'git_*.py')))
github_py_scripts = list(glob.glob(join(dpath, 'github_*.py')))
git_scripts = git_py_scripts + git_sh_scripts + github_py_scripts
for fpath in git_scripts:
fname = basename(fpath)
script_text = (SCRIPT_HEADER + '\n\n' +
SCRIPT_FOOTER_FMT.format(fname=fname) + '\n')
new_fname = splitext(fname)[0].replace('_', '-')
new_fpath = join(dpath, 'scripts', new_fname)
print('writing script {!r}'.format(new_fname))
ub.writeto(new_fpath, script_text)
ub.cmd('chmod +x ' + new_fpath)
ub.cmd('chmod +x ' + fpath)
def test_modify_directory_symlinks():
dpath = ub.ensure_app_cache_dir('ubelt', 'test_modify_symlinks')
ub.delete(dpath, verbose=2)
ub.ensuredir(dpath, verbose=2)
happy_dpath = join(dpath, 'happy_dpath')
happy_dlink = join(dpath, 'happy_dlink')
ub.ensuredir(happy_dpath, verbose=2)
ub.symlink(happy_dpath, happy_dlink, verbose=2)
# Test file inside directory symlink
file_path1 = join(happy_dpath, 'file.txt')
file_path2 = join(happy_dlink, 'file.txt')
ub.touch(file_path1, verbose=2)
assert exists(file_path1)
assert exists(file_path2)
ub.writeto(file_path1, 'foo')
assert ub.readfrom(file_path1) == 'foo'
assert ub.readfrom(file_path2) == 'foo'
ub.writeto(file_path2, 'bar')
assert ub.readfrom(file_path1) == 'bar'
assert ub.readfrom(file_path2) == 'bar'
ub.delete(file_path2, verbose=2)
assert not exists(file_path1)
assert not exists(file_path2)
# Test directory inside directory symlink
dir_path1 = join(happy_dpath, 'dir')
dir_path2 = join(happy_dlink, 'dir')
ub.ensuredir(dir_path1, verbose=2)
assert exists(dir_path1)
assert exists(dir_path2)
subfile_path1 = join(dir_path1, 'subfile.txt')
subfile_path2 = join(dir_path2, 'subfile.txt')
ub.writeto(subfile_path2, 'foo')
assert ub.readfrom(subfile_path1) == 'foo'
assert ub.readfrom(subfile_path2) == 'foo'
ub.writeto(subfile_path1, 'bar')
assert ub.readfrom(subfile_path1) == 'bar'
assert ub.readfrom(subfile_path2) == 'bar'
ub.delete(dir_path1, verbose=2)
assert not exists(dir_path1)
assert not exists(dir_path2)
def make_input_file(im_paths, gt_paths=None, ext='.png', dpath=None):
"""
Example:
>>> from pysseg import util
>>> imdir = '~/sseg/virat/imall'
>>> gtdir = '~/sseg/virat/gtall'
>>> im_paths = util.load_image_paths(imdir)
>>> ext = '.png'
"""
input_fpath = join(dpath, 'inputs.txt')
if gt_paths is not None:
assert fnameutil.check_aligned(im_paths, gt_paths), (
'image names should correspond')
text = '\n'.join([
'{} {}'.format(xpath, ypath)
for xpath, ypath in zip(im_paths, gt_paths)
])
else:
text = '\n'.join(im_paths)
ub.writeto(input_fpath, text)
return input_fpath
def test_modify_file_symlinks():
"""
CommandLine:
python -m ubelt.tests.test_links test_modify_symlinks
"""
# TODO: test that we handle broken links
dpath = ub.ensure_app_cache_dir('ubelt', 'test_modify_symlinks')
happy_fpath = join(dpath, 'happy_fpath.txt')
happy_flink = join(dpath, 'happy_flink.txt')
ub.touch(happy_fpath, verbose=2)
ub.symlink(happy_fpath, happy_flink, verbose=2)
# Test file symlink
ub.writeto(happy_fpath, 'foo')
assert ub.readfrom(happy_fpath) == 'foo'
assert ub.readfrom(happy_flink) == 'foo'
ub.writeto(happy_flink, 'bar')
assert ub.readfrom(happy_fpath) == 'bar'
assert ub.readfrom(happy_flink) == 'bar'
def modify_conf():
"""
pip install redbaron
"""
import redbaron
import ubelt as ub
conf_path = 'docs/conf.py'
source = ub.readfrom(conf_path)
red = redbaron.RedBaron(source)
# Insert custom extensions
extra_extensions = ['"sphinxcontrib.napoleon"']
ext_node = red.find('name', value='extensions').parent
ext_node.value.value.extend(extra_extensions)
# Overwrite theme to read-the-docs
theme_node = red.find('name', value='html_theme').parent
theme_node.value.value = '"sphinx_rtd_theme"'
ub.writeto(conf_path, red.dumps())
def check_fsspec():
import ubelt as ub
from os.path import join
dpath = ub.ensure_app_cache_dir('ubelt/simple_server')
info = ub.cmd(['python', '-m', 'http.server', '--directory', dpath], detatch=True)
fnames = ['file_{}.txt'.format(i) for i in range(100)]
for fname in fnames:
ub.writeto(join(dpath, fname), ub.hash_data(fname))
# info = ub.cmd('python -m http.server --directory "{}"'.format(dpath), verbose=3)
# proc = info['proc']
# TODO: ub.cmd return with some object that can tee the output on demand?
# _proc_iteroutput = ub.util_cmd._proc_iteroutput_thread(proc)
# line = next(_proc_iteroutput)
urls = ['http://localhost:8000/{}'.format(fname) for fname in fnames]
import fsspec
file = fsspec.open(urls[0]).open().read()
with ub.Timer(label='fsspec.cat', verbose=1):
fs = fsspec.filesystem("http")
out = fs.cat(urls) # fetches data concurrently
with ub.Timer(label='ub.DownloadManager', verbose=1):
dpath = ub.ensure_app_cache_dir('ubelt/simple_download_root')
dman = ub.DownloadManager(dpath)
for url in urls:
dman.submit(url)
results = []
for future in dman.as_completed(prog=True):
fpath = future.result()
results.append(fpath)
# print('fpath = {!r}'.format(fpath))
proc.terminate()
def benchmark_write(xxd, proto):
dpath = dpaths[xxd]
args = {'proto': proto, 'xxd': xxd}
fpath = join(dpath, 'test_{}.pkl'.format(proto))
for timer in ub.Timerit(10, label='save {}'.format(args)):
ub.delete(fpath)
ub.writeto(fpath, 'junkdata')
ub.delete(fpath)
with timer:
with open(fpath, 'wb') as file:
pickle.dump(data, file, protocol=proto)
result = args.copy()
result['write_time'] = timer.ellapsed
for timer in ub.Timerit(10, label='read {}'.format(args)):
with timer:
with open(fpath, 'rb') as file:
pickle.load(file)
result['read_time'] = timer.ellapsed
return result
def write_research(r, to_write, rate=-5):
fname = join(split(__file__)[0], 'to_speak.txt')
import ubelt as ub
ub.writeto(fname, to_write)
def make_dummy_package(dpath, pkgname='mkinit_dummy_module'):
"""
Creates a dummy package structure with or without __init__ files
"""
root = ub.ensuredir(join(dpath, pkgname))
ub.delete(root)
ub.ensuredir(root)
paths = {
'root': root,
'submod1': ub.touch(join(root, 'submod1.py')),
'submod2': ub.touch(join(root, 'submod2.py')),
'subdir1': ub.ensuredir(join(root, 'subdir1')),
'subdir2': ub.ensuredir(join(root, 'subdir2')),
}
paths['subdir1_init'] = ub.touch(join(paths['subdir1'], '__init__.py'))
paths['subdir2_init'] = ub.touch(join(paths['subdir2'], '__init__.py'))
paths['root_init'] = ub.touch(join(paths['root'], '__init__.py'))
ub.writeto(paths['subdir1_init'], ub.codeblock(
'''
simple_subattr1 = "hello world"
simple_subattr2 = "hello world"
_private_attr = "hello world"
'''))
ub.writeto(paths['subdir2_init'], ub.codeblock(
'''
__all__ = ['public_attr']
public_attr = "hello world"
private_attr = "hello world"
'''))
ub.writeto(paths['submod1'], ub.codeblock(
'''
import six
attr1 = True
attr2 = six.moves.zip
# ------------------------
if True:
good_attr_01 = None
if False:
bad_attr_false1 = None
if None:
bad_attr_none1 = None
# ------------------------
if True:
good_attr_02 = None
else:
bad_attr_true2 = None
if False:
bad_attr_false2 = None
else:
good_attr_03 = None
if None:
bad_attr_none2 = None
else:
good_attr_04 = None
# ------------------------
if True:
good_attr_05 = None
elif False:
bad_attr3 = None
else:
bad_attr3 = None
if False:
bad_attr_elif_True3_0 = None
elif True:
good_attr_06 = None
else:
bad_attr_elif_True3_1 = None
# ------------------------
import sys
if sys.version_info.major == 3:
good_attr_07 = 'py3'
bad_attr_uncommon4_1 = None
else:
good_attr_07 = 'py2'
bad_attr_uncommon4_0 = None
# ------------------------
# This is static, so maybe another_val exists as a global
if sys.version_info.major == good_attr_07:
good_attr_08 = None
bad_attr_uncommon5_1 = None
bad_attr_uncommon5_0 = None
elif sys:
good_attr_08 = None
bad_attr_uncommon5_1 = None
else:
good_attr_08 = None
bad_attr_uncommon5_0 = None
# ------------------------
flag1 = sys.version_info.major < 10
flag2 = sys.version_info.major > 10
flag3 = sys.version_info.major > 10
if flag1:
bad_attr_num6 = 1
elif flag2:
bad_attr_num6 = 1
elif flag3:
bad_attr_num6 = 1
if flag1:
bad_attr_num6_0 = 1
elif 0:
bad_attr_num0 = 1
elif 1:
bad_attr_09 = 1
else:
bad_attr13 = 1
if flag1:
good_attr_09 = 1
elif 1:
good_attr_09 = 1
bad_attr_09_1 = 1
elif 2 == 3:
pass
# ------------------------
if 'foobar':
good_attr_10 = 1
if False:
bad_attr_str7 = 1
elif (1, 2):
good_attr_11 = 1
elif True:
bad_attr_true8 = 1
# ------------------------
if flag1 != flag2:
good_attr_12 = None
else:
bad_attr_12 = None
raise Exception
# ------------------------
try:
good_attr_13 = None
bad_attr_13 = None
except Exception:
good_attr_13 = None
# ------------------------
try:
good_attr_14 = None
except Exception:
bad_attr_14 = None
raise
# ------------------------
def func1():
pass
class class1():
pass
if __name__ == '__main__':
bad_attr_main = None
if __name__ == 'something_else':
bad_something_else = None
'''))
return paths
def make_dummy_package(dpath, pkgname="mkinit_dummy_module", side_effects=0):
"""
Creates a dummy package structure with or without __init__ files
ANY EXISTING FILES ARE DELETED
"""
root = ub.ensuredir(join(dpath, pkgname))
ub.delete(root)
ub.ensuredir(root)
paths = {
"root":
root,
"submod1":
ub.touch(join(root, "submod1.py")),
"submod2":
ub.touch(join(root, "submod2.py")),
"subdir1":
ub.ensuredir(join(root, "subdir1")),
"subdir2":
ub.ensuredir(join(root, "subdir2")),
"longsubdir":
ub.ensuredir(join(root, "avery/long/subdir/that/goes/over80chars")),
}
paths["subdir1_init"] = ub.touch(join(paths["subdir1"], "__init__.py"))
paths["subdir2_init"] = ub.touch(join(paths["subdir2"], "__init__.py"))
paths["root_init"] = ub.touch(join(paths["root"], "__init__.py"))
paths["long_subdir_init"] = ub.touch(
join(paths["longsubdir"], "__init__.py"))
paths["long_submod"] = ub.touch(join(paths["longsubdir"],
"long_submod.py"))
ub.writeto(
paths["subdir1_init"],
ub.codeblock("""
simple_subattr1 = "hello world"
simple_subattr2 = "hello world"
_private_attr = "hello world"
"""),
)
ub.writeto(
paths["subdir2_init"],
ub.codeblock("""
__all__ = ['public_attr']
public_attr = "hello world"
private_attr = "hello world"
"""),
)
ub.writeto(
paths["submod1"],
ub.codeblock("""
import six
attr1 = True
attr2 = six.moves.zip
# ------------------------
if True:
good_attr_01 = None
if False:
bad_attr_false1 = None
if None:
bad_attr_none1 = None
# ------------------------
if True:
good_attr_02 = None
else:
bad_attr_true2 = None
if False:
bad_attr_false2 = None
else:
good_attr_03 = None
if None:
bad_attr_none2 = None
else:
good_attr_04 = None
# ------------------------
if True:
good_attr_05 = None
elif False:
bad_attr3 = None
else:
bad_attr3 = None
if False:
bad_attr_elif_True3_0 = None
elif True:
good_attr_06 = None
else:
bad_attr_elif_True3_1 = None
# ------------------------
import sys
if sys.version_info.major == 3:
good_attr_07 = 'py3'
bad_attr_uncommon4_1 = None
else:
good_attr_07 = 'py2'
bad_attr_uncommon4_0 = None
# ------------------------
# This is static, so maybe another_val exists as a global
if sys.version_info.major == good_attr_07:
good_attr_08 = None
bad_attr_uncommon5_1 = None
bad_attr_uncommon5_0 = None
elif sys:
good_attr_08 = None
bad_attr_uncommon5_1 = None
else:
good_attr_08 = None
bad_attr_uncommon5_0 = None
# ------------------------
flag1 = sys.version_info.major < 10
flag2 = sys.version_info.major > 10
flag3 = sys.version_info.major > 10
if flag1:
bad_attr_num6 = 1
elif flag2:
bad_attr_num6 = 1
elif flag3:
bad_attr_num6 = 1
if flag1:
bad_attr_num6_0 = 1
elif 0:
bad_attr_num0 = 1
elif 1:
bad_attr_09 = 1
else:
bad_attr13 = 1
if flag1:
good_attr_09 = 1
elif 1:
good_attr_09 = 1
bad_attr_09_1 = 1
elif 2 == 3:
pass
# ------------------------
if 'foobar':
good_attr_10 = 1
if False:
bad_attr_str7 = 1
elif (1, 2):
good_attr_11 = 1
elif True:
bad_attr_true8 = 1
# ------------------------
if flag1 != flag2:
good_attr_12 = None
else:
bad_attr_12 = None
raise Exception
# ------------------------
try:
good_attr_13 = None
bad_attr_13 = None
except Exception:
good_attr_13 = None
# ------------------------
try:
good_attr_14 = None
except Exception:
bad_attr_14 = None
raise
# ------------------------
def func1():
pass
class class1():
pass
if __name__ == '__main__':
bad_attr_main = None
if __name__ == 'something_else':
bad_something_else = None
"""),
)
# Ensure that each submodule has an __init__
# (do we need this with PEP 420 anymore?)
root = paths["root"]
from os.path import relpath, exists, dirname
import os
dpath = dirname(root)
init_fpaths = set()
for key, path in paths.items():
relative = relpath(path, dpath)
suffix = []
parts = relative.split(os.sep)
for part in parts:
if "." not in part:
suffix.append(part)
middir = join(dpath, os.sep.join(suffix))
fpath = join(middir, "__init__.py")
init_fpaths.add(fpath)
for fpath in init_fpaths:
if not exists(fpath):
ub.touch(fpath)
ub.writeto(
paths["long_submod"],
ub.codeblock("""
def a_very_nested_function():
print('a_very_nested_function in {}'.format(__file__))
def another_func1():
pass
def another_func2():
pass
def another_func3():
pass
def another_func4():
pass
"""),
)
if side_effects:
with open(paths["long_submod"], "a") as file:
file.write(
ub.codeblock("""
print('NESTED SIDE EFFECT')
"""))
return paths
def main():
from os.path import join
import six
import pytest
import ubelt as ub
import os
from xdoctest import static_analysis as static
dpath = ub.ensure_app_cache_dir('import_abi_test')
text = ub.codeblock('''
a = 42
''')
os.chdir(dpath)
modpath = join(dpath, 'foo.py')
ub.writeto(modpath, text)
import foo
assert foo.a == 42
tag = static._extension_module_tags()[0]
# Q: Can augment .py files with an ABI tag
# A: No
modpath = join(dpath, 'bar.' + tag + '.py')
ub.writeto(modpath, text)
with pytest.raises(ImportError, match=".*No module named .?bar.*"):
import bar # NOQA
# Q: Can we augment .so files with an ABI tag
# A: Yes (note its not a valid so so it fails)
modpath = join(dpath, 'baz.' + tag + '.so')
ub.writeto(modpath, text)
with pytest.raises(ImportError, match=".*file too short"):
import baz # NOQA
# Q: Can we augment .so files with an **arbitrary** tag?
# A: No
modpath = join(dpath, 'buz.' + 'junktag' + '.so')
ub.writeto(modpath, text)
with pytest.raises(ImportError, match="No module named .*"):
import buz # NOQA
# Q: Can we augment .so files with an "abi3" tag?
# A: Yes, but not in python2
modpath = join(dpath, 'biz.' + 'abi3' + '.so')
ub.writeto(modpath, text)
if six.PY3:
with pytest.raises(ImportError, match=".*file too short"):
import biz # NOQA
else:
with pytest.raises(ImportError, match="No module named .*"):
import biz # NOQA
# Q: Can we augment .so files with a semi-nice tag?
# A: No
modpath = join(dpath, 'spam.' + 'linux' + '.so')
ub.writeto(modpath, text)
with pytest.raises(ImportError, match="No module named .*"):
import spam # NOQA
modpath = join(dpath, 'eggs.' + 'cp3-abi3-linux_x86_64' + '.so')
ub.writeto(modpath, text)
with pytest.raises(ImportError, match="No module named .*"):
import eggs # NOQA
modpath = join(dpath, 'ham.' + 'py3-none-any' + '.so')
ub.writeto(modpath, text)
with pytest.raises(ImportError, match="No module named .*"):
import ham # NOQA
def write_submission_file(arch_to_paths, params, output_file, arches,
ensemble_weights=None):
"""
TODO: take in multiple prediction paths for an enemble
# Ensemble these two predictions together
params = {}
params = {'alpha': 0.8800, 'mask_thresh': 0.7870, 'min_seed_size': 85.1641, 'min_size': 64.0634, 'seed_thresh': 0.4320}
alpha = params.get('alpha')
stitched_dpath1 = '/home/local/KHQ/jon.crall/data/work/urban_mapper2/eval/input_18600-vvhcfmyo/solver_52200-fqljkqlk_unet2_ybypbjtw_smvuzfkv_a=1,c=RGB,n_ch=6,n_cl=4/_epoch_00000000/stitched'
stitched_dpath2 = '/home/local/KHQ/jon.crall/data/work/urban_mapper4/eval/input_18600-vkmqiooh/solver_25800-phpjjsqu_dense_unet_mmavmuou_zeosddyf_a=1,c=RGB,n_ch=6,n_cl=4/_epoch_00000026/stitched'
dump_dpath = ub.ensuredir('/home/local/KHQ/jon.crall/data/work/urban_mapper_ensemble')
stitched_dpaths = [stitched_dpath1, stitched_dpath2]
ensemble_weights = [alpha, 1 - alpha]
"""
# prob_paths = pharn._restitch_type('probs', blend='avew', force=False)
# prob1_paths = pharn._restitch_type('probs1', blend='avew', force=False)
ensemble_paths = list(ub.take(arch_to_paths, arches))
if ensemble_weights is None:
assert len(ensemble_paths[0]) == 1
ensemble_weights = [1]
def seeded_predictions(**params):
# Convert to submission output format
n_scenes = len(ensemble_paths[0]['probs'])
for ix in tqdm.tqdm(list(range(n_scenes)), desc='classifying'):
probs = 0
probs1 = 0
for paths, w in zip(ensemble_paths, ensemble_weights):
path = paths['probs'][ix]
probs = probs + w * util.read_arr(paths['probs'][ix])
probs1 = probs1 + w * util.read_arr(paths['probs1'][ix])
INNER_BUILDING_ID = 1
BUILDING_ID = 1
seed_prob = probs[:, :, INNER_BUILDING_ID]
mask_prob = probs1[:, :, BUILDING_ID]
pred = seeded_instance_label_from_probs(seed_prob, mask_prob,
**params)
tile_id = splitext(basename(path))[0]
yield tile_id, pred
lines = []
for tile_id, pred in seeded_predictions(**params):
(width, height), runlen = util.run_length_encoding(pred)
lines.append(tile_id)
lines.append('{},{}'.format(width, height))
lines.append(','.join(list(map(str, runlen))))
text = '\n'.join(lines)
print('output_file = {!r}'.format(output_file))
fpath = output_file + '.txt'
ub.writeto(fpath, text)
def prepare_center_stats(self,
task,
nan_value=-32767.0,
colorspace='RGB',
with_im=False,
stride=1):
"""
Ignore:
>>> from clab.torch.sseg_train import *
>>> #task = get_task('urban_mapper_3d')
>>> task = get_task('camvid')
>>> self, test = next(task.xval_splits())
>>> self.base_dpath = ub.ensuredir((task.workdir, 'inputs'))
>>> nan_value = -32767.0
>>> colorspace = 'RGB'
>>> colorspace = 'LAB'
>>> self.prepare_center_stats(task, colorspace=colorspace)
"""
# TODO: handle different color spaces
# colorspace = 'RGB'
import pickle
import copy
from clab.util import jsonutil
from clab.torch.transforms import NaNInputer
from clab.torch import im_loaders
self.prepare_input()
fpath = join(self.input_dpath, 'center_stats{}.pkl'.format(colorspace))
simple_fpath = join(self.input_dpath,
'center_stats_simple{}.json'.format(colorspace))
print('Checking intensity stats')
if not exists(fpath):
print('Need to compute intensity stats')
im_paths = self.im_paths
class IntensityStats(object):
def __init__(p):
p.run = imutil.RunningStats()
p.scalar_internals = imutil.InternalRunningStats()
p.channel_internals = imutil.InternalRunningStats(axis=(0,
1))
def update(p, im):
p.run.update(im)
p.scalar_internals.update(im)
p.channel_internals.update(im)
def info(p):
image_simple_info = {
'desc':
'statistics about the per channel/entire intensity averaged across the dataset',
'channel': p.run.simple(axis=(0, 1)),
'image': p.run.simple(),
}
image_detail_info = {
'desc':
'statistics about the per pixel intensity averaged across the dataset',
'pixel': p.run.detail(),
}
image_internal_info = {
'desc':
'statistics about the average internal mean/med/mad/std intensity within an image',
'image': p.scalar_internals.info(),
'channel': p.channel_internals.info(),
}
return {
'simple': image_simple_info,
'detail': image_detail_info,
'internal': image_internal_info,
}
if with_im:
run_im = IntensityStats()
im_paths = im_paths[::stride]
for path in ub.ProgIter(im_paths,
label='computing mean image',
verbose=1):
im = im_loaders.np_loader(path, colorspace=colorspace)
run_im.update(im)
im_info = run_im.info()
im_info['colorspace'] = colorspace
else:
im_info = None
if self.aux_paths:
# nan_value = -32767.0
nan_inputer = NaNInputer(fill='median', nan_value=nan_value)
run_aux = IntensityStats()
prog = ub.ProgIter(label='aux stats', verbose=1)
paths_list = list(zip(*self.aux_paths.values()))[::stride]
for aux_paths in prog(paths_list):
aux = np.dstack(
[im_loaders.np_loader(path) for path in aux_paths])
aux = nan_inputer(aux)
run_aux.update(aux)
aux_channel_names = list(self.aux_paths.keys())
aux_info = run_aux.info()
aux_info['channel_names'] = aux_channel_names
else:
aux_info = None
detail_info = {
'aux': aux_info,
'image': im_info,
}
print('writing to fpath = {!r}'.format(fpath))
with open(fpath, 'wb') as file:
pickle.dump(detail_info, file)
simple_info = copy.deepcopy(detail_info)
# Write a simpler version of the file
if simple_info['aux']:
simple_info['aux'].pop('detail')
if simple_info['image']:
simple_info['image'].pop('detail')
# pretty writing of json stats
json_text = json.dumps(simple_info,
cls=jsonutil.NumpyAwareJSONEncoder,
indent=4)
print('writing to simple_fpath = {!r}'.format(simple_fpath))
ub.writeto(simple_fpath, json_text)
print('reading fpath = {!r}'.format(fpath))
with open(fpath, 'rb') as file:
info = pickle.load(file)
return info
def make_baseline_truthfiles():
work_dir = ub.truepath('~/work')
data_dir = ub.truepath('~/data')
challenge_data_dir = join(data_dir, 'viame-challenge-2018')
challenge_work_dir = join(work_dir, 'viame-challenge-2018')
ub.ensuredir(challenge_work_dir)
img_root = join(challenge_data_dir, 'phase0-imagery')
annot_dir = join(challenge_data_dir, 'phase0-annotations')
fpaths = list(glob.glob(join(annot_dir, '*.json')))
# ignore the non-bounding box nwfsc and afsc datasets for now
# exclude = ('nwfsc', 'afsc', 'mouss', 'habcam')
# exclude = ('nwfsc', 'afsc', 'mouss',)
# fpaths = [p for p in fpaths if not basename(p).startswith(exclude)]
import json
dsets = ub.odict()
for fpath in fpaths:
key = basename(fpath).split('.')[0]
dsets[key] = json.load(open(fpath, 'r'))
print('Merging')
merged = coco_union(dsets)
merged_fpath = join(challenge_work_dir, 'phase0-merged.mscoco.json')
with open(merged_fpath, 'w') as fp:
json.dump(merged, fp, indent=4)
import copy
self = CocoDataset(copy.deepcopy(merged), img_root=img_root, autobuild=False)
self._build_index()
self.run_fixes()
if True:
# remove all point annotations
print('Remove point annotations')
to_remove = []
for ann in self.dataset['annotations']:
if ann['roi_shape'] == 'point':
to_remove.append(ann)
for ann in to_remove:
self.dataset['annotations'].remove(ann)
self._build_index()
# remove empty images
print('Remove empty images')
to_remove = []
for gid in self.imgs.keys():
aids = self.gid_to_aids.get(gid, [])
if not aids:
to_remove.append(self.imgs[gid])
for img in to_remove:
self.dataset['images'].remove(img)
self._build_index()
print('# self.anns = {!r}'.format(len(self.anns)))
print('# self.imgs = {!r}'.format(len(self.imgs)))
print('# self.cats = {!r}'.format(len(self.cats)))
catname_to_nannots = ub.map_keys(lambda x: self.cats[x]['name'],
ub.map_vals(len, self.cid_to_aids))
catname_to_nannots = ub.odict(sorted(catname_to_nannots.items(),
key=lambda kv: kv[1]))
print(ub.repr2(catname_to_nannots))
if False:
# aid = list(self.anns.values())[0]['id']
# self.show_annotation(aid)
gids = sorted([gid for gid, aids in self.gid_to_aids.items() if aids])
# import utool as ut
# for gid in ut.InteractiveIter(gids):
for gid in gids:
from matplotlib import pyplot as plt
fig = plt.figure(1)
fig.clf()
self.show_annotation(gid=gid)
fig.canvas.draw()
for ann in self.anns.values():
primary_aid = ann['id']
print('primary_aid = {!r}'.format(primary_aid))
print(len(self.gid_to_aids[ann['image_id']]))
if 'roi_shape' not in ann:
ann['roi_shape'] = 'bounding_box'
if ann['roi_shape'] == 'boundingBox':
pass
if ann['roi_shape'] == 'point':
primary_aid = ann['id']
print('primary_aid = {!r}'.format(primary_aid))
print(len(self.gid_to_aids[ann['image_id']]))
break
# Split into train / test set
print('Splitting')
skf = StratifiedGroupKFold(n_splits=2)
groups = [ann['image_id'] for ann in self.anns.values()]
y = [ann['category_id'] for ann in self.anns.values()]
X = [ann['id'] for ann in self.anns.values()]
split = list(skf.split(X=X, y=y, groups=groups))[0]
train_idx, test_idx = split
print('Taking subsets')
aid_to_gid = {aid: ann['image_id'] for aid, ann in self.anns.items()}
train_aids = list(ub.take(X, train_idx))
test_aids = list(ub.take(X, test_idx))
train_gids = sorted(set(ub.take(aid_to_gid, train_aids)))
test_gids = sorted(set(ub.take(aid_to_gid, test_aids)))
train_dset = self.subset(train_gids)
test_dset = self.subset(test_gids)
print('---------')
print('# train_dset.anns = {!r}'.format(len(train_dset.anns)))
print('# train_dset.imgs = {!r}'.format(len(train_dset.imgs)))
print('# train_dset.cats = {!r}'.format(len(train_dset.cats)))
print('---------')
print('# test_dset.anns = {!r}'.format(len(test_dset.anns)))
print('# test_dset.imgs = {!r}'.format(len(test_dset.imgs)))
print('# test_dset.cats = {!r}'.format(len(test_dset.cats)))
train_dset._ensure_imgsize()
test_dset._ensure_imgsize()
print('Writing')
with open(join(challenge_work_dir, 'phase0-merged-train.mscoco.json'), 'w') as fp:
json.dump(train_dset.dataset, fp, indent=4)
with open(join(challenge_work_dir, 'phase0-merged-test.mscoco.json'), 'w') as fp:
json.dump(test_dset.dataset, fp, indent=4)
# Make a detectron yaml file
config_text = ub.codeblock(
"""
MODEL:
TYPE: generalized_rcnn
CONV_BODY: ResNet.add_ResNet50_conv4_body
NUM_CLASSES: {num_classes}
FASTER_RCNN: True
NUM_GPUS: 1
SOLVER:
WEIGHT_DECAY: 0.0001
LR_POLICY: steps_with_decay
BASE_LR: 0.01
GAMMA: 0.1
# 1x schedule (note TRAIN.IMS_PER_BATCH: 1)
MAX_ITER: 180000
STEPS: [0, 120000, 160000]
RPN:
SIZES: (32, 64, 128, 256, 512)
FAST_RCNN:
ROI_BOX_HEAD: ResNet.add_ResNet_roi_conv5_head
ROI_XFORM_METHOD: RoIAlign
TRAIN:
WEIGHTS: https://s3-us-west-2.amazonaws.com/detectron/ImageNetPretrained/MSRA/R-50.pkl
DATASETS: ('/work/viame-challenge-2018/phase0-merged-train.mscoco.json',)
IM_DIR: '/data/viame-challenge-2018/phase0-imagery'
SCALES: (800,)
MAX_SIZE: 1333
IMS_PER_BATCH: 1
BATCH_SIZE_PER_IM: 512
TEST:
DATASETS: ('/work/viame-challenge-2018/phase0-merged-test.mscoco.json',)
IM_DIR: '/data/viame-challenge-2018/phase0-imagery'
SCALES: (800,)
MAX_SIZE: 1333
NMS: 0.5
RPN_PRE_NMS_TOP_N: 6000
RPN_POST_NMS_TOP_N: 1000
OUTPUT_DIR: /work/viame-challenge-2018/output
""")
config_text = config_text.format(
num_classes=len(self.cats),
)
ub.writeto(join(challenge_work_dir, 'phase0-faster-rcnn.yaml'), config_text)
docker_cmd = ('nvidia-docker run '
'-v {work_dir}:/work -v {data_dir}:/data '
'-it detectron:c2-cuda9-cudnn7 bash').format(
work_dir=work_dir, data_dir=data_dir)
train_cmd = ('python2 tools/train_net.py '
'--cfg /work/viame-challenge-2018/phase0-faster-rcnn.yaml '
'OUTPUT_DIR /work/viame-challenge-2018/output')
hacks = ub.codeblock(
"""
git remote add Erotemic https://github.com/Erotemic/Detectron.git
git fetch --all
git checkout general_dataset
# curl https://github.com/Erotemic/Detectron/blob/42d44b2d155c775dc509b6a44518d0c582f8cdf5/tools/train_net.py
# wget https://github.com/Erotemic/Detectron/blob/42d44b2d155c775dc509b6a44518d0c582f8cdf5/lib/core/config.py
""")
print(docker_cmd)
print(train_cmd)
def bench_local_versus_global_import():
"""
Write two python files that loop over a test function that uses some
external module.
One version imports the dependency globally at startup, the other does a
lazy import of the module inside the function.
We time how long this takes over several tests where we varry the number of
times this inner function is looped over (and thus the number of times we
will run over the lazy import versus accessing the global import).
It should be noted that startup time of the interpreter will play a
considerable role in these measurements. Any ideas for mitigating that
would be appreciated.
"""
import ubelt as ub
from os.path import join
import timerit
ti = timerit.Timerit(30, bestof=3, verbose=2)
for num in [0, 1, 10, 1000, 1000, 10000]:
fmtkw = {
# 'modname': 'numpy',
# 'attrname': 'array',
# 'modname': 'ubelt',
# 'attrname': 'take',
'modname': 'networkx',
'attrname': 'Graph',
'num': 100000,
}
global_codeblock = ub.codeblock('''
import {modname}
def testfunc():
return {modname}.{attrname}
def main():
for _ in range({num}):
testfunc()
if __name__ == '__main__':
testfunc()
''').format(**fmtkw)
local_codeblock = ub.codeblock('''
def testfunc():
import {modname}
return {modname}.{attrname}
def main():
for _ in range({num}):
testfunc()
if __name__ == '__main__':
testfunc()
''').format(**fmtkw)
dpath = ub.ensure_app_cache_dir('ubelt/bench')
local_modpath = join(dpath, 'local_import_test.py')
global_modpath = join(dpath, 'global_import_test.py')
ub.writeto(local_modpath, local_codeblock)
ub.writeto(global_modpath, global_codeblock)
ub.cmd('python ' + global_modpath)
for timer in ti.reset('local imports @ {}'.format(num)):
with timer:
ub.cmd('python ' + local_modpath)
for timer in ti.reset('global imports @ {}'.format(num)):
with timer:
ub.cmd('python ' + global_modpath)
print('ti.rankings = {}'.format(
ub.repr2(ti.rankings, nl=2, precision=4, align=':')))
