def initial_imagenet_weights():
# import os
import ubelt as ub
try:
darknet_weight_fpath = ub.grabdata(
'https://pjreddie.com/media/files/darknet19_448.conv.23',
appname='netharn',
hash_prefix='8016f5b7ddc15c5d7dad2315')
torch_fpath = darknet_weight_fpath + '_lntf.pt'
import os
if not os.path.exists(torch_fpath):
import lightnet.models
# hack to transform initial state
model = lightnet.models.Yolo(num_classes=20)
model.load_weights(darknet_weight_fpath)
torch.save(model.state_dict(), torch_fpath)
except ImportError:
# Maybe this had a weird bad init state?
torch_fpath = ub.grabdata(
'https://data.kitware.com/api/v1/item/5b16b81b8d777f15ebe1ffcd/download',
fname='darknet19_448.conv.23.pt',
appname='netharn',
# hash_prefix='fd2b99b9f66bb4',
hash_prefix='f38968224a81a')
return torch_fpath
def demo_voc_weights(key='lightnet'):
"""
Demo weights for Pascal VOC dataset
"""
if key == 'lightnet':
# url = 'https://gitlab.com/EAVISE/lightnet/raw/master/examples/yolo-voc/lightnet_weights.pt'
# hash_prefix = 'c4597fed8eb1b01da3495'
fpath = ub.grabdata(
'https://data.kitware.com/api/v1/file/5c2e6e1a8d777f072bf2dc65/download',
fname='lightnet_weights.pt',
appname='netharn',
hasher='sha512',
hash_prefix='c4597fed8eb1b01')
return fpath
elif key == 'darknet':
url = 'https://pjreddie.com/media/files/yolo-voc.weights'
hash_prefix = '3033f5f510c25ab3ff6b9'
fpath = ub.grabdata(url, appname='netharn', hash_prefix=hash_prefix)
return fpath
else:
raise KeyError(key)
# import lightnet
# from os.path import dirname, join
# dpath = dirname(dirname(lightnet.__file__))
# fpath = join(dpath, 'examples', 'yolo-voc', 'lightnet_weights.pt')
return fpath
def parse_mscoco():
# Test that our implementation can handle the real mscoco data
root = ub.expandpath('~/data/standard_datasets/mscoco/')
fpath = join(root, 'annotations/instances_val2014.json')
img_root = normpath(ub.ensuredir((root, 'images', 'val2014')))
# fpath = join(root, 'annotations/stuff_val2017.json')
# img_root = normpath(ub.ensuredir((root, 'images', 'val2017')))
import ujson
dataset = ujson.load(open(fpath, 'rb'))
import ndsampler
dset = ndsampler.CocoDataset(dataset)
dset.img_root = img_root
gid_iter = iter(dset.imgs.keys())
gid = ub.peek(gid_iter)
for gid in ub.ProgIter(gid_iter):
img = dset.imgs[gid]
ub.grabdata(img['coco_url'], dpath=img_root, verbose=0)
anns = [dset.anns[aid] for aid in dset.gid_to_aids[gid]]
dset.show_image(gid=gid)
ann = anns[0]
segmentation = ann['segmentation']
from PIL import Image
gpath = join(dset.img_root, img['file_name'])
with Image.open(gpath) as pil_img:
np_img = np.array(pil_img)
def test_grabdata_value_error():
"""
Check where the url is downloaded to when fpath is not specified.
"""
# url = 'http://i.imgur.com/rqwaDag.png'
# if not ub.argflag('--network'):
# pytest.skip('not running network tests')
url = _demo_url()
dpath = ub.ensure_app_cache_dir('ubelt')
fname = basename(url)
fpath = join(dpath, fname)
with pytest.raises(ValueError):
ub.grabdata(url, fname=fname, fpath=fpath, dpath=dpath)
with pytest.raises(ValueError):
ub.grabdata(url, fname=fname, fpath=fpath)
with pytest.raises(ValueError):
ub.grabdata(url, dpath=dpath, fpath=fpath)
with pytest.raises(ValueError):
ub.grabdata(url, fpath=fpath, appname='foobar')
with pytest.raises(ValueError):
ub.grabdata(url, dpath=dpath, appname='foobar')
def grab_test_image_fpath(key='astro'):
"""
Args:
key (str): which test image to grab. Valid choices are:
astro - an astronaught
carl - Carl Sagan
paraview - ParaView logo
stars - picture of stars in the sky
Example:
>>> for key in grab_test_image.keys():
... grab_test_image_fpath(key)
"""
try:
item = _TEST_IMAGES[key]
except KeyError:
valid_keys = sorted(_TEST_IMAGES.keys())
raise KeyError('Unknown key={!r}. Valid keys are {!r}'.format(
key, valid_keys))
if not isinstance(item, dict):
item = {'url': item}
if 'sha1' in item:
fpath = ub.grabdata(item['url'],
hash_prefix=item['sha1'],
appname='netharn')
else:
fpath = ub.grabdata(item['url'], appname='netharn')
return fpath
def grab_camvid_train_test_val_splits(coco_dset, mode='segnet'):
# Use the split from SegNet: https://github.com/alexgkendall/SegNet-Tutorial
split_files = {
'train':
ub.grabdata(
'https://raw.githubusercontent.com/alexgkendall/SegNet-Tutorial/master/CamVid/train.txt'
),
'vali':
ub.grabdata(
'https://raw.githubusercontent.com/alexgkendall/SegNet-Tutorial/master/CamVid/val.txt'
),
'test':
ub.grabdata(
'https://raw.githubusercontent.com/alexgkendall/SegNet-Tutorial/master/CamVid/test.txt'
),
}
gid_subsets = {}
for tag, fpath in split_files.items():
text = open(fpath, 'r').read()
parts = text.replace('\n', ' ').split(' ')
parts = [p for p in parts if p]
from os.path import basename
names = sorted(set(basename(p) for p in parts))
gids = [
coco_dset.index.file_name_to_img['701_StillsRaw_full/' +
name]['id'] for name in names
]
gid_subsets[tag] = gids
return gid_subsets
def ensure_voc_data(dpath=None, force=False, years=[2007, 2012]):
"""
Download the Pascal VOC data if it does not already exist.
Example:
>>> # xdoctest: +REQUIRES(--download)
>>> devkit_dpath = ensure_voc_data()
"""
if dpath is None:
dpath = ub.expandpath('~/data/VOC')
devkit_dpath = join(dpath, 'VOCdevkit')
# if force or not exists(devkit_dpath):
ub.ensuredir(dpath)
def extract_tarfile(fpath, dpath='.'):
# Old way
# ub.cmd('tar xvf "{}" -C "{}"'.format(fpath1, dpath), verbout=1)
import tarfile
try:
tar = tarfile.open(fpath1)
tar.extractall(dpath)
finally:
tar.close()
fpath1 = ub.grabdata(
'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCdevkit_08-Jun-2007.tar',
dpath=dpath)
if force or not exists(join(dpath, 'VOCdevkit', 'VOCcode')):
extract_tarfile(fpath1, dpath)
if 2007 in years:
# VOC 2007 train+validation data
fpath2 = ub.grabdata(
'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar',
dpath=dpath)
if force or not exists(
join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main',
'bird_trainval.txt')):
extract_tarfile(fpath2, dpath)
# VOC 2007 test data
fpath3 = ub.grabdata(
'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar',
dpath=dpath)
if force or not exists(
join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main',
'bird_test.txt')):
extract_tarfile(fpath3, dpath)
if 2012 in years:
# VOC 2012 train+validation data
fpath4 = ub.grabdata(
'https://pjreddie.com/media/files/VOCtrainval_11-May-2012.tar',
dpath=dpath)
if force or not exists(
join(dpath, 'VOCdevkit', 'VOC2012', 'ImageSets', 'Main',
'bird_trainval.txt')):
extract_tarfile(fpath4, dpath)
return devkit_dpath
def test_grabdata_fpath_and_dpath():
# url = 'http://i.imgur.com/rqwaDag.png'
# if not ub.argflag('--network'):
# pytest.skip('not running network tests')
url = _demo_url()
with pytest.raises(ValueError):
ub.grabdata(url, fpath='foo', dpath='bar')
def test_grabdata_delete_hash_stamp():
import ubelt as ub
fname = 'foo3.bar'
url = _demo_url(128 * 12)
prefix1 = '43f92597d7eb08b57c88b636'
fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
stamp_fpath = fpath + '.sha512.hash'
ub.delete(stamp_fpath)
fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
def ensure_voc_data(VOCDataset,
dpath=None,
force=False,
years=[2007, 2012]):
"""
Download the Pascal VOC 2007 data if it does not already exist.
CommandLine:
python -m netharn.data.voc VOCDataset.ensure_voc_data
Example:
>>> # SCRIPT
>>> # xdoc: +REQUIRES(--voc)
>>> from netharn.data.voc import * # NOQA
>>> VOCDataset.ensure_voc_data()
"""
if dpath is None:
dpath = ub.expandpath('~/data/VOC')
devkit_dpath = join(dpath, 'VOCdevkit')
# if force or not exists(devkit_dpath):
ub.ensuredir(dpath)
fpath1 = ub.grabdata(
'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCdevkit_08-Jun-2007.tar',
dpath=dpath)
if force or not exists(join(dpath, 'VOCdevkit', 'VOCcode')):
ub.cmd('tar xvf "{}" -C "{}"'.format(fpath1, dpath), verbout=1)
if 2007 in years:
# VOC 2007 train+validation data
fpath2 = ub.grabdata(
'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar',
dpath=dpath)
if force or not exists(
join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main',
'bird_trainval.txt')):
ub.cmd('tar xvf "{}" -C "{}"'.format(fpath2, dpath), verbout=1)
# VOC 2007 test data
fpath3 = ub.grabdata(
'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar',
dpath=dpath)
if force or not exists(
join(dpath, 'VOCdevkit', 'VOC2007', 'ImageSets', 'Main',
'bird_test.txt')):
ub.cmd('tar xvf "{}" -C "{}"'.format(fpath3, dpath), verbout=1)
if 2012 in years:
# VOC 2012 train+validation data
fpath4 = ub.grabdata(
'https://pjreddie.com/media/files/VOCtrainval_11-May-2012.tar',
dpath=dpath)
if force or not exists(
join(dpath, 'VOCdevkit', 'VOC2012', 'ImageSets', 'Main',
'bird_trainval.txt')):
ub.cmd('tar xvf "{}" -C "{}"'.format(fpath4, dpath), verbout=1)
return devkit_dpath
def test_grabdata_hash_typo():
"""
CommandLine:
xdoctest ~/code/ubelt/ubelt/tests/test_download.py test_grabdata_hash_typo --network
"""
# url = 'https://www.dropbox.com/s/jl506apezj42zjz/ibeis-win32-setup-ymd_hm-2015-08-01_16-28.exe?dl=1'
import hashlib
# url = 'http://i.imgur.com/rqwaDag.png'
# if not ub.argflag('--network'):
# pytest.skip('not running network tests')
url = _demo_url()
dpath = ub.ensure_app_cache_dir('ubelt')
fname = basename(url)
fpath = join(dpath, fname)
for verbose in [0]:
ub.delete(fpath)
ub.delete(fpath + '.md5.hash')
assert not exists(fpath)
print('[STEP1] Downloading file, but we have a typo in the hash')
with pytest.raises(RuntimeError):
got_fpath = ub.grabdata(
url,
hash_prefix='545e3a51404f-typo-4e46aa65a70948e126',
hasher=hashlib.md5(),
verbose=verbose)
assert exists(fpath)
print(
'[STEP2] Fixing the typo recomputes the hash, but does not redownload the file'
)
got_fpath = ub.grabdata(url,
hash_prefix='22d42eb002cefa81e9ad604ea57bc01d',
hasher=hashlib.md5(),
verbose=verbose)
assert got_fpath == fpath
assert exists(fpath)
# If we delete the .hash file we will simply recompute
ub.delete(fpath + '.md5.hash')
print('[STEP3] Deleting the hash file recomputes the hash')
got_fpath = ub.grabdata(url,
fpath=fpath,
hash_prefix='22d42eb002cefa81e9ad604ea57bc01d',
hasher=hashlib.md5(),
verbose=verbose)
assert exists(fpath + '.md5.hash')
def grab_test_imgpath(key='astro.png', allow_external=True, verbose=True):
"""
Gets paths to standard / fun test images.
Downloads them if they dont exits
Args:
key (str): one of the standard test images, e.g. astro.png, carl.jpg, ...
allow_external (bool): if True you can specify existing fpaths
Returns:
str: testimg_fpath - filepath to the downloaded or cached test image.
Example:
>>> testimg_fpath = grab_test_imgpath('carl.jpg')
>>> assert exists(testimg_fpath)
"""
if allow_external and key not in TESTIMG_URL_DICT:
testimg_fpath = key
if not exists(testimg_fpath):
raise AssertionError(
'testimg_fpath={!r} not found did you mean on of {!r}' % (
testimg_fpath, sorted(TESTIMG_URL_DICT.keys())))
else:
testimg_fname = key
testimg_url = TESTIMG_URL_DICT[key]
testimg_fpath = ub.grabdata(testimg_url, fname=testimg_fname, verbose=verbose)
return testimg_fpath
def master():
master_fpath = ub.grabdata(
'https://raw.githubusercontent.com/pokemongo-dev-contrib/pokemongo-game-master/master/versions/latest/V2_GAME_MASTER.json',
expires=24 * 60 * 60)
with open(master_fpath) as file:
master = json.load(file)
master.keys()
def item_type(item):
data = item['data']
if 'move' in data:
return 'move'
if 'pokemon' in data:
return 'pokemon'
type_to_items = ub.group_items(master['template'], key=item_type)
pokemon_items = type_to_items['pokemon'] # NOQA
move_items = type_to_items['move']
for item in move_items:
uid = item['data']['move']['uniqueId']
if 'MOONBLAST' in uid:
print('item = {}'.format(ub.repr2(item, nl=3)))
def ensure_att_dataset():
def unzip(zip_fpath, dpath=None, verbose=1):
"""
Extracts all members of a zipfile.
Args:
zip_fpath (str): path of zip file to unzip.
dpath (str): directory to unzip to. If not specified, it defaults
to a folder parallel to the zip file (excluding the extension).
verbose (int): verbosity level
"""
import zipfile
from os.path import splitext
from ubelt import progiter
if dpath is None:
dpath = splitext(zip_fpath)[0]
with zipfile.ZipFile(zip_fpath, 'r') as zf:
members = zf.namelist()
prog = progiter.ProgIter(members,
verbose=verbose,
label='unzipping')
for zipinfo in prog:
zf.extract(zipinfo, path=dpath, pwd=None)
return dpath
faces_zip_fpath = ub.grabdata(
'http://www.cl.cam.ac.uk/Research/DTG/attarchive/pub/data/att_faces.zip'
)
from os.path import splitext
dpath = splitext(faces_zip_fpath)[0]
if not os.path.exists(dpath):
dpath = unzip(faces_zip_fpath, dpath=dpath)
return dpath
def main():
"""
Checks that the latest wheels on pypi agree with the gpg key
"""
import requests
package_name = 'ubelt'
url = "https://pypi.python.org/pypi/{}/json".format(package_name)
package = requests.get(url).json()
max_ver = max(package["releases"].keys())
# ... check compatibility
latest_wheel_info_list = package['releases'][max_ver]
for wheel_info in latest_wheel_info_list:
import ubelt as ub
whl_fpath = ub.grabdata(wheel_info['url'],
hash_prefix=wheel_info['digests']['sha256'],
hasher='sha256')
if not wheel_info['has_sig']:
raise ValueError('info says no sig')
sig_fpath = ub.download(wheel_info['url'] + '.asc', )
info = ub.cmd('gpg --verify {} {}'.format(sig_fpath, whl_fpath),
verbose=3)
assert info['ret'] == 0
def grab_nitfs():
base = 'https://gwg.nga.mil/ntb/baseline/software/testfile/Nitfv2_1/'
urls = [base + fname for fname in NITF_TEST_NAMES]
nitf_fpaths = []
for url in urls:
fpath = ub.grabdata(url)
nitf_fpaths.append(fpath)
return nitf_fpaths
def test_grabdata_cache():
"""
Check where the url is downloaded to when fpath is not specified.
"""
url = 'http://i.imgur.com/rqwaDag.png'
dpath = ub.ensure_app_cache_dir('ubelt')
fname = basename(url)
fpath = join(dpath, fname)
got_fpath = ub.grabdata(url)
assert got_fpath == fpath
assert exists(fpath)
ub.delete(fpath)
assert not exists(fpath)
ub.grabdata(url)
assert exists(fpath)
def test_grabdata_fname_only():
url = 'http://i.imgur.com/rqwaDag.png'
dpath = ub.ensure_app_cache_dir('ubelt')
fname = 'mario.png'
fpath = join(dpath, fname)
got_fpath = ub.grabdata(url, fname=fname)
assert got_fpath == fpath
assert exists(fpath)
def test_grabdata_dpath_only():
url = 'http://i.imgur.com/rqwaDag.png'
dpath = ub.ensure_app_cache_dir('ubelt', 'test')
fname = basename(url)
fpath = join(dpath, fname)
got_fpath = ub.grabdata(url, dpath=dpath)
assert got_fpath == fpath
assert exists(fpath)
def ensure_lightnet_initial_weights():
import os
weight_fpath = ub.grabdata(
'https://pjreddie.com/media/files/darknet19_448.conv.23', appname='clab')
torch_fpath = weight_fpath + '.pt'
if not os.path.exists(torch_fpath):
# hack to transform initial state
model = light_yolo.Yolo(num_classes=1000)
model.load_weights(weight_fpath)
torch.save(model.state_dict(), torch_fpath)
return torch_fpath
def demo_voc_weights():
"""
Demo weights for Pascal VOC dataset
"""
import ubelt as ub
url = 'https://gitlab.com/EAVISE/lightnet/raw/master/examples/yolo-voc/lightnet_weights.pt'
fpath = ub.grabdata(url, appname='netharn')
# import lightnet
# from os.path import dirname, join
# dpath = dirname(dirname(lightnet.__file__))
# fpath = join(dpath, 'examples', 'yolo-voc', 'lightnet_weights.pt')
return fpath
def grab_test_image(key='astro', space='rgb'):
from clab.util.imutil import convert_colorspace
import cv2
if key == 'astro':
url = 'https://i.imgur.com/KXhKM72.png'
elif key == 'carl':
url = 'https://i.imgur.com/oHGsmvF.png'
else:
raise KeyError(key)
fpath = ub.grabdata(url)
bgr = cv2.imread(fpath)
image = convert_colorspace(bgr, space, src_space='bgr')
return image
def test_grabdata_dpath_only():
# url = 'http://i.imgur.com/rqwaDag.png'
# if not ub.argflag('--network'):
# pytest.skip('not running network tests')
url = _demo_url()
dpath = ub.ensure_app_cache_dir('ubelt', 'test')
fname = basename(url)
fpath = join(dpath, fname)
got_fpath = ub.grabdata(url, dpath=dpath)
assert got_fpath == fpath
assert exists(fpath)
def test_grabdata_with_fpath():
"""
Check where the url is downloaded to when fpath is not specified.
"""
# url = 'http://i.imgur.com/rqwaDag.png'
# if not ub.argflag('--network'):
# pytest.skip('not running network tests')
url = _demo_url()
dpath = ub.ensure_app_cache_dir('ubelt')
fname = basename(url)
fpath = join(dpath, fname)
got_fpath = ub.grabdata(url, fpath=fpath, verbose=3)
assert got_fpath == fpath
assert exists(fpath)
ub.delete(fpath)
assert not exists(fpath)
ub.grabdata(url, fpath=fpath, verbose=3)
assert exists(fpath)
def test_grabdata_fname_only():
url = 'http://i.imgur.com/rqwaDag.png'
if not ub.argflag('--network'):
pytest.skip('not running network tests')
dpath = ub.ensure_app_cache_dir('ubelt')
fname = 'mario.png'
fpath = join(dpath, fname)
got_fpath = ub.grabdata(url, fname=fname)
assert got_fpath == fpath
assert exists(fpath)
def demo_deployed(self):
"""
Returns a path to a netharn deployed model
Returns:
str: file path to a scallop detector
"""
import ubelt as ub
url = 'https://data.kitware.com/api/v1/file/5dd3eb8eaf2e2eed3508d604/download'
deployed_fpath = ub.grabdata(
url, fname='deploy_MM_CascadeRCNN_myovdqvi_035_MVKVVR_fix3.zip',
appname='viame', hash_prefix='22a1eeb18c9e5706f6578e66abda1e97',
hasher='sha512')
return deployed_fpath
def demodata_input(dataset='demo'):
"""
Specifies the input files for testing and demos
"""
if dataset == 'demo':
import zipfile
from os.path import commonprefix
dpath = ub.ensure_app_cache_dir('camtrawl')
try:
demodata_zip = ub.grabdata(
'http://acidalia:8000/data/camtrawl_demodata.zip', dpath=dpath)
except Exception:
raise ValueError(
'Demo data is currently only available on Kitware VPN')
with zipfile.ZipFile(demodata_zip) as zfile:
dname = commonprefix(zfile.namelist())
data_fpath = join(dpath, dname)
if not exists(data_fpath):
zfile.extractall(dpath)
cal_fpath = join(data_fpath, 'cal.npz')
img_path1 = join(data_fpath, 'left')
img_path2 = join(data_fpath, 'right')
elif dataset == 'test':
data_fpath = expanduser('~/data/autoprocess_test_set')
cal_fpath = join(data_fpath, 'cal_201608.mat')
img_path1 = join(data_fpath, 'image_data/left')
img_path2 = join(data_fpath, 'image_data/right')
elif dataset == 'haul83-small':
data_fpath = expanduser('~/data/camtrawl_stereo_sample_data_small')
cal_fpath = join(data_fpath,
'201608_calibration_data/selected/Camtrawl_2016.npz')
img_path1 = join(data_fpath, 'Haul_83/left')
img_path2 = join(data_fpath, 'Haul_83/right')
elif dataset == 'haul83':
data_fpath = expanduser('~/data/camtrawl_stereo_sample_data/')
cal_fpath = join(data_fpath,
'201608_calibration_data/selected/Camtrawl_2016.npz')
img_path1 = join(
data_fpath,
'Haul_83/D20160709-T021759/images/AB-800GE_00-0C-DF-06-40-BF'
) # left
img_path2 = join(
data_fpath,
'Haul_83/D20160709-T021759/images/AM-800GE_00-0C-DF-06-20-47'
) # right
else:
raise ValueError('unknown dataset={!r}'.format(dataset))
return img_path1, img_path2, cal_fpath
def grab_test_image_fpath(key='astro'):
"""
Ensures that the test image exists (this might use the network) and returns
the cached filepath to the requested image.
Args:
key (str): which test image to grab. Valid choices are:
astro - an astronaught
carl - Carl Sagan
paraview - ParaView logo
stars - picture of stars in the sky
Returns:
str: path to the requested image
CommandLine:
python -c "import kwimage; print(kwimage.grab_test_image_fpath('airport'))"
Example:
>>> for key in grab_test_image.keys():
... grab_test_image_fpath(key)
"""
try:
item = _TEST_IMAGES[key]
except KeyError:
valid_keys = sorted(_TEST_IMAGES.keys())
raise KeyError(
'Unknown key={!r}. Valid keys are {!r}'.format(
key, valid_keys))
if not isinstance(item, dict):
item = {'url': item}
grabkw = {
'appname': 'kwimage/demodata',
}
hasher_priority = ['sha1']
for hasher in hasher_priority:
if hasher in item:
grabkw.update({
'hash_prefix': item[hasher],
'hasher': hasher,
})
break
if 'fname' in item:
grabkw['fname'] = item['fname']
fpath = ub.grabdata(item['url'], **grabkw)
return fpath
def test_grabdata_url_only():
"""
Check where the url is downloaded to when fpath is not specified.
"""
url = 'http://i.imgur.com/rqwaDag.png'
if not ub.argflag('--network'):
pytest.skip('not running network tests')
dpath = ub.ensure_app_cache_dir('ubelt')
fname = basename(url)
fpath = join(dpath, fname)
got_fpath = ub.grabdata(url)
assert got_fpath == fpath
assert exists(fpath)
def demo_deployed(self):
"""
Returns a path to a netharn deployed model
Returns:
str: file path to a scallop detector
"""
import ubelt as ub
url = 'https://data.kitware.com/api/v1/file/5dcf2e68af2e2eed35fb5b13/download'
deployed_fpath = ub.grabdata(
url,
fname='deploy_scallop_MM_CascadeRCNN_myovdqvi_035_MVKVVR_fix2.zip',
appname='viame',
hash_prefix='267e20c39baacad10893bc7befcebce85',
hasher='sha512')
return deployed_fpath
def grab_zipped_url(url):
import ubelt as ub
def unzip_file(zip_fpath, force_commonprefix=True, output_dir=None,
prefix=None, dryrun=False, overwrite=None, verbose=1):
import zipfile
zip_file = zipfile.ZipFile(zip_fpath)
if output_dir is None:
output_dir = dirname(zip_fpath)
archive_namelist = zip_file.namelist()
# force extracted components into a subdirectory if force_commonprefix is
if prefix is not None:
output_dir = join(output_dir, prefix)
ub.ensuredir(output_dir)
archive_basename, ext = splitext(basename(zip_fpath))
if force_commonprefix and commonprefix(archive_namelist) == '':
# use the archivename as the default common prefix
output_dir = join(output_dir, archive_basename)
ub.ensuredir(output_dir)
for member in archive_namelist:
(dname, fname) = split(member)
dpath = join(output_dir, dname)
ub.ensuredir(dpath)
if verbose:
print('Unarchive ' + fname + ' in ' + dpath)
if not dryrun:
if overwrite is False:
if exists(join(output_dir, member)):
continue
zip_file.extract(member, path=output_dir)
zip_file.close()
# hack
return join(output_dir, archive_basename)
zip_fpath = ub.grabdata(url)
dpath = unzip_file(zip_fpath)
return dpath
def demodata_input(dataset='demo'):
"""
Specifies the input files for testing and demos
"""
if dataset == 'demo':
import zipfile
from os.path import commonprefix
dpath = ub.ensure_app_cache_dir('camtrawl')
try:
demodata_zip = ub.grabdata('http://acidalia:8000/data/camtrawl_demodata.zip', dpath=dpath)
except Exception:
raise ValueError(
'Demo data is currently only available on Kitware VPN')
with zipfile.ZipFile(demodata_zip) as zfile:
dname = commonprefix(zfile.namelist())
data_fpath = join(dpath, dname)
if not exists(data_fpath):
zfile.extractall(dpath)
cal_fpath = join(data_fpath, 'cal.npz')
img_path1 = join(data_fpath, 'left')
img_path2 = join(data_fpath, 'right')
elif dataset == 'test':
data_fpath = expanduser('~/data/autoprocess_test_set')
cal_fpath = join(data_fpath, 'cal_201608.mat')
img_path1 = join(data_fpath, 'image_data/left')
img_path2 = join(data_fpath, 'image_data/right')
elif dataset == 'haul83-small':
data_fpath = expanduser('~/data/camtrawl_stereo_sample_data_small')
cal_fpath = join(data_fpath, '201608_calibration_data/selected/Camtrawl_2016.npz')
img_path1 = join(data_fpath, 'Haul_83/left')
img_path2 = join(data_fpath, 'Haul_83/right')
elif dataset == 'haul83':
data_fpath = expanduser('~/data/camtrawl_stereo_sample_data/')
cal_fpath = join(data_fpath, '201608_calibration_data/selected/Camtrawl_2016.npz')
img_path1 = join(data_fpath, 'Haul_83/D20160709-T021759/images/AB-800GE_00-0C-DF-06-40-BF') # left
img_path2 = join(data_fpath, 'Haul_83/D20160709-T021759/images/AM-800GE_00-0C-DF-06-20-47') # right
else:
raise ValueError('unknown dataset={!r}'.format(dataset))
return img_path1, img_path2, cal_fpath