def download_filelist(collection_name, fname, root_dir):
if collection_name not in precomputed_urls:
return False
else:
url = precomputed_urls[collection_name]['paths']
log.info('[cpuvisor] Downloading dataset filelist...')
log.info('[cpuvisor] URL is: %s' % url)
assert(os.path.splitext(urlparse.urlparse(url).path)[1] == os.path.splitext(fname)[1])
utils.subproc_call_check(['wget -O %s %s' % (fname, url)], shell=True)
# re-generate and check for consistency
with utils.make_temp_directory() as temp_dir:
regen_fname = os.path.join(temp_dir, os.path.split(fname)[1])
generate_filelist(regen_fname, root_dir, True)
if filecmp.cmp(fname, regen_fname):
raise RuntimeError('Downloaded filelist for dataset <%s>: %s '
'is inconsistent with images found in '
'dataset directory: %s' %
(collection_name, fname, root_dir))
return True
def download_feats(collection_name, fname):
if collection_name not in precomputed_urls:
return False
else:
url = precomputed_urls[collection_name]['feats']
log.info('[cpuvisor] Downloading features for dataset...')
log.info('[cpuvisor] URL is: %s' % url)
(target_path, target_fname) = os.path.split(fname)
with utils.make_temp_directory() as temp_dir:
tarball_file_ext = os.path.splitext(urlparse.urlparse(url).path)[1]
tarball_fname = os.path.join(temp_dir, 'feats' + tarball_file_ext)
utils.subproc_call_check(['wget -O %s %s' % (tarball_fname, url)], shell=True)
with tarfile.open(tarball_fname) as tar:
tar_ifos = {x.name: x for x in tar.getmembers()}
if target_fname not in tar_ifos:
raise RuntimeError('Precomputed feature tarball does not contain required file')
else:
tar.extractall(target_path, [tar_ifos[target_fname]])
return True
