def download_filelist(collection_name, fname, root_dir): if collection_name not in precomputed_urls: return False else: url = precomputed_urls[collection_name]['paths'] log.info('[cpuvisor] Downloading dataset filelist...') log.info('[cpuvisor] URL is: %s' % url) assert(os.path.splitext(urlparse.urlparse(url).path)[1] == os.path.splitext(fname)[1]) utils.subproc_call_check(['wget -O %s %s' % (fname, url)], shell=True) # re-generate and check for consistency with utils.make_temp_directory() as temp_dir: regen_fname = os.path.join(temp_dir, os.path.split(fname)[1]) generate_filelist(regen_fname, root_dir, True) if filecmp.cmp(fname, regen_fname): raise RuntimeError('Downloaded filelist for dataset <%s>: %s ' 'is inconsistent with images found in ' 'dataset directory: %s' % (collection_name, fname, root_dir)) return True
def download_feats(collection_name, fname): if collection_name not in precomputed_urls: return False else: url = precomputed_urls[collection_name]['feats'] log.info('[cpuvisor] Downloading features for dataset...') log.info('[cpuvisor] URL is: %s' % url) (target_path, target_fname) = os.path.split(fname) with utils.make_temp_directory() as temp_dir: tarball_file_ext = os.path.splitext(urlparse.urlparse(url).path)[1] tarball_fname = os.path.join(temp_dir, 'feats' + tarball_file_ext) utils.subproc_call_check(['wget -O %s %s' % (tarball_fname, url)], shell=True) with tarfile.open(tarball_fname) as tar: tar_ifos = {x.name: x for x in tar.getmembers()} if target_fname not in tar_ifos: raise RuntimeError('Precomputed feature tarball does not contain required file') else: tar.extractall(target_path, [tar_ifos[target_fname]]) return True