def _download_and_uncompress_dataset(self, destination: str, url: str):
"""
Downloads dataset and uncompresses it.
Args:
destination (:obj:`str`): The dataset cached directory.
url (:obj: str): The link to be downloaded a dataset.
"""
if not os.path.exists(destination):
dataset_package = download(url=url, path=DATA_HOME)
if is_xarfile(dataset_package):
unarchive(dataset_package, DATA_HOME)
else:
logger.info("Dataset {} already cached.".format(destination))
def execute(self, argv: List) -> bool:
if not argv:
print("ERROR: You must give at least one module to install.")
return False
manager = LocalModuleManager()
for _arg in argv:
if os.path.exists(_arg) and os.path.isdir(_arg):
manager.install(directory=_arg)
elif os.path.exists(_arg) and xarfile.is_xarfile(_arg):
manager.install(archive=_arg)
else:
_arg = _arg.split('==')
name = _arg[0]
version = None if len(_arg) == 1 else _arg[1]
CacheUpdater("hub_install", name, version).start()
manager.install(name=name, version=version)
return True
def download(name: str, save_path: str, version: str = None):
'''The download interface provided to PaddleX for downloading the specified model and resource files.'''
file = os.path.join(save_path, name)
file = os.path.realpath(file)
if os.path.exists(file):
return
resources = module_server.search_resouce(name=name,
version=version,
type='Model')
if not resources:
raise ResourceNotFoundError(name, version)
for item in resources:
if item['name'] == name and utils.Version(
item['version']).match(version):
url = item['url']
break
else:
raise ResourceNotFoundError(name, version)
with utils.generate_tempdir() as _dir:
if not os.path.exists(save_path):
os.makedirs(save_path)
with log.ProgressBar('Download {}'.format(url)) as _bar:
for savefile, dsize, tsize in utils.download_with_progress(
url, _dir):
_bar.update(float(dsize / tsize))
if xarfile.is_xarfile(savefile):
with log.ProgressBar('Decompress {}'.format(savefile)) as _bar:
for savefile, usize, tsize in xarfile.unarchive_with_progress(
savefile, _dir):
_bar.update(float(usize / tsize))
savefile = os.path.join(_dir, savefile.split(os.sep)[0])
shutil.move(savefile, file)
def execute(self, argv: List) -> bool:
if not argv:
print("ERROR: You must give at least one module to install.")
return False
options = [arg for arg in argv if arg.startswith('-')]
argv = [arg for arg in argv if not arg.startswith('-')]
args = self.parser.parse_args(options)
manager = LocalModuleManager()
for _arg in argv:
if os.path.exists(_arg) and os.path.isdir(_arg):
manager.install(directory=_arg)
elif os.path.exists(_arg) and xarfile.is_xarfile(_arg):
manager.install(archive=_arg)
else:
_arg = _arg.split('==')
name = _arg[0]
version = None if len(_arg) == 1 else _arg[1]
CacheUpdater("hub_install", name, version).start()
manager.install(name=name,
version=version,
ignore_env_mismatch=args.ignore_env_mismatch)
return True
