def download_file_and_uncompress(self, url: str, save_path: str, print_progress: bool):
with utils.generate_tempdir() as _dir:
if print_progress:
with log.ProgressBar('Download {}'.format(url)) as bar:
for path, ds, ts in utils.download_with_progress(url=url, path=_dir):
bar.update(float(ds) / ts)
else:
path = utils.download(url=url, path=_dir)
if print_progress:
with log.ProgressBar('Decompress {}'.format(path)) as bar:
for path, ds, ts in xarfile.unarchive_with_progress(name=path, path=save_path):
bar.update(float(ds) / ts)
else:
path = xarfile.unarchive(name=path, path=save_path)
def _install_from_url(self, url: str) -> HubModule:
'''Install HubModule from url'''
with utils.generate_tempdir() as _tdir:
with log.ProgressBar('Download {}'.format(url)) as bar:
for file, ds, ts in utils.download_with_progress(url, _tdir):
bar.update(float(ds) / ts)
return self._install_from_archive(file)
def _install_from_archive(self, archive: str) -> HubModule:
'''Install HubModule from archive file (eg xxx.tar.gz)'''
with utils.generate_tempdir() as _tdir:
with log.ProgressBar('Decompress {}'.format(archive)) as bar:
for path, ds, ts in xarfile.unarchive_with_progress(archive, _tdir):
bar.update(float(ds) / ts)
# Sometimes the path contains '.'
path = os.path.normpath(path)
directory = os.path.join(_tdir, path.split(os.sep)[0])
return self._install_from_directory(directory)
def download(name: str, save_path: str, version: str = None):
'''The download interface provided to PaddleX for downloading the specified model and resource files.'''
file = os.path.join(save_path, name)
file = os.path.realpath(file)
if os.path.exists(file):
return
resources = module_server.search_resouce(name=name,
version=version,
type='Model')
if not resources:
raise ResourceNotFoundError(name, version)
for item in resources:
if item['name'] == name and utils.Version(
item['version']).match(version):
url = item['url']
break
else:
raise ResourceNotFoundError(name, version)
with utils.generate_tempdir() as _dir:
if not os.path.exists(save_path):
os.makedirs(save_path)
with log.ProgressBar('Download {}'.format(url)) as _bar:
for savefile, dsize, tsize in utils.download_with_progress(
url, _dir):
_bar.update(float(dsize / tsize))
if xarfile.is_xarfile(savefile):
with log.ProgressBar('Decompress {}'.format(savefile)) as _bar:
for savefile, usize, tsize in xarfile.unarchive_with_progress(
savefile, _dir):
_bar.update(float(usize / tsize))
savefile = os.path.join(_dir, savefile.split(os.sep)[0])
shutil.move(savefile, file)
def execute(self, argv: List) -> bool:
if not argv:
print("ERROR: You must give at least one module to download.")
return False
for _arg in argv:
result = module_server.search_module(_arg)
CacheUpdater("hub_download", _arg).start()
if result:
url = result[0]['url']
with log.ProgressBar('Download {}'.format(url)) as bar:
for file, ds, ts in utils.download_with_progress(url):
bar.update(float(ds) / ts)
else:
print(
'ERROR: Could not find a HubModule named {}'.format(_arg))
return True
