def _install_from_directory(self, directory: str) -> HubModule:
'''Install a HubModule from directory containing module.py'''
module_info = HubModule.load_module_info(directory)
# A temporary directory is copied here for two purposes:
# 1. Avoid affecting user-specified directory (for example, a __pycache__
# directory will be generated).
# 2. HubModule is essentially a python package. When internal package
# references are made in it, the correct package name is required.
with utils.generate_tempdir() as _dir:
tempdir = os.path.join(_dir, module_info.name)
tempdir = self._get_normalized_name(tempdir)
shutil.copytree(directory, tempdir)
# Uninstall local module
if os.path.exists(self._get_normalized_path(module_info.name)):
self.uninstall(module_info.name)
shutil.copytree(directory, self._get_normalized_path(module_info.name))
# Install python package requirements, This behavior needs to occur before Module.load,
# otherwise the model will fail to load due to missing dependencies.
self._install_module_requirements(directory)
hub_module_cls = HubModule.load(self._get_normalized_path(module_info.name))
self._local_modules[module_info.name] = hub_module_cls
log.logger.info('Successfully installed {}-{}'.format(hub_module_cls.name, hub_module_cls.version))
return hub_module_cls
def _install_from_url(self, url: str) -> HubModule:
'''Install HubModule from url'''
with utils.generate_tempdir() as _tdir:
with log.ProgressBar('Download {}'.format(url)) as bar:
for file, ds, ts in utils.download_with_progress(url, _tdir):
bar.update(float(ds) / ts)
return self._install_from_archive(file)
def _install_from_archive(self, archive: str) -> HubModule:
'''Install HubModule from archive file (eg xxx.tar.gz)'''
with utils.generate_tempdir() as _tdir:
with log.ProgressBar('Decompress {}'.format(archive)) as bar:
for path, ds, ts in xarfile.unarchive_with_progress(archive, _tdir):
bar.update(float(ds) / ts)
# Sometimes the path contains '.'
path = os.path.normpath(path)
directory = os.path.join(_tdir, path.split(os.sep)[0])
return self._install_from_directory(directory)
def download_file_and_uncompress(self, url: str, save_path: str, print_progress: bool):
with utils.generate_tempdir() as _dir:
if print_progress:
with log.ProgressBar('Download {}'.format(url)) as bar:
for path, ds, ts in utils.download_with_progress(url=url, path=_dir):
bar.update(float(ds) / ts)
else:
path = utils.download(url=url, path=_dir)
if print_progress:
with log.ProgressBar('Decompress {}'.format(path)) as bar:
for path, ds, ts in xarfile.unarchive_with_progress(name=path, path=save_path):
bar.update(float(ds) / ts)
else:
path = xarfile.unarchive(name=path, path=save_path)
def _create_predictor(self) -> paddle.fluid.core.PaddlePredictor:
'''
create high-performance predictor for predict.
Returns:
PaddlePredictor: the high-performance predictor
'''
with generate_tempdir() as _dir:
self.save_inference_model(dirname=_dir)
predictor_config = paddle.fluid.core.AnalysisConfig(_dir)
predictor_config.disable_glog_info()
if self.config.use_cuda:
predictor_config.enable_use_gpu(100, 0)
predictor_config.switch_ir_optim(True)
else:
predictor_config.disable_gpu()
predictor_config.enable_memory_optim()
return paddle.fluid.core.create_paddle_predictor(predictor_config)
def download(name: str, save_path: str, version: str = None):
'''The download interface provided to PaddleX for downloading the specified model and resource files.'''
file = os.path.join(save_path, name)
file = os.path.realpath(file)
if os.path.exists(file):
return
resources = module_server.search_resouce(name=name,
version=version,
type='Model')
if not resources:
raise ResourceNotFoundError(name, version)
for item in resources:
if item['name'] == name and utils.Version(
item['version']).match(version):
url = item['url']
break
else:
raise ResourceNotFoundError(name, version)
with utils.generate_tempdir() as _dir:
if not os.path.exists(save_path):
os.makedirs(save_path)
with log.ProgressBar('Download {}'.format(url)) as _bar:
for savefile, dsize, tsize in utils.download_with_progress(
url, _dir):
_bar.update(float(dsize / tsize))
if xarfile.is_xarfile(savefile):
with log.ProgressBar('Decompress {}'.format(savefile)) as _bar:
for savefile, usize, tsize in xarfile.unarchive_with_progress(
savefile, _dir):
_bar.update(float(usize / tsize))
savefile = os.path.join(_dir, savefile.split(os.sep)[0])
shutil.move(savefile, file)