def download(name,
save_path,
version=None,
decompress=True,
resource_type='Model',
extra={}):
file = os.path.join(save_path, name)
file = os.path.realpath(file)
if os.path.exists(file):
return
if not hub.HubServer()._server_check():
raise ServerConnectionError
search_result = hub.HubServer().get_resource_url(
name, resource_type=resource_type, version=version, extra=extra)
if not search_result:
raise ResourceNotFoundError(name, version)
CacheUpdater("x_download", name, version).start()
url = search_result['url']
with tmp_dir() as _dir:
if not os.path.exists(save_path):
os.makedirs(save_path)
_, _, savefile = default_downloader.download_file(url=url,
save_path=_dir,
print_progress=True)
if tarfile.is_tarfile(savefile) and decompress:
_, _, savefile = default_downloader.uncompress(file=savefile,
print_progress=True)
shutil.move(savefile, file)
def install_module(self,
module_name,
module_version=None,
upgrade=False,
extra=None):
self.all_modules(update=True)
module_info = self.modules_dict.get(module_name, None)
if module_info:
if not module_version or module_version == self.modules_dict[
module_name][1]:
module_dir = self.modules_dict[module_name][0]
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
tips = "Module %s already installed in %s" % (module_tag,
module_dir)
return True, tips, self.modules_dict[module_name]
search_result = hub.default_hub_server.get_module_url(
module_name, version=module_version, extra=extra)
name = search_result.get('name', None)
url = search_result.get('url', None)
md5_value = search_result.get('md5', None)
installed_module_version = search_result.get('version', None)
if not url or (module_version is not None and installed_module_version
!= module_version) or (name != module_name):
tips = "Can't find module %s" % module_name
if module_version:
tips += " with version %s" % module_version
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
return False, tips, None
result, tips, module_zip_file = default_downloader.download_file(
url=url,
save_path=hub.CACHE_HOME,
save_name=module_name,
replace=True,
print_progress=True)
result, tips, module_dir = default_downloader.uncompress(
file=module_zip_file,
dirname=MODULE_HOME,
delete_file=True,
print_progress=True)
if module_dir:
with open(os.path.join(MODULE_HOME, module_dir, "md5.txt"),
"w") as fp:
fp.write(md5_value)
save_path = os.path.join(MODULE_HOME, module_name)
if os.path.exists(save_path):
shutil.rmtree(save_path)
shutil.move(module_dir, save_path)
module_dir = save_path
tips = "Successfully installed %s" % module_name
if installed_module_version:
tips += "-%s" % installed_module_version
return True, tips, (module_dir, installed_module_version)
tips = "Download %s-%s failed" % (module_name, module_version)
return False, tips, module_dir
def execute(self, argv):
if not argv:
print("ERROR: Please provide the model/module name\n")
self.help()
return False
mod_name = argv[0]
mod_version = None if "==" not in mod_name else mod_name.split("==")[1]
mod_name = mod_name if "==" not in mod_name else mod_name.split(
"==")[0]
self.args = self.parser.parse_args(argv[1:])
self.args.type = self.check_type(self.args.type)
if self.args.type in ["Module", "Model"]:
search_result = default_hub_server.get_resource_url(
mod_name, resource_type=self.args.type, version=mod_version)
else:
search_result = default_hub_server.get_resource_url(
mod_name, resource_type="Module", version=mod_version)
self.args.type = "Module"
if search_result == {}:
search_result = default_hub_server.get_resource_url(
mod_name, resource_type="Model", version=mod_version)
self.args.type = "Model"
url = search_result.get('url', None)
except_md5_value = search_result.get('md5', None)
if not url:
tips = "PaddleHub can't find model/module named %s" % mod_name
if mod_version:
tips += " with version %s" % mod_version
tips += ". Please use the 'hub search' command to find the correct model/module name."
print(tips)
return True
need_to_download_file = True
file_name = os.path.basename(url)
file = os.path.join(self.args.output_path, file_name)
if os.path.exists(file):
print("File %s already existed\nWait to check the MD5 value" %
file_name)
file_md5_value = utils.md5_of_file(file)
if except_md5_value == file_md5_value:
print("MD5 check pass.")
need_to_download_file = False
else:
print("MD5 check failed!\nDelete invalid file.")
os.remove(file)
if need_to_download_file:
result, tips, file = default_downloader.download_file(
url=url, save_path=self.args.output_path, print_progress=True)
if not result:
print(tips)
return False
if self.args.uncompress:
result, tips, file = default_downloader.uncompress(
file=file,
dirname=self.args.output_path,
delete_file=True,
print_progress=True)
print(tips)
if self.args.type == "Model":
os.rename(file, "./" + mod_name)
return True
def install_module(self,
module_name=None,
module_dir=None,
module_package=None,
module_version=None,
upgrade=False,
extra=None):
md5_value = installed_module_version = None
from_user_dir = True if module_dir else False
with tmp_dir() as _dir:
if module_name:
self.all_modules(update=True)
module_info = self.modules_dict.get(module_name, None)
if module_info:
if not module_version or module_version == self.modules_dict[
module_name][1]:
module_dir = self.modules_dict[module_name][0]
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
tips = "Module %s already installed in %s" % (
module_tag, module_dir)
return True, tips, self.modules_dict[module_name]
search_result = hub.HubServer().get_module_url(
module_name, version=module_version, extra=extra)
name = search_result.get('name', None)
url = search_result.get('url', None)
md5_value = search_result.get('md5', None)
installed_module_version = search_result.get('version', None)
if not url or (module_version is not None
and installed_module_version != module_version
) or (name != module_name):
if hub.HubServer()._server_check() is False:
tips = "Request Hub-Server unsuccessfully, please check your network."
return False, tips, None
module_versions_info = hub.HubServer().search_module_info(
module_name)
if module_versions_info is None:
tips = "Can't find module %s, please check your spelling." \
% (module_name)
elif module_version is not None and module_version not in [
item[1] for item in module_versions_info
]:
tips = "Can't find module %s with version %s, all versions are listed below." \
% (module_name, module_version)
tips += paint_modules_info(module_versions_info)
else:
tips = "The version of PaddlePaddle(%s) or PaddleHub(%s) can not match module, please upgrade your PaddlePaddle or PaddleHub according to the form below." \
% (sys_paddle_version, sys_hub_verion)
tips += paint_modules_info(module_versions_info)
return False, tips, None
result, tips, module_zip_file = default_downloader.download_file(
url=url,
save_path=_dir,
save_name=module_name,
replace=True,
print_progress=True)
result, tips, module_dir = default_downloader.uncompress(
file=module_zip_file,
dirname=os.path.join(_dir, "tmp_module"),
delete_file=True,
print_progress=True)
if module_package:
with tarfile.open(module_package, "r:gz") as tar:
file_names = tar.getnames()
size = len(file_names) - 1
module_name = file_names[0]
module_dir = os.path.join(_dir, module_name)
for index, file_name in enumerate(file_names):
tar.extract(file_name, _dir)
if "-" in module_name:
module_name = module_name.replace("-", "_")
new_module_dir = os.path.join(_dir, module_name)
shutil.move(module_dir, new_module_dir)
module_dir = new_module_dir
module_name = hub.Module(directory=module_dir).name
if from_user_dir:
module_name = hub.Module(directory=module_dir).name
module_version = hub.Module(directory=module_dir).version
self.all_modules(update=False)
module_info = self.modules_dict.get(module_name, None)
if module_info:
if module_version == module_info[1]:
module_dir = self.modules_dict[module_name][0]
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
tips = "Module %s already installed in %s" % (
module_tag, module_dir)
return True, tips, self.modules_dict[module_name]
if module_dir:
if md5_value:
with open(os.path.join(MODULE_HOME, module_dir, "md5.txt"),
"w") as fp:
fp.write(md5_value)
save_path = os.path.join(MODULE_HOME,
module_name.replace("-", "_"))
if save_path != module_dir:
if os.path.exists(save_path):
shutil.rmtree(save_path)
if from_user_dir:
shutil.copytree(module_dir, save_path)
else:
shutil.move(module_dir, save_path)
module_dir = save_path
tips = "Successfully installed %s" % module_name
if installed_module_version:
tips += "-%s" % installed_module_version
return True, tips, (module_dir, installed_module_version)
tips = "Download %s-%s failed" % (module_name, module_version)
return False, tips, module_dir
def install_module(self,
module_name=None,
module_dir=None,
module_package=None,
module_version=None,
upgrade=False,
extra=None):
md5_value = installed_module_version = None
from_user_dir = True if module_dir else False
with tmp_dir() as _dir:
if module_name:
self.all_modules(update=True)
module_info = self.modules_dict.get(module_name, None)
if module_info:
if not module_version or module_version == self.modules_dict[
module_name][1]:
module_dir = self.modules_dict[module_name][0]
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
tips = "Module %s already installed in %s" % (
module_tag, module_dir)
return True, tips, self.modules_dict[module_name]
search_result = hub.HubServer().get_module_url(
module_name, version=module_version, extra=extra)
name = search_result.get('name', None)
url = search_result.get('url', None)
md5_value = search_result.get('md5', None)
installed_module_version = search_result.get('version', None)
if not url or (module_version is not None
and installed_module_version != module_version
) or (name != module_name):
if hub.HubServer()._server_check() is False:
tips = "Request Hub-Server unsuccessfully, please check your network."
return False, tips, None
module_versions_info = hub.HubServer().search_module_info(
module_name)
if module_versions_info is not None and len(
module_versions_info) > 0:
if utils.is_windows():
placeholders = [20, 8, 14, 14]
else:
placeholders = [30, 8, 16, 16]
tp = TablePrinter(titles=[
"ResourceName", "Version", "PaddlePaddle",
"PaddleHub"
],
placeholders=placeholders)
module_versions_info.sort(
key=cmp_to_key(utils.sort_version_key))
for resource_name, resource_version, paddle_version, \
hub_version in module_versions_info:
colors = ["yellow", None, None, None]
tp.add_line(contents=[
resource_name, resource_version,
utils.strflist_version(paddle_version),
utils.strflist_version(hub_version)
],
colors=colors)
tips = "The version of PaddlePaddle or PaddleHub " \
"can not match module, please upgrade your " \
"PaddlePaddle or PaddleHub according to the form " \
"below." + tp.get_text()
else:
tips = "Can't find module %s" % module_name
if module_version:
tips += " with version %s" % module_version
return False, tips, None
result, tips, module_zip_file = default_downloader.download_file(
url=url,
save_path=_dir,
save_name=module_name,
replace=True,
print_progress=True)
result, tips, module_dir = default_downloader.uncompress(
file=module_zip_file,
dirname=MODULE_HOME,
delete_file=True,
print_progress=True)
if module_package:
with tarfile.open(module_package, "r:gz") as tar:
file_names = tar.getnames()
size = len(file_names) - 1
module_dir = os.path.split(file_names[0])[0]
module_dir = os.path.join(_dir, module_dir)
for index, file_name in enumerate(file_names):
tar.extract(file_name, _dir)
if module_dir:
if not module_name:
module_name = hub.Module(directory=module_dir).name
self.all_modules(update=False)
module_info = self.modules_dict.get(module_name, None)
if module_info:
module_dir = self.modules_dict[module_name][0]
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
tips = "Module %s already installed in %s" % (module_tag,
module_dir)
return True, tips, self.modules_dict[module_name]
if module_dir:
if md5_value:
with open(os.path.join(MODULE_HOME, module_dir, "md5.txt"),
"w") as fp:
fp.write(md5_value)
save_path = os.path.join(MODULE_HOME, module_name)
if os.path.exists(save_path):
shutil.move(save_path)
if from_user_dir:
shutil.copytree(module_dir, save_path)
else:
shutil.move(module_dir, save_path)
module_dir = save_path
tips = "Successfully installed %s" % module_name
if installed_module_version:
tips += "-%s" % installed_module_version
return True, tips, (module_dir, installed_module_version)
tips = "Download %s-%s failed" % (module_name, module_version)
return False, tips, module_dir