def execute(self, argv): args = self.parser.parse_args() if not args.module_name or not args.model_dir: ConvertCommand.show_help() return False self.module = args.module_name self.version = args.module_version if args.module_version is not None else '1.0.0' self.src = args.model_dir if not os.path.isdir(self.src): print('`{}` is not exists or not a directory path'.format(self.src)) return False self.dest = args.output_dir if args.output_dir is not None else os.path.join( '{}_{}'.format(self.module, str(time.time()))) CacheUpdater("hub_convert", self.module, self.version).start() os.makedirs(self.dest) with tmp_dir() as _dir: self._tmp_dir = _dir self.create_module_py() self.create_init_py() self.create_serving_demo_py() self.create_module_tar() print('The converted module is stored in `{}`.'.format(self.dest)) return True
def download(name, save_path, version=None, decompress=True, resource_type='Model', extra={}): file = os.path.join(save_path, name) file = os.path.realpath(file) if os.path.exists(file): return if not hub.HubServer()._server_check(): raise ServerConnectionError search_result = hub.HubServer().get_resource_url( name, resource_type=resource_type, version=version, extra=extra) if not search_result: raise ResourceNotFoundError(name, version) CacheUpdater("x_download", name, version).start() url = search_result['url'] with tmp_dir() as _dir: if not os.path.exists(save_path): os.makedirs(save_path) _, _, savefile = default_downloader.download_file(url=url, save_path=_dir, print_progress=True) if tarfile.is_tarfile(savefile) and decompress: _, _, savefile = default_downloader.uncompress(file=savefile, print_progress=True) shutil.move(savefile, file)
def create_module(directory, name, author, email, module_type, summary, version): save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX) with tmp_dir() as base_dir: # package the module with tarfile.open(save_file, "w:gz") as tar: module_dir = os.path.join(base_dir, name) shutil.copytree(directory, module_dir) # record module info and serialize desc = module_desc_pb2.ModuleDesc() attr = desc.attr attr.type = module_desc_pb2.MAP module_info = attr.map.data['module_info'] module_info.type = module_desc_pb2.MAP utils.from_pyobj_to_module_attr(name, module_info.map.data['name']) utils.from_pyobj_to_module_attr(author, module_info.map.data['author']) utils.from_pyobj_to_module_attr( email, module_info.map.data['author_email']) utils.from_pyobj_to_module_attr(module_type, module_info.map.data['type']) utils.from_pyobj_to_module_attr(summary, module_info.map.data['summary']) utils.from_pyobj_to_module_attr(version, module_info.map.data['version']) module_desc_path = os.path.join(module_dir, "module_desc.pb") with open(module_desc_path, "wb") as f: f.write(desc.SerializeToString()) # generate check info checker = ModuleChecker(module_dir) checker.generate_check_info() # add __init__ module_init = os.path.join(module_dir, "__init__.py") with open(module_init, "a") as file: file.write("") _cwd = os.getcwd() os.chdir(base_dir) for dirname, _, files in os.walk(module_dir): for file in files: tar.add(os.path.join(dirname, file).replace(base_dir, ".")) os.chdir(_cwd)
def run(self, module, version, src, dest): self.module = module self.version = version self.src = src self.dest = dest os.makedirs(self.dest) with tmp_dir() as _dir: self._tmp_dir = _dir self.create_module_py() self.create_init_py() self.create_serving_demo_py() self.create_module_tar() return True
def __init__(self, name=None, directory=None, module_dir=None, version=None, max_seq_len=128, **kwargs): if not directory: return super(TransformerModule, self).__init__( name=name, directory=directory, module_dir=module_dir, version=version, **kwargs) self.max_seq_len = max_seq_len if version_compare(paddle.__version__, '1.8'): with tmp_dir() as _dir: input_dict, output_dict, program = self.context( max_seq_len=max_seq_len) fluid.io.save_inference_model( dirname=_dir, main_program=program, feeded_var_names=[ input_dict['input_ids'].name, input_dict['position_ids'].name, input_dict['segment_ids'].name, input_dict['input_mask'].name ], target_vars=[ output_dict["pooled_output"], output_dict["sequence_output"] ], executor=fluid.Executor(fluid.CPUPlace())) with fluid.dygraph.guard(): self.model_runner = fluid.dygraph.StaticModelRunner(_dir)
def install_module(self, module_name=None, module_dir=None, module_package=None, module_version=None, upgrade=False, extra=None): md5_value = installed_module_version = None from_user_dir = True if module_dir else False with tmp_dir() as _dir: if module_name: self.all_modules(update=True) module_info = self.modules_dict.get(module_name, None) if module_info: if not module_version or module_version == self.modules_dict[ module_name][1]: module_dir = self.modules_dict[module_name][0] module_tag = module_name if not module_version else '%s-%s' % ( module_name, module_version) tips = "Module %s already installed in %s" % ( module_tag, module_dir) return True, tips, self.modules_dict[module_name] search_result = hub.HubServer().get_module_url( module_name, version=module_version, extra=extra) name = search_result.get('name', None) url = search_result.get('url', None) md5_value = search_result.get('md5', None) installed_module_version = search_result.get('version', None) if not url or (module_version is not None and installed_module_version != module_version ) or (name != module_name): if hub.HubServer()._server_check() is False: tips = "Request Hub-Server unsuccessfully, please check your network." return False, tips, None module_versions_info = hub.HubServer().search_module_info( module_name) if module_versions_info is None: tips = "Can't find module %s, please check your spelling." \ % (module_name) elif module_version is not None and module_version not in [ item[1] for item in module_versions_info ]: tips = "Can't find module %s with version %s, all versions are listed below." \ % (module_name, module_version) tips += paint_modules_info(module_versions_info) else: tips = "The version of PaddlePaddle(%s) or PaddleHub(%s) can not match module, please upgrade your PaddlePaddle or PaddleHub according to the form below." \ % (sys_paddle_version, sys_hub_verion) tips += paint_modules_info(module_versions_info) return False, tips, None result, tips, module_zip_file = default_downloader.download_file( url=url, save_path=_dir, save_name=module_name, replace=True, print_progress=True) result, tips, module_dir = default_downloader.uncompress( file=module_zip_file, dirname=os.path.join(_dir, "tmp_module"), delete_file=True, print_progress=True) if module_package: with tarfile.open(module_package, "r:gz") as tar: file_names = tar.getnames() size = len(file_names) - 1 module_name = file_names[0] module_dir = os.path.join(_dir, module_name) for index, file_name in enumerate(file_names): tar.extract(file_name, _dir) if "-" in module_name: module_name = module_name.replace("-", "_") new_module_dir = os.path.join(_dir, module_name) shutil.move(module_dir, new_module_dir) module_dir = new_module_dir module_name = hub.Module(directory=module_dir).name if from_user_dir: module_name = hub.Module(directory=module_dir).name module_version = hub.Module(directory=module_dir).version self.all_modules(update=False) module_info = self.modules_dict.get(module_name, None) if module_info: if module_version == module_info[1]: module_dir = self.modules_dict[module_name][0] module_tag = module_name if not module_version else '%s-%s' % ( module_name, module_version) tips = "Module %s already installed in %s" % ( module_tag, module_dir) return True, tips, self.modules_dict[module_name] if module_dir: if md5_value: with open(os.path.join(MODULE_HOME, module_dir, "md5.txt"), "w") as fp: fp.write(md5_value) save_path = os.path.join(MODULE_HOME, module_name.replace("-", "_")) if save_path != module_dir: if os.path.exists(save_path): shutil.rmtree(save_path) if from_user_dir: shutil.copytree(module_dir, save_path) else: shutil.move(module_dir, save_path) module_dir = save_path tips = "Successfully installed %s" % module_name if installed_module_version: tips += "-%s" % installed_module_version return True, tips, (module_dir, installed_module_version) tips = "Download %s-%s failed" % (module_name, module_version) return False, tips, module_dir
def install_module(self, module_name=None, module_dir=None, module_package=None, module_version=None, upgrade=False, extra=None): md5_value = installed_module_version = None from_user_dir = True if module_dir else False with tmp_dir() as _dir: if module_name: self.all_modules(update=True) module_info = self.modules_dict.get(module_name, None) if module_info: if not module_version or module_version == self.modules_dict[ module_name][1]: module_dir = self.modules_dict[module_name][0] module_tag = module_name if not module_version else '%s-%s' % ( module_name, module_version) tips = "Module %s already installed in %s" % ( module_tag, module_dir) return True, tips, self.modules_dict[module_name] search_result = hub.HubServer().get_module_url( module_name, version=module_version, extra=extra) name = search_result.get('name', None) url = search_result.get('url', None) md5_value = search_result.get('md5', None) installed_module_version = search_result.get('version', None) if not url or (module_version is not None and installed_module_version != module_version ) or (name != module_name): if hub.HubServer()._server_check() is False: tips = "Request Hub-Server unsuccessfully, please check your network." return False, tips, None module_versions_info = hub.HubServer().search_module_info( module_name) if module_versions_info is not None and len( module_versions_info) > 0: if utils.is_windows(): placeholders = [20, 8, 14, 14] else: placeholders = [30, 8, 16, 16] tp = TablePrinter(titles=[ "ResourceName", "Version", "PaddlePaddle", "PaddleHub" ], placeholders=placeholders) module_versions_info.sort( key=cmp_to_key(utils.sort_version_key)) for resource_name, resource_version, paddle_version, \ hub_version in module_versions_info: colors = ["yellow", None, None, None] tp.add_line(contents=[ resource_name, resource_version, utils.strflist_version(paddle_version), utils.strflist_version(hub_version) ], colors=colors) tips = "The version of PaddlePaddle or PaddleHub " \ "can not match module, please upgrade your " \ "PaddlePaddle or PaddleHub according to the form " \ "below." + tp.get_text() else: tips = "Can't find module %s" % module_name if module_version: tips += " with version %s" % module_version return False, tips, None result, tips, module_zip_file = default_downloader.download_file( url=url, save_path=_dir, save_name=module_name, replace=True, print_progress=True) result, tips, module_dir = default_downloader.uncompress( file=module_zip_file, dirname=MODULE_HOME, delete_file=True, print_progress=True) if module_package: with tarfile.open(module_package, "r:gz") as tar: file_names = tar.getnames() size = len(file_names) - 1 module_dir = os.path.split(file_names[0])[0] module_dir = os.path.join(_dir, module_dir) for index, file_name in enumerate(file_names): tar.extract(file_name, _dir) if module_dir: if not module_name: module_name = hub.Module(directory=module_dir).name self.all_modules(update=False) module_info = self.modules_dict.get(module_name, None) if module_info: module_dir = self.modules_dict[module_name][0] module_tag = module_name if not module_version else '%s-%s' % ( module_name, module_version) tips = "Module %s already installed in %s" % (module_tag, module_dir) return True, tips, self.modules_dict[module_name] if module_dir: if md5_value: with open(os.path.join(MODULE_HOME, module_dir, "md5.txt"), "w") as fp: fp.write(md5_value) save_path = os.path.join(MODULE_HOME, module_name) if os.path.exists(save_path): shutil.move(save_path) if from_user_dir: shutil.copytree(module_dir, save_path) else: shutil.move(module_dir, save_path) module_dir = save_path tips = "Successfully installed %s" % module_name if installed_module_version: tips += "-%s" % installed_module_version return True, tips, (module_dir, installed_module_version) tips = "Download %s-%s failed" % (module_name, module_version) return False, tips, module_dir
def create_module(directory, name, author, email, module_type, summary, version): save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX) with tmp_dir() as base_dir: # package the module with tarfile.open(save_file, "w:gz") as tar: module_dir = os.path.join(base_dir, name) shutil.copytree(directory, module_dir) # record module info and serialize desc = module_desc_pb2.ModuleDesc() attr = desc.attr attr.type = module_desc_pb2.MAP module_info = attr.map.data['module_info'] module_info.type = module_desc_pb2.MAP utils.from_pyobj_to_module_attr(name, module_info.map.data['name']) utils.from_pyobj_to_module_attr(author, module_info.map.data['author']) utils.from_pyobj_to_module_attr( email, module_info.map.data['author_email']) utils.from_pyobj_to_module_attr(module_type, module_info.map.data['type']) utils.from_pyobj_to_module_attr(summary, module_info.map.data['summary']) utils.from_pyobj_to_module_attr(version, module_info.map.data['version']) module_desc_path = os.path.join(module_dir, "module_desc.pb") with open(module_desc_path, "wb") as f: f.write(desc.SerializeToString()) # generate check info checker = ModuleChecker(module_dir) checker.generate_check_info() # add __init__ module_init = os.path.join(module_dir, "__init__.py") with open(module_init, "a") as file: file.write("") _cwd = os.getcwd() os.chdir(base_dir) module_dir = module_dir.replace(base_dir, ".") tar.add(module_dir, recursive=False) files = [] for dirname, _, subfiles in os.walk(module_dir): for file in subfiles: # if file.startswith("."): # continue files.append(os.path.join(dirname, file)) total_length = len(files) print("Create Module {}-{}".format(name, version)) for index, file in enumerate(files): done = int(float(index) / total_length * 50) progress("[%-50s] %.2f%%" % ('=' * done, float(index / total_length * 100))) tar.add(file) progress("[%-50s] %.2f%%" % ('=' * 50, 100), end=True) print("Module package saved as {}".format(save_file)) os.chdir(_cwd)