def init_with_directory(cls, directory): desc_file = os.path.join(directory, MODULE_DESC_PBNAME) checker = ModuleChecker(directory) checker.check() module_code_version = checker.module_code_version if module_code_version == "v2": basename = os.path.split(directory)[-1] dirname = os.path.join(*list(os.path.split(directory)[:-1])) sys.path.append(dirname) user_module = importlib.import_module("{}.module".format(basename)) return user_module.HubModule(directory=directory) return ModuleV1(directory=directory)
def create_module(directory, name, author, email, module_type, summary, version): save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX) with tmp_dir() as base_dir: # package the module with tarfile.open(save_file, "w:gz") as tar: module_dir = os.path.join(base_dir, name) shutil.copytree(directory, module_dir) # record module info and serialize desc = module_desc_pb2.ModuleDesc() attr = desc.attr attr.type = module_desc_pb2.MAP module_info = attr.map.data['module_info'] module_info.type = module_desc_pb2.MAP utils.from_pyobj_to_module_attr(name, module_info.map.data['name']) utils.from_pyobj_to_module_attr(author, module_info.map.data['author']) utils.from_pyobj_to_module_attr( email, module_info.map.data['author_email']) utils.from_pyobj_to_module_attr(module_type, module_info.map.data['type']) utils.from_pyobj_to_module_attr(summary, module_info.map.data['summary']) utils.from_pyobj_to_module_attr(version, module_info.map.data['version']) module_desc_path = os.path.join(module_dir, "module_desc.pb") with open(module_desc_path, "wb") as f: f.write(desc.SerializeToString()) # generate check info checker = ModuleChecker(module_dir) checker.generate_check_info() # add __init__ module_init = os.path.join(module_dir, "__init__.py") with open(module_init, "a") as file: file.write("") _cwd = os.getcwd() os.chdir(base_dir) for dirname, _, files in os.walk(module_dir): for file in files: tar.add(os.path.join(dirname, file).replace(base_dir, ".")) os.chdir(_cwd)
def _init_with_module_file(self, module_dir): checker = ModuleChecker(module_dir) checker.check() self.helper = ModuleHelper(module_dir) with open(self.helper.module_desc_path(), "rb") as fi: self.desc.ParseFromString(fi.read()) exe = fluid.Executor(fluid.CPUPlace()) self.program, _, _ = fluid.io.load_inference_model( self.helper.model_path(), executor=exe) for block in self.program.blocks: for op in block.ops: if "op_callstack" in op.all_attrs(): op._set_attr("op_callstack", [""]) self._load_processor() self._load_assets() self._recover_from_desc() self._generate_sign_attr() self._generate_extra_info() self._restore_parameter(self.program) self._recover_variable_info(self.program)
def init_with_directory(cls, directory, **kwargs): desc_file = os.path.join(directory, MODULE_DESC_PBNAME) if os.path.exists(desc_file): checker = ModuleChecker(directory) checker.check() return ModuleV1(directory=directory, **kwargs) if directory.endswith(os.sep): directory = directory[:-1] basename = os.path.split(directory)[-1] dirname = os.path.join(*list(os.path.split(directory)[:-1])) sys.path.insert(0, dirname) _module = importlib.import_module("{}.module".format(basename)) for _item, _cls in inspect.getmembers(_module, inspect.isclass): _item = _module.__dict__[_item] _file = os.path.realpath(sys.modules[_item.__module__].__file__) _module_path = os.path.realpath( os.path.join(directory, "module.py")) if issubclass(_item, Module) and _file.startswith(_module_path): user_module = _item(directory=directory, **kwargs) break sys.path.pop(0) return user_module
def serialize_to_path(self, path=None, exe=None): self._check_signatures() self._generate_desc() # create module path for saving if path is None: path = os.path.join(".", self.name) self.helper = ModuleHelper(path) utils.mkdir(self.helper.module_dir) # create module pb module_desc = module_desc_pb2.ModuleDesc() logger.info("PaddleHub version = %s" % version.hub_version) logger.info("PaddleHub Module proto version = %s" % version.module_proto_version) logger.info("Paddle version = %s" % paddle.__version__) feeded_var_names = [ input.name for key, sign in self.signatures.items() for input in sign.inputs ] target_vars = [ output for key, sign in self.signatures.items() for output in sign.outputs ] feeded_var_names = list(set(feeded_var_names)) target_vars = list(set(target_vars)) # save inference program program = self.program.clone() for block in program.blocks: for op in block.ops: if "op_callstack" in op.all_attrs(): op._set_attr("op_callstack", [""]) if not exe: place = fluid.CPUPlace() exe = fluid.Executor(place=place) utils.mkdir(self.helper.model_path()) fluid.io.save_inference_model( self.helper.model_path(), feeded_var_names=list(feeded_var_names), target_vars=list(target_vars), main_program=program, executor=exe) with open(os.path.join(self.helper.model_path(), "__model__"), "rb") as file: program_desc_str = file.read() rename_program = fluid.framework.Program.parse_from_string( program_desc_str) varlist = { var: block for block in rename_program.blocks for var in block.vars if self.get_name_prefix() not in var } for var, block in varlist.items(): old_name = var new_name = self.get_var_name_with_prefix(old_name) block._rename_var(old_name, new_name) utils.mkdir(self.helper.model_path()) with open( os.path.join(self.helper.model_path(), "__model__"), "wb") as f: f.write(rename_program.desc.serialize_to_string()) for file in os.listdir(self.helper.model_path()): if (file == "__model__" or self.get_name_prefix() in file): continue os.rename( os.path.join(self.helper.model_path(), file), os.path.join(self.helper.model_path(), self.get_var_name_with_prefix(file))) # create processor file if self.processor: self._dump_processor() # create assets self._dump_assets() # create check info checker = ModuleChecker(self.helper.module_dir) checker.generate_check_info() # Serialize module_desc pb module_pb = self.desc.SerializeToString() with open(self.helper.module_desc_path(), "wb") as f: f.write(module_pb)
def create_module(directory, name, author, email, module_type, summary, version): save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX) with tmp_dir() as base_dir: # package the module with tarfile.open(save_file, "w:gz") as tar: module_dir = os.path.join(base_dir, name) shutil.copytree(directory, module_dir) # record module info and serialize desc = module_desc_pb2.ModuleDesc() attr = desc.attr attr.type = module_desc_pb2.MAP module_info = attr.map.data['module_info'] module_info.type = module_desc_pb2.MAP utils.from_pyobj_to_module_attr(name, module_info.map.data['name']) utils.from_pyobj_to_module_attr(author, module_info.map.data['author']) utils.from_pyobj_to_module_attr( email, module_info.map.data['author_email']) utils.from_pyobj_to_module_attr(module_type, module_info.map.data['type']) utils.from_pyobj_to_module_attr(summary, module_info.map.data['summary']) utils.from_pyobj_to_module_attr(version, module_info.map.data['version']) module_desc_path = os.path.join(module_dir, "module_desc.pb") with open(module_desc_path, "wb") as f: f.write(desc.SerializeToString()) # generate check info checker = ModuleChecker(module_dir) checker.generate_check_info() # add __init__ module_init = os.path.join(module_dir, "__init__.py") with open(module_init, "a") as file: file.write("") _cwd = os.getcwd() os.chdir(base_dir) module_dir = module_dir.replace(base_dir, ".") tar.add(module_dir, recursive=False) files = [] for dirname, _, subfiles in os.walk(module_dir): for file in subfiles: # if file.startswith("."): # continue files.append(os.path.join(dirname, file)) total_length = len(files) print("Create Module {}-{}".format(name, version)) for index, file in enumerate(files): done = int(float(index) / total_length * 50) progress("[%-50s] %.2f%%" % ('=' * done, float(index / total_length * 100))) tar.add(file) progress("[%-50s] %.2f%%" % ('=' * 50, 100), end=True) print("Module package saved as {}".format(save_file)) os.chdir(_cwd)