def __init__(self,
name=None,
module_dir=None,
signatures=None,
module_info=None,
assets=None,
processor=None,
extra_info=None,
version=None):
self.desc = module_desc_pb2.ModuleDesc()
self.program = None
self.assets = []
self.helper = None
self.signatures = {}
self.default_signature = None
self.module_info = None
self.processor = None
self.extra_info = {} if extra_info is None else extra_info
if not isinstance(self.extra_info, dict):
raise TypeError(
"The extra_info should be an instance of python dict")
# cache data
self.last_call_name = None
self.cache_feed_dict = None
self.cache_fetch_dict = None
self.cache_program = None
fp_lock = open(os.path.join(CONF_HOME, 'config.json'))
lock.flock(fp_lock, lock.LOCK_EX)
if name:
self._init_with_name(name=name, version=version)
lock.flock(fp_lock, lock.LOCK_UN)
elif module_dir:
self._init_with_module_file(module_dir=module_dir[0])
lock.flock(fp_lock, lock.LOCK_UN)
name = module_dir[0].split("/")[-1]
version = module_dir[1]
elif signatures:
if processor:
if not issubclass(processor, BaseProcessor):
raise TypeError(
"Processor shoule be an instance of paddlehub.BaseProcessor"
)
if assets:
self.assets = utils.to_list(assets)
# for asset in assets:
# utils.check_path(assets)
self.processor = processor
self._generate_module_info(module_info)
self._init_with_signature(signatures=signatures)
lock.flock(fp_lock, lock.LOCK_UN)
else:
lock.flock(fp_lock, lock.LOCK_UN)
raise ValueError("Module initialized parameter is empty")
CacheUpdater(name, version).start()
def __init__(self, name=None, directory=None, module_dir=None,
version=None):
if not directory:
return
super(ModuleV1, self).__init__(name, directory, module_dir, version)
self.program = None
self.assets = []
self.helper = None
self.signatures = {}
self.default_signature = None
self.processor = None
self.extra_info = {}
self._code_version = "v1"
# parse desc
self.module_desc_path = os.path.join(self.directory, MODULE_DESC_PBNAME)
self._desc = module_desc_pb2.ModuleDesc()
with open(self.module_desc_path, "rb") as file:
self._desc.ParseFromString(file.read())
module_info = self.desc.attr.map.data['module_info']
self._name = utils.from_module_attr_to_pyobj(
module_info.map.data['name'])
self._author = utils.from_module_attr_to_pyobj(
module_info.map.data['author'])
self._author_email = utils.from_module_attr_to_pyobj(
module_info.map.data['author_email'])
self._version = utils.from_module_attr_to_pyobj(
module_info.map.data['version'])
self._type = utils.from_module_attr_to_pyobj(
module_info.map.data['type'])
self._summary = utils.from_module_attr_to_pyobj(
module_info.map.data['summary'])
# cache data
self.last_call_name = None
self.cache_feed_dict = None
self.cache_fetch_dict = None
self.cache_program = None
self.helper = ModuleHelper(directory)
exe = fluid.Executor(fluid.CPUPlace())
self.program, _, _ = fluid.io.load_inference_model(
self.helper.model_path(), executor=exe)
for block in self.program.blocks:
for op in block.ops:
if "op_callstack" in op.all_attrs():
op._set_attr("op_callstack", [""])
self._load_processor()
self._load_assets()
self._recover_from_desc()
self._generate_sign_attr()
self._generate_extra_info()
self._restore_parameter(self.program)
self._recover_variable_info(self.program)
def check_module_valid(self, module_path):
#TODO(wuzewu): code
info = {}
try:
desc_pb_path = os.path.join(module_path, 'module_desc.pb')
if os.path.exists(desc_pb_path) and os.path.isfile(desc_pb_path):
desc = module_desc_pb2.ModuleDesc()
with open(desc_pb_path, "rb") as fp:
desc.ParseFromString(fp.read())
info['version'] = desc.attr.map.data["module_info"].map.data[
"version"].s
except:
return False, None
return True, info
def __init__(self,
name=None,
module_dir=None,
signatures=None,
module_info=None,
assets=None,
processor=None,
extra_info=None,
version=None):
self.desc = module_desc_pb2.ModuleDesc()
self.program = None
self.assets = []
self.helper = None
self.signatures = {}
self.default_signature = None
self.module_info = None
self.processor = None
self.extra_info = {} if extra_info is None else extra_info
if not isinstance(self.extra_info, dict):
raise TypeError(
"The extra_info should be an instance of python dict")
# cache data
self.last_call_name = None
self.cache_feed_dict = None
self.cache_fetch_dict = None
self.cache_program = None
# TODO(wuzewu): print more module loading info log
if name:
self._init_with_name(name=name, version=version)
elif module_dir:
self._init_with_module_file(module_dir=module_dir[0])
elif signatures:
if processor:
if not issubclass(processor, BaseProcessor):
raise TypeError(
"Processor shoule be an instance of paddlehub.BaseProcessor"
)
if assets:
self.assets = utils.to_list(assets)
# for asset in assets:
# utils.check_path(assets)
self.processor = processor
self._generate_module_info(module_info)
self._init_with_signature(signatures=signatures)
else:
raise ValueError("Module initialized parameter is empty")
def create_module(directory, name, author, email, module_type, summary,
version):
save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX)
with tmp_dir() as base_dir:
# package the module
with tarfile.open(save_file, "w:gz") as tar:
module_dir = os.path.join(base_dir, name)
shutil.copytree(directory, module_dir)
# record module info and serialize
desc = module_desc_pb2.ModuleDesc()
attr = desc.attr
attr.type = module_desc_pb2.MAP
module_info = attr.map.data['module_info']
module_info.type = module_desc_pb2.MAP
utils.from_pyobj_to_module_attr(name, module_info.map.data['name'])
utils.from_pyobj_to_module_attr(author,
module_info.map.data['author'])
utils.from_pyobj_to_module_attr(
email, module_info.map.data['author_email'])
utils.from_pyobj_to_module_attr(module_type,
module_info.map.data['type'])
utils.from_pyobj_to_module_attr(summary,
module_info.map.data['summary'])
utils.from_pyobj_to_module_attr(version,
module_info.map.data['version'])
module_desc_path = os.path.join(module_dir, "module_desc.pb")
with open(module_desc_path, "wb") as f:
f.write(desc.SerializeToString())
# generate check info
checker = ModuleChecker(module_dir)
checker.generate_check_info()
# add __init__
module_init = os.path.join(module_dir, "__init__.py")
with open(module_init, "a") as file:
file.write("")
_cwd = os.getcwd()
os.chdir(base_dir)
for dirname, _, files in os.walk(module_dir):
for file in files:
tar.add(os.path.join(dirname, file).replace(base_dir, "."))
os.chdir(_cwd)
def check_module_valid(self, module_path):
try:
desc_pb_path = os.path.join(module_path, 'module_desc.pb')
if os.path.exists(desc_pb_path) and os.path.isfile(desc_pb_path):
info = {}
desc = module_desc_pb2.ModuleDesc()
with open(desc_pb_path, "rb") as fp:
desc.ParseFromString(fp.read())
info['version'] = desc.attr.map.data["module_info"].map.data[
"version"].s
return True, info
else:
logger.warning(
"%s does not exist, the module will be reinstalled" %
desc_pb_path)
except:
pass
return False, None
def check_module_valid(self, module_path):
try:
desc_pb_path = os.path.join(module_path, 'module_desc.pb')
if os.path.exists(desc_pb_path) and os.path.isfile(desc_pb_path):
info = {}
desc = module_desc_pb2.ModuleDesc()
with open(desc_pb_path, "rb") as fp:
desc.ParseFromString(fp.read())
info['version'] = desc.attr.map.data["module_info"].map.data[
"version"].s
info['name'] = desc.attr.map.data["module_info"].map.data[
"name"].s
return True, info
else:
module_file = os.path.realpath(
os.path.join(module_path, 'module.py'))
if os.path.exists(module_file):
basename = os.path.split(module_path)[-1]
dirname = os.path.join(
*list(os.path.split(module_path)[:-1]))
sys.path.insert(0, dirname)
_module = importlib.import_module(
"{}.module".format(basename))
for _item, _cls in inspect.getmembers(
_module, inspect.isclass):
_item = _module.__dict__[_item]
_file = os.path.realpath(
sys.modules[_item.__module__].__file__)
if issubclass(
_item,
hub.Module) and _file.startswith(module_file):
version = _item._version
break
sys.path.pop(0)
return True, {'version': version, 'name': _item._name}
logger.warning(
"%s does not exist, the module will be reinstalled" %
desc_pb_path)
except:
pass
return False, None
def __init__(self,
name=None,
directory=None,
module_dir=None,
version=None):
# Avoid module being initialized multiple times
if not directory or id(self) in Module._record:
return
Module._record[id(self)] = True
mod = self.__class__.__module__ + "." + self.__class__.__name__
if mod in _module_runnable_func:
_run_func_name = _module_runnable_func[mod]
self._run_func = getattr(self, _run_func_name)
else:
self._run_func = None
self._code_version = "v2"
self._directory = directory
self.module_desc_path = os.path.join(self.directory,
MODULE_DESC_PBNAME)
self._desc = module_desc_pb2.ModuleDesc()
with open(self.module_desc_path, "rb") as file:
self._desc.ParseFromString(file.read())
module_info = self.desc.attr.map.data['module_info']
self._name = utils.from_module_attr_to_pyobj(
module_info.map.data['name'])
self._author = utils.from_module_attr_to_pyobj(
module_info.map.data['author'])
self._author_email = utils.from_module_attr_to_pyobj(
module_info.map.data['author_email'])
self._version = utils.from_module_attr_to_pyobj(
module_info.map.data['version'])
self._type = utils.from_module_attr_to_pyobj(
module_info.map.data['type'])
self._summary = utils.from_module_attr_to_pyobj(
module_info.map.data['summary'])
self._initialize()
def serialize_to_path(self, path=None, exe=None):
self._check_signatures()
self._generate_desc()
# create module path for saving
if path is None:
path = os.path.join(".", self.name)
self.helper = ModuleHelper(path)
utils.mkdir(self.helper.module_dir)
# create module pb
module_desc = module_desc_pb2.ModuleDesc()
logger.info("PaddleHub version = %s" % version.hub_version)
logger.info("PaddleHub Module proto version = %s" %
version.module_proto_version)
logger.info("Paddle version = %s" % paddle.__version__)
feeded_var_names = [
input.name for key, sign in self.signatures.items()
for input in sign.inputs
]
target_vars = [
output for key, sign in self.signatures.items()
for output in sign.outputs
]
feeded_var_names = list(set(feeded_var_names))
target_vars = list(set(target_vars))
# save inference program
program = self.program.clone()
for block in program.blocks:
for op in block.ops:
if "op_callstack" in op.all_attrs():
op._set_attr("op_callstack", [""])
if not exe:
place = fluid.CPUPlace()
exe = fluid.Executor(place=place)
utils.mkdir(self.helper.model_path())
fluid.io.save_inference_model(
self.helper.model_path(),
feeded_var_names=list(feeded_var_names),
target_vars=list(target_vars),
main_program=program,
executor=exe)
with open(os.path.join(self.helper.model_path(), "__model__"),
"rb") as file:
program_desc_str = file.read()
rename_program = fluid.framework.Program.parse_from_string(
program_desc_str)
varlist = {
var: block
for block in rename_program.blocks for var in block.vars
if self.get_name_prefix() not in var
}
for var, block in varlist.items():
old_name = var
new_name = self.get_var_name_with_prefix(old_name)
block._rename_var(old_name, new_name)
utils.mkdir(self.helper.model_path())
with open(
os.path.join(self.helper.model_path(), "__model__"),
"wb") as f:
f.write(rename_program.desc.serialize_to_string())
for file in os.listdir(self.helper.model_path()):
if (file == "__model__" or self.get_name_prefix() in file):
continue
os.rename(
os.path.join(self.helper.model_path(), file),
os.path.join(self.helper.model_path(),
self.get_var_name_with_prefix(file)))
# create processor file
if self.processor:
self._dump_processor()
# create assets
self._dump_assets()
# create check info
checker = ModuleChecker(self.helper.module_dir)
checker.generate_check_info()
# Serialize module_desc pb
module_pb = self.desc.SerializeToString()
with open(self.helper.module_desc_path(), "wb") as f:
f.write(module_pb)
def create_module(directory, name, author, email, module_type, summary,
version):
save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX)
with tmp_dir() as base_dir:
# package the module
with tarfile.open(save_file, "w:gz") as tar:
module_dir = os.path.join(base_dir, name)
shutil.copytree(directory, module_dir)
# record module info and serialize
desc = module_desc_pb2.ModuleDesc()
attr = desc.attr
attr.type = module_desc_pb2.MAP
module_info = attr.map.data['module_info']
module_info.type = module_desc_pb2.MAP
utils.from_pyobj_to_module_attr(name, module_info.map.data['name'])
utils.from_pyobj_to_module_attr(author,
module_info.map.data['author'])
utils.from_pyobj_to_module_attr(
email, module_info.map.data['author_email'])
utils.from_pyobj_to_module_attr(module_type,
module_info.map.data['type'])
utils.from_pyobj_to_module_attr(summary,
module_info.map.data['summary'])
utils.from_pyobj_to_module_attr(version,
module_info.map.data['version'])
module_desc_path = os.path.join(module_dir, "module_desc.pb")
with open(module_desc_path, "wb") as f:
f.write(desc.SerializeToString())
# generate check info
checker = ModuleChecker(module_dir)
checker.generate_check_info()
# add __init__
module_init = os.path.join(module_dir, "__init__.py")
with open(module_init, "a") as file:
file.write("")
_cwd = os.getcwd()
os.chdir(base_dir)
module_dir = module_dir.replace(base_dir, ".")
tar.add(module_dir, recursive=False)
files = []
for dirname, _, subfiles in os.walk(module_dir):
for file in subfiles:
# if file.startswith("."):
# continue
files.append(os.path.join(dirname, file))
total_length = len(files)
print("Create Module {}-{}".format(name, version))
for index, file in enumerate(files):
done = int(float(index) / total_length * 50)
progress("[%-50s] %.2f%%" %
('=' * done, float(index / total_length * 100)))
tar.add(file)
progress("[%-50s] %.2f%%" % ('=' * 50, 100), end=True)
print("Module package saved as {}".format(save_file))
os.chdir(_cwd)