def _recover_variable_info(self, program):
var_infos = self.desc.attr.map.data['var_infos']
for var_info in var_infos.map.data:
idx = utils.from_module_attr_to_pyobj(
var_infos.map.data[var_info].map.data['block_id'])
stop_gradient = utils.from_module_attr_to_pyobj(
var_infos.map.data[var_info].map.data['stop_gradient'])
block = program.blocks[idx]
var_name = self.get_var_name_with_prefix(var_info)
if var_name in block.vars:
var = block.vars[var_name]
var.stop_gradient = stop_gradient
def _recover_from_desc(self):
# recover signature
for sign, module_var in self.desc.sign2var.items():
inputs = []
outputs = []
feed_names = []
fetch_names = []
for var in module_var.feed_desc:
variable = self.program.global_block().vars[var.var_name]
inputs.append(variable)
feed_names.append(var.alias)
for var in module_var.fetch_desc:
variable = self.program.global_block().vars[var.var_name]
outputs.append(variable)
fetch_names.append(var.alias)
self.signatures[sign] = create_signature(
sign,
inputs=inputs,
outputs=outputs,
feed_names=feed_names,
fetch_names=fetch_names)
# recover default signature
default_signature_name = utils.from_module_attr_to_pyobj(
self.desc.attr.map.data['default_signature'])
self.default_signature = self.signatures[
default_signature_name] if default_signature_name else None
# recover module info
module_info = self.desc.attr.map.data['module_info']
self.name = utils.from_module_attr_to_pyobj(
module_info.map.data['name'])
self.author = utils.from_module_attr_to_pyobj(
module_info.map.data['author'])
self.author_email = utils.from_module_attr_to_pyobj(
module_info.map.data['author_email'])
self.version = utils.from_module_attr_to_pyobj(
module_info.map.data['version'])
self.type = utils.from_module_attr_to_pyobj(
module_info.map.data['type'])
self.summary = utils.from_module_attr_to_pyobj(
module_info.map.data['summary'])
# recover extra info
extra_info = self.desc.attr.map.data['extra_info']
self.extra_info = {}
for key, value in extra_info.map.data.items():
self.extra_info[key] = utils.from_module_attr_to_pyobj(value)
# recover name prefix
self.name_prefix = utils.from_module_attr_to_pyobj(
self.desc.attr.map.data["name_prefix"])
def _load_processor(self):
processor_path = self.helper.processor_path()
if os.path.exists(processor_path):
sys.path.append(processor_path)
processor_name = utils.from_module_attr_to_pyobj(
self.desc.attr.map.data['processor_info'])
self.processor = __import__(processor_name).Processor(module=self)
else:
self.processor = None
def from_module_attr_to_param(module_attr):
param = {'gradient_clip_attr': None, 'regularizer': None}
param['trainable'] = from_module_attr_to_pyobj(
module_attr.map.data['trainable'])
param['do_model_average'] = from_module_attr_to_pyobj(
module_attr.map.data['do_model_average'])
# do not recover learning rate
#param['optimize_attr'] = from_module_attr_to_pyobj(
# module_attr.map.data['optimize_attr'])
if module_attr.map.data['regularizer'].type != module_desc_pb2.NONE:
regularizer_type = module_attr.map.data['regularizer'].name
regularization_coeff = from_module_attr_to_pyobj(
module_attr.map.data['regularizer'].object.
data['_regularization_coeff'])
param['regularizer'] = eval(
"fluid.regularizer.%s(regularization_coeff = %f)" %
(regularizer_type, regularization_coeff))
if module_attr.map.data['gradient_clip_attr'].type != module_desc_pb2.NONE:
clip_type = module_attr.map.data['gradient_clip_attr'].name
if clip_type == "ErrorClipByValue" or clip_type == "GradientClipByValue":
max = from_module_attr_to_pyobj(
module_attr.map.data['gradient_clip_attr'].object.data['max'])
min = from_module_attr_to_pyobj(
module_attr.map.data['gradient_clip_attr'].object.data['min'])
param['gradient_clip_attr'] = eval(
"fluid.clip.%s(max = %f, min = %f)" % (clip_type, max, min))
if clip_type == "GradientClipByNorm":
clip_norm = from_module_attr_to_pyobj(
module_attr.map.data['gradient_clip_attr'].object.
data['clip_norm'])
param['gradient_clip_attr'] = eval(
"fluid.clip.%s(clip_norm = %f)" % (clip_type, clip_norm))
if clip_type == "GradientClipByGlobalNorm":
clip_norm = from_module_attr_to_pyobj(
module_attr.map.data['gradient_clip_attr'].object.
data['clip_norm'])
group_name = from_module_attr_to_pyobj(
module_attr.map.data['gradient_clip_attr'].object.
data['group_name'])
param['gradient_clip_attr'] = eval(
"fluid.clip.%s(clip_norm = %f, group_name = \"%s\")" %
(clip_type, clip_norm, group_name))
return param
def __init__(self, name=None, directory=None, module_dir=None,
version=None):
if not directory:
return
super(ModuleV1, self).__init__(name, directory, module_dir, version)
self.program = None
self.assets = []
self.helper = None
self.signatures = {}
self.default_signature = None
self.processor = None
self.extra_info = {}
self._code_version = "v1"
# parse desc
self.module_desc_path = os.path.join(self.directory, MODULE_DESC_PBNAME)
self._desc = module_desc_pb2.ModuleDesc()
with open(self.module_desc_path, "rb") as file:
self._desc.ParseFromString(file.read())
module_info = self.desc.attr.map.data['module_info']
self._name = utils.from_module_attr_to_pyobj(
module_info.map.data['name'])
self._author = utils.from_module_attr_to_pyobj(
module_info.map.data['author'])
self._author_email = utils.from_module_attr_to_pyobj(
module_info.map.data['author_email'])
self._version = utils.from_module_attr_to_pyobj(
module_info.map.data['version'])
self._type = utils.from_module_attr_to_pyobj(
module_info.map.data['type'])
self._summary = utils.from_module_attr_to_pyobj(
module_info.map.data['summary'])
# cache data
self.last_call_name = None
self.cache_feed_dict = None
self.cache_fetch_dict = None
self.cache_program = None
self.helper = ModuleHelper(directory)
exe = fluid.Executor(fluid.CPUPlace())
self.program, _, _ = fluid.io.load_inference_model(
self.helper.model_path(), executor=exe)
for block in self.program.blocks:
for op in block.ops:
if "op_callstack" in op.all_attrs():
op._set_attr("op_callstack", [""])
self._load_processor()
self._load_assets()
self._recover_from_desc()
self._generate_sign_attr()
self._generate_extra_info()
self._restore_parameter(self.program)
self._recover_variable_info(self.program)
def __init__(self,
name=None,
directory=None,
module_dir=None,
version=None):
# Avoid module being initialized multiple times
if not directory or id(self) in Module._record:
return
Module._record[id(self)] = True
mod = self.__class__.__module__ + "." + self.__class__.__name__
if mod in _module_runnable_func:
_run_func_name = _module_runnable_func[mod]
self._run_func = getattr(self, _run_func_name)
else:
self._run_func = None
self._code_version = "v2"
self._directory = directory
self.module_desc_path = os.path.join(self.directory,
MODULE_DESC_PBNAME)
self._desc = module_desc_pb2.ModuleDesc()
with open(self.module_desc_path, "rb") as file:
self._desc.ParseFromString(file.read())
module_info = self.desc.attr.map.data['module_info']
self._name = utils.from_module_attr_to_pyobj(
module_info.map.data['name'])
self._author = utils.from_module_attr_to_pyobj(
module_info.map.data['author'])
self._author_email = utils.from_module_attr_to_pyobj(
module_info.map.data['author_email'])
self._version = utils.from_module_attr_to_pyobj(
module_info.map.data['version'])
self._type = utils.from_module_attr_to_pyobj(
module_info.map.data['type'])
self._summary = utils.from_module_attr_to_pyobj(
module_info.map.data['summary'])
self._initialize()
