def _recover_variable_info(self, program): var_infos = self.desc.attr.map.data['var_infos'] for var_info in var_infos.map.data: idx = utils.from_module_attr_to_pyobj( var_infos.map.data[var_info].map.data['block_id']) stop_gradient = utils.from_module_attr_to_pyobj( var_infos.map.data[var_info].map.data['stop_gradient']) block = program.blocks[idx] var_name = self.get_var_name_with_prefix(var_info) if var_name in block.vars: var = block.vars[var_name] var.stop_gradient = stop_gradient
def _recover_from_desc(self): # recover signature for sign, module_var in self.desc.sign2var.items(): inputs = [] outputs = [] feed_names = [] fetch_names = [] for var in module_var.feed_desc: variable = self.program.global_block().vars[var.var_name] inputs.append(variable) feed_names.append(var.alias) for var in module_var.fetch_desc: variable = self.program.global_block().vars[var.var_name] outputs.append(variable) fetch_names.append(var.alias) self.signatures[sign] = create_signature( sign, inputs=inputs, outputs=outputs, feed_names=feed_names, fetch_names=fetch_names) # recover default signature default_signature_name = utils.from_module_attr_to_pyobj( self.desc.attr.map.data['default_signature']) self.default_signature = self.signatures[ default_signature_name] if default_signature_name else None # recover module info module_info = self.desc.attr.map.data['module_info'] self.name = utils.from_module_attr_to_pyobj( module_info.map.data['name']) self.author = utils.from_module_attr_to_pyobj( module_info.map.data['author']) self.author_email = utils.from_module_attr_to_pyobj( module_info.map.data['author_email']) self.version = utils.from_module_attr_to_pyobj( module_info.map.data['version']) self.type = utils.from_module_attr_to_pyobj( module_info.map.data['type']) self.summary = utils.from_module_attr_to_pyobj( module_info.map.data['summary']) # recover extra info extra_info = self.desc.attr.map.data['extra_info'] self.extra_info = {} for key, value in extra_info.map.data.items(): self.extra_info[key] = utils.from_module_attr_to_pyobj(value) # recover name prefix self.name_prefix = utils.from_module_attr_to_pyobj( self.desc.attr.map.data["name_prefix"])
def _load_processor(self): processor_path = self.helper.processor_path() if os.path.exists(processor_path): sys.path.append(processor_path) processor_name = utils.from_module_attr_to_pyobj( self.desc.attr.map.data['processor_info']) self.processor = __import__(processor_name).Processor(module=self) else: self.processor = None
def from_module_attr_to_param(module_attr): param = {'gradient_clip_attr': None, 'regularizer': None} param['trainable'] = from_module_attr_to_pyobj( module_attr.map.data['trainable']) param['do_model_average'] = from_module_attr_to_pyobj( module_attr.map.data['do_model_average']) # do not recover learning rate #param['optimize_attr'] = from_module_attr_to_pyobj( # module_attr.map.data['optimize_attr']) if module_attr.map.data['regularizer'].type != module_desc_pb2.NONE: regularizer_type = module_attr.map.data['regularizer'].name regularization_coeff = from_module_attr_to_pyobj( module_attr.map.data['regularizer'].object. data['_regularization_coeff']) param['regularizer'] = eval( "fluid.regularizer.%s(regularization_coeff = %f)" % (regularizer_type, regularization_coeff)) if module_attr.map.data['gradient_clip_attr'].type != module_desc_pb2.NONE: clip_type = module_attr.map.data['gradient_clip_attr'].name if clip_type == "ErrorClipByValue" or clip_type == "GradientClipByValue": max = from_module_attr_to_pyobj( module_attr.map.data['gradient_clip_attr'].object.data['max']) min = from_module_attr_to_pyobj( module_attr.map.data['gradient_clip_attr'].object.data['min']) param['gradient_clip_attr'] = eval( "fluid.clip.%s(max = %f, min = %f)" % (clip_type, max, min)) if clip_type == "GradientClipByNorm": clip_norm = from_module_attr_to_pyobj( module_attr.map.data['gradient_clip_attr'].object. data['clip_norm']) param['gradient_clip_attr'] = eval( "fluid.clip.%s(clip_norm = %f)" % (clip_type, clip_norm)) if clip_type == "GradientClipByGlobalNorm": clip_norm = from_module_attr_to_pyobj( module_attr.map.data['gradient_clip_attr'].object. data['clip_norm']) group_name = from_module_attr_to_pyobj( module_attr.map.data['gradient_clip_attr'].object. data['group_name']) param['gradient_clip_attr'] = eval( "fluid.clip.%s(clip_norm = %f, group_name = \"%s\")" % (clip_type, clip_norm, group_name)) return param
def __init__(self, name=None, directory=None, module_dir=None, version=None): if not directory: return super(ModuleV1, self).__init__(name, directory, module_dir, version) self.program = None self.assets = [] self.helper = None self.signatures = {} self.default_signature = None self.processor = None self.extra_info = {} self._code_version = "v1" # parse desc self.module_desc_path = os.path.join(self.directory, MODULE_DESC_PBNAME) self._desc = module_desc_pb2.ModuleDesc() with open(self.module_desc_path, "rb") as file: self._desc.ParseFromString(file.read()) module_info = self.desc.attr.map.data['module_info'] self._name = utils.from_module_attr_to_pyobj( module_info.map.data['name']) self._author = utils.from_module_attr_to_pyobj( module_info.map.data['author']) self._author_email = utils.from_module_attr_to_pyobj( module_info.map.data['author_email']) self._version = utils.from_module_attr_to_pyobj( module_info.map.data['version']) self._type = utils.from_module_attr_to_pyobj( module_info.map.data['type']) self._summary = utils.from_module_attr_to_pyobj( module_info.map.data['summary']) # cache data self.last_call_name = None self.cache_feed_dict = None self.cache_fetch_dict = None self.cache_program = None self.helper = ModuleHelper(directory) exe = fluid.Executor(fluid.CPUPlace()) self.program, _, _ = fluid.io.load_inference_model( self.helper.model_path(), executor=exe) for block in self.program.blocks: for op in block.ops: if "op_callstack" in op.all_attrs(): op._set_attr("op_callstack", [""]) self._load_processor() self._load_assets() self._recover_from_desc() self._generate_sign_attr() self._generate_extra_info() self._restore_parameter(self.program) self._recover_variable_info(self.program)
def __init__(self, name=None, directory=None, module_dir=None, version=None): # Avoid module being initialized multiple times if not directory or id(self) in Module._record: return Module._record[id(self)] = True mod = self.__class__.__module__ + "." + self.__class__.__name__ if mod in _module_runnable_func: _run_func_name = _module_runnable_func[mod] self._run_func = getattr(self, _run_func_name) else: self._run_func = None self._code_version = "v2" self._directory = directory self.module_desc_path = os.path.join(self.directory, MODULE_DESC_PBNAME) self._desc = module_desc_pb2.ModuleDesc() with open(self.module_desc_path, "rb") as file: self._desc.ParseFromString(file.read()) module_info = self.desc.attr.map.data['module_info'] self._name = utils.from_module_attr_to_pyobj( module_info.map.data['name']) self._author = utils.from_module_attr_to_pyobj( module_info.map.data['author']) self._author_email = utils.from_module_attr_to_pyobj( module_info.map.data['author_email']) self._version = utils.from_module_attr_to_pyobj( module_info.map.data['version']) self._type = utils.from_module_attr_to_pyobj( module_info.map.data['type']) self._summary = utils.from_module_attr_to_pyobj( module_info.map.data['summary']) self._initialize()