def __init__(self, name=None, module_dir=None, signatures=None, module_info=None, assets=None, processor=None, extra_info=None, version=None): self.desc = module_desc_pb2.ModuleDesc() self.program = None self.assets = [] self.helper = None self.signatures = {} self.default_signature = None self.module_info = None self.processor = None self.extra_info = {} if extra_info is None else extra_info if not isinstance(self.extra_info, dict): raise TypeError( "The extra_info should be an instance of python dict") # cache data self.last_call_name = None self.cache_feed_dict = None self.cache_fetch_dict = None self.cache_program = None fp_lock = open(os.path.join(CONF_HOME, 'config.json')) lock.flock(fp_lock, lock.LOCK_EX) if name: self._init_with_name(name=name, version=version) lock.flock(fp_lock, lock.LOCK_UN) elif module_dir: self._init_with_module_file(module_dir=module_dir[0]) lock.flock(fp_lock, lock.LOCK_UN) name = module_dir[0].split("/")[-1] version = module_dir[1] elif signatures: if processor: if not issubclass(processor, BaseProcessor): raise TypeError( "Processor shoule be an instance of paddlehub.BaseProcessor" ) if assets: self.assets = utils.to_list(assets) # for asset in assets: # utils.check_path(assets) self.processor = processor self._generate_module_info(module_info) self._init_with_signature(signatures=signatures) lock.flock(fp_lock, lock.LOCK_UN) else: lock.flock(fp_lock, lock.LOCK_UN) raise ValueError("Module initialized parameter is empty") CacheUpdater(name, version).start()
def __init__(self, name, inputs, outputs, feed_names=None, fetch_names=None, for_predict=False): inputs = to_list(inputs) outputs = to_list(outputs) if not feed_names: feed_names = [""] * len(inputs) feed_names = to_list(feed_names) if len(inputs) != len(feed_names): raise ValueError( "The length of feed_names must be same with inputs") if not fetch_names: fetch_names = [""] * len(outputs) fetch_names = to_list(fetch_names) if len(outputs) != len(fetch_names): raise ValueError( "the length of fetch_names must be same with outputs") self.name = name for item in inputs: if not isinstance(item, Variable): raise TypeError( "Item in inputs list shoule be an instance of fluid.framework.Variable" ) for item in outputs: if not isinstance(item, Variable): raise TypeError( "Item in outputs list shoule be an instance of fluid.framework.Variable" ) self.inputs = inputs self.outputs = outputs self.feed_names = feed_names self.fetch_names = fetch_names self.for_predict = for_predict
def create_module(sign_arr, module_dir, processor=None, assets=None, module_info=None, exe=None, extra_info=None): sign_arr = utils.to_list(sign_arr) module = Module(signatures=sign_arr, processor=processor, assets=assets, module_info=module_info, extra_info=extra_info) module.serialize_to_path(path=module_dir, exe=exe)
def __init__(self, name=None, module_dir=None, signatures=None, module_info=None, assets=None, processor=None, extra_info=None, version=None): self.desc = module_desc_pb2.ModuleDesc() self.program = None self.assets = [] self.helper = None self.signatures = {} self.default_signature = None self.module_info = None self.processor = None self.extra_info = {} if extra_info is None else extra_info if not isinstance(self.extra_info, dict): raise TypeError( "The extra_info should be an instance of python dict") # cache data self.last_call_name = None self.cache_feed_dict = None self.cache_fetch_dict = None self.cache_program = None # TODO(wuzewu): print more module loading info log if name: self._init_with_name(name=name, version=version) elif module_dir: self._init_with_module_file(module_dir=module_dir[0]) elif signatures: if processor: if not issubclass(processor, BaseProcessor): raise TypeError( "Processor shoule be an instance of paddlehub.BaseProcessor" ) if assets: self.assets = utils.to_list(assets) # for asset in assets: # utils.check_path(assets) self.processor = processor self._generate_module_info(module_info) self._init_with_signature(signatures=signatures) else: raise ValueError("Module initialized parameter is empty")
def _build_env(self): if self.env.is_inititalized: return self._build_env_start_event() self.env.is_inititalized = True self.env.main_program = clone_program(self._base_main_program, for_test=False) self.env.startup_program = fluid.Program() with fluid.program_guard(self.env.main_program, self._base_startup_program): with fluid.unique_name.guard(self.env.UNG): self.env.outputs = self._build_net() if self.is_train_phase or self.is_test_phase: self.env.labels = self._add_label() self.env.loss = self._add_loss() self.env.metrics = self._add_metrics() if self.is_predict_phase or self.is_test_phase: self.env.main_program = clone_program(self.env.main_program, for_test=True) hub.common.paddle_helper.set_op_attr(self.env.main_program, is_test=True) if self.config.use_pyreader: t_program = fluid.Program() with fluid.program_guard(t_program, self.env.startup_program): self.env.py_reader = fluid.layers.py_reader( capacity=64, shapes=[var.shape for var in self.feed_var_list], dtypes=[ dtype_map[var.dtype] for var in self.feed_var_list ], lod_levels=[var.lod_level for var in self.feed_var_list], use_double_buffer=False) feed_var_list = self.feed_var_list py_vars = fluid.layers.read_file(self.env.py_reader) py_vars = to_list(py_vars) input_dict = { feed_var_list[index].name: py_var for index, py_var in enumerate(py_vars) } hub.connect_program(pre_program=t_program, next_program=self.env.main_program, input_dict=input_dict, need_log=False) self.env.main_program = t_program if not self.is_predict_phase: self.env.loss = self.env.main_program.global_block().vars[ self.env.loss.name] metrics_name = [var.name for var in self.env.metrics] self.env.metrics = [ self.env.main_program.global_block().vars[name] for name in metrics_name ] outputs_name = [var.name for var in self.env.outputs] self.env.outputs = [ self.env.main_program.global_block().vars[name] for name in outputs_name ] if self.config.enable_memory_optim: for var_name in self.fetch_list: var = self.env.main_program.global_block().vars[var_name] var.persistable = True if self.is_train_phase: with fluid.program_guard(self.env.main_program, self._base_startup_program): with fluid.unique_name.guard(self.env.UNG): self.config.strategy.execute(self.loss, self._base_data_reader, self.config) if self.is_train_phase: loss_name = self.env.loss.name share_vars_from = None else: loss_name = None if self._base_compiled_program is None: share_vars_from = None else: share_vars_from = self._base_compiled_program if not self.config.use_data_parallel: if self.config.enable_memory_optim: fluid.memory_optimize(self.env.main_program) self.env.main_program_compiled = None else: self.env.main_program_compiled = fluid.CompiledProgram( self.env.main_program).with_data_parallel( loss_name=loss_name, share_vars_from=share_vars_from, build_strategy=self.build_strategy) if self._base_compiled_program is None: self._base_compiled_program = self.env.main_program_compiled self.exe.run(self.env.startup_program) self._build_env_end_event()
def _build_env(self): if self.env.is_inititalized: return self._build_env_start_event() self.env.is_inititalized = True self.env.main_program = clone_program(self.base_main_program, for_test=False) self.env.startup_program = fluid.Program() with fluid.program_guard(self.env.main_program, self._base_startup_program): with fluid.unique_name.guard(self.env.UNG): self.env.outputs = self._build_net() if self.is_train_phase or self.is_test_phase: self.env.labels = self._add_label() self.env.loss = self._add_loss() self.env.metrics = self._add_metrics() if self.is_predict_phase or self.is_test_phase: # Todo: paddle.fluid.core_avx.EnforceNotMet: Getting 'tensor_desc' is not supported by the type of var kCUDNNFwdAlgoCache. at # self.env.main_program = clone_program( # self.env.main_program, for_test=True) hub.common.paddle_helper.set_op_attr(self.env.main_program, is_test=True) if self.config.use_pyreader: t_program = fluid.Program() with fluid.program_guard(t_program, self.env.startup_program): self.env.py_reader = fluid.layers.py_reader( capacity=64, shapes=[var.shape for var in self.feed_var_list], dtypes=[ dtype_map[var.dtype] for var in self.feed_var_list ], lod_levels=[var.lod_level for var in self.feed_var_list], use_double_buffer=False) feed_var_list = self.feed_var_list py_vars = fluid.layers.read_file(self.env.py_reader) py_vars = to_list(py_vars) input_dict = { feed_var_list[index].name: py_var for index, py_var in enumerate(py_vars) } hub.connect_program(pre_program=t_program, next_program=self.env.main_program, input_dict=input_dict, need_log=False) self.env.main_program = t_program if not self.is_predict_phase: self.env.loss = self.env.main_program.global_block().vars[ self.env.loss.name] metrics_name = [var.name for var in self.env.metrics] self.env.metrics = [ self.env.main_program.global_block().vars[name] for name in metrics_name ] outputs_name = [var.name for var in self.env.outputs] self.env.outputs = [ self.env.main_program.global_block().vars[name] for name in outputs_name ] if self.config.enable_memory_optim: for var_name in self.fetch_list: var = self.env.main_program.global_block().vars[var_name] var.persistable = True # to avoid to print logger two times in result of the logger usage of paddle-fluid 1.6 for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) if self.is_train_phase: with fluid.program_guard(self.env.main_program, self._base_startup_program): with fluid.unique_name.guard(self.env.UNG): self.scheduled_lr, self.max_train_steps = self.config.strategy.execute( self.loss, self._base_data_reader, self.config, self.device_count) if self.is_train_phase: loss_name = self.env.loss.name else: loss_name = None share_vars_from = self._base_compiled_program if not self.config.use_data_parallel: self.env.main_program_compiled = None else: self.env.main_program_compiled = fluid.CompiledProgram( self.env.main_program).with_data_parallel( loss_name=loss_name, share_vars_from=share_vars_from, build_strategy=self.build_strategy) self.exe.run(self.env.startup_program) # to avoid to print logger two times in result of the logger usage of paddle-fluid 1.5 for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) self._build_env_end_event()