def dygraph2program(layer, inputs, feed_prefix='feed_', fetch_prefix='fetch_', tmp_prefix='t_', extract_inputs_fn=None, extract_outputs_fn=None, dtypes=None): print(type(layer)) assert isinstance(layer, Layer) extract_inputs_fn = extract_inputs_fn if extract_inputs_fn is not None else extract_vars extract_outputs_fn = extract_outputs_fn if extract_outputs_fn is not None else extract_vars if os.environ.get("FLAGS_enable_eager_mode") == "1": return _dy2prog(layer, inputs, feed_prefix, fetch_prefix, tmp_prefix, extract_inputs_fn, extract_outputs_fn, dtypes) tracer = _dygraph_tracer()._get_program_desc_tracer() with program_desc_tracing_guard(True): if _is_shape(inputs): shapes = [inputs] inputs = _create_tensors(shapes, dtypes=dtypes) input_var_list = inputs elif _is_shapes(inputs): inputs = _create_tensors(inputs, dtypes=dtypes) input_var_list = inputs else: inputs = to_variables(inputs) input_var_list = extract_inputs_fn(inputs) original_outputs = layer(*inputs) # 'original_outputs' may be dict, so we should convert it to list of varibles. # And should not create new varibles in 'extract_vars'. out_var_list = extract_outputs_fn(original_outputs) program_desc, feed_names, fetch_names, parameters = tracer.create_program_desc( input_var_list, feed_prefix, out_var_list, fetch_prefix, tmp_prefix) tracer.reset() with _dygraph_guard(None): program = Program() program.desc = program_desc program.blocks = [Block(program, 0)] program._sync_with_cpp() return program
def _construct_grad_program_from_forward(self, fwd_program, grad_op_desc, op_grad_to_var): """Generate grad_program which contains the grad_op. Args: fwd_program (tuple): The program that contains grad_op_desc's corresponding forward op. grad_op_desc (OpDesc): The OpDesc of grad op. op_grad_to_var (dict): The relation of variables in grad op and its forward op. Returns: grad_program (program): The program which contains the grad_op. """ grad_program = Program() grad_block = grad_program.global_block() new_op_desc = grad_block.desc.append_op() new_op_desc.copy_from(grad_op_desc) grad_program._sync_with_cpp() # Create grad vars based on fwd vars (shape and dtype) for arg in grad_op_desc.input_arg_names( ) + grad_op_desc.output_arg_names(): fwd_var_name = op_grad_to_var.get(arg, None) if fwd_var_name is None: fwd_var_name = arg fwd_var = fwd_program.global_block().vars.get(fwd_var_name) assert fwd_var is not None, "{} cannot be found".format( fwd_var_name) grad_var = grad_block.create_var(name=arg, dtype=fwd_var.dtype, shape=fwd_var.shape, type=fwd_var.type, persistable=False) # Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op, # and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]). # Set persistable for those variables in order to get them from global_scope for inplace grad test directly other than feed them, # since feed op calls check_memory_size() which fails when tensor's holder_ is NULL. if 0 in grad_var.shape: grad_var.persistable = True grad_program._sync_with_cpp() return grad_program
def test__remove_op(self): program = Program() program_desc = program.desc self.assertIsNotNone(program_desc) block = program_desc.block(0) self.assertIsNotNone(block) op0 = block.append_op() op1 = block.append_op() op2 = block.append_op() op0.set_type("test") op1.set_type("test") op2.set_type("test") block._remove_op(1, 2) program._sync_with_cpp() all_ops = [] for idx in range(0, block.op_size()): all_ops.append(block.op(idx)) self.assertEqual(all_ops, [op0, op2])
def create_program_from_desc(program_desc): program = Program() program.desc = program_desc program.blocks = [Block(program, 0)] program._sync_with_cpp() return program
def save_vars(executor, dirname, main_program=None, vars=None, predicate=None, filename=None): """ This API saves specific variables in the `Program` to files. There are two ways to specify the variables to be saved: set variables in a list and assign it to the `vars`, or use the `predicate` function to select variables that make `predicate(variable) == True`. The first way has a higher priority. The `dirname` is used to specify the folder where to save variables. If you prefer to save variables in separate files in the `dirname` floder, do not set `filename`. If you prefer to save all variables in a single file, use `filename` to specify it. Args: executor(Executor): The executor to run for saving variables. dirname(str, optional): The folder where to save variables. When you need to save the parameter to the memory, set it to None. main_program(Program, optional): The program whose variables will be saved. If it is None, the default main program will be used automatically. Default: None vars(list[Variable], optional): The list contains all variables to be saved. Default: None predicate(function, optional): The function selects the variables that make `predicate(variable) == True`. Default: None filename(str, optional): If you prefer to save all variables in a single file, use `filename` to specify it. Otherwise, let `filename` be None. Default: None Returns: str: When saving parameters to a file, returns None. When saving parameters to memory, returns a binary string containing parameters. Raises: TypeError: If `main_program` is not an instance of Program nor None. Examples: .. code-block:: python import paddle.fluid as fluid main_prog = fluid.Program() startup_prog = fluid.Program() with fluid.program_guard(main_prog, startup_prog): data = fluid.layers.data(name="img", shape=[64, 784], append_batch_size=False) w = fluid.layers.create_parameter(shape=[784, 200], dtype='float32', name='fc_w') b = fluid.layers.create_parameter(shape=[200], dtype='float32', name='fc_b') hidden_w = fluid.layers.matmul(x=data, y=w) hidden_b = fluid.layers.elementwise_add(hidden_w, b) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_prog) # The first usage: use `vars` to set the saved variables. var_list = [w, b] path = "./my_paddle_vars" fluid.io.save_vars(executor=exe, dirname=path, vars=var_list, filename="vars_file") # w and b will be save in a file named "var_file". # The second usage: use `predicate` to select the saved variable. def name_has_fc(var): res = "fc" in var.name return res param_path = "./my_paddle_model" fluid.io.save_vars(executor=exe, dirname=param_path, main_program=main_prog, vars=None, predicate = name_has_fc) # all variables whose names contain "fc " are saved. """ save_to_memory = False if dirname is None and filename is None: save_to_memory = True main_program = _get_valid_program(main_program) if vars is None: return save_vars( executor, main_program=main_program, dirname=dirname, vars=list(filter(predicate, main_program.list_vars())), filename=filename) else: params_var_name = unique_name.generate("saved_params") # give warning when there is no var in model if len(list(vars)) == 0: warnings.warn( "no variable in your model, please ensure there are any variables in your model to save" ) return None save_program = Program() save_block = save_program.global_block() save_var_map = {} for each_var in vars: # NOTE: don't save the variable which type is RAW if each_var.type == core.VarDesc.VarType.RAW: continue new_var = _clone_var_in_block_(save_block, each_var) if filename is None and save_to_memory is False: save_file_path = os.path.join( os.path.normpath(dirname), new_var.name) save_block.append_op( type='save', inputs={'X': [new_var]}, outputs={}, attrs={'file_path': os.path.normpath(save_file_path)}) else: save_var_map[new_var.name] = new_var if filename is not None or save_to_memory: save_var_list = [] for name in sorted(save_var_map.keys()): save_var_list.append(save_var_map[name]) save_path = str() if save_to_memory is False: save_path = os.path.join(os.path.normpath(dirname), filename) saved_params = save_block.create_var( type=core.VarDesc.VarType.RAW, name=params_var_name) saved_params.desc.set_persistable(True) save_block.append_op( type='save_combine', inputs={'X': save_var_list}, outputs={'Y': saved_params}, attrs={ 'file_path': save_path, 'save_to_memory': save_to_memory }) #NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc, # which leads to diff on save_program and its desc. Call _sync_with_cpp # to keep consistency. save_program._sync_with_cpp() executor.run(save_program) if save_to_memory: return global_scope().find_var(params_var_name).get_bytes()
def get_pserver_program(self, endpoint): """ Get parameter server side program. Args: endpoint (str): current parameter server endpoint. Returns: Program: the program for current parameter server to run. """ # TODO(panyx0718): Revisit this assumption. what if #blocks > #pservers. # NOTE: assume blocks of the same variable is not distributed # on the same pserver, only change param/grad varnames for # trainers to fetch. sys.stderr.write( "get_pserver_program() is deprecated, call get_pserver_programs() to get pserver main and startup in a single call.\n" ) # step1 pserver_program = Program() pserver_program.random_seed = self.origin_program.random_seed pserver_program._copy_dist_param_info_from(self.origin_program) # step2: Create vars to receive vars at parameter servers. recv_inputs = [] for v in self.param_grad_ep_mapping[endpoint]["params"]: self._clone_var(pserver_program.global_block(), v) for v in self.param_grad_ep_mapping[endpoint]["opti"]: # create vars for each trainer in global scope, so # we don't need to create them when grad arrives. # change client side var name to origin name by # removing ".trainer_%d" suffix suff_idx = v.name.find(".opti.trainer_") if suff_idx >= 0: orig_var_name = v.name[:suff_idx] # NOTE: single_trainer_var must be created for multi-trainer # case to merge grads from multiple trainers single_trainer_var = pserver_program.global_block().var( orig_var_name) if self.sync_mode and self.trainer_num > 1: for trainer_id in range(self.trainer_num): var = pserver_program.global_block().create_var( name="%s.opti.trainer_%d" % (orig_var_name, trainer_id), persistable=False, type=v.type, dtype=v.dtype, shape=v.shape) recv_inputs.append(var) # step 3 # Create a union-find data structure from optimize ops, # If two ops are connected, we could add these two ops # into one set. ufind = self._create_ufind(self.optimize_ops) # step 3.2 # Iterate through the ops and append optimize op which # located on current pserver opt_op_on_pserver = [] for _, op in enumerate(self.optimize_ops): if self._is_optimizer_op(op) and self._is_opt_op_on_pserver( endpoint, op): opt_op_on_pserver.append(op) # step 3.4 # Iterate through the ops, and if an op and the optimize ops # which located on current pserver are in one set, then # append it into the sub program. global_ops = [] # sparse grad name to param name sparse_grad_to_param = [] # append lr decay ops to the child block if exists lr_ops = self._get_lr_ops() # record optimize blocks and we can run them on pserver parallel opti_blocks = [] # append op to the current block grad_to_block_id = [] pre_block_idx = pserver_program.num_blocks - 1 for idx, opt_op in enumerate(self._opti_var_list): per_opt_block = pserver_program._create_block(pre_block_idx) opti_blocks.append(per_opt_block) optimize_target_param_name = self._opti_to_param[opt_op] pserver_block = per_opt_block.program.global_block() # append grad merging ops before clip and weight decay # e.g. merge grad -> L2Decay op -> clip op -> optimize merged_var = pserver_block.vars[optimize_target_param_name] if self.sync_mode and self.trainer_num > 1: vars2merge = [] for i in range(self.trainer_num): per_trainer_name = "%s.opti.trainer_%d" % \ (optimize_target_param_name, i) vars2merge.append(pserver_block.vars[per_trainer_name]) per_opt_block.append_op(type="sum", inputs={"X": vars2merge}, outputs={"Out": merged_var}, attrs={"use_mkldnn": False}) per_opt_block.append_op( type="scale", inputs={"X": merged_var}, outputs={"Out": merged_var}, attrs={"scale": 1.0 / float(self.trainer_num)}) # In some case, some parameter server will have no parameter to optimize # So we give an empty optimize block to parameter server. attrs = { "optimize_blocks": opti_blocks, "endpoint": endpoint, "Fanin": self.trainer_num, "sync_mode": self.sync_mode, } # step5 append the listen_and_serv op pserver_program.global_block().append_op(type="fl_listen_and_serv", inputs={'X': recv_inputs}, outputs={}, attrs=attrs) pserver_program._sync_with_cpp() # save pserver program to generate pserver side startup relatively. self.pserver_program = pserver_program return pserver_program