def _get_var_size(self, block, name, batch): """ input: - block: where the var info is stored - name: var name - batch: batch size when training """ dtype_to_size = { core.VarDesc.VarType.FP16: 2, core.VarDesc.VarType.FP32: 4, core.VarDesc.VarType.FP64: 8, core.VarDesc.VarType.INT16: 2, core.VarDesc.VarType.INT32: 4, core.VarDesc.VarType.INT64: 8, core.VarDesc.VarType.BOOL: 1, core.VarDesc.VarType.UINT8: 1, } if not block.desc.find_var(cpt.to_bytes(name)): print("get var size failed, can not find var name %s" % name) return 0, [0] #print(block.desc.var(cpt.to_bytes(name))) if block.desc.var( cpt.to_bytes(name)).type() != core.VarDesc.VarType.LOD_TENSOR: print("not lod tensor var, var name %s" % name) return 0, [0] var = block.var(name) if var.shape[0] == -1: res = -reduce(lambda x, y: x * y, var.shape) * batch * dtype_to_size[var.dtype] else: res = reduce(lambda x, y: x * y, var.shape) * dtype_to_size[var.dtype] if res < 0: return 0, [0] return res, var.shape
def test_add_var(self): program_desc = core.ProgramDesc() self.assertIsNotNone(program_desc) block = program_desc.block(0) self.assertIsNotNone(block) var1 = block.var(cpt.to_bytes("var1")) var2 = block.var(cpt.to_bytes("var2")) var3 = block.var(cpt.to_bytes("var3")) all_vars = block.all_vars() self.assertEqual(set(all_vars), {var1, var2, var3}) var2_re = block.find_var(cpt.to_bytes("var2")) self.assertEqual(var2_re, var2)
def fill_block_desc(self, block_desc): for name in self.vars: var_desc = block_desc.var(cpt.to_bytes(name)) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) if self.vars_lod_level is not None and name in self.vars_lod_level.keys( ): var_desc.set_lod_level(self.vars_lod_level[name]) if self.vars_var_type is not None and name in self.vars_var_type.keys( ): if self.vars_var_type[name] == VarType.LOD_TENSOR_ARRAY: var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR_ARRAY) elif self.vars_var_type[name] == VarType.STEP_SCOPES: var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES) continue var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) if self.vars_dtype is not None and name in self.vars_dtype.keys(): var_desc.set_dtype( convert_np_dtype_to_dtype_(self.vars_dtype[name])) for op_config in self.ops: op_desc = block_desc.append_op() op_desc.set_type(op_config.type) for name, values in op_config.inputs.items(): op_desc.set_input(name, values) for name, values in op_config.attrs.items(): op_desc._set_attr(name, values) for name, values in op_config.outputs.items(): op_desc.set_output(name, values) for v in values: if block_desc.has_var_recursive(cpt.to_bytes(v)): continue var_desc = block_desc.var(cpt.to_bytes(v)) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) if op_config.outputs_var_type is not None and v in op_config.outputs_var_type.keys( ): if op_config.outputs_var_type[ v] == VarType.LOD_TENSOR_ARRAY: var_desc.set_type( core.VarDesc.VarType.LOD_TENSOR_ARRAY) elif op_config.outputs_var_type[ v] == VarType.STEP_SCOPES: var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES) continue var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys( ): var_desc.set_dtype( convert_np_dtype_to_dtype_( op_config.outputs_dtype[v])) if op_config.type not in _OP_WITHOUT_KERNEL_SET: op_desc.infer_var_type(block_desc) op_desc.infer_shape(block_desc) op_desc.check_attrs()
def _get_grad_vartype(self, name): assert self.program_desc is not None grad_name = name + core.grad_var_suffix() for i in six.moves.range(self.program_desc.num_blocks()): block = self.program_desc.block(i) var_desc = block.find_var_recursive(cpt.to_bytes(grad_name)) return var_desc.type() if var_desc is not None else None
def test_dtype(self): program_desc = core.ProgramDesc() block = program_desc.block(0) var = block.var(cpt.to_bytes('my_var')) var.set_type(core.VarDesc.VarType.LOD_TENSOR) var.set_dtype(core.VarDesc.VarType.INT32) self.assertEqual(core.VarDesc.VarType.INT32, var.dtype()) self.assertEqual(core.VarDesc.VarType.LOD_TENSOR, var.type())
def test_multiple_lod_level(self): program_desc = core.ProgramDesc() block = program_desc.block(0) var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_types = [3, 1, 2] var.set_lod_levels(src_types) self.assertEqual(src_types, var.lod_levels()) self.assertEqual(core.VarDesc.VarType.READER, var.type())
def test_multiple_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_shapes = [[2, 3, 3], [4, 5], [6, 7, 8, 9]] var.set_shapes(src_shapes) res_shapes = var.shapes() self.assertEqual(src_shapes, res_shapes) self.assertEqual(core.VarDesc.VarType.READER, var.type())
def test_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) var = block.var(cpt.to_bytes('my_var')) var.set_type(core.VarDesc.VarType.SELECTED_ROWS) src_shape = [3, 2, 10, 8] var.set_shape(src_shape) res_shape = var.shape() self.assertEqual(src_shape, res_shape) self.assertEqual(core.VarDesc.VarType.SELECTED_ROWS, var.type())
def erase_dots(self, vars_to_erase): for var in vars_to_erase: if id(var) in self.vars: del self.vars[id(var)] self.dot2bar.delete_keyvars(vars_to_erase) self.var2dot.delete_valuevars(vars_to_erase) block = self.block for var in vars_to_erase: name = var.name block.desc._remove_var(cpt.to_bytes(name)) del block.vars[name] block._sync_with_cpp()
def test_multiple_dtype(self): program_desc = core.ProgramDesc() block = program_desc.block(0) var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_types = [ core.VarDesc.VarType.INT32, core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32 ] var.set_dtypes(src_types) self.assertEqual(src_types, var.dtypes()) self.assertEqual(core.VarDesc.VarType.READER, var.type())
def __build_dict(tar_file, dict_size, save_path, lang): word_dict = defaultdict(int) with tarfile.open(tar_file, mode="r") as f: for line in f.extractfile("wmt16/train"): line = cpt.to_text(line) line_split = line.strip().split("\t") if len(line_split) != 2: continue sen = line_split[0] if lang == "en" else line_split[1] for w in sen.split(): word_dict[w] += 1 with open(save_path, "wb") as fout: fout.write( cpt.to_bytes("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK))) for idx, word in enumerate( sorted(six.iteritems(word_dict), key=lambda x: x[1], reverse=True)): if idx + 3 == dict_size: break fout.write(cpt.to_bytes(word[0])) fout.write(cpt.to_bytes('\n'))
def _set_grad_type(self, params, train_program): # NOTE: if user set sparse gradient mode, the param's gradient # will be SelectedRows, not LoDTensor. But tracer will just # set param grad VarBase by forward VarBase(LoDTensor) # If we don't change grad_var type here, RunProgramOp need # transform SelectedRows to LoDTensor forcibly, it may not # be user wanted result. for param in params: grad_name = param.name + core.grad_var_suffix() grad_var = train_program.desc.block(0).find_var( cpt.to_bytes(grad_name)) # NOTE: cannot find var desc maybe no problem, such as in batch_norm if grad_var is None: continue param._set_grad_type(grad_var.type())
def _lower(block, reverse): # Some functions which are only used in _lower. def bind(args, to_bind, value_table): for i in range(len(args)): if isinstance(args[i], list): bind(args[i], to_bind, value_table) elif args[i] is not None and args[i].name in to_bind: args[i] = value_table[to_bind[args[i].name]] def bind_name(names, to_bind): return_list = [] for name in names: if isinstance(name, list): return_list.append(bind_name(name, to_bind)) else: return_list.append(to_bind[name] if name in to_bind else name) return return_list def expand_nested_list(xs): return_list = [] for x in xs: if isinstance(x, list): return_list = return_list + expand_nested_list(x) else: return_list.append(x) return return_list # Step1: Do some preparatory work for lower lower_fn = _prim2orig if reverse else _orig2prim lookup_fn = lookup_prim2orig if reverse else lookup_orig2prim value_table = {} to_bind = {} to_bind_rev = {} for var in block.desc.all_vars(): value_table[var.name()] = block.var(var.name()) ops_to_remove = [] vars_to_remove = set() # Step2: Process all ops in the target block for op_idx in range(len(block.ops)): op = block.ops[op_idx] ops_to_remove.append(op_idx) if lookup_fn(op.type) is not None: input_args = get_input_var_list(op) bind(input_args, to_bind, value_table) for orig_out, new_out in zip( expand_nested_list(get_output_var_list(op)), expand_nested_list(to_tensors(lower_fn(op, *input_args)))): assert not (orig_out is None) ^ ( new_out is None), "orig_out and new_out should match." vars_to_remove.add(new_out.name) value_table[new_out.name] = new_out to_bind[orig_out.name] = new_out.name to_bind_rev[new_out.name] = orig_out.name else: inputs = {} for i in range(len(op.input_names)): inputs[op.input_names[i]] = bind_name( op.input(op.input_names[i]), to_bind) outputs = {} for i in range(len(op.output_names)): outputs[op.output_names[i]] = op.output(op.output_names[i]) attrs = {} for name in sorted(op.attr_names): attrs[name] = op.attr(name) from paddle.fluid.dygraph.base import param_guard new_op_desc = block.desc.append_op() with param_guard(inputs), param_guard(outputs): op = Operator(block=block, desc=new_op_desc, type=op.type, inputs=inputs, outputs=outputs, attrs=attrs) block.ops.append(op) # Step3: Do some post-processing work for op_idx in reversed(ops_to_remove): block.desc._remove_op(op_idx, op_idx + 1) del block.ops[op_idx] block._sync_with_cpp() for op_idx in range(len(block.ops)): op = block.ops[op_idx] for in_name in op.input_arg_names: if in_name in to_bind_rev: op._rename_input(in_name, to_bind_rev[in_name]) for out_name in op.output_arg_names: if out_name in to_bind_rev: op._rename_output(out_name, to_bind_rev[out_name]) for var_name in sorted(vars_to_remove): assert var_name in to_bind_rev, 'var_name "{}" is not in to_bind_rev.'.format( var_name) if var_name != to_bind_rev[var_name]: block.desc._remove_var(cpt.to_bytes(var_name)) del block.vars[var_name] block._sync_with_cpp()
def __impl__(self, *input): # 1. prepare inputs, outputs, attrs input_vars = [] for i, value in enumerate(input): if not isinstance(value, (np.ndarray, core.VarBase)): raise TypeError( "The type of input in TranslatedLayer must be numpy array or Variable(VarBase), but received %s." % type(value)) # NOTE: In order to unify the API, firstly convert the input to VarBase if isinstance(value, np.ndarray): var = core.VarBase( value=value, name=program_holder.input_names[i], persistable=False, place=framework._current_expected_place(), zero_copy=True) else: var = value # NOTE: we changed var name here, # but it may be an important name set by user var.name = program_holder.input_names[i] input_vars.append(var) persistable_vars = [] for var_name in program_holder.persistable_names: dy_var_name = self._persistable_var_name_dict[var_name] if dy_var_name in self._parameters: persistable_vars.append(self._parameters[dy_var_name]) elif dy_var_name in self._buffers: persistable_vars.append(self._buffers[dy_var_name]) else: raise ValueError( "The persistable variable %s is not exists in current TranslatedLayer." % var_name) output_vars = [] for var_desc in program_holder.output_decs: var = core.VarBase(var_desc.dtype(), var_desc.shape(), var_desc.name(), var_desc.type(), False) output_vars.append(var) # hold forward variables tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [], "program_out_scope", core.VarDesc.VarType.STEP_SCOPES, True) tmp_scope_vec.value().set_scope(program_holder.scope) # 2. run program by op trace_program = program_holder.infer_program if self._is_test else program_holder.train_program end_op_index = program_holder.infer_program.block(0).op_size() framework._dygraph_tracer().trace_op(type='run_program', inputs={ 'X': input_vars, 'Params': persistable_vars }, outputs={ 'Out': output_vars, 'OutScope': tmp_scope_vec }, attrs={ 'global_block': trace_program.block(0), 'start_op_index': 0, 'end_op_index': end_op_index, 'is_test': self._is_test }) # NOTE: [ why need set param's gradient type here ] # if user set sparse gradient mode, the param's gradient # will be SelectedRows, not LoDTensor. But tracer will just # set param grad VarBase by forward VarBase(LoDTensor) # If we don't change grad_var type here, RunProgramOp need # transform SelectedRows to LoDTensor forcibly, it may not # be user wanted result. for persistable_var in persistable_vars: grad_var_name = var.name + core.grad_var_suffix() grad_var = trace_program.block(0).find_var( cpt.to_bytes(grad_var_name)) # NOTE: cannot find var desc maybe not problem, # such as in batch_norm if grad_var is None: continue persistable_var._set_grad_type(grad_var.type()) # 3. prepare output, keep same form with inputs outs = output_vars if len(output_vars) == 1: outs = output_vars[0] return outs
def test_to_bytes(self): # Only support python2.x and python3.x now self.assertTrue(six.PY2 | six.PY3) if six.PY2: # check None self.assertIsNone(cpt.to_bytes(None)) # check all string related types self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes)) self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes)) self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) self.assertEqual(b"", cpt.to_bytes(str(""))) self.assertEqual(b"123", cpt.to_bytes(str("123"))) self.assertEqual(b"", cpt.to_bytes(b"")) self.assertEqual(b"123", cpt.to_bytes(b"123")) self.assertEqual(b"", cpt.to_bytes(u"")) self.assertEqual(b"123", cpt.to_bytes(u"123")) # check list types, not inplace l = [""] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertEqual(l, l2) self.assertEqual([b""], l2) l = ["", "123"] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123"], l2) l = ["", b'123', u"321"] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123", b"321"], l2) for i in l2: self.assertTrue(isinstance(i, bytes)) # check list types, inplace l = [""] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b""], l2) l = ["", "123"] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123"], l2) l = ["", b"123", u"321"] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123", b"321"], l2) # check set types, not inplace l = set("") l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertEqual(l, l2) self.assertEqual(set(b""), l2) l = set([b"", b"123"]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123"]), l2) l = set(["", b"123", u"321"]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123", b"321"]), l2) for i in l2: self.assertTrue(isinstance(i, bytes)) # check set types, inplace l = set("") l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set(b""), l2) l = set([b"", b"123"]) l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123"]), l2) l = set(["", b"123", u"321"]) l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123", b"321"]), l2) elif six.PY3: self.assertIsNone(cpt.to_bytes(None)) self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes)) self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes)) self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) self.assertEqual(b"", cpt.to_bytes(str(""))) self.assertEqual(b"123", cpt.to_bytes(str("123"))) self.assertEqual(b"", cpt.to_bytes(b"")) self.assertEqual(b"123", cpt.to_bytes(b"123")) self.assertEqual(b"", cpt.to_bytes(u"")) self.assertEqual(b"123", cpt.to_bytes(u"123")) # check list types, not inplace l = [""] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual([b""], l2) l = ["", "123"] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual([b"", b"123"], l2) l = ["", b"123", u"321"] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual([b"", b"123", b"321"], l2) # check list types, inplace l = [""] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b""], l2) l = ["", b"123"] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123"], l2) l = ["", b"123", u"321"] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123", b"321"], l2) for i in l2: self.assertTrue(isinstance(i, bytes)) # check set types, not inplace l = set([""]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual(set([b""]), l2) l = set([u"", u"123"]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual(set([b"", b"123"]), l2) l = set(["", b"123", u"321"]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual(set([b"", b"123", b"321"]), l2) # check set types, inplace l = set("") l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set(b""), l2) l = set([u"", u"123"]) l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123"]), l2) l = set(["", b"123", u"321"]) l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123", b"321"]), l2) for i in l2: self.assertTrue(isinstance(i, bytes))
def get_attr_default_value(op_name): return core.get_op_attrs_default_value(cpt.to_bytes(op_name))
def test_get_op_attrs_default_value(self): core.get_op_attrs_default_value(cpt.to_bytes("fill_constant"))
def __init__(self, block, type=core.VarDesc.VarType.LOD_TENSOR, name=None, shape=None, dtype=None, lod_level=None, capacity=None, persistable=None, error_clip=None, stop_gradient=False, is_data=False, need_check_feed=False, belong_to_optimizer=False, **kwargs): self.block = block if name is None: name = unique_name.generate('_generated_var') if dtype is not None: if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) self.belong_to_optimizer = belong_to_optimizer self.error_clip = error_clip is_new_var = False name = cpt.to_text(name) self.desc = self.block.desc.find_var(cpt.to_bytes(name)) if self.desc is None: self.desc = self.block.desc.var(cpt.to_bytes(name)) is_new_var = True if is_new_var: self.desc.set_type(type) elif self.desc.type() != type: raise ValueError("MpcVariable {0} has been created before. The " "previous type is {1}; the new type is {2}. They" " are not matched".format(self.name, self.desc.type(), type)) if shape is not None: if is_new_var: # resize the shape for MpcVariable mpc_shape = list(shape) mpc_shape.insert(0, 2) self.desc.set_shape(mpc_shape) else: old_shape = self.shape shape = tuple(shape) if shape != old_shape: raise ValueError( "MpcVariable {0} has been created before. the previous " "shape is {1}; the new shape is {2}. They are not " "matched.".format(self.name, old_shape, shape)) if dtype is not None: if is_new_var: self.desc.set_dtype(dtype) else: old_dtype = self.dtype if dtype != old_dtype: raise ValueError( "MpcVariable {0} has been created before. " "The previous data type is {1}; the new " "data type is {2}. They are not " "matched.".format(self.name, old_dtype, dtype)) if lod_level is not None: if is_new_var: self.desc.set_lod_level(lod_level) else: if lod_level != self.lod_level: raise ValueError( "MpcVariable {0} has been created before. " "The previous lod_level is {1}; the new " "lod_level is {2}. They are not " "matched".format(self.name, self.lod_level, lod_level)) if persistable is not None: if is_new_var: self.desc.set_persistable(persistable) else: if persistable != self.persistable: raise ValueError( "MpcVariable {0} has been created before." "The previous persistable is {1}; the new " "persistable is {2}. They are not matched".format( self.name, self.persistable, persistable)) if need_check_feed and is_new_var: self.desc.set_need_check_feed(need_check_feed) if capacity is not None: if is_new_var: self.desc.set_capacity(capacity) else: # TODO(abhinavarora) by Paddle 1.7: Compare with set capacity once, # get_capacity is implemented pass self.block.vars[name] = self self.op = None self._stop_gradient = stop_gradient self.is_data = is_data
def create_fake_model(program_config): ''' Create a Paddle model(in memory) according to the given config. ''' paddle.enable_static() main_program_desc = core.ProgramDesc() util_program = fluid.Program() main_block_desc = main_program_desc.block(0) var_desc = main_block_desc.var(cpt.to_bytes("feed")) var_desc.set_type(core.VarDesc.VarType.FEED_MINIBATCH) var_desc.set_persistable(True) index = 0 for name, tensor_config in program_config.inputs.items(): var_desc = main_block_desc.var(cpt.to_bytes(name)) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype)) var_desc.set_shape(tensor_config.shape) var_desc.set_need_check_feed(True) if tensor_config.lod is not None: var_desc.set_lod_level(len(tensor_config.lod)) op_desc = main_block_desc._prepend_op() op_desc.set_type("feed") op_desc.set_input('X', ["feed"]) op_desc.set_output('Out', [name]) op_desc._set_attr("col", index) index = index + 1 save_var_map = {} for name, tensor_config in program_config.weights.items(): var_desc = main_block_desc.var(cpt.to_bytes(name)) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype)) var_desc.set_shape(tensor_config.shape) var_desc.set_persistable(True) save_var_map[name] = util_program.global_block().create_parameter( dtype=tensor_config.dtype, shape=tensor_config.shape, type=core.VarDesc.VarType.LOD_TENSOR, name=name, initializer=NumpyArrayInitializer(tensor_config.data)) in_vars = [] for name in sorted(save_var_map.keys()): in_vars.append(save_var_map[name]) out_var = util_program.global_block().create_var( type=core.VarDesc.VarType.RAW, name="out_var_0") out_var.desc.set_persistable(True) util_program.global_block().append_op(type='save_combine', inputs={'X': in_vars}, outputs={'Y': out_var}, attrs={ 'file_path': '', 'save_to_memory': True }) for op_config in program_config.ops: op_desc = main_block_desc.append_op() op_desc.set_type(op_config.type) for name, values in op_config.inputs.items(): op_desc.set_input(name, values) for name, values in op_config.attrs.items(): op_desc._set_attr(name, values) for name, values in op_config.outputs.items(): op_desc.set_output(name, values) for v in values: var_desc = main_block_desc.var(cpt.to_bytes(v)) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys( ): var_desc.set_dtype( convert_np_dtype_to_dtype_(op_config.outputs_dtype[v])) op_desc.infer_var_type(main_block_desc) op_desc.infer_shape(main_block_desc) op_desc.check_attrs() for index, name in enumerate(program_config.outputs): var_desc = main_block_desc.var(cpt.to_bytes("fetch")) var_desc.set_type(core.VarDesc.VarType.FETCH_LIST) var_desc.set_need_check_feed(True) op_desc = main_block_desc.append_op() op_desc.set_type("fetch") op_desc.set_input('X', [name]) op_desc.set_output('Out', ["fetch"]) op_desc._set_attr("col", index) main_program_desc._set_version() paddle.fluid.core.save_op_version_info(main_program_desc) model = main_program_desc.serialize_to_string() util_program._sync_with_cpp() place = fluid.CPUPlace() executor = fluid.Executor(place) scope = fluid.Scope() with fluid.scope_guard(scope): executor.run(util_program) params = scope.find_var("out_var_0").get_bytes() return model, params
def find_var_by_name(block, name): return block.desc.var(cpt.to_bytes(name))
def _preprocess(self, program_desc): # 1. Prune original program # remove feed, fetch and scale-1 op, remove op_callstack attr ops_to_remove = [] root_block = program_desc.block(0) for i in six.moves.range(root_block.op_size()): op = root_block.op(i) if op.type() == 'feed': ops_to_remove.append(i) feed_var_name = cpt.to_bytes(op.input('X')[0]) root_block._remove_var(feed_var_name) self._input_names.append(cpt.to_bytes(op.output('Out')[0])) elif op.type() == 'scale' and op.output('Out')[0].startswith( 'save_infer_model/scale_'): ops_to_remove.append(i) out_var_name = cpt.to_bytes(op.output('Out')[0]) root_block._remove_var(out_var_name) self._output_descs.append( root_block.find_var(cpt.to_bytes(op.input('X')[0]))) elif op.type() == 'fetch': ops_to_remove.append(i) fetch_var_name = cpt.to_bytes(op.output('Out')[0]) root_block._remove_var(fetch_var_name) # NOTE: some old pre-train models have no extra scale_op if not op.input('X')[0].startswith('save_infer_model/scale_'): self._output_descs.append( root_block.find_var(cpt.to_bytes(op.input('X')[0]))) else: if op.has_attr("op_callstack"): op.remove_attr("op_callstack") for op_idx in reversed(ops_to_remove): root_block._remove_op(op_idx, op_idx + 1) # 2. Input processing, reverse feed vars self._input_names.reverse() # 3. Output processing, add scale for outputs tmp_program = _build_program_by_desc(program_desc) # NOTE: [why need append scale for outputs] # When dealing with some more complex pre-training models, there # will be situations where the pre-training model has multiple # fetch outputs. In the scenario of multiple fetch outputs, # there is a special case where multiple outputs of the model # may be on the same branch. According to the user's subsequent # use, multiple outputs may be associated with multiple branches. # These subsequent operations are added in TranslatedLayer is # agnostic during initialization, which results in subsequent # gradient accumulation operations that are required on the # output node in the middle of the branch will not be performed, # resulting in error, details see pull request: # [https://github.com/PaddlePaddle/Paddle/pull/24627] self._append_scale_to_output(tmp_program) # 4. Persistable vars processing # - append loaded suffix to persistable vars # NOTE: [why need to append suffix to persistable vars] # Dygraph and static graph mode use the same naming mechanism. # If users want to load the model fine-tune, it is possible # to add the existing Layer in the loaded model to enhance # the network. For example, the original saved model has linear, # and later after loading, a new linear is added. At this time, # there will be a problem of duplicate names, so here is unified # to add the LOADED suffix to the parameters of the model loaded self._suffix_varname_dict = _append_loaded_suffix_to_var(program_desc) # - get persistable var self._persistable_names = _get_persistable_var_names(program_desc) return program_desc
def test_to_bytes(self): self.assertIsNone(cpt.to_bytes(None)) self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes)) self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes)) self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) self.assertEqual(b"", cpt.to_bytes(str(""))) self.assertEqual(b"123", cpt.to_bytes(str("123"))) self.assertEqual(b"", cpt.to_bytes(b"")) self.assertEqual(b"123", cpt.to_bytes(b"123")) self.assertEqual(b"", cpt.to_bytes(u"")) self.assertEqual(b"123", cpt.to_bytes(u"123")) # check list types, not inplace l = [""] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual([b""], l2) l = ["", "123"] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual([b"", b"123"], l2) l = ["", b"123", u"321"] l2 = cpt.to_bytes(l) self.assertTrue(isinstance(l2, list)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual([b"", b"123", b"321"], l2) # check list types, inplace l = [""] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b""], l2) l = ["", b"123"] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123"], l2) l = ["", b"123", u"321"] l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, list)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual([b"", b"123", b"321"], l2) for i in l2: self.assertTrue(isinstance(i, bytes)) # check set types, not inplace l = set([""]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual(set([b""]), l2) l = set([u"", u"123"]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual(set([b"", b"123"]), l2) l = set(["", b"123", u"321"]) l2 = cpt.to_bytes(l, inplace=False) self.assertTrue(isinstance(l2, set)) self.assertFalse(l is l2) self.assertNotEqual(l, l2) self.assertEqual(set([b"", b"123", b"321"]), l2) # check set types, inplace l = set("") l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set(b""), l2) l = set([u"", u"123"]) l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123"]), l2) l = set(["", b"123", u"321"]) l2 = cpt.to_bytes(l, inplace=True) self.assertTrue(isinstance(l2, set)) self.assertTrue(l is l2) self.assertEqual(l, l2) self.assertEqual(set([b"", b"123", b"321"]), l2) for i in l2: self.assertTrue(isinstance(i, bytes))