def _get_var_size(self, block, name, batch):
"""
input:
- block: where the var info is stored
- name: var name
- batch: batch size when training
"""
dtype_to_size = {
core.VarDesc.VarType.FP16: 2,
core.VarDesc.VarType.FP32: 4,
core.VarDesc.VarType.FP64: 8,
core.VarDesc.VarType.INT16: 2,
core.VarDesc.VarType.INT32: 4,
core.VarDesc.VarType.INT64: 8,
core.VarDesc.VarType.BOOL: 1,
core.VarDesc.VarType.UINT8: 1,
}
if not block.desc.find_var(cpt.to_bytes(name)):
print("get var size failed, can not find var name %s" % name)
return 0, [0]
#print(block.desc.var(cpt.to_bytes(name)))
if block.desc.var(
cpt.to_bytes(name)).type() != core.VarDesc.VarType.LOD_TENSOR:
print("not lod tensor var, var name %s" % name)
return 0, [0]
var = block.var(name)
if var.shape[0] == -1:
res = -reduce(lambda x, y: x * y,
var.shape) * batch * dtype_to_size[var.dtype]
else:
res = reduce(lambda x, y: x * y,
var.shape) * dtype_to_size[var.dtype]
if res < 0:
return 0, [0]
return res, var.shape
def test_add_var(self):
program_desc = core.ProgramDesc()
self.assertIsNotNone(program_desc)
block = program_desc.block(0)
self.assertIsNotNone(block)
var1 = block.var(cpt.to_bytes("var1"))
var2 = block.var(cpt.to_bytes("var2"))
var3 = block.var(cpt.to_bytes("var3"))
all_vars = block.all_vars()
self.assertEqual(set(all_vars), {var1, var2, var3})
var2_re = block.find_var(cpt.to_bytes("var2"))
self.assertEqual(var2_re, var2)
def fill_block_desc(self, block_desc):
for name in self.vars:
var_desc = block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
if self.vars_lod_level is not None and name in self.vars_lod_level.keys(
):
var_desc.set_lod_level(self.vars_lod_level[name])
if self.vars_var_type is not None and name in self.vars_var_type.keys(
):
if self.vars_var_type[name] == VarType.LOD_TENSOR_ARRAY:
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR_ARRAY)
elif self.vars_var_type[name] == VarType.STEP_SCOPES:
var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES)
continue
var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32))
if self.vars_dtype is not None and name in self.vars_dtype.keys():
var_desc.set_dtype(
convert_np_dtype_to_dtype_(self.vars_dtype[name]))
for op_config in self.ops:
op_desc = block_desc.append_op()
op_desc.set_type(op_config.type)
for name, values in op_config.inputs.items():
op_desc.set_input(name, values)
for name, values in op_config.attrs.items():
op_desc._set_attr(name, values)
for name, values in op_config.outputs.items():
op_desc.set_output(name, values)
for v in values:
if block_desc.has_var_recursive(cpt.to_bytes(v)):
continue
var_desc = block_desc.var(cpt.to_bytes(v))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
if op_config.outputs_var_type is not None and v in op_config.outputs_var_type.keys(
):
if op_config.outputs_var_type[
v] == VarType.LOD_TENSOR_ARRAY:
var_desc.set_type(
core.VarDesc.VarType.LOD_TENSOR_ARRAY)
elif op_config.outputs_var_type[
v] == VarType.STEP_SCOPES:
var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES)
continue
var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32))
if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys(
):
var_desc.set_dtype(
convert_np_dtype_to_dtype_(
op_config.outputs_dtype[v]))
if op_config.type not in _OP_WITHOUT_KERNEL_SET:
op_desc.infer_var_type(block_desc)
op_desc.infer_shape(block_desc)
op_desc.check_attrs()
def _get_grad_vartype(self, name):
assert self.program_desc is not None
grad_name = name + core.grad_var_suffix()
for i in six.moves.range(self.program_desc.num_blocks()):
block = self.program_desc.block(i)
var_desc = block.find_var_recursive(cpt.to_bytes(grad_name))
return var_desc.type() if var_desc is not None else None
def test_dtype(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_var'))
var.set_type(core.VarDesc.VarType.LOD_TENSOR)
var.set_dtype(core.VarDesc.VarType.INT32)
self.assertEqual(core.VarDesc.VarType.INT32, var.dtype())
self.assertEqual(core.VarDesc.VarType.LOD_TENSOR, var.type())
def test_multiple_lod_level(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_reader'))
var.set_type(core.VarDesc.VarType.READER)
src_types = [3, 1, 2]
var.set_lod_levels(src_types)
self.assertEqual(src_types, var.lod_levels())
self.assertEqual(core.VarDesc.VarType.READER, var.type())
def test_multiple_shape(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_reader'))
var.set_type(core.VarDesc.VarType.READER)
src_shapes = [[2, 3, 3], [4, 5], [6, 7, 8, 9]]
var.set_shapes(src_shapes)
res_shapes = var.shapes()
self.assertEqual(src_shapes, res_shapes)
self.assertEqual(core.VarDesc.VarType.READER, var.type())
def test_shape(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_var'))
var.set_type(core.VarDesc.VarType.SELECTED_ROWS)
src_shape = [3, 2, 10, 8]
var.set_shape(src_shape)
res_shape = var.shape()
self.assertEqual(src_shape, res_shape)
self.assertEqual(core.VarDesc.VarType.SELECTED_ROWS, var.type())
def erase_dots(self, vars_to_erase):
for var in vars_to_erase:
if id(var) in self.vars:
del self.vars[id(var)]
self.dot2bar.delete_keyvars(vars_to_erase)
self.var2dot.delete_valuevars(vars_to_erase)
block = self.block
for var in vars_to_erase:
name = var.name
block.desc._remove_var(cpt.to_bytes(name))
del block.vars[name]
block._sync_with_cpp()
def test_multiple_dtype(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_reader'))
var.set_type(core.VarDesc.VarType.READER)
src_types = [
core.VarDesc.VarType.INT32, core.VarDesc.VarType.FP64,
core.VarDesc.VarType.FP32
]
var.set_dtypes(src_types)
self.assertEqual(src_types, var.dtypes())
self.assertEqual(core.VarDesc.VarType.READER, var.type())
def __build_dict(tar_file, dict_size, save_path, lang):
word_dict = defaultdict(int)
with tarfile.open(tar_file, mode="r") as f:
for line in f.extractfile("wmt16/train"):
line = cpt.to_text(line)
line_split = line.strip().split("\t")
if len(line_split) != 2: continue
sen = line_split[0] if lang == "en" else line_split[1]
for w in sen.split():
word_dict[w] += 1
with open(save_path, "wb") as fout:
fout.write(
cpt.to_bytes("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)))
for idx, word in enumerate(
sorted(six.iteritems(word_dict),
key=lambda x: x[1],
reverse=True)):
if idx + 3 == dict_size: break
fout.write(cpt.to_bytes(word[0]))
fout.write(cpt.to_bytes('\n'))
def _set_grad_type(self, params, train_program):
# NOTE: if user set sparse gradient mode, the param's gradient
# will be SelectedRows, not LoDTensor. But tracer will just
# set param grad VarBase by forward VarBase(LoDTensor)
# If we don't change grad_var type here, RunProgramOp need
# transform SelectedRows to LoDTensor forcibly, it may not
# be user wanted result.
for param in params:
grad_name = param.name + core.grad_var_suffix()
grad_var = train_program.desc.block(0).find_var(
cpt.to_bytes(grad_name))
# NOTE: cannot find var desc maybe no problem, such as in batch_norm
if grad_var is None:
continue
param._set_grad_type(grad_var.type())
def _lower(block, reverse):
# Some functions which are only used in _lower.
def bind(args, to_bind, value_table):
for i in range(len(args)):
if isinstance(args[i], list):
bind(args[i], to_bind, value_table)
elif args[i] is not None and args[i].name in to_bind:
args[i] = value_table[to_bind[args[i].name]]
def bind_name(names, to_bind):
return_list = []
for name in names:
if isinstance(name, list):
return_list.append(bind_name(name, to_bind))
else:
return_list.append(to_bind[name] if name in to_bind else name)
return return_list
def expand_nested_list(xs):
return_list = []
for x in xs:
if isinstance(x, list):
return_list = return_list + expand_nested_list(x)
else:
return_list.append(x)
return return_list
# Step1: Do some preparatory work for lower
lower_fn = _prim2orig if reverse else _orig2prim
lookup_fn = lookup_prim2orig if reverse else lookup_orig2prim
value_table = {}
to_bind = {}
to_bind_rev = {}
for var in block.desc.all_vars():
value_table[var.name()] = block.var(var.name())
ops_to_remove = []
vars_to_remove = set()
# Step2: Process all ops in the target block
for op_idx in range(len(block.ops)):
op = block.ops[op_idx]
ops_to_remove.append(op_idx)
if lookup_fn(op.type) is not None:
input_args = get_input_var_list(op)
bind(input_args, to_bind, value_table)
for orig_out, new_out in zip(
expand_nested_list(get_output_var_list(op)),
expand_nested_list(to_tensors(lower_fn(op, *input_args)))):
assert not (orig_out is None) ^ (
new_out is None), "orig_out and new_out should match."
vars_to_remove.add(new_out.name)
value_table[new_out.name] = new_out
to_bind[orig_out.name] = new_out.name
to_bind_rev[new_out.name] = orig_out.name
else:
inputs = {}
for i in range(len(op.input_names)):
inputs[op.input_names[i]] = bind_name(
op.input(op.input_names[i]), to_bind)
outputs = {}
for i in range(len(op.output_names)):
outputs[op.output_names[i]] = op.output(op.output_names[i])
attrs = {}
for name in sorted(op.attr_names):
attrs[name] = op.attr(name)
from paddle.fluid.dygraph.base import param_guard
new_op_desc = block.desc.append_op()
with param_guard(inputs), param_guard(outputs):
op = Operator(block=block,
desc=new_op_desc,
type=op.type,
inputs=inputs,
outputs=outputs,
attrs=attrs)
block.ops.append(op)
# Step3: Do some post-processing work
for op_idx in reversed(ops_to_remove):
block.desc._remove_op(op_idx, op_idx + 1)
del block.ops[op_idx]
block._sync_with_cpp()
for op_idx in range(len(block.ops)):
op = block.ops[op_idx]
for in_name in op.input_arg_names:
if in_name in to_bind_rev:
op._rename_input(in_name, to_bind_rev[in_name])
for out_name in op.output_arg_names:
if out_name in to_bind_rev:
op._rename_output(out_name, to_bind_rev[out_name])
for var_name in sorted(vars_to_remove):
assert var_name in to_bind_rev, 'var_name "{}" is not in to_bind_rev.'.format(
var_name)
if var_name != to_bind_rev[var_name]:
block.desc._remove_var(cpt.to_bytes(var_name))
del block.vars[var_name]
block._sync_with_cpp()
def __impl__(self, *input):
# 1. prepare inputs, outputs, attrs
input_vars = []
for i, value in enumerate(input):
if not isinstance(value, (np.ndarray, core.VarBase)):
raise TypeError(
"The type of input in TranslatedLayer must be numpy array or Variable(VarBase), but received %s."
% type(value))
# NOTE: In order to unify the API, firstly convert the input to VarBase
if isinstance(value, np.ndarray):
var = core.VarBase(
value=value,
name=program_holder.input_names[i],
persistable=False,
place=framework._current_expected_place(),
zero_copy=True)
else:
var = value
# NOTE: we changed var name here,
# but it may be an important name set by user
var.name = program_holder.input_names[i]
input_vars.append(var)
persistable_vars = []
for var_name in program_holder.persistable_names:
dy_var_name = self._persistable_var_name_dict[var_name]
if dy_var_name in self._parameters:
persistable_vars.append(self._parameters[dy_var_name])
elif dy_var_name in self._buffers:
persistable_vars.append(self._buffers[dy_var_name])
else:
raise ValueError(
"The persistable variable %s is not exists in current TranslatedLayer."
% var_name)
output_vars = []
for var_desc in program_holder.output_decs:
var = core.VarBase(var_desc.dtype(), var_desc.shape(),
var_desc.name(), var_desc.type(), False)
output_vars.append(var)
# hold forward variables
tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [],
"program_out_scope",
core.VarDesc.VarType.STEP_SCOPES,
True)
tmp_scope_vec.value().set_scope(program_holder.scope)
# 2. run program by op
trace_program = program_holder.infer_program if self._is_test else program_holder.train_program
end_op_index = program_holder.infer_program.block(0).op_size()
framework._dygraph_tracer().trace_op(type='run_program',
inputs={
'X': input_vars,
'Params': persistable_vars
},
outputs={
'Out': output_vars,
'OutScope': tmp_scope_vec
},
attrs={
'global_block':
trace_program.block(0),
'start_op_index':
0,
'end_op_index':
end_op_index,
'is_test':
self._is_test
})
# NOTE: [ why need set param's gradient type here ]
# if user set sparse gradient mode, the param's gradient
# will be SelectedRows, not LoDTensor. But tracer will just
# set param grad VarBase by forward VarBase(LoDTensor)
# If we don't change grad_var type here, RunProgramOp need
# transform SelectedRows to LoDTensor forcibly, it may not
# be user wanted result.
for persistable_var in persistable_vars:
grad_var_name = var.name + core.grad_var_suffix()
grad_var = trace_program.block(0).find_var(
cpt.to_bytes(grad_var_name))
# NOTE: cannot find var desc maybe not problem,
# such as in batch_norm
if grad_var is None:
continue
persistable_var._set_grad_type(grad_var.type())
# 3. prepare output, keep same form with inputs
outs = output_vars
if len(output_vars) == 1:
outs = output_vars[0]
return outs
def test_to_bytes(self):
# Only support python2.x and python3.x now
self.assertTrue(six.PY2 | six.PY3)
if six.PY2:
# check None
self.assertIsNone(cpt.to_bytes(None))
# check all string related types
self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes))
self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes))
self.assertTrue(isinstance(cpt.to_bytes(b""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(b""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(u""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(u""), bytes))
self.assertEqual(b"", cpt.to_bytes(str("")))
self.assertEqual(b"123", cpt.to_bytes(str("123")))
self.assertEqual(b"", cpt.to_bytes(b""))
self.assertEqual(b"123", cpt.to_bytes(b"123"))
self.assertEqual(b"", cpt.to_bytes(u""))
self.assertEqual(b"123", cpt.to_bytes(u"123"))
# check list types, not inplace
l = [""]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b""], l2)
l = ["", "123"]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123"], l2)
l = ["", b'123', u"321"]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123", b"321"], l2)
for i in l2:
self.assertTrue(isinstance(i, bytes))
# check list types, inplace
l = [""]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b""], l2)
l = ["", "123"]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123"], l2)
l = ["", b"123", u"321"]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123", b"321"], l2)
# check set types, not inplace
l = set("")
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set(b""), l2)
l = set([b"", b"123"])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123"]), l2)
l = set(["", b"123", u"321"])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123", b"321"]), l2)
for i in l2:
self.assertTrue(isinstance(i, bytes))
# check set types, inplace
l = set("")
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set(b""), l2)
l = set([b"", b"123"])
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123"]), l2)
l = set(["", b"123", u"321"])
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123", b"321"]), l2)
elif six.PY3:
self.assertIsNone(cpt.to_bytes(None))
self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes))
self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes))
self.assertTrue(isinstance(cpt.to_bytes(b""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(b""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(u""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(u""), bytes))
self.assertEqual(b"", cpt.to_bytes(str("")))
self.assertEqual(b"123", cpt.to_bytes(str("123")))
self.assertEqual(b"", cpt.to_bytes(b""))
self.assertEqual(b"123", cpt.to_bytes(b"123"))
self.assertEqual(b"", cpt.to_bytes(u""))
self.assertEqual(b"123", cpt.to_bytes(u"123"))
# check list types, not inplace
l = [""]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual([b""], l2)
l = ["", "123"]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual([b"", b"123"], l2)
l = ["", b"123", u"321"]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual([b"", b"123", b"321"], l2)
# check list types, inplace
l = [""]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b""], l2)
l = ["", b"123"]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123"], l2)
l = ["", b"123", u"321"]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123", b"321"], l2)
for i in l2:
self.assertTrue(isinstance(i, bytes))
# check set types, not inplace
l = set([""])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual(set([b""]), l2)
l = set([u"", u"123"])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual(set([b"", b"123"]), l2)
l = set(["", b"123", u"321"])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual(set([b"", b"123", b"321"]), l2)
# check set types, inplace
l = set("")
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set(b""), l2)
l = set([u"", u"123"])
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123"]), l2)
l = set(["", b"123", u"321"])
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123", b"321"]), l2)
for i in l2:
self.assertTrue(isinstance(i, bytes))
def get_attr_default_value(op_name):
return core.get_op_attrs_default_value(cpt.to_bytes(op_name))
def test_get_op_attrs_default_value(self):
core.get_op_attrs_default_value(cpt.to_bytes("fill_constant"))
def __init__(self,
block,
type=core.VarDesc.VarType.LOD_TENSOR,
name=None,
shape=None,
dtype=None,
lod_level=None,
capacity=None,
persistable=None,
error_clip=None,
stop_gradient=False,
is_data=False,
need_check_feed=False,
belong_to_optimizer=False,
**kwargs):
self.block = block
if name is None:
name = unique_name.generate('_generated_var')
if dtype is not None:
if not isinstance(dtype, core.VarDesc.VarType):
dtype = convert_np_dtype_to_dtype_(dtype)
self.belong_to_optimizer = belong_to_optimizer
self.error_clip = error_clip
is_new_var = False
name = cpt.to_text(name)
self.desc = self.block.desc.find_var(cpt.to_bytes(name))
if self.desc is None:
self.desc = self.block.desc.var(cpt.to_bytes(name))
is_new_var = True
if is_new_var:
self.desc.set_type(type)
elif self.desc.type() != type:
raise ValueError("MpcVariable {0} has been created before. The "
"previous type is {1}; the new type is {2}. They"
" are not matched".format(self.name,
self.desc.type(), type))
if shape is not None:
if is_new_var:
# resize the shape for MpcVariable
mpc_shape = list(shape)
mpc_shape.insert(0, 2)
self.desc.set_shape(mpc_shape)
else:
old_shape = self.shape
shape = tuple(shape)
if shape != old_shape:
raise ValueError(
"MpcVariable {0} has been created before. the previous "
"shape is {1}; the new shape is {2}. They are not "
"matched.".format(self.name, old_shape, shape))
if dtype is not None:
if is_new_var:
self.desc.set_dtype(dtype)
else:
old_dtype = self.dtype
if dtype != old_dtype:
raise ValueError(
"MpcVariable {0} has been created before. "
"The previous data type is {1}; the new "
"data type is {2}. They are not "
"matched.".format(self.name, old_dtype, dtype))
if lod_level is not None:
if is_new_var:
self.desc.set_lod_level(lod_level)
else:
if lod_level != self.lod_level:
raise ValueError(
"MpcVariable {0} has been created before. "
"The previous lod_level is {1}; the new "
"lod_level is {2}. They are not "
"matched".format(self.name, self.lod_level, lod_level))
if persistable is not None:
if is_new_var:
self.desc.set_persistable(persistable)
else:
if persistable != self.persistable:
raise ValueError(
"MpcVariable {0} has been created before."
"The previous persistable is {1}; the new "
"persistable is {2}. They are not matched".format(
self.name, self.persistable, persistable))
if need_check_feed and is_new_var:
self.desc.set_need_check_feed(need_check_feed)
if capacity is not None:
if is_new_var:
self.desc.set_capacity(capacity)
else:
# TODO(abhinavarora) by Paddle 1.7: Compare with set capacity once,
# get_capacity is implemented
pass
self.block.vars[name] = self
self.op = None
self._stop_gradient = stop_gradient
self.is_data = is_data
def create_fake_model(program_config):
''' Create a Paddle model(in memory) according to the given config. '''
paddle.enable_static()
main_program_desc = core.ProgramDesc()
util_program = fluid.Program()
main_block_desc = main_program_desc.block(0)
var_desc = main_block_desc.var(cpt.to_bytes("feed"))
var_desc.set_type(core.VarDesc.VarType.FEED_MINIBATCH)
var_desc.set_persistable(True)
index = 0
for name, tensor_config in program_config.inputs.items():
var_desc = main_block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype))
var_desc.set_shape(tensor_config.shape)
var_desc.set_need_check_feed(True)
if tensor_config.lod is not None:
var_desc.set_lod_level(len(tensor_config.lod))
op_desc = main_block_desc._prepend_op()
op_desc.set_type("feed")
op_desc.set_input('X', ["feed"])
op_desc.set_output('Out', [name])
op_desc._set_attr("col", index)
index = index + 1
save_var_map = {}
for name, tensor_config in program_config.weights.items():
var_desc = main_block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype))
var_desc.set_shape(tensor_config.shape)
var_desc.set_persistable(True)
save_var_map[name] = util_program.global_block().create_parameter(
dtype=tensor_config.dtype,
shape=tensor_config.shape,
type=core.VarDesc.VarType.LOD_TENSOR,
name=name,
initializer=NumpyArrayInitializer(tensor_config.data))
in_vars = []
for name in sorted(save_var_map.keys()):
in_vars.append(save_var_map[name])
out_var = util_program.global_block().create_var(
type=core.VarDesc.VarType.RAW, name="out_var_0")
out_var.desc.set_persistable(True)
util_program.global_block().append_op(type='save_combine',
inputs={'X': in_vars},
outputs={'Y': out_var},
attrs={
'file_path': '',
'save_to_memory': True
})
for op_config in program_config.ops:
op_desc = main_block_desc.append_op()
op_desc.set_type(op_config.type)
for name, values in op_config.inputs.items():
op_desc.set_input(name, values)
for name, values in op_config.attrs.items():
op_desc._set_attr(name, values)
for name, values in op_config.outputs.items():
op_desc.set_output(name, values)
for v in values:
var_desc = main_block_desc.var(cpt.to_bytes(v))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32))
if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys(
):
var_desc.set_dtype(
convert_np_dtype_to_dtype_(op_config.outputs_dtype[v]))
op_desc.infer_var_type(main_block_desc)
op_desc.infer_shape(main_block_desc)
op_desc.check_attrs()
for index, name in enumerate(program_config.outputs):
var_desc = main_block_desc.var(cpt.to_bytes("fetch"))
var_desc.set_type(core.VarDesc.VarType.FETCH_LIST)
var_desc.set_need_check_feed(True)
op_desc = main_block_desc.append_op()
op_desc.set_type("fetch")
op_desc.set_input('X', [name])
op_desc.set_output('Out', ["fetch"])
op_desc._set_attr("col", index)
main_program_desc._set_version()
paddle.fluid.core.save_op_version_info(main_program_desc)
model = main_program_desc.serialize_to_string()
util_program._sync_with_cpp()
place = fluid.CPUPlace()
executor = fluid.Executor(place)
scope = fluid.Scope()
with fluid.scope_guard(scope):
executor.run(util_program)
params = scope.find_var("out_var_0").get_bytes()
return model, params
def find_var_by_name(block, name):
return block.desc.var(cpt.to_bytes(name))
def _preprocess(self, program_desc):
# 1. Prune original program
# remove feed, fetch and scale-1 op, remove op_callstack attr
ops_to_remove = []
root_block = program_desc.block(0)
for i in six.moves.range(root_block.op_size()):
op = root_block.op(i)
if op.type() == 'feed':
ops_to_remove.append(i)
feed_var_name = cpt.to_bytes(op.input('X')[0])
root_block._remove_var(feed_var_name)
self._input_names.append(cpt.to_bytes(op.output('Out')[0]))
elif op.type() == 'scale' and op.output('Out')[0].startswith(
'save_infer_model/scale_'):
ops_to_remove.append(i)
out_var_name = cpt.to_bytes(op.output('Out')[0])
root_block._remove_var(out_var_name)
self._output_descs.append(
root_block.find_var(cpt.to_bytes(op.input('X')[0])))
elif op.type() == 'fetch':
ops_to_remove.append(i)
fetch_var_name = cpt.to_bytes(op.output('Out')[0])
root_block._remove_var(fetch_var_name)
# NOTE: some old pre-train models have no extra scale_op
if not op.input('X')[0].startswith('save_infer_model/scale_'):
self._output_descs.append(
root_block.find_var(cpt.to_bytes(op.input('X')[0])))
else:
if op.has_attr("op_callstack"):
op.remove_attr("op_callstack")
for op_idx in reversed(ops_to_remove):
root_block._remove_op(op_idx, op_idx + 1)
# 2. Input processing, reverse feed vars
self._input_names.reverse()
# 3. Output processing, add scale for outputs
tmp_program = _build_program_by_desc(program_desc)
# NOTE: [why need append scale for outputs]
# When dealing with some more complex pre-training models, there
# will be situations where the pre-training model has multiple
# fetch outputs. In the scenario of multiple fetch outputs,
# there is a special case where multiple outputs of the model
# may be on the same branch. According to the user's subsequent
# use, multiple outputs may be associated with multiple branches.
# These subsequent operations are added in TranslatedLayer is
# agnostic during initialization, which results in subsequent
# gradient accumulation operations that are required on the
# output node in the middle of the branch will not be performed,
# resulting in error, details see pull request:
# [https://github.com/PaddlePaddle/Paddle/pull/24627]
self._append_scale_to_output(tmp_program)
# 4. Persistable vars processing
# - append loaded suffix to persistable vars
# NOTE: [why need to append suffix to persistable vars]
# Dygraph and static graph mode use the same naming mechanism.
# If users want to load the model fine-tune, it is possible
# to add the existing Layer in the loaded model to enhance
# the network. For example, the original saved model has linear,
# and later after loading, a new linear is added. At this time,
# there will be a problem of duplicate names, so here is unified
# to add the LOADED suffix to the parameters of the model loaded
self._suffix_varname_dict = _append_loaded_suffix_to_var(program_desc)
# - get persistable var
self._persistable_names = _get_persistable_var_names(program_desc)
return program_desc
def test_to_bytes(self):
self.assertIsNone(cpt.to_bytes(None))
self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes))
self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes))
self.assertTrue(isinstance(cpt.to_bytes(b""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(b""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(u""), bytes))
self.assertTrue(isinstance(cpt.to_bytes(u""), bytes))
self.assertEqual(b"", cpt.to_bytes(str("")))
self.assertEqual(b"123", cpt.to_bytes(str("123")))
self.assertEqual(b"", cpt.to_bytes(b""))
self.assertEqual(b"123", cpt.to_bytes(b"123"))
self.assertEqual(b"", cpt.to_bytes(u""))
self.assertEqual(b"123", cpt.to_bytes(u"123"))
# check list types, not inplace
l = [""]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual([b""], l2)
l = ["", "123"]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual([b"", b"123"], l2)
l = ["", b"123", u"321"]
l2 = cpt.to_bytes(l)
self.assertTrue(isinstance(l2, list))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual([b"", b"123", b"321"], l2)
# check list types, inplace
l = [""]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b""], l2)
l = ["", b"123"]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123"], l2)
l = ["", b"123", u"321"]
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, list))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual([b"", b"123", b"321"], l2)
for i in l2:
self.assertTrue(isinstance(i, bytes))
# check set types, not inplace
l = set([""])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual(set([b""]), l2)
l = set([u"", u"123"])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual(set([b"", b"123"]), l2)
l = set(["", b"123", u"321"])
l2 = cpt.to_bytes(l, inplace=False)
self.assertTrue(isinstance(l2, set))
self.assertFalse(l is l2)
self.assertNotEqual(l, l2)
self.assertEqual(set([b"", b"123", b"321"]), l2)
# check set types, inplace
l = set("")
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set(b""), l2)
l = set([u"", u"123"])
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123"]), l2)
l = set(["", b"123", u"321"])
l2 = cpt.to_bytes(l, inplace=True)
self.assertTrue(isinstance(l2, set))
self.assertTrue(l is l2)
self.assertEqual(l, l2)
self.assertEqual(set([b"", b"123", b"321"]), l2)
for i in l2:
self.assertTrue(isinstance(i, bytes))
