def setUp(self):
'''
Test eye op with specified shape
'''
self.set_npu()
self.place = paddle.NPUPlace(0)
self.op_type = "eye"
self.inputs = {}
self.num_rows = 0
self.num_columns = 0
self.dtype = np.float32
self.initTestCase()
if self.num_columns == 0:
self.attrs = {
'num_rows': self.num_rows,
'dtype': framework.convert_np_dtype_to_dtype_(self.dtype)
}
self.outputs = {'Out': np.eye(self.num_rows, dtype=self.dtype)}
else:
self.attrs = {
'num_rows': self.num_rows,
'num_columns': self.num_columns,
'dtype': framework.convert_np_dtype_to_dtype_(self.dtype)
}
self.outputs = {
'Out': np.eye(self.num_rows,
self.num_columns,
dtype=self.dtype)
}
def setUp(self):
self.op_type = "reduce_sum"
self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")}
self.outputs = {'Out': self.inputs['X'].sum().astype('float64')}
self.attrs = {'reduce_all': True}
self.attrs.update({
'in_dtype':
int(convert_np_dtype_to_dtype_(np.float32)),
'out_dtype':
int(convert_np_dtype_to_dtype_(np.float64))
})
def setUp(self):
self.op_type = "reduce_sum"
self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")}
self.outputs = {'Out': self.inputs['X'].sum(axis=1, keepdims=True)}
self.attrs = {'dim': [1], 'keep_dim': True}
self.attrs.update({
'in_dtype':
int(convert_np_dtype_to_dtype_(np.float32)),
'out_dtype':
int(convert_np_dtype_to_dtype_(np.float64))
})
def fill_block_desc(self, block_desc):
for name in self.vars:
var_desc = block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
if self.vars_lod_level is not None and name in self.vars_lod_level.keys(
):
var_desc.set_lod_level(self.vars_lod_level[name])
if self.vars_var_type is not None and name in self.vars_var_type.keys(
):
if self.vars_var_type[name] == VarType.LOD_TENSOR_ARRAY:
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR_ARRAY)
elif self.vars_var_type[name] == VarType.STEP_SCOPES:
var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES)
continue
var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32))
if self.vars_dtype is not None and name in self.vars_dtype.keys():
var_desc.set_dtype(
convert_np_dtype_to_dtype_(self.vars_dtype[name]))
for op_config in self.ops:
op_desc = block_desc.append_op()
op_desc.set_type(op_config.type)
for name, values in op_config.inputs.items():
op_desc.set_input(name, values)
for name, values in op_config.attrs.items():
op_desc._set_attr(name, values)
for name, values in op_config.outputs.items():
op_desc.set_output(name, values)
for v in values:
if block_desc.has_var_recursive(cpt.to_bytes(v)):
continue
var_desc = block_desc.var(cpt.to_bytes(v))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
if op_config.outputs_var_type is not None and v in op_config.outputs_var_type.keys(
):
if op_config.outputs_var_type[
v] == VarType.LOD_TENSOR_ARRAY:
var_desc.set_type(
core.VarDesc.VarType.LOD_TENSOR_ARRAY)
elif op_config.outputs_var_type[
v] == VarType.STEP_SCOPES:
var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES)
continue
var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32))
if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys(
):
var_desc.set_dtype(
convert_np_dtype_to_dtype_(
op_config.outputs_dtype[v]))
if op_config.type not in _OP_WITHOUT_KERNEL_SET:
op_desc.infer_var_type(block_desc)
op_desc.infer_shape(block_desc)
op_desc.check_attrs()
def test_from_numpy(self):
x_numpy = np.ones([10, 12])
x_np_spec = InputSpec.from_numpy(x_numpy)
self.assertEqual(x_np_spec.dtype,
convert_np_dtype_to_dtype_(x_numpy.dtype))
self.assertEqual(x_np_spec.shape, x_numpy.shape)
self.assertEqual(x_np_spec.name, None)
x_numpy2 = np.array([1, 2, 3, 4]).astype('int64')
x_np_spec2 = InputSpec.from_numpy(x_numpy2, name='x_np_int64')
self.assertEqual(x_np_spec2.dtype,
convert_np_dtype_to_dtype_(x_numpy2.dtype))
self.assertEqual(x_np_spec2.shape, x_numpy2.shape)
self.assertEqual(x_np_spec2.name, 'x_np_int64')
def setUp(self):
self.op_type = "fill_zeros_like2"
self.dtype = np.float32
self.init_dtype()
self.inputs = {'X': np.random.random((219, 232)).astype(self.dtype)}
self.outputs = {'Out': np.zeros_like(self.inputs["X"])}
self.attrs = {'dtype': convert_np_dtype_to_dtype_(self.dtype)}
def init_config(self):
shape = [500, 3]
dtype = 'bool'
dtype_inner = convert_np_dtype_to_dtype_(dtype)
self.attrs = {'shape': shape, 'dtype': dtype_inner}
self.inputs = {}
self.outputs = {'Out': np.zeros(shape).astype(dtype)}
def init_config(self):
self.shape = [500, 3]
dtype = 'float32'
dtype_inner = convert_np_dtype_to_dtype_(dtype)
self.attrs = {'dtype': dtype_inner}
self.inputs = {"ShapeTensor": np.array(self.shape).astype("int32")}
self.outputs = {'Out': np.zeros(self.shape).astype(dtype)}
def setUp(self):
self.op_type = "assign_value"
self.inputs = {}
self.attrs = {}
self.init_data()
self.attrs["shape"] = self.value.shape
self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_(
self.value.dtype)
self.outputs = {"Out": self.value}
def __init__(self, shape, dtype='float32', name=None):
# replace `None` in shape with -1
self.shape = self._verify(shape)
# convert dtype into united represention
if dtype is not None:
if not isinstance(dtype, core.VarDesc.VarType):
dtype = convert_np_dtype_to_dtype_(dtype)
self.dtype = dtype
self.name = name
def setUp(self):
self.initDefaultParameters()
self.initParameters()
if not isinstance(self.x, np.ndarray):
self.x = np.array(self.x)
self.inputs = {'X': self.x, 'MaxLenTensor': self.maxlen_tensor}
self.outputs = {'Y': self.calc_ground_truth_mask()}
self.attrs = {'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype)}
def setUp(self):
self.op_type = "assign_value"
x = numpy.random.random(size=(2, 5)).astype(numpy.float32)
self.inputs = {}
self.outputs = {'Out': x}
self.attrs = {
'shape': x.shape,
'dtype': framework.convert_np_dtype_to_dtype_(x.dtype),
'fp32_values': [float(v) for v in x.flat]
}
def unique_segment(data, dtype="int64"):
"""Return Segment Id from data
"""
if in_dygraph_mode():
attr_dtype = convert_np_dtype_to_dtype_(dtype)
unique, index, _ = core.ops.unique_with_counts(data, "dtype",
attr_dtype)
return unique, index
else:
unique, index, _ = L.unique_with_counts(data)
return unique, index
def setUp(self):
'''
Test eye op with specified shape
'''
self.op_type = "eye"
self.inputs = {}
self.attrs = {
'num_rows': 219,
'num_columns': 319,
'dtype': framework.convert_np_dtype_to_dtype_(np.int32)
}
self.outputs = {'Out': np.eye(219, 319, dtype=np.int32)}
def compare(self):
a = np.array([[1.0 + 1.0j, 2.0 + 1.0j],
[3.0 + 1.0j, 4.0 + 1.0j]]).astype(self._dtype)
b = np.array([[1.0 + 1.0j, 1.0 + 1.0j]]).astype(self._dtype)
with dg.guard():
x = dg.to_variable(a, "x")
y = dg.to_variable(b)
out = paddle.fluid.layers.elementwise_add(x, y)
self.assertIsNotNone("{}".format(out))
self.assertTrue(np.allclose(out.numpy(), a + b))
self.assertEqual(out.dtype, convert_np_dtype_to_dtype_(self._dtype))
self.assertEqual(out.shape, x.shape)
def setUp(self):
self.op_type = "fill_any_like"
self.python_api = paddle.full_like
self.init_data()
x = np.zeros(self.shape)
out = np.full_like(x, self.fill_value, self.dtype)
self.inputs = {'X': x}
self.outputs = {'Out': out}
self.attrs = {
'value': self.fill_value,
'dtype': convert_np_dtype_to_dtype_(self.dtype)
}
def init_config(self):
self.shape = [123, 92]
self.infer_shape = [-1, 92]
dtype = 'float32'
dtype_inner = convert_np_dtype_to_dtype_(dtype)
shape_tensor_list = []
for index, ele in enumerate(self.shape):
shape_tensor_list.append(("x" + str(index), np.ones(
(1)).astype('int32') * ele))
self.inputs = {"ShapeTensorList": shape_tensor_list}
self.attrs = {'shape': self.infer_shape, 'dtype': dtype_inner}
self.outputs = {'Out': np.zeros(self.shape).astype(dtype)}
def setUp(self):
self.op_type = "fill_constant_batch_size_like"
self.python_api = fill_constant_batch_size_like
self.init_data()
input = np.zeros(self.shape)
out = np.full_like(input, self.value, self.dtype)
self.inputs = {'Input': input}
self.outputs = {'Out': out}
self.attrs = {
'shape': self.shape,
'dtype': convert_np_dtype_to_dtype_(self.dtype),
'value': self.value,
'input_dim_idx': self.input_dim_idx,
'output_dim_idx': self.output_dim_idx,
'force_cpu': self.force_cpu
}
def test_shape_with_none(self):
tensor_spec = InputSpec([None, 4, None], dtype='int8', name='x_spec')
self.assertEqual(tensor_spec.dtype, convert_np_dtype_to_dtype_('int8'))
self.assertEqual(tensor_spec.name, 'x_spec')
self.assertEqual(tensor_spec.shape, (-1, 4, -1))
def test_default(self):
tensor_spec = InputSpec([3, 4])
self.assertEqual(tensor_spec.dtype,
convert_np_dtype_to_dtype_('float32'))
self.assertEqual(tensor_spec.name, None)
def create_fake_model(program_config):
''' Create a Paddle model(in memory) according to the given config. '''
paddle.enable_static()
main_program_desc = core.ProgramDesc()
util_program = fluid.Program()
main_block_desc = main_program_desc.block(0)
var_desc = main_block_desc.var(cpt.to_bytes("feed"))
var_desc.set_type(core.VarDesc.VarType.FEED_MINIBATCH)
var_desc.set_persistable(True)
index = 0
for name, tensor_config in program_config.inputs.items():
var_desc = main_block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype))
var_desc.set_shape(tensor_config.shape)
var_desc.set_need_check_feed(True)
if tensor_config.lod is not None:
var_desc.set_lod_level(len(tensor_config.lod))
op_desc = main_block_desc._prepend_op()
op_desc.set_type("feed")
op_desc.set_input('X', ["feed"])
op_desc.set_output('Out', [name])
op_desc._set_attr("col", index)
index = index + 1
save_var_map = {}
for name, tensor_config in program_config.weights.items():
var_desc = main_block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype))
var_desc.set_shape(tensor_config.shape)
var_desc.set_persistable(True)
save_var_map[name] = util_program.global_block().create_parameter(
dtype=tensor_config.dtype,
shape=tensor_config.shape,
type=core.VarDesc.VarType.LOD_TENSOR,
name=name,
initializer=NumpyArrayInitializer(tensor_config.data))
in_vars = []
for name in sorted(save_var_map.keys()):
in_vars.append(save_var_map[name])
out_var = util_program.global_block().create_var(
type=core.VarDesc.VarType.RAW, name="out_var_0")
out_var.desc.set_persistable(True)
util_program.global_block().append_op(type='save_combine',
inputs={'X': in_vars},
outputs={'Y': out_var},
attrs={
'file_path': '',
'save_to_memory': True
})
for op_config in program_config.ops:
op_desc = main_block_desc.append_op()
op_desc.set_type(op_config.type)
for name, values in op_config.inputs.items():
op_desc.set_input(name, values)
for name, values in op_config.attrs.items():
op_desc._set_attr(name, values)
for name, values in op_config.outputs.items():
op_desc.set_output(name, values)
for v in values:
var_desc = main_block_desc.var(cpt.to_bytes(v))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32))
if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys(
):
var_desc.set_dtype(
convert_np_dtype_to_dtype_(op_config.outputs_dtype[v]))
op_desc.infer_var_type(main_block_desc)
op_desc.infer_shape(main_block_desc)
op_desc.check_attrs()
for index, name in enumerate(program_config.outputs):
var_desc = main_block_desc.var(cpt.to_bytes("fetch"))
var_desc.set_type(core.VarDesc.VarType.FETCH_LIST)
var_desc.set_need_check_feed(True)
op_desc = main_block_desc.append_op()
op_desc.set_type("fetch")
op_desc.set_input('X', [name])
op_desc.set_output('Out', ["fetch"])
op_desc._set_attr("col", index)
main_program_desc._set_version()
paddle.fluid.core.save_op_version_info(main_program_desc)
model = main_program_desc.serialize_to_string()
util_program._sync_with_cpp()
place = fluid.CPUPlace()
executor = fluid.Executor(place)
scope = fluid.Scope()
with fluid.scope_guard(scope):
executor.run(util_program)
params = scope.find_var("out_var_0").get_bytes()
return model, params
def __init__(self,
block,
type=core.VarDesc.VarType.LOD_TENSOR,
name=None,
shape=None,
dtype=None,
lod_level=None,
capacity=None,
persistable=None,
error_clip=None,
stop_gradient=False,
is_data=False,
need_check_feed=False,
belong_to_optimizer=False,
**kwargs):
self.block = block
if name is None:
name = unique_name.generate('_generated_var')
if dtype is not None:
if not isinstance(dtype, core.VarDesc.VarType):
dtype = convert_np_dtype_to_dtype_(dtype)
self.belong_to_optimizer = belong_to_optimizer
self.error_clip = error_clip
is_new_var = False
name = cpt.to_text(name)
self.desc = self.block.desc.find_var(cpt.to_bytes(name))
if self.desc is None:
self.desc = self.block.desc.var(cpt.to_bytes(name))
is_new_var = True
if is_new_var:
self.desc.set_type(type)
elif self.desc.type() != type:
raise ValueError("MpcVariable {0} has been created before. The "
"previous type is {1}; the new type is {2}. They"
" are not matched".format(self.name,
self.desc.type(), type))
if shape is not None:
if is_new_var:
# resize the shape for MpcVariable
mpc_shape = list(shape)
mpc_shape.insert(0, 2)
self.desc.set_shape(mpc_shape)
else:
old_shape = self.shape
shape = tuple(shape)
if shape != old_shape:
raise ValueError(
"MpcVariable {0} has been created before. the previous "
"shape is {1}; the new shape is {2}. They are not "
"matched.".format(self.name, old_shape, shape))
if dtype is not None:
if is_new_var:
self.desc.set_dtype(dtype)
else:
old_dtype = self.dtype
if dtype != old_dtype:
raise ValueError(
"MpcVariable {0} has been created before. "
"The previous data type is {1}; the new "
"data type is {2}. They are not "
"matched.".format(self.name, old_dtype, dtype))
if lod_level is not None:
if is_new_var:
self.desc.set_lod_level(lod_level)
else:
if lod_level != self.lod_level:
raise ValueError(
"MpcVariable {0} has been created before. "
"The previous lod_level is {1}; the new "
"lod_level is {2}. They are not "
"matched".format(self.name, self.lod_level, lod_level))
if persistable is not None:
if is_new_var:
self.desc.set_persistable(persistable)
else:
if persistable != self.persistable:
raise ValueError(
"MpcVariable {0} has been created before."
"The previous persistable is {1}; the new "
"persistable is {2}. They are not matched".format(
self.name, self.persistable, persistable))
if need_check_feed and is_new_var:
self.desc.set_need_check_feed(need_check_feed)
if capacity is not None:
if is_new_var:
self.desc.set_capacity(capacity)
else:
# TODO(abhinavarora) by Paddle 1.7: Compare with set capacity once,
# get_capacity is implemented
pass
self.block.vars[name] = self
self.op = None
self._stop_gradient = stop_gradient
self.is_data = is_data
def tdm_child(x, node_nums, child_nums, param_attr=None, dtype='int32'):
"""
**Tdm Child**
According to the input node_id on the given tree, return the corresponding child node_id and
whether child is a leaf node by leaf_mask value.
.. code-block:: text
Given:
tree[[0], [1, 2], [3, 4], [5, 6]] # A binary tree with seven nodes
x = [[2], [3]]
node_nums = 7
child_nums = 2
we get:
child = [[5, 6],
[0, 0]]
leaf_mask = [[1, 1],
[0, 0]]
Args:
x(Variable): Variable contained the node_id information, dtype support int32/int64.
node_nums(int): Number of total nodes.
child_nums(int): Maximum number of child nodes per node.
param_attr(ParamAttr): To specify the tdm-tree-info parameter property. Default: None, which means the
default weight parameter property is used. See usage for details in: ref: `api_fluid_ParamAttr`, should
has shape(node_nums, 3 + child_nums), dtype support int32/int64.
The dimension[1] of tdm-tree-info contains the following:
1. Item_id(int, shape(1)), if node is a leaf node, give its item_id corresponding to node_id, else give 0.
2. Layer_id(int, shape(1)), indicates which layer the node is on.
3. Parent_id(int, shape(1)), node's parent node.
4. Child_id(int, shape(child_nums)), all child node's node_id of this node should be given.
If the number of child nodes is insufficient, padding 0 until child nums equal to child_nums
dtype(str): The data type of output child and leaf_mask, support int32/int64.
Returns:
tuple: A tuple including input node's child(Variable) and leaf_mask(Variable).
If child is a leaf node, leaf_mask equal ot 1, otherwise equal to 0.
Examples:
.. code-block:: python
import paddle.fluid as fluid
import numpy as np
x = fluid.data(name="x", shape=[None, 1], dtype="int32", lod_level=1)
tree_info = [[0,0,0,1,2],
[0,1,0,3,4],[0,1,0,5,6],
[0,2,1,0,0],[1,2,1,0,0],[2,2,2,0,0],[3,2,2,0,0]]
tree_info_np = np.array(tree_info)
tree_info_np = np.reshape(tree_info_np, (7,5))
node_nums = 7
child_nums = 2
child, leaf_mask = fluid.contrib.layers.tdm_child(x, node_nums, child_nums,
param_attr=fluid.ParamAttr(
initializer=fluid.initializer.NumpyArrayInitializer(
tree_info_np)))
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
xx = np.array([[2],[3]]).reshape((2,1)).astype("int32")
child_res, leaf_mask_res = exe.run(feed={"x":xx}, fetch_list=[child, leaf_mask])
"""
helper = LayerHelper("tdm_child", **locals())
check_dtype(dtype, 'dtype', ['int32', 'int64'],
'fluid.contrib.layers.tdm_child')
c_dtype = convert_np_dtype_to_dtype_(dtype)
tree_info = helper.create_parameter(attr=helper.param_attr,
shape=[node_nums, 3 + child_nums],
dtype=dtype,
default_initializer=Constant(0))
tree_info.stop_gradient = True
child = helper.create_variable_for_type_inference(dtype=dtype)
leaf_mask = helper.create_variable_for_type_inference(dtype=dtype)
helper.append_op(type='tdm_child',
inputs={
'X': x,
'TreeInfo': tree_info
},
outputs={
'Child': child,
'LeafMask': leaf_mask
},
attrs={
'child_nums': child_nums,
'dtype': c_dtype
},
stop_gradient=True)
return (child, leaf_mask)
def test_convert_np_dtype_to_dtype(self):
self.assertEqual(convert_np_dtype_to_dtype_(np.complex64),
core.VarDesc.VarType.COMPLEX64)
self.assertEqual(convert_np_dtype_to_dtype_(np.complex64),
core.VarDesc.VarType.COMPLEX64)
