def test_instance(self):
program_desc = core.ProgramDesc()
self.assertIsNotNone(program_desc)
del program_desc
program_desc = core.ProgramDesc()
self.assertIsNotNone(program_desc)
self.assertIsNotNone(program_desc.block(0))
del program_desc
def test_sum_op(self):
prog = core.ProgramDesc()
self.assertIsNotNone(prog)
block = prog.block(0)
self.assertIsNotNone(block)
shape = [10, 20]
# prepare input/output
x1 = block.var(six.b("x1"))
x1.set_type(core.VarDesc.VarType.LOD_TENSOR)
x1.set_shape(shape)
x2 = block.var(six.b("x2"))
x2.set_type(core.VarDesc.VarType.LOD_TENSOR)
x2.set_shape(shape)
out = block.var(six.b("out"))
out.set_type(core.VarDesc.VarType.LOD_TENSOR)
# prepare the operator
sum_op_desc = block.append_op()
sum_op_desc.set_type("sum")
sum_op_desc.set_input("X", ["x1", "x2"])
sum_op_desc.set_output("Out", ["out"])
sum_op_desc.check_attrs()
sum_op_desc.infer_shape(block)
self.assertEqual(out.shape(), shape)
def test_mul_op(self):
prog = core.ProgramDesc()
self.assertIsNotNone(prog)
block = prog.block(0)
self.assertIsNotNone(block)
x_shape = [10, 20]
y_shape = [20, 30]
# prepare input/output
x1 = block.var(six.b("x"))
x1.set_type(core.VarDesc.VarType.LOD_TENSOR)
x1.set_shape(x_shape)
x2 = block.var(six.b("y"))
x2.set_type(core.VarDesc.VarType.LOD_TENSOR)
x2.set_shape(y_shape)
out = block.var(six.b("out"))
out.set_type(core.VarDesc.VarType.LOD_TENSOR)
# prepare the operator
mul_op_desc = block.append_op()
mul_op_desc.set_type("mul")
mul_op_desc.set_input("X", ["x"])
mul_op_desc.set_input("Y", ["y"])
mul_op_desc.set_output("Out", ["out"])
mul_op_desc._set_attr("x_num_col_dims", 1)
mul_op_desc._set_attr("y_num_col_dims", 1)
mul_op_desc.check_attrs()
mul_op_desc.infer_shape(block)
self.assertEqual(out.shape(), [x_shape[0], y_shape[1]])
def test_expand_op(self):
prog = core.ProgramDesc()
self.assertIsNotNone(prog)
block = prog.block(0)
self.assertIsNotNone(block)
shape = [-1, 20]
expand_times = [3, 1]
# prepare input/output
x1 = block.var(six.b("x"))
x1.set_type(core.VarDesc.VarType.LOD_TENSOR)
x1.set_shape(shape)
out = block.var(six.b("out"))
out.set_type(core.VarDesc.VarType.LOD_TENSOR)
# prepare the operator
sum_op_desc = block.append_op()
sum_op_desc.set_type("expand")
sum_op_desc.set_input("X", ["x"])
sum_op_desc.set_input('expand_times_tensor', [])
sum_op_desc.set_output("Out", ["out"])
sum_op_desc._set_attr('expand_times', expand_times)
sum_op_desc.check_attrs()
sum_op_desc.infer_shape(block)
self.assertEqual(out.shape(), shape)
def test_dtype(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_var'))
var.set_type(core.VarDesc.VarType.LOD_TENSOR)
var.set_dtype(core.VarDesc.VarType.INT32)
self.assertEqual(core.VarDesc.VarType.INT32, var.dtype())
self.assertEqual(core.VarDesc.VarType.LOD_TENSOR, var.type())
def test_multiple_lod_level(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_reader'))
var.set_type(core.VarDesc.VarType.READER)
src_types = [3, 1, 2]
var.set_lod_levels(src_types)
self.assertEqual(src_types, var.lod_levels())
self.assertEqual(core.VarDesc.VarType.READER, var.type())
def test_op_desc(self):
program_desc = core.ProgramDesc()
self.assertIsNotNone(program_desc)
block = program_desc.block(0)
self.assertIsNotNone(block)
op = block.append_op()
self.assertIsNotNone(op)
op.set_type("test")
self.assertEqual("test", op.type())
op.set_input("X", ["a", "b", "c"])
self.assertEqual(["a", "b", "c"], op.input("X"))
self.assertEqual(["X"], op.input_names())
op.set_output("Out", ["z"])
self.assertEqual(['z'], op.output("Out"))
self.assertEqual(["Out"], op.output_names())
op._set_attr("int_attr", 1)
self.assertEqual(1, op.attr("int_attr"))
self.assertTrue(op.has_attr("int_attr"))
self.assertEqual(core.AttrType.INT, op.attr_type("int_attr"))
op._set_attr("float_attr", -1.32)
self.assertAlmostEqual(-1.32, op.attr("float_attr"), delta=1e-4)
self.assertTrue(op.has_attr("float_attr"))
op._set_attr("bool_attr", False)
self.assertFalse(op.attr("bool_attr"))
op._set_attr("string_attr", "abc")
self.assertEqual("abc", op.attr("string_attr"))
self.assertTrue(op.has_attr("string_attr"))
op._set_attr("ints_attr", [1, 2, 3])
self.assertEqual([1, 2, 3], op.attr("ints_attr"))
expected = [1.2, 2.3, 3.4]
op._set_attr("floats_attr", expected)
for e, a in zip(expected, op.attr("floats_attr")):
self.assertAlmostEqual(e, a, delta=1e-4)
op._set_attr("strings_attr", ["a", "b", "c"])
self.assertEqual(["a", "b", "c"], op.attr("strings_attr"))
op._set_attr("bools_attr", [True, False, True])
self.assertEqual([True, False, True], op.attr("bools_attr"))
self.assertEqual(8, len(op.attr_names()))
op.set_block_attr("_block_attr", program_desc.block(0))
self.assertEqual(0, op._block_attr_id("_block_attr"))
mul_op = block.append_op()
mul_op.set_type("mul")
mul_op.check_attrs()
self.assertEqual(mul_op.attr("x_num_col_dims"), 1)
self.assertEqual(mul_op.attr("y_num_col_dims"), 1)
def test_get_valid_program_error(self):
# case 1: CompiledProgram no program
graph = core.Graph(core.ProgramDesc())
compiled_program = fluid.CompiledProgram(graph)
with self.assertRaises(TypeError):
fluid.io._get_valid_program(compiled_program)
# case 2: main_program type error
with self.assertRaises(TypeError):
fluid.io._get_valid_program("program")
def test_multiple_shape(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_reader'))
var.set_type(core.VarDesc.VarType.READER)
src_shapes = [[2, 3, 3], [4, 5], [6, 7, 8, 9]]
var.set_shapes(src_shapes)
res_shapes = var.shapes()
self.assertEqual(src_shapes, res_shapes)
self.assertEqual(core.VarDesc.VarType.READER, var.type())
def test_shape(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_var'))
var.set_type(core.VarDesc.VarType.SELECTED_ROWS)
src_shape = [3, 2, 10, 8]
var.set_shape(src_shape)
res_shape = var.shape()
self.assertEqual(src_shape, res_shape)
self.assertEqual(core.VarDesc.VarType.SELECTED_ROWS, var.type())
def load_program_desc(model_file_path):
# 1. parse program desc
with open(model_file_path, "rb") as f:
program_desc_str = f.read()
program_desc = core.ProgramDesc(program_desc_str)
if not core._is_program_version_supported(program_desc._version()):
raise ValueError("Unsupported program version: %d\n" %
program_desc._version())
return program_desc
def test_multiple_dtype(self):
program_desc = core.ProgramDesc()
block = program_desc.block(0)
var = block.var(cpt.to_bytes('my_reader'))
var.set_type(core.VarDesc.VarType.READER)
src_types = [
core.VarDesc.VarType.INT32, core.VarDesc.VarType.FP64,
core.VarDesc.VarType.FP32
]
var.set_dtypes(src_types)
self.assertEqual(src_types, var.dtypes())
self.assertEqual(core.VarDesc.VarType.READER, var.type())
def test_add_var(self):
program_desc = core.ProgramDesc()
self.assertIsNotNone(program_desc)
block = program_desc.block(0)
self.assertIsNotNone(block)
var1 = block.var(cpt.to_bytes("var1"))
var2 = block.var(cpt.to_bytes("var2"))
var3 = block.var(cpt.to_bytes("var3"))
all_vars = block.all_vars()
self.assertEqual(set(all_vars), {var1, var2, var3})
var2_re = block.find_var(cpt.to_bytes("var2"))
self.assertEqual(var2_re, var2)
def test_add_op(self):
program_desc = core.ProgramDesc()
self.assertIsNotNone(program_desc)
block = program_desc.block(0)
self.assertIsNotNone(block)
op1 = block.append_op()
op2 = block.append_op()
op0 = block._prepend_op()
all_ops = []
for idx in range(0, block.op_size()):
all_ops.append(block.op(idx))
self.assertEqual(all_ops, [op0, op1, op2])
def test_append_block(self):
program_desc = core.ProgramDesc()
self.assertIsNotNone(program_desc)
block_root = program_desc.block(0)
self.assertIsNotNone(block_root)
self.assertEqual(block_root.id, 0)
block1 = program_desc.append_block(block_root)
block2 = program_desc.append_block(block1)
self.assertIsNotNone(block1)
self.assertEqual(block1.id, block2.parent)
self.assertEqual(block_root.id, block1.parent)
block3 = program_desc.append_block(block_root)
self.assertEqual(block3.parent, block_root.id)
self.assertEqual(program_desc.block(1).id, 1)
self.assertEqual(4, program_desc.num_blocks())
def _append_backward_desc(self, infer_program_desc):
program_desc_copy = core.ProgramDesc(infer_program_desc)
# 1. set all `is_test` attributes to False
_change_is_test_status(program_desc_copy, False)
# 2. prepare program and related var
# NOTE: To reuse backward interfaces, build Program firstly.
# Originally, there is no need to build a program, but need to almost
# rewrite a series of methods for append_backward for program_desc.
# Therefore, in order to reuse the method of backward.py, build the program here.
program = _build_program_by_desc(program_desc_copy)
targets = []
for out in self._output_descs:
targets.append(program.global_block().var(out.name()))
# 3. append backward
backward.gradients(targets=targets, inputs=[])
return program.desc
def create_fake_model(program_config):
''' Create a Paddle model(in memory) according to the given config. '''
paddle.enable_static()
main_program_desc = core.ProgramDesc()
util_program = fluid.Program()
main_block_desc = main_program_desc.block(0)
var_desc = main_block_desc.var(cpt.to_bytes("feed"))
var_desc.set_type(core.VarDesc.VarType.FEED_MINIBATCH)
var_desc.set_persistable(True)
index = 0
for name, tensor_config in program_config.inputs.items():
var_desc = main_block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype))
var_desc.set_shape(tensor_config.shape)
var_desc.set_need_check_feed(True)
if tensor_config.lod is not None:
var_desc.set_lod_level(len(tensor_config.lod))
op_desc = main_block_desc._prepend_op()
op_desc.set_type("feed")
op_desc.set_input('X', ["feed"])
op_desc.set_output('Out', [name])
op_desc._set_attr("col", index)
index = index + 1
save_var_map = {}
for name, tensor_config in program_config.weights.items():
var_desc = main_block_desc.var(cpt.to_bytes(name))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(tensor_config.dtype))
var_desc.set_shape(tensor_config.shape)
var_desc.set_persistable(True)
save_var_map[name] = util_program.global_block().create_parameter(
dtype=tensor_config.dtype,
shape=tensor_config.shape,
type=core.VarDesc.VarType.LOD_TENSOR,
name=name,
initializer=NumpyArrayInitializer(tensor_config.data))
in_vars = []
for name in sorted(save_var_map.keys()):
in_vars.append(save_var_map[name])
out_var = util_program.global_block().create_var(
type=core.VarDesc.VarType.RAW, name="out_var_0")
out_var.desc.set_persistable(True)
util_program.global_block().append_op(type='save_combine',
inputs={'X': in_vars},
outputs={'Y': out_var},
attrs={
'file_path': '',
'save_to_memory': True
})
for op_config in program_config.ops:
op_desc = main_block_desc.append_op()
op_desc.set_type(op_config.type)
for name, values in op_config.inputs.items():
op_desc.set_input(name, values)
for name, values in op_config.attrs.items():
op_desc._set_attr(name, values)
for name, values in op_config.outputs.items():
op_desc.set_output(name, values)
for v in values:
var_desc = main_block_desc.var(cpt.to_bytes(v))
var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR)
var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32))
if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys(
):
var_desc.set_dtype(
convert_np_dtype_to_dtype_(op_config.outputs_dtype[v]))
op_desc.infer_var_type(main_block_desc)
op_desc.infer_shape(main_block_desc)
op_desc.check_attrs()
for index, name in enumerate(program_config.outputs):
var_desc = main_block_desc.var(cpt.to_bytes("fetch"))
var_desc.set_type(core.VarDesc.VarType.FETCH_LIST)
var_desc.set_need_check_feed(True)
op_desc = main_block_desc.append_op()
op_desc.set_type("fetch")
op_desc.set_input('X', [name])
op_desc.set_output('Out', ["fetch"])
op_desc._set_attr("col", index)
main_program_desc._set_version()
paddle.fluid.core.save_op_version_info(main_program_desc)
model = main_program_desc.serialize_to_string()
util_program._sync_with_cpp()
place = fluid.CPUPlace()
executor = fluid.Executor(place)
scope = fluid.Scope()
with fluid.scope_guard(scope):
executor.run(util_program)
params = scope.find_var("out_var_0").get_bytes()
return model, params
