def main(self, tensor, expect_array, expect_lod, expect_max_len, level=0):
place = self.place()
program = Program()
x = layers.data(name='x', shape=[10], main_program=program)
x.persistable = True
table = layers.lod_rank_table(x, level=level, main_program=program)
max_len = layers.max_sequence_len(table, main_program=program)
max_len.persistable = True
array = layers.lod_tensor_to_array(x, table, main_program=program)
array.persistable = True
result = layers.array_to_lod_tensor(array, table, main_program=program)
result.persistable = True
exe = Executor(place)
scope = core.Scope()
exe.run(program, feed={'x': tensor}, scope=scope)
var = scope.find_var(array.name)
array = var.get_lod_tensor_array()
if expect_array is not None and expect_lod is not None:
self.check_array_same(array, expect_array, expect_lod)
self.check_tensor_same(
scope.find_var(result.name).get_tensor(), tensor)
self.assertEqual(
numpy.array(scope.find_var(max_len.name).get_tensor())[0],
expect_max_len)
def test_get_set(self):
scope = core.Scope()
arr = scope.var('tmp_lod_tensor_array')
tensor_array = arr.get_lod_tensor_array()
self.assertEqual(0, len(tensor_array))
cpu = core.CPUPlace()
for i in xrange(10):
t = core.LoDTensor()
t.set(numpy.array([i], dtype='float32'), cpu)
t.set_lod([[0, 1]])
tensor_array.append(t)
self.assertEqual(10, len(tensor_array))
for i in xrange(10):
t = tensor_array[i]
self.assertEqual(numpy.array(t), numpy.array([i], dtype='float32'))
self.assertEqual([[0, 1]], t.lod())
t = core.LoDTensor()
t.set(numpy.array([i + 10], dtype='float32'), cpu)
t.set_lod([[0, 2]])
tensor_array[i] = t
t = tensor_array[i]
self.assertEqual(numpy.array(t),
numpy.array([i + 10], dtype='float32'))
self.assertEqual([[0, 2]], t.lod())
def forward(self):
self.scope = core.Scope()
self.create_global_variables()
self.create_cond_op()
self.create_sub_net()
ctx = core.DeviceContext.create(core.CPUPlace())
self.condop.run(self.scope, ctx)
return np.array(self.scope.find_var("Out").get_tensor())
def setUp(self):
self.scope = core.Scope()
self.ctx = core.DeviceContext.create(core.CPUPlace())
self._create_ids()
self._create_scores()
self._create_pre_ids()
self.scope.var('selected_ids')
self.scope.var('selected_scores')
def check_with_place(self, place):
scope = core.Scope()
# create and initialize Grad Variable
height = 10
rows = [0, 4, 7]
row_numel = 12
grad_selected_rows = scope.var('Grad').get_selected_rows()
grad_selected_rows.set_height(height)
grad_selected_rows.set_rows(rows)
np_array = np.ones((len(rows), row_numel)).astype("float32")
np_array[0, 0] = 2.0
np_array[2, 8] = 4.0
grad_tensor = grad_selected_rows.get_tensor()
grad_tensor.set(np_array, place)
# create and initialize Param Variable
param = scope.var('Param').get_tensor()
param_array = np.full((height, row_numel), 5.0).astype("float32")
param.set(param_array, place)
# create and initialize LeraningRate Variable
lr = scope.var('LearningRate').get_tensor()
lr_array = np.full((1), 2.0).astype("float32")
lr.set(lr_array, place)
# create and run sgd operator
sgd_op = Operator("sgd",
Param='Param',
Grad='Grad',
ParamOut='Param',
LearningRate='LearningRate')
ctx = core.DeviceContext.create(place)
sgd_op.run(scope, ctx)
# get and compare result
result_array = np.array(param)
# rows[0] = 0, 5.0 - 2.0 * 2.0
self.assertAlmostEqual(1.0, result_array[rows[0], 0])
# rows[0] = 0, 5.0 - 2.0 * 1.0
self.assertAlmostEqual(3.0, result_array[rows[0], 2])
# 5.0 - 2.0 * 0.0
self.assertAlmostEqual(5.0, result_array[1, 0])
# rows[1] = 4, 5.0 - 2.0 * 1.0
self.assertAlmostEqual(3.0, result_array[rows[1], 10])
# 5.0 - 2.0 * 0.0
self.assertAlmostEqual(5.0, result_array[5, 8])
# rows[2] = 7, 5.0 - 2.0 * 1.0
self.assertAlmostEqual(3.0, result_array[rows[2], 1])
# rows[2] = 7, 5.0 - 2.0 * 4.0
self.assertAlmostEqual(-3.0, result_array[rows[2], 8])
def setUp(self):
self.scope = core.Scope()
# create input variables
np_data0 = np.array([0, 1, 2])
create_tensor(self.scope, "X0", np_data0)
np_data1 = np.array([1])
t = create_tensor(self.scope, "X1", np_data1)
t.set_dims([0])
# create output variables
self.scope.var("out")
def main(self,
tensor,
mask,
expect_true,
expect_false,
expect_out,
level=0):
place = self.place()
program = Program()
x = layers.data(name='x', shape=[1], main_program=program)
x.persistable = True
y = layers.data(name='y', shape=[1], main_program=program)
y.persistable = True
out_true, out_false = layers.split_lod_tensor(input=x,
mask=y,
level=level,
main_program=program)
out_true.persistable = True
out_false.persistable = True
out = layers.merge_lod_tensor(in_true=out_true,
in_false=out_false,
mask=y,
x=x,
level=level,
main_program=program)
out.persistable = True
exe = Executor(place)
scope = core.Scope()
exe.run(program,
feed={
'x': tensor,
'y': mask
},
scope=scope,
return_numpy=False)
var_true = scope.find_var(out_true.name).get_tensor()
var_false = scope.find_var(out_false.name).get_tensor()
var_out = scope.find_var(out.name).get_tensor()
self.check_tensor_same(var_true, expect_true)
self.check_tensor_same(var_false, expect_false)
self.check_tensor_same(var_out, expect_out)
def test_lod_rank_table(self):
x = data(name='x', shape=[100])
cpu = core.CPUPlace()
rank_table = lod_rank_table(x=x, level=1)
rank_table.persistable = True
exe = Executor(cpu)
scope = core.Scope()
tensor = core.LoDTensor()
tensor.set(numpy.random.random(size=(17, 100)), cpu)
tensor.set_lod([[0, 1, 3], [0, 5, 6, 7], [0, 3, 4, 9, 10, 13, 16, 17]])
exe.run(scope=scope, feed={'x': tensor})
var = scope.find_var(rank_table.name)
table = var.get_lod_rank_table()
self.assertEqual([(0, 5), (1, 1), (2, 1)], table.items())
def uniform_random_test(self, place):
scope = core.Scope()
scope.var('X').get_tensor()
op = Operator("uniform_random",
Out='X',
shape=[1000, 784],
min=-5.0,
max=10.0,
seed=10)
ctx = core.DeviceContext.create(place)
op.run(scope, ctx)
tensor = numpy.array(scope.find_var('X').get_tensor())
self.assertAlmostEqual(tensor.mean(), 2.5, delta=0.1)
def gaussian_random_test(self, place):
scope = core.Scope()
scope.var('Out').get_tensor()
op = Operator("gaussian_random",
Out='Out',
shape=[1000, 784],
mean=.0,
std=1.,
seed=10)
context = core.DeviceContext.create(place)
op.run(scope, context)
tensor = numpy.array(scope.find_var('Out').get_tensor())
self.assertAlmostEqual(numpy.mean(tensor), .0, delta=0.1)
self.assertAlmostEqual(numpy.std(tensor), 1., delta=0.1)
def check_grad(self,
inputs_to_check,
output_names,
no_grad_set=None,
numeric_grad_delta=0.005,
in_place=False,
max_relative_error=0.005,
user_defined_grads=None):
self.scope = core.Scope()
op_inputs = self.inputs if hasattr(self, "inputs") else dict()
op_outputs = self.outputs if hasattr(self, "outputs") else dict()
op_attrs = self.attrs if hasattr(self, "attrs") else dict()
self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs,
op_attrs)
if no_grad_set is None:
no_grad_set = set()
if not type(output_names) is list:
output_names = [output_names]
numeric_grads = user_defined_grads or [
get_numeric_gradient(
self.scope,
self.op,
self.inputs,
input_to_check,
output_names,
delta=numeric_grad_delta,
in_place=in_place) for input_to_check in inputs_to_check
]
cpu_place = core.CPUPlace()
cpu_analytic_grads = self._get_gradient(inputs_to_check, cpu_place,
output_names, no_grad_set)
self.__assert_is_close(numeric_grads, cpu_analytic_grads,
inputs_to_check, max_relative_error,
"Gradient Check On %s" % str(cpu_place))
if core.is_compile_gpu() and self.op.support_gpu():
gpu_place = core.GPUPlace(0)
gpu_analytic_grads = self._get_gradient(inputs_to_check, gpu_place,
output_names, no_grad_set)
self.__assert_is_close(numeric_grads, gpu_analytic_grads,
inputs_to_check, max_relative_error,
"Gradient Check On %s" % str(gpu_place))
def test_lod_tensor_init(self):
scope = core.Scope()
place = core.CPUPlace()
lod_py = [[0, 2, 5], [0, 2, 4, 5]]
lod_tensor = core.LoDTensor(lod_py)
lod_tensor.set_dims([5, 2, 3, 4])
lod_tensor.alloc_float(place)
tensor_array = numpy.array(lod_tensor)
tensor_array[0, 0, 0, 0] = 1.0
tensor_array[0, 0, 0, 1] = 2.0
lod_tensor.set(tensor_array, place)
lod_v = numpy.array(lod_tensor)
self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0])
self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1])
self.assertListEqual(lod_py, lod_tensor.lod())
def test_int_tensor(self):
scope = core.Scope()
var = scope.var("test_tensor")
place = core.CPUPlace()
tensor = var.get_tensor()
tensor.set_dims([1000, 784])
tensor.alloc_int(place)
tensor_array = numpy.array(tensor)
self.assertEqual((1000, 784), tensor_array.shape)
tensor_array[3, 9] = 1
tensor_array[19, 11] = 2
tensor.set(tensor_array, place)
tensor_array_2 = numpy.array(tensor)
self.assertEqual(1, tensor_array_2[3, 9])
self.assertEqual(2, tensor_array_2[19, 11])
def test_int_lod_tensor(self):
place = core.CPUPlace()
scope = core.Scope()
var_lod = scope.var("test_lod_tensor")
lod_tensor = var_lod.get_tensor()
lod_tensor.set_dims([4, 4, 6])
lod_tensor.alloc_int(place)
array = numpy.array(lod_tensor)
array[0, 0, 0] = 3
array[3, 3, 5] = 10
lod_tensor.set(array, place)
lod_tensor.set_lod([[0, 2, 4]])
lod_v = numpy.array(lod_tensor)
self.assertTrue(numpy.alltrue(array == lod_v))
lod = lod_tensor.lod()
self.assertEqual(0, lod[0][0])
self.assertEqual(2, lod[0][1])
self.assertEqual(4, lod[0][2])
def test_feed_fetch(self):
scope = core.Scope()
place = core.CPUPlace()
input_array = np.ones((4, 4, 6)).astype("float32")
input_array[0, 0, 0] = 3
input_array[3, 3, 5] = 10
input_tensor = core.LoDTensor([[0, 2, 4]])
input_tensor.set(input_array, place)
core.set_feed_variable(scope, input_tensor, "feed", 0)
output_tensor = core.get_fetch_variable(scope, "feed", 0)
output_lod = output_tensor.lod()
self.assertEqual(0, output_lod[0][0])
self.assertEqual(2, output_lod[0][1])
self.assertEqual(4, output_lod[0][2])
output_array = np.array(output_tensor)
self.assertEqual(3, output_array[0, 0, 0])
self.assertEqual(10, output_array[3, 3, 5])
def test_float_lod_tensor(self):
place = core.CPUPlace()
scope = core.Scope()
var_lod = scope.var("test_lod_tensor")
lod_tensor = var_lod.get_tensor()
lod_tensor.set_dims([5, 2, 3, 4])
lod_tensor.alloc_float(place)
tensor_array = numpy.array(lod_tensor)
self.assertEqual((5, 2, 3, 4), tensor_array.shape)
tensor_array[0, 0, 0, 0] = 1.0
tensor_array[0, 0, 0, 1] = 2.0
lod_tensor.set(tensor_array, place)
lod_v = numpy.array(lod_tensor)
self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0])
self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1])
self.assertEqual(len(lod_tensor.lod()), 0)
lod_py = [[0, 2, 5], [0, 2, 4, 5]]
lod_tensor.set_lod(lod_py)
lod = lod_tensor.lod()
self.assertListEqual(lod_py, lod)
def test_grad(self):
place = core.CPUPlace()
program = Program()
x = layers.data(name='x',
shape=[1],
dtype='float32',
main_program=program,
stop_gradient=False)
y = layers.data(name='y',
shape=[1],
dtype='bool',
main_program=program,
stop_gradient=False)
level = 0
out_true, out_false = layers.split_lod_tensor(input=x,
mask=y,
level=level,
main_program=program)
out = layers.merge_lod_tensor(in_true=out_true,
in_false=out_false,
mask=y,
x=x,
level=level,
main_program=program)
mean = layers.mean(x=out, main_program=program)
append_backward_ops(mean)
tensor = core.LoDTensor()
tensor.set(np.arange(10).reshape(10, 1).astype('float32'), place)
tensor.set_lod([[0, 3, 9, 10]])
mask_np = np.array([0, 1, 0]).astype('bool')
mask_np = np.expand_dims(mask_np, axis=1)
mask = core.LoDTensor()
mask.set(mask_np, place)
exe = Executor(place)
scope = core.Scope()
g_vars = program.global_block().var(x.name + "@GRAD")
g_out = [
item.sum() for item in map(
np.array,
exe.run(program,
feed={
'x': tensor,
'y': mask
},
fetch_list=[g_vars],
scope=scope,
return_numpy=False))
]
g_out_sum = np.array(g_out).sum()
self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)
def setUp(self):
self.scope = core.Scope()
self.cpu_place = core.CPUPlace()
def test_with_place(place, tensor_format, shape):
# attr
epsilon = 0.00001
momentum = 0.9
if len(shape) == 2:
x_shape = shape
c = shape[1]
else:
# n, h, w, c = 2, 3, 4, 2
n, h, w, c = shape[0], shape[1], shape[2], shape[3]
if data_format == "NHWC":
x_shape = [n, h, w, c]
elif data_format == "NCHW":
x_shape = [n, c, h, w]
else:
raise ValueError("Unknown data type.")
scale_shape = [c]
x_val = np.random.random_sample(x_shape).astype(np.float32)
scale_val = np.random.random_sample(scale_shape).astype(np.float32)
bias_val = np.random.random_sample(scale_shape).astype(np.float32)
mean = np.zeros(scale_shape).astype(np.float32)
variance = np.ones(scale_shape).astype(np.float32)
# run forward
y_out, saved_mean, var_ref = _reference_training(
x_val, scale_val, bias_val, epsilon, data_format)
# update moving mean and variance
mean_out = saved_mean * (1. - momentum) + momentum * mean
variance_out = var_ref * (1. - momentum) + momentum * variance
saved_variance = 1. / np.sqrt(var_ref + epsilon)
# for gradient test
# y_grad = np.ones(x_shape).astype(np.float32)
y_grad = np.zeros(x_shape).astype(np.float32)
if len(y_grad.shape) == 2:
y_grad[0, 0] = 1.
else:
y_grad[0, 0, 0, 0] = 1.
# y_grad = np.random.random_sample(x_shape).astype(np.float32)
x_grad_ref, scale_grad_ref, bias_grad_ref = _reference_grad(
x_val, y_grad, scale_val, saved_mean, var_ref, epsilon,
data_format)
scope = core.Scope()
# create input
x_tensor = create_or_get_tensor(scope, "x_val", x_val, place)
scale_tensor = create_or_get_tensor(scope, "scale_val", scale_val,
place)
bias_tensor = create_or_get_tensor(scope, "bias_val", bias_val,
place)
mean_tensor = create_or_get_tensor(scope, "mean", mean, place)
variance_tensor = create_or_get_tensor(scope, "variance", variance,
place)
# create output
y_tensor = create_or_get_tensor(scope, "y_out", None, place)
saved_mean_tensor = create_or_get_tensor(scope, "saved_mean", None,
place)
saved_variance_tensor = create_or_get_tensor(
scope, "saved_variance", None, place)
mean_out_tensor = mean_tensor
variance_out_tensor = variance_tensor
batch_norm_op = Operator(
"batch_norm",
# inputs
X="x_val",
Scale="scale_val",
Bias="bias_val",
Mean="mean",
Variance="variance",
# outputs
Y="y_out",
MeanOut="mean",
VarianceOut="variance",
SavedMean="saved_mean",
SavedVariance="saved_variance",
# attrs
is_test=False,
tensor_format=tensor_format,
momentum=momentum,
epsilon=epsilon)
ctx = core.DeviceContext.create(place)
batch_norm_op.run(scope, ctx)
# check forward result
self.__assert_close(y_tensor, y_out, "y_out")
self.__assert_close(saved_mean_tensor, saved_mean, "saved_mean")
self.__assert_close(saved_variance_tensor, saved_variance,
"saved_variance")
self.__assert_close(mean_out_tensor, mean_out, "mean_out")
if isinstance(place, core.GPUPlace):
atol = 5e-2
else:
atol = 1e-4
self.__assert_close(variance_out_tensor, variance_out,
"variance_out", atol)
print "op test forward passed: ", str(place), tensor_format
# run backward
batch_norm_op_grad = get_backward_op(scope, batch_norm_op, set())
set_output_grad(
scope,
["y_out", "mean", "variance", "saved_mean", "saved_variance"],
place,
feed_dict={"y_out": y_grad})
batch_norm_op_grad.run(scope, ctx)
x_grad_tensor = create_or_get_tensor(scope, grad_var_name("x_val"),
None, place)
scale_grad_tensor = create_or_get_tensor(
scope, grad_var_name("scale_val"), None, place)
bias_grad_tensor = create_or_get_tensor(scope,
grad_var_name("bias_val"),
None, place)
# check gradient output
self.__assert_close(x_grad_tensor, x_grad_ref, "x_grad")
self.__assert_close(scale_grad_tensor, scale_grad_ref,
"scale_grad")
self.__assert_close(bias_grad_tensor, bias_grad_ref, "bias_grad")
print "op test backward passed: ", str(place), tensor_format
def test_read_write(self):
x = [
layers.data(name='x0', shape=[100]),
layers.data(name='x1', shape=[100]),
layers.data(name='x2', shape=[100])
]
for each_x in x:
each_x.stop_gradient = False
i = layers.zeros(shape=[1], dtype='int64')
i.stop_gradient = False
arr = layers.array_write(x=x[0], i=i)
i = layers.increment(x=i)
arr = layers.array_write(x=x[1], i=i, array=arr)
i = layers.increment(x=i)
arr = layers.array_write(x=x[2], i=i, array=arr)
i = layers.zeros(shape=[1], dtype='int64')
i.stop_gradient = False
a0 = layers.array_read(array=arr, i=i)
i = layers.increment(x=i)
a1 = layers.array_read(array=arr, i=i)
i = layers.increment(x=i)
a2 = layers.array_read(array=arr, i=i)
mean_a0 = layers.mean(x=a0)
mean_a1 = layers.mean(x=a1)
mean_a2 = layers.mean(x=a2)
a_sum = layers.sums(input=[mean_a0, mean_a1, mean_a2])
mean_x0 = layers.mean(x=x[0])
mean_x1 = layers.mean(x=x[1])
mean_x2 = layers.mean(x=x[2])
x_sum = layers.sums(input=[mean_x0, mean_x1, mean_x2])
scope = core.Scope()
cpu = core.CPUPlace()
exe = Executor(cpu)
tensor = numpy.random.random(size=(100, 100)).astype('float32')
outs = exe.run(feed={
'x0': tensor,
'x1': tensor,
'x2': tensor
},
fetch_list=[a_sum, x_sum],
scope=scope)
self.assertEqual(outs[0], outs[1])
total_sum = layers.sums(input=[a_sum, x_sum])
total_sum_scaled = layers.scale(x=total_sum, scale=1 / 6.0)
append_backward_ops(total_sum_scaled)
g_vars = map(default_main_program().global_block().var,
[each_x.name + "@GRAD" for each_x in x])
g_out = [
item.sum() for item in exe.run(feed={
'x0': tensor,
'x1': tensor,
'x2': tensor
},
fetch_list=g_vars)
]
g_out_sum = numpy.array(g_out).sum()
# since our final gradient is 1 and the neural network are all linear
# with mean_op.
# the input gradient should also be 1
self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)
def check_with_place(self, place):
scope = core.Scope()
# create and initialize Grad Variable
height = 10
rows = [0, 4, 7, 4]
row_numel = 12
grad_selected_rows = scope.var('Grad').get_selected_rows()
grad_selected_rows.set_height(height)
grad_selected_rows.set_rows(rows)
np_array = np.ones((len(rows), row_numel)).astype("float32")
np_array[0, 0] = 2.0
np_array[2, 8] = 4.0
grad_tensor = grad_selected_rows.get_tensor()
grad_tensor.set(np_array, place)
# create and initialize Param Variable
param = scope.var('Param').get_tensor()
param_array = np.full((height, row_numel), 5.0).astype("float32")
param.set(param_array, place)
# create and initialize LeraningRate Variable
lr = scope.var('LearningRate').get_tensor()
lr_array = np.full((1), 2.0).astype("float32")
lr.set(lr_array, place)
# create and initialize moment Variable
moment = scope.var('Moment').get_tensor()
moment_np_array = np.full((height, row_numel), 2.0).astype("float32")
moment.set(moment_np_array, place)
# create and run sgd operator
adagrad_op = Operator("adagrad",
Param='Param',
Grad='Grad',
ParamOut='Param',
Moment='Moment',
MomentOut='Moment',
LearningRate='LearningRate',
epsilon=2.0)
ctx = core.DeviceContext.create(place)
adagrad_op.run(scope, ctx)
# get and compare moment result
moment_result_array = np.array(moment)
self.assertAlmostEqual(6.0, moment_result_array[rows[0], 0])
self.assertAlmostEqual(3.0, moment_result_array[rows[0], 2])
self.assertAlmostEqual(2.0, moment_result_array[1, 0])
# 2.0 + (1.0 + 1.0)^2
self.assertAlmostEqual(6.0, moment_result_array[rows[1], 10])
self.assertAlmostEqual(6.0, moment_result_array[rows[3], 4])
self.assertAlmostEqual(2.0, moment_result_array[5, 8])
self.assertAlmostEqual(3.0, moment_result_array[rows[2], 1])
self.assertAlmostEqual(18.0, moment_result_array[rows[2], 8])
# get and compare param result
result_array = np.array(param)
def get_out(param, lr, grad, m, epsilon):
return param - lr * grad / (math.sqrt(m) + epsilon)
self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0),
result_array[rows[0], 0],
places=5)
self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0),
result_array[rows[0], 2],
places=5)
self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0),
result_array[1, 0],
places=5)
# grad_merge = 1.0 + 1.0
# m = 6.0
self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0),
result_array[rows[1], 10],
places=5)
self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0),
result_array[5, 8],
places=5)
self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0),
result_array[rows[2], 1],
places=5)
self.assertAlmostEqual(get_out(5.0, 2.0, 4.0, 18.0, 2.0),
result_array[rows[2], 8],
places=5)