def main(self, tensor, expect_array, expect_lod, expect_max_len, level=0): place = self.place() program = Program() x = layers.data(name='x', shape=[10], main_program=program) x.persistable = True table = layers.lod_rank_table(x, level=level, main_program=program) max_len = layers.max_sequence_len(table, main_program=program) max_len.persistable = True array = layers.lod_tensor_to_array(x, table, main_program=program) array.persistable = True result = layers.array_to_lod_tensor(array, table, main_program=program) result.persistable = True exe = Executor(place) scope = core.Scope() exe.run(program, feed={'x': tensor}, scope=scope) var = scope.find_var(array.name) array = var.get_lod_tensor_array() if expect_array is not None and expect_lod is not None: self.check_array_same(array, expect_array, expect_lod) self.check_tensor_same( scope.find_var(result.name).get_tensor(), tensor) self.assertEqual( numpy.array(scope.find_var(max_len.name).get_tensor())[0], expect_max_len)
def test_get_set(self): scope = core.Scope() arr = scope.var('tmp_lod_tensor_array') tensor_array = arr.get_lod_tensor_array() self.assertEqual(0, len(tensor_array)) cpu = core.CPUPlace() for i in xrange(10): t = core.LoDTensor() t.set(numpy.array([i], dtype='float32'), cpu) t.set_lod([[0, 1]]) tensor_array.append(t) self.assertEqual(10, len(tensor_array)) for i in xrange(10): t = tensor_array[i] self.assertEqual(numpy.array(t), numpy.array([i], dtype='float32')) self.assertEqual([[0, 1]], t.lod()) t = core.LoDTensor() t.set(numpy.array([i + 10], dtype='float32'), cpu) t.set_lod([[0, 2]]) tensor_array[i] = t t = tensor_array[i] self.assertEqual(numpy.array(t), numpy.array([i + 10], dtype='float32')) self.assertEqual([[0, 2]], t.lod())
def forward(self): self.scope = core.Scope() self.create_global_variables() self.create_cond_op() self.create_sub_net() ctx = core.DeviceContext.create(core.CPUPlace()) self.condop.run(self.scope, ctx) return np.array(self.scope.find_var("Out").get_tensor())
def setUp(self): self.scope = core.Scope() self.ctx = core.DeviceContext.create(core.CPUPlace()) self._create_ids() self._create_scores() self._create_pre_ids() self.scope.var('selected_ids') self.scope.var('selected_scores')
def check_with_place(self, place): scope = core.Scope() # create and initialize Grad Variable height = 10 rows = [0, 4, 7] row_numel = 12 grad_selected_rows = scope.var('Grad').get_selected_rows() grad_selected_rows.set_height(height) grad_selected_rows.set_rows(rows) np_array = np.ones((len(rows), row_numel)).astype("float32") np_array[0, 0] = 2.0 np_array[2, 8] = 4.0 grad_tensor = grad_selected_rows.get_tensor() grad_tensor.set(np_array, place) # create and initialize Param Variable param = scope.var('Param').get_tensor() param_array = np.full((height, row_numel), 5.0).astype("float32") param.set(param_array, place) # create and initialize LeraningRate Variable lr = scope.var('LearningRate').get_tensor() lr_array = np.full((1), 2.0).astype("float32") lr.set(lr_array, place) # create and run sgd operator sgd_op = Operator("sgd", Param='Param', Grad='Grad', ParamOut='Param', LearningRate='LearningRate') ctx = core.DeviceContext.create(place) sgd_op.run(scope, ctx) # get and compare result result_array = np.array(param) # rows[0] = 0, 5.0 - 2.0 * 2.0 self.assertAlmostEqual(1.0, result_array[rows[0], 0]) # rows[0] = 0, 5.0 - 2.0 * 1.0 self.assertAlmostEqual(3.0, result_array[rows[0], 2]) # 5.0 - 2.0 * 0.0 self.assertAlmostEqual(5.0, result_array[1, 0]) # rows[1] = 4, 5.0 - 2.0 * 1.0 self.assertAlmostEqual(3.0, result_array[rows[1], 10]) # 5.0 - 2.0 * 0.0 self.assertAlmostEqual(5.0, result_array[5, 8]) # rows[2] = 7, 5.0 - 2.0 * 1.0 self.assertAlmostEqual(3.0, result_array[rows[2], 1]) # rows[2] = 7, 5.0 - 2.0 * 4.0 self.assertAlmostEqual(-3.0, result_array[rows[2], 8])
def setUp(self): self.scope = core.Scope() # create input variables np_data0 = np.array([0, 1, 2]) create_tensor(self.scope, "X0", np_data0) np_data1 = np.array([1]) t = create_tensor(self.scope, "X1", np_data1) t.set_dims([0]) # create output variables self.scope.var("out")
def main(self, tensor, mask, expect_true, expect_false, expect_out, level=0): place = self.place() program = Program() x = layers.data(name='x', shape=[1], main_program=program) x.persistable = True y = layers.data(name='y', shape=[1], main_program=program) y.persistable = True out_true, out_false = layers.split_lod_tensor(input=x, mask=y, level=level, main_program=program) out_true.persistable = True out_false.persistable = True out = layers.merge_lod_tensor(in_true=out_true, in_false=out_false, mask=y, x=x, level=level, main_program=program) out.persistable = True exe = Executor(place) scope = core.Scope() exe.run(program, feed={ 'x': tensor, 'y': mask }, scope=scope, return_numpy=False) var_true = scope.find_var(out_true.name).get_tensor() var_false = scope.find_var(out_false.name).get_tensor() var_out = scope.find_var(out.name).get_tensor() self.check_tensor_same(var_true, expect_true) self.check_tensor_same(var_false, expect_false) self.check_tensor_same(var_out, expect_out)
def test_lod_rank_table(self): x = data(name='x', shape=[100]) cpu = core.CPUPlace() rank_table = lod_rank_table(x=x, level=1) rank_table.persistable = True exe = Executor(cpu) scope = core.Scope() tensor = core.LoDTensor() tensor.set(numpy.random.random(size=(17, 100)), cpu) tensor.set_lod([[0, 1, 3], [0, 5, 6, 7], [0, 3, 4, 9, 10, 13, 16, 17]]) exe.run(scope=scope, feed={'x': tensor}) var = scope.find_var(rank_table.name) table = var.get_lod_rank_table() self.assertEqual([(0, 5), (1, 1), (2, 1)], table.items())
def uniform_random_test(self, place): scope = core.Scope() scope.var('X').get_tensor() op = Operator("uniform_random", Out='X', shape=[1000, 784], min=-5.0, max=10.0, seed=10) ctx = core.DeviceContext.create(place) op.run(scope, ctx) tensor = numpy.array(scope.find_var('X').get_tensor()) self.assertAlmostEqual(tensor.mean(), 2.5, delta=0.1)
def gaussian_random_test(self, place): scope = core.Scope() scope.var('Out').get_tensor() op = Operator("gaussian_random", Out='Out', shape=[1000, 784], mean=.0, std=1., seed=10) context = core.DeviceContext.create(place) op.run(scope, context) tensor = numpy.array(scope.find_var('Out').get_tensor()) self.assertAlmostEqual(numpy.mean(tensor), .0, delta=0.1) self.assertAlmostEqual(numpy.std(tensor), 1., delta=0.1)
def check_grad(self, inputs_to_check, output_names, no_grad_set=None, numeric_grad_delta=0.005, in_place=False, max_relative_error=0.005, user_defined_grads=None): self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs, op_attrs) if no_grad_set is None: no_grad_set = set() if not type(output_names) is list: output_names = [output_names] numeric_grads = user_defined_grads or [ get_numeric_gradient( self.scope, self.op, self.inputs, input_to_check, output_names, delta=numeric_grad_delta, in_place=in_place) for input_to_check in inputs_to_check ] cpu_place = core.CPUPlace() cpu_analytic_grads = self._get_gradient(inputs_to_check, cpu_place, output_names, no_grad_set) self.__assert_is_close(numeric_grads, cpu_analytic_grads, inputs_to_check, max_relative_error, "Gradient Check On %s" % str(cpu_place)) if core.is_compile_gpu() and self.op.support_gpu(): gpu_place = core.GPUPlace(0) gpu_analytic_grads = self._get_gradient(inputs_to_check, gpu_place, output_names, no_grad_set) self.__assert_is_close(numeric_grads, gpu_analytic_grads, inputs_to_check, max_relative_error, "Gradient Check On %s" % str(gpu_place))
def test_lod_tensor_init(self): scope = core.Scope() place = core.CPUPlace() lod_py = [[0, 2, 5], [0, 2, 4, 5]] lod_tensor = core.LoDTensor(lod_py) lod_tensor.set_dims([5, 2, 3, 4]) lod_tensor.alloc_float(place) tensor_array = numpy.array(lod_tensor) tensor_array[0, 0, 0, 0] = 1.0 tensor_array[0, 0, 0, 1] = 2.0 lod_tensor.set(tensor_array, place) lod_v = numpy.array(lod_tensor) self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0]) self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1]) self.assertListEqual(lod_py, lod_tensor.lod())
def test_int_tensor(self): scope = core.Scope() var = scope.var("test_tensor") place = core.CPUPlace() tensor = var.get_tensor() tensor.set_dims([1000, 784]) tensor.alloc_int(place) tensor_array = numpy.array(tensor) self.assertEqual((1000, 784), tensor_array.shape) tensor_array[3, 9] = 1 tensor_array[19, 11] = 2 tensor.set(tensor_array, place) tensor_array_2 = numpy.array(tensor) self.assertEqual(1, tensor_array_2[3, 9]) self.assertEqual(2, tensor_array_2[19, 11])
def test_int_lod_tensor(self): place = core.CPUPlace() scope = core.Scope() var_lod = scope.var("test_lod_tensor") lod_tensor = var_lod.get_tensor() lod_tensor.set_dims([4, 4, 6]) lod_tensor.alloc_int(place) array = numpy.array(lod_tensor) array[0, 0, 0] = 3 array[3, 3, 5] = 10 lod_tensor.set(array, place) lod_tensor.set_lod([[0, 2, 4]]) lod_v = numpy.array(lod_tensor) self.assertTrue(numpy.alltrue(array == lod_v)) lod = lod_tensor.lod() self.assertEqual(0, lod[0][0]) self.assertEqual(2, lod[0][1]) self.assertEqual(4, lod[0][2])
def test_feed_fetch(self): scope = core.Scope() place = core.CPUPlace() input_array = np.ones((4, 4, 6)).astype("float32") input_array[0, 0, 0] = 3 input_array[3, 3, 5] = 10 input_tensor = core.LoDTensor([[0, 2, 4]]) input_tensor.set(input_array, place) core.set_feed_variable(scope, input_tensor, "feed", 0) output_tensor = core.get_fetch_variable(scope, "feed", 0) output_lod = output_tensor.lod() self.assertEqual(0, output_lod[0][0]) self.assertEqual(2, output_lod[0][1]) self.assertEqual(4, output_lod[0][2]) output_array = np.array(output_tensor) self.assertEqual(3, output_array[0, 0, 0]) self.assertEqual(10, output_array[3, 3, 5])
def test_float_lod_tensor(self): place = core.CPUPlace() scope = core.Scope() var_lod = scope.var("test_lod_tensor") lod_tensor = var_lod.get_tensor() lod_tensor.set_dims([5, 2, 3, 4]) lod_tensor.alloc_float(place) tensor_array = numpy.array(lod_tensor) self.assertEqual((5, 2, 3, 4), tensor_array.shape) tensor_array[0, 0, 0, 0] = 1.0 tensor_array[0, 0, 0, 1] = 2.0 lod_tensor.set(tensor_array, place) lod_v = numpy.array(lod_tensor) self.assertAlmostEqual(1.0, lod_v[0, 0, 0, 0]) self.assertAlmostEqual(2.0, lod_v[0, 0, 0, 1]) self.assertEqual(len(lod_tensor.lod()), 0) lod_py = [[0, 2, 5], [0, 2, 4, 5]] lod_tensor.set_lod(lod_py) lod = lod_tensor.lod() self.assertListEqual(lod_py, lod)
def test_grad(self): place = core.CPUPlace() program = Program() x = layers.data(name='x', shape=[1], dtype='float32', main_program=program, stop_gradient=False) y = layers.data(name='y', shape=[1], dtype='bool', main_program=program, stop_gradient=False) level = 0 out_true, out_false = layers.split_lod_tensor(input=x, mask=y, level=level, main_program=program) out = layers.merge_lod_tensor(in_true=out_true, in_false=out_false, mask=y, x=x, level=level, main_program=program) mean = layers.mean(x=out, main_program=program) append_backward_ops(mean) tensor = core.LoDTensor() tensor.set(np.arange(10).reshape(10, 1).astype('float32'), place) tensor.set_lod([[0, 3, 9, 10]]) mask_np = np.array([0, 1, 0]).astype('bool') mask_np = np.expand_dims(mask_np, axis=1) mask = core.LoDTensor() mask.set(mask_np, place) exe = Executor(place) scope = core.Scope() g_vars = program.global_block().var(x.name + "@GRAD") g_out = [ item.sum() for item in map( np.array, exe.run(program, feed={ 'x': tensor, 'y': mask }, fetch_list=[g_vars], scope=scope, return_numpy=False)) ] g_out_sum = np.array(g_out).sum() self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)
def setUp(self): self.scope = core.Scope() self.cpu_place = core.CPUPlace()
def test_with_place(place, tensor_format, shape): # attr epsilon = 0.00001 momentum = 0.9 if len(shape) == 2: x_shape = shape c = shape[1] else: # n, h, w, c = 2, 3, 4, 2 n, h, w, c = shape[0], shape[1], shape[2], shape[3] if data_format == "NHWC": x_shape = [n, h, w, c] elif data_format == "NCHW": x_shape = [n, c, h, w] else: raise ValueError("Unknown data type.") scale_shape = [c] x_val = np.random.random_sample(x_shape).astype(np.float32) scale_val = np.random.random_sample(scale_shape).astype(np.float32) bias_val = np.random.random_sample(scale_shape).astype(np.float32) mean = np.zeros(scale_shape).astype(np.float32) variance = np.ones(scale_shape).astype(np.float32) # run forward y_out, saved_mean, var_ref = _reference_training( x_val, scale_val, bias_val, epsilon, data_format) # update moving mean and variance mean_out = saved_mean * (1. - momentum) + momentum * mean variance_out = var_ref * (1. - momentum) + momentum * variance saved_variance = 1. / np.sqrt(var_ref + epsilon) # for gradient test # y_grad = np.ones(x_shape).astype(np.float32) y_grad = np.zeros(x_shape).astype(np.float32) if len(y_grad.shape) == 2: y_grad[0, 0] = 1. else: y_grad[0, 0, 0, 0] = 1. # y_grad = np.random.random_sample(x_shape).astype(np.float32) x_grad_ref, scale_grad_ref, bias_grad_ref = _reference_grad( x_val, y_grad, scale_val, saved_mean, var_ref, epsilon, data_format) scope = core.Scope() # create input x_tensor = create_or_get_tensor(scope, "x_val", x_val, place) scale_tensor = create_or_get_tensor(scope, "scale_val", scale_val, place) bias_tensor = create_or_get_tensor(scope, "bias_val", bias_val, place) mean_tensor = create_or_get_tensor(scope, "mean", mean, place) variance_tensor = create_or_get_tensor(scope, "variance", variance, place) # create output y_tensor = create_or_get_tensor(scope, "y_out", None, place) saved_mean_tensor = create_or_get_tensor(scope, "saved_mean", None, place) saved_variance_tensor = create_or_get_tensor( scope, "saved_variance", None, place) mean_out_tensor = mean_tensor variance_out_tensor = variance_tensor batch_norm_op = Operator( "batch_norm", # inputs X="x_val", Scale="scale_val", Bias="bias_val", Mean="mean", Variance="variance", # outputs Y="y_out", MeanOut="mean", VarianceOut="variance", SavedMean="saved_mean", SavedVariance="saved_variance", # attrs is_test=False, tensor_format=tensor_format, momentum=momentum, epsilon=epsilon) ctx = core.DeviceContext.create(place) batch_norm_op.run(scope, ctx) # check forward result self.__assert_close(y_tensor, y_out, "y_out") self.__assert_close(saved_mean_tensor, saved_mean, "saved_mean") self.__assert_close(saved_variance_tensor, saved_variance, "saved_variance") self.__assert_close(mean_out_tensor, mean_out, "mean_out") if isinstance(place, core.GPUPlace): atol = 5e-2 else: atol = 1e-4 self.__assert_close(variance_out_tensor, variance_out, "variance_out", atol) print "op test forward passed: ", str(place), tensor_format # run backward batch_norm_op_grad = get_backward_op(scope, batch_norm_op, set()) set_output_grad( scope, ["y_out", "mean", "variance", "saved_mean", "saved_variance"], place, feed_dict={"y_out": y_grad}) batch_norm_op_grad.run(scope, ctx) x_grad_tensor = create_or_get_tensor(scope, grad_var_name("x_val"), None, place) scale_grad_tensor = create_or_get_tensor( scope, grad_var_name("scale_val"), None, place) bias_grad_tensor = create_or_get_tensor(scope, grad_var_name("bias_val"), None, place) # check gradient output self.__assert_close(x_grad_tensor, x_grad_ref, "x_grad") self.__assert_close(scale_grad_tensor, scale_grad_ref, "scale_grad") self.__assert_close(bias_grad_tensor, bias_grad_ref, "bias_grad") print "op test backward passed: ", str(place), tensor_format
def test_read_write(self): x = [ layers.data(name='x0', shape=[100]), layers.data(name='x1', shape=[100]), layers.data(name='x2', shape=[100]) ] for each_x in x: each_x.stop_gradient = False i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = False arr = layers.array_write(x=x[0], i=i) i = layers.increment(x=i) arr = layers.array_write(x=x[1], i=i, array=arr) i = layers.increment(x=i) arr = layers.array_write(x=x[2], i=i, array=arr) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = False a0 = layers.array_read(array=arr, i=i) i = layers.increment(x=i) a1 = layers.array_read(array=arr, i=i) i = layers.increment(x=i) a2 = layers.array_read(array=arr, i=i) mean_a0 = layers.mean(x=a0) mean_a1 = layers.mean(x=a1) mean_a2 = layers.mean(x=a2) a_sum = layers.sums(input=[mean_a0, mean_a1, mean_a2]) mean_x0 = layers.mean(x=x[0]) mean_x1 = layers.mean(x=x[1]) mean_x2 = layers.mean(x=x[2]) x_sum = layers.sums(input=[mean_x0, mean_x1, mean_x2]) scope = core.Scope() cpu = core.CPUPlace() exe = Executor(cpu) tensor = numpy.random.random(size=(100, 100)).astype('float32') outs = exe.run(feed={ 'x0': tensor, 'x1': tensor, 'x2': tensor }, fetch_list=[a_sum, x_sum], scope=scope) self.assertEqual(outs[0], outs[1]) total_sum = layers.sums(input=[a_sum, x_sum]) total_sum_scaled = layers.scale(x=total_sum, scale=1 / 6.0) append_backward_ops(total_sum_scaled) g_vars = map(default_main_program().global_block().var, [each_x.name + "@GRAD" for each_x in x]) g_out = [ item.sum() for item in exe.run(feed={ 'x0': tensor, 'x1': tensor, 'x2': tensor }, fetch_list=g_vars) ] g_out_sum = numpy.array(g_out).sum() # since our final gradient is 1 and the neural network are all linear # with mean_op. # the input gradient should also be 1 self.assertAlmostEqual(1.0, g_out_sum, delta=0.1)
def check_with_place(self, place): scope = core.Scope() # create and initialize Grad Variable height = 10 rows = [0, 4, 7, 4] row_numel = 12 grad_selected_rows = scope.var('Grad').get_selected_rows() grad_selected_rows.set_height(height) grad_selected_rows.set_rows(rows) np_array = np.ones((len(rows), row_numel)).astype("float32") np_array[0, 0] = 2.0 np_array[2, 8] = 4.0 grad_tensor = grad_selected_rows.get_tensor() grad_tensor.set(np_array, place) # create and initialize Param Variable param = scope.var('Param').get_tensor() param_array = np.full((height, row_numel), 5.0).astype("float32") param.set(param_array, place) # create and initialize LeraningRate Variable lr = scope.var('LearningRate').get_tensor() lr_array = np.full((1), 2.0).astype("float32") lr.set(lr_array, place) # create and initialize moment Variable moment = scope.var('Moment').get_tensor() moment_np_array = np.full((height, row_numel), 2.0).astype("float32") moment.set(moment_np_array, place) # create and run sgd operator adagrad_op = Operator("adagrad", Param='Param', Grad='Grad', ParamOut='Param', Moment='Moment', MomentOut='Moment', LearningRate='LearningRate', epsilon=2.0) ctx = core.DeviceContext.create(place) adagrad_op.run(scope, ctx) # get and compare moment result moment_result_array = np.array(moment) self.assertAlmostEqual(6.0, moment_result_array[rows[0], 0]) self.assertAlmostEqual(3.0, moment_result_array[rows[0], 2]) self.assertAlmostEqual(2.0, moment_result_array[1, 0]) # 2.0 + (1.0 + 1.0)^2 self.assertAlmostEqual(6.0, moment_result_array[rows[1], 10]) self.assertAlmostEqual(6.0, moment_result_array[rows[3], 4]) self.assertAlmostEqual(2.0, moment_result_array[5, 8]) self.assertAlmostEqual(3.0, moment_result_array[rows[2], 1]) self.assertAlmostEqual(18.0, moment_result_array[rows[2], 8]) # get and compare param result result_array = np.array(param) def get_out(param, lr, grad, m, epsilon): return param - lr * grad / (math.sqrt(m) + epsilon) self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0), result_array[rows[0], 0], places=5) self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0), result_array[rows[0], 2], places=5) self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0), result_array[1, 0], places=5) # grad_merge = 1.0 + 1.0 # m = 6.0 self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0), result_array[rows[1], 10], places=5) self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0), result_array[5, 8], places=5) self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0), result_array[rows[2], 1], places=5) self.assertAlmostEqual(get_out(5.0, 2.0, 4.0, 18.0, 2.0), result_array[rows[2], 8], places=5)