def func(self, place): prog = fluid.Program() with fluid.program_guard(prog): x = layers.create_parameter(dtype="float64", shape=[2, 8], name='x') y = layers.create_parameter(dtype="float64", shape=[8, 4], name='y') z = layers.mul(x=x, y=y) gradient_checker.grad_check([x, y], z, place=place)
def test_mul(self): a = data(name='a', shape=[784], dtype='float32') b = data( name='b', shape=[784, 100], dtype='float32', append_batch_size=False) out = mul(x=a, y=b) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) a_np = numpy.random.random((100, 784)).astype('float32') b_np = numpy.random.random((784, 100)).astype('float32') self.assertEqual(0, core.get_mem_usage(0)) exe = Executor(place) outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[out]) out = outs[0] #disable this assert since ctest will ignore the os.environ setting #self.assertGreater(core.get_mem_usage(0), 0) raised = False try: core.print_mem_usage() except: raised = True self.assertFalse(raised, 'Exception raised')
def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2) self.feed_vars.append( fluid.data( name="data2", shape=[128, 128], dtype=dtype)) # subgraph with 2 op nodes tmp_0 = self.feed_vars[0] * self.feed_vars[1] tmp_1 = layers.cast(tmp_0, dtype="float16") zero = layers.fill_constant(shape=[128], dtype="float16", value=0) # TODO(xreki): fix precision problem when using softmax of float16. # tmp_2 = layers.softmax(tmp_1) tmp_2 = layers.elementwise_add(tmp_1, zero) tmp_3 = layers.mul(tmp_0, self.feed_vars[2]) # subgraph with 4 op nodes tmp_3 = layers.cast(tmp_2, dtype="float16") tmp_4 = layers.relu(tmp_1 + tmp_3) tmp_5 = layers.cast(tmp_4, dtype=dtype) tmp_3 = layers.cast(tmp_2, dtype=dtype) self.append_gradients(tmp_5) self.num_fused_ops = 4 self.fetch_list = [tmp_5, self.grad(tmp_0)]
def test_mul(self): i = zeros(shape=[1], dtype='int64') a = data(name='a', shape=[784], dtype='float32') array = array_write(x=a, i=i) i = increment(i) b = data( name='b', shape=[784, 100], dtype='float32', append_batch_size=False) array_write(x=b, i=i, array=array) i = increment(i) out = mul(x=a, y=b) array_write(x=out, i=i, array=array) a_np = numpy.random.random((100, 784)).astype('float32') b_np = numpy.random.random((784, 100)).astype('float32') exe = Executor() res, res_array = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[out, array]) self.assertEqual((100, 100), res.shape) self.assertTrue(numpy.allclose(res, numpy.dot(a_np, b_np))) self.assertTrue(numpy.allclose(res_array[0], a_np)) self.assertTrue(numpy.allclose(res_array[1], b_np)) self.assertTrue(numpy.allclose(res_array[2], res))
def test_calc_gradient(self): x = layers.create_parameter(dtype="float32", shape=[5, 10]) y = layers.create_parameter(dtype="float32", shape=[10, 8]) mul_out = layers.mul(x=x, y=y) mean_out = layers.mean(mul_out) a = calc_gradient(mean_out, mul_out) b = calc_gradient(mean_out, x) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) exe.run(fluid.default_main_program(), feed={}, fetch_list=[a, b])
def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( fluid.data(name="data3", shape=[128, 32], dtype=dtype)) # subgraph with 3 op node tmp_0 = self.feed_vars[0] - self.feed_vars[1] tmp_1 = tmp_0 * self.feed_vars[2] tmp_2 = layers.assign(tmp_1, output=tmp_0) tmp_3 = layers.mul(tmp_2, self.feed_vars[3]) self.num_fused_ops = 1 self.fetch_list = [tmp_3]
def test_calc_gradient(self): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): x = layers.create_parameter(dtype="float32", shape=[5, 10]) y = layers.create_parameter(dtype="float32", shape=[10, 8]) mul_out = layers.mul(x=x, y=y) mean_out = layers.mean(mul_out) a = calc_gradient(mean_out, mul_out) b = calc_gradient(mean_out, x) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) exe.run(main, feed={}, fetch_list=[a, b])
def test_mul(self): a = data(name='a', shape=[784], dtype='float32') b = data(name='b', shape=[784, 100], dtype='float32', append_batch_size=False) out = mul(x=a, y=b) place = core.CPUPlace() a_np = numpy.random.random((100, 784)).astype('float32') b_np = numpy.random.random((784, 100)).astype('float32') exe = Executor(place) outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[out]) out = outs[0] self.assertEqual((100, 100), out.shape) self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( fluid.data(name="data3", shape=[128, 32], dtype=dtype)) # subgraph with 3 op node tmp_0 = self.feed_vars[0] + self.feed_vars[1] tmp_1 = layers.relu(self.feed_vars[2] * tmp_0) # subgraph with 2 op nodes tmp_2 = layers.relu(layers.sigmoid(self.feed_vars[3])) tmp_3 = layers.mul(tmp_1, tmp_2) self.append_gradients(tmp_3) self.num_fused_ops = 2 self.fetch_list = [tmp_3, self.grad(tmp_1)]
def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2) self.feed_vars.append( fluid.data(name="data2", shape=[128, 128], dtype=dtype)) # subgraph with only 1 op node tmp_0 = self.feed_vars[0] * self.feed_vars[1] tmp_1 = layers.mul(tmp_0, self.feed_vars[2]) # subgraph with 2 op nodes tmp_2 = layers.relu(tmp_0 + tmp_1) self.append_gradients(tmp_2) self.num_fused_ops = 2 self.fetch_list = [tmp_2, self.grad(tmp_1)]
def test_mul(self): a = data(name='a', shape=[784], dtype='float32') b = data( name='b', shape=[784, 100], dtype='float32', append_batch_size=False) out = mul(x=a, y=b) place = core.CPUPlace() a_np = numpy.random.random((100, 784)).astype('float32') b_np = numpy.random.random((784, 100)).astype('float32') exe = Executor(place) outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[out]) out = outs[0] self.assertEqual((100, 100), out.shape) self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
def func(self, place): # the shape of input variable shoule be clearly specified, not inlcude -1. x_shape = [7, 11] y_shape = [11, 9] eps = 0.005 dtype = np.float64 x = layers.data('x', x_shape, False, dtype) x.persistable = True y = layers.data('y', y_shape, False, dtype) y.persistable = True out = layers.mul(x, y) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, y_shape).astype(dtype) gradient_checker.double_grad_check( [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps)
def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( fluid.data(name="data3", shape=[128, 32], dtype=dtype)) # subgraph with 3 op node tmp_0 = self.feed_vars[0] + self.feed_vars[1] tmp_1 = layers.relu(self.feed_vars[2] * tmp_0) # subgraph with 2 op nodes tmp_2 = layers.relu(layers.sigmoid(self.feed_vars[3])) tmp_3 = layers.mul(tmp_1, tmp_2) # TODO(wangchaochaohu): support the case when some vars are set # stop_gradient = True. self.num_fused_ops = 2 self.fetch_list = [tmp_3]
def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( fluid.data(name="data3", shape=[128, 128], dtype=dtype)) # subgraph with 2 op nodes tmp_0 = layers.sum( [self.feed_vars[0], self.feed_vars[1], self.feed_vars[2]]) tmp_1 = layers.sqrt(tmp_0) tmp_2 = layers.mul(tmp_0, self.feed_vars[3]) # subgraph with 2 op nodes tmp_3 = layers.square(layers.sum([tmp_1, tmp_2])) self.append_gradients(tmp_3) self.num_fused_ops = 4 self.fetch_list = [tmp_3, self.grad(tmp_0)]
def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2) self.feed_vars.append( fluid.data(name="data2", shape=[128, 128], dtype=dtype)) # subgraph with 2 op nodes tmp_0 = self.feed_vars[0] * self.feed_vars[1] tmp_1 = layers.softmax(layers.cast(tmp_0, dtype="float16")) tmp_2 = layers.mul(tmp_0, self.feed_vars[2]) # subgraph with 4 op nodes tmp_3 = layers.cast(tmp_2, dtype="float16") tmp_4 = layers.relu(tmp_1 + tmp_3) tmp_5 = layers.cast(tmp_4, dtype=dtype) self.append_gradients(tmp_5) self.num_fused_ops = 4 self.fetch_list = [tmp_5, self.grad(tmp_0)]
def subsample_labels(labels, num_samples, positive_fraction): positive = torch.nonzero(mul((labels != config.ignore_label).cast('int'), (labels != 0).cast('int')).cast('bool'), as_tuple=False).squeeze(1) negative = torch.nonzero(labels == 0, as_tuple=False).squeeze(1) num_pos = int(num_samples * positive_fraction) num_pos = min(positive.numel(), num_pos) num_neg = num_samples - num_pos num_neg = min(negative.numel(), num_neg) # randomly select positive and negative examples if type(num_pos) == torch.Tensor: num_pos = num_pos.numpy().item() if type(num_neg) == torch.Tensor: num_neg = num_neg.numpy().item() perm1 = torch.randperm(positive.numel())[:num_pos] perm2 = torch.randperm(negative.numel())[:num_neg] pos_idx = torch.gather(positive, perm1) neg_idx = torch.gather(negative, perm2) return pos_idx, neg_idx
def test_mul(self): a = data(name='a', shape=[784], dtype='float32') b = data( name='b', shape=[784, 100], dtype='float32', append_batch_size=False) output = mul(x=a, y=b) place = core.CPUPlace() a_np = numpy.random.random((100, 784)).astype('float32') b_np = numpy.random.random((784, 100)).astype('float32') exe = Executor(place) import time use_cache = True step_num = 3 run_time = 0.0 for i in range(step_num): begin = time.time() outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[output.name], use_program_cache=use_cache) end = time.time() run_time += end - begin out = outs[0] self.assertEqual((100, 100), out.shape) self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np))) print("run time %f" % run_time) use_cache = False run_time = 0.0 for i in range(step_num): begin = time.time() outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[output.name], use_program_cache=use_cache) end = time.time() run_time += end - begin out = outs[0] self.assertEqual((100, 100), out.shape) self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np))) print("run time %f" % run_time) use_cache = True run_time = 0.0 for i in range(step_num): begin = time.time() outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[output.name], use_program_cache=use_cache) end = time.time() run_time += end - begin out = outs[0] self.assertEqual((100, 100), out.shape) self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np))) print("run time %f" % run_time) use_cache = True run_time = 0.0 for i in range(step_num): begin = time.time() outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[output], use_program_cache=use_cache) end = time.time() run_time += end - begin out = outs[0] self.assertEqual((100, 100), out.shape) self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np))) print("run time %f" % run_time)