def func(self, place):
prog = fluid.Program()
with fluid.program_guard(prog):
x = layers.create_parameter(dtype="float64", shape=[2, 8], name='x')
y = layers.create_parameter(dtype="float64", shape=[8, 4], name='y')
z = layers.mul(x=x, y=y)
gradient_checker.grad_check([x, y], z, place=place)
def test_mul(self):
a = data(name='a', shape=[784], dtype='float32')
b = data(
name='b',
shape=[784, 100],
dtype='float32',
append_batch_size=False)
out = mul(x=a, y=b)
if core.is_compiled_with_cuda():
place = core.CUDAPlace(0)
a_np = numpy.random.random((100, 784)).astype('float32')
b_np = numpy.random.random((784, 100)).astype('float32')
self.assertEqual(0, core.get_mem_usage(0))
exe = Executor(place)
outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[out])
out = outs[0]
#disable this assert since ctest will ignore the os.environ setting
#self.assertGreater(core.get_mem_usage(0), 0)
raised = False
try:
core.print_mem_usage()
except:
raised = True
self.assertFalse(raised, 'Exception raised')
def build_program(self, dtype):
with fluid.program_guard(self.main_program, self.startup_program):
self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2)
self.feed_vars.append(
fluid.data(
name="data2", shape=[128, 128], dtype=dtype))
# subgraph with 2 op nodes
tmp_0 = self.feed_vars[0] * self.feed_vars[1]
tmp_1 = layers.cast(tmp_0, dtype="float16")
zero = layers.fill_constant(shape=[128], dtype="float16", value=0)
# TODO(xreki): fix precision problem when using softmax of float16.
# tmp_2 = layers.softmax(tmp_1)
tmp_2 = layers.elementwise_add(tmp_1, zero)
tmp_3 = layers.mul(tmp_0, self.feed_vars[2])
# subgraph with 4 op nodes
tmp_3 = layers.cast(tmp_2, dtype="float16")
tmp_4 = layers.relu(tmp_1 + tmp_3)
tmp_5 = layers.cast(tmp_4, dtype=dtype)
tmp_3 = layers.cast(tmp_2, dtype=dtype)
self.append_gradients(tmp_5)
self.num_fused_ops = 4
self.fetch_list = [tmp_5, self.grad(tmp_0)]
def test_mul(self):
i = zeros(shape=[1], dtype='int64')
a = data(name='a', shape=[784], dtype='float32')
array = array_write(x=a, i=i)
i = increment(i)
b = data(
name='b',
shape=[784, 100],
dtype='float32',
append_batch_size=False)
array_write(x=b, i=i, array=array)
i = increment(i)
out = mul(x=a, y=b)
array_write(x=out, i=i, array=array)
a_np = numpy.random.random((100, 784)).astype('float32')
b_np = numpy.random.random((784, 100)).astype('float32')
exe = Executor()
res, res_array = exe.run(feed={'a': a_np,
'b': b_np},
fetch_list=[out, array])
self.assertEqual((100, 100), res.shape)
self.assertTrue(numpy.allclose(res, numpy.dot(a_np, b_np)))
self.assertTrue(numpy.allclose(res_array[0], a_np))
self.assertTrue(numpy.allclose(res_array[1], b_np))
self.assertTrue(numpy.allclose(res_array[2], res))
def test_calc_gradient(self):
x = layers.create_parameter(dtype="float32", shape=[5, 10])
y = layers.create_parameter(dtype="float32", shape=[10, 8])
mul_out = layers.mul(x=x, y=y)
mean_out = layers.mean(mul_out)
a = calc_gradient(mean_out, mul_out)
b = calc_gradient(mean_out, x)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
exe.run(fluid.default_main_program(), feed={}, fetch_list=[a, b])
def build_program(self, dtype):
with fluid.program_guard(self.main_program, self.startup_program):
self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3)
self.feed_vars.append(
fluid.data(name="data3", shape=[128, 32], dtype=dtype))
# subgraph with 3 op node
tmp_0 = self.feed_vars[0] - self.feed_vars[1]
tmp_1 = tmp_0 * self.feed_vars[2]
tmp_2 = layers.assign(tmp_1, output=tmp_0)
tmp_3 = layers.mul(tmp_2, self.feed_vars[3])
self.num_fused_ops = 1
self.fetch_list = [tmp_3]
def test_calc_gradient(self):
main = fluid.Program()
startup = fluid.Program()
with fluid.program_guard(main, startup):
x = layers.create_parameter(dtype="float32", shape=[5, 10])
y = layers.create_parameter(dtype="float32", shape=[10, 8])
mul_out = layers.mul(x=x, y=y)
mean_out = layers.mean(mul_out)
a = calc_gradient(mean_out, mul_out)
b = calc_gradient(mean_out, x)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup)
exe.run(main, feed={}, fetch_list=[a, b])
def test_mul(self):
a = data(name='a', shape=[784], dtype='float32')
b = data(name='b',
shape=[784, 100],
dtype='float32',
append_batch_size=False)
out = mul(x=a, y=b)
place = core.CPUPlace()
a_np = numpy.random.random((100, 784)).astype('float32')
b_np = numpy.random.random((784, 100)).astype('float32')
exe = Executor(place)
outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[out])
out = outs[0]
self.assertEqual((100, 100), out.shape)
self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
def build_program(self, dtype):
with fluid.program_guard(self.main_program, self.startup_program):
self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3)
self.feed_vars.append(
fluid.data(name="data3", shape=[128, 32], dtype=dtype))
# subgraph with 3 op node
tmp_0 = self.feed_vars[0] + self.feed_vars[1]
tmp_1 = layers.relu(self.feed_vars[2] * tmp_0)
# subgraph with 2 op nodes
tmp_2 = layers.relu(layers.sigmoid(self.feed_vars[3]))
tmp_3 = layers.mul(tmp_1, tmp_2)
self.append_gradients(tmp_3)
self.num_fused_ops = 2
self.fetch_list = [tmp_3, self.grad(tmp_1)]
def build_program(self, dtype):
with fluid.program_guard(self.main_program, self.startup_program):
self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2)
self.feed_vars.append(
fluid.data(name="data2", shape=[128, 128], dtype=dtype))
# subgraph with only 1 op node
tmp_0 = self.feed_vars[0] * self.feed_vars[1]
tmp_1 = layers.mul(tmp_0, self.feed_vars[2])
# subgraph with 2 op nodes
tmp_2 = layers.relu(tmp_0 + tmp_1)
self.append_gradients(tmp_2)
self.num_fused_ops = 2
self.fetch_list = [tmp_2, self.grad(tmp_1)]
def test_mul(self):
a = data(name='a', shape=[784], dtype='float32')
b = data(
name='b',
shape=[784, 100],
dtype='float32',
append_batch_size=False)
out = mul(x=a, y=b)
place = core.CPUPlace()
a_np = numpy.random.random((100, 784)).astype('float32')
b_np = numpy.random.random((784, 100)).astype('float32')
exe = Executor(place)
outs = exe.run(feed={'a': a_np, 'b': b_np}, fetch_list=[out])
out = outs[0]
self.assertEqual((100, 100), out.shape)
self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
def func(self, place):
# the shape of input variable shoule be clearly specified, not inlcude -1.
x_shape = [7, 11]
y_shape = [11, 9]
eps = 0.005
dtype = np.float64
x = layers.data('x', x_shape, False, dtype)
x.persistable = True
y = layers.data('y', y_shape, False, dtype)
y.persistable = True
out = layers.mul(x, y)
x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype)
y_arr = np.random.uniform(-1, 1, y_shape).astype(dtype)
gradient_checker.double_grad_check(
[x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps)
def build_program(self, dtype):
with fluid.program_guard(self.main_program, self.startup_program):
self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3)
self.feed_vars.append(
fluid.data(name="data3", shape=[128, 32], dtype=dtype))
# subgraph with 3 op node
tmp_0 = self.feed_vars[0] + self.feed_vars[1]
tmp_1 = layers.relu(self.feed_vars[2] * tmp_0)
# subgraph with 2 op nodes
tmp_2 = layers.relu(layers.sigmoid(self.feed_vars[3]))
tmp_3 = layers.mul(tmp_1, tmp_2)
# TODO(wangchaochaohu): support the case when some vars are set
# stop_gradient = True.
self.num_fused_ops = 2
self.fetch_list = [tmp_3]
def build_program(self, dtype):
with fluid.program_guard(self.main_program, self.startup_program):
self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3)
self.feed_vars.append(
fluid.data(name="data3", shape=[128, 128], dtype=dtype))
# subgraph with 2 op nodes
tmp_0 = layers.sum(
[self.feed_vars[0], self.feed_vars[1], self.feed_vars[2]])
tmp_1 = layers.sqrt(tmp_0)
tmp_2 = layers.mul(tmp_0, self.feed_vars[3])
# subgraph with 2 op nodes
tmp_3 = layers.square(layers.sum([tmp_1, tmp_2]))
self.append_gradients(tmp_3)
self.num_fused_ops = 4
self.fetch_list = [tmp_3, self.grad(tmp_0)]
def build_program(self, dtype):
with fluid.program_guard(self.main_program, self.startup_program):
self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2)
self.feed_vars.append(
fluid.data(name="data2", shape=[128, 128], dtype=dtype))
# subgraph with 2 op nodes
tmp_0 = self.feed_vars[0] * self.feed_vars[1]
tmp_1 = layers.softmax(layers.cast(tmp_0, dtype="float16"))
tmp_2 = layers.mul(tmp_0, self.feed_vars[2])
# subgraph with 4 op nodes
tmp_3 = layers.cast(tmp_2, dtype="float16")
tmp_4 = layers.relu(tmp_1 + tmp_3)
tmp_5 = layers.cast(tmp_4, dtype=dtype)
self.append_gradients(tmp_5)
self.num_fused_ops = 4
self.fetch_list = [tmp_5, self.grad(tmp_0)]
def subsample_labels(labels, num_samples, positive_fraction):
positive = torch.nonzero(mul((labels != config.ignore_label).cast('int'),
(labels != 0).cast('int')).cast('bool'),
as_tuple=False).squeeze(1)
negative = torch.nonzero(labels == 0, as_tuple=False).squeeze(1)
num_pos = int(num_samples * positive_fraction)
num_pos = min(positive.numel(), num_pos)
num_neg = num_samples - num_pos
num_neg = min(negative.numel(), num_neg)
# randomly select positive and negative examples
if type(num_pos) == torch.Tensor:
num_pos = num_pos.numpy().item()
if type(num_neg) == torch.Tensor:
num_neg = num_neg.numpy().item()
perm1 = torch.randperm(positive.numel())[:num_pos]
perm2 = torch.randperm(negative.numel())[:num_neg]
pos_idx = torch.gather(positive, perm1)
neg_idx = torch.gather(negative, perm2)
return pos_idx, neg_idx
def test_mul(self):
a = data(name='a', shape=[784], dtype='float32')
b = data(
name='b',
shape=[784, 100],
dtype='float32',
append_batch_size=False)
output = mul(x=a, y=b)
place = core.CPUPlace()
a_np = numpy.random.random((100, 784)).astype('float32')
b_np = numpy.random.random((784, 100)).astype('float32')
exe = Executor(place)
import time
use_cache = True
step_num = 3
run_time = 0.0
for i in range(step_num):
begin = time.time()
outs = exe.run(feed={'a': a_np,
'b': b_np},
fetch_list=[output.name],
use_program_cache=use_cache)
end = time.time()
run_time += end - begin
out = outs[0]
self.assertEqual((100, 100), out.shape)
self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
print("run time %f" % run_time)
use_cache = False
run_time = 0.0
for i in range(step_num):
begin = time.time()
outs = exe.run(feed={'a': a_np,
'b': b_np},
fetch_list=[output.name],
use_program_cache=use_cache)
end = time.time()
run_time += end - begin
out = outs[0]
self.assertEqual((100, 100), out.shape)
self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
print("run time %f" % run_time)
use_cache = True
run_time = 0.0
for i in range(step_num):
begin = time.time()
outs = exe.run(feed={'a': a_np,
'b': b_np},
fetch_list=[output.name],
use_program_cache=use_cache)
end = time.time()
run_time += end - begin
out = outs[0]
self.assertEqual((100, 100), out.shape)
self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
print("run time %f" % run_time)
use_cache = True
run_time = 0.0
for i in range(step_num):
begin = time.time()
outs = exe.run(feed={'a': a_np,
'b': b_np},
fetch_list=[output],
use_program_cache=use_cache)
end = time.time()
run_time += end - begin
out = outs[0]
self.assertEqual((100, 100), out.shape)
self.assertTrue(numpy.allclose(out, numpy.dot(a_np, b_np)))
print("run time %f" % run_time)
