def test_gpu_node_dot(a, b):
set_cuda_active(True)
g1 = Variable(a)
g2 = Variable(b)
g3 = dot(g1, g2)
g4 = rm.sum(g3)
g = g4.grad()
g_g1 = g.get(g1)
g_g2 = g.get(g2)
g_g3 = g.get(g3)
g3.to_cpu()
g4.to_cpu()
set_cuda_active(False)
c3 = dot(g1, g2)
c4 = rm.sum(c3)
c = c4.grad()
c_g1 = c.get(g1)
c_g2 = c.get(g2)
c_c3 = c.get(c3)
close(g3, c3)
close(g4, c4)
close(c_g1, g_g1)
close(c_g2, g_g2)
close(c_c3, g_g3)
def test_layer_normalize(node, use_gpu):
node = Variable(node * 50)
assert_cuda_active(use_gpu)
layer = LayerNormalize()
layer2 = Dense(4)
layer3 = Conv2d(channel=3)
def func(node):
ret = layer(node)
if len(ret.shape) > 2:
return sum(layer3(ret))
else:
return sum(layer2(ret))
a = 1e-5
r = 1e-3
if use_gpu:
a = 1e-2
r = 1e-3
for trial in range(3):
try:
compare(func, node, node, atol=a, rtol=r)
compare(func, layer.params["gain"], node)
compare(func, layer.params["bias"], node)
return
except:
node = Variable(rand(node.shape))
assert False
def test_max_unpoolnd(node, use_gpu):
assert_cuda_active(use_gpu)
node = Variable(node)
l0 = MaxPoolNd(kernel=2, padding=1, stride=1)
l1 = MaxUnPoolNd()
l2 = Dense(2)
np.set_printoptions(suppress=True)
def func(node):
ret = node
reta = l0(node)
ret = l1(reta, reta)
ret = l2(ret.reshape(ret.shape[0], -1))
ret = sum(ret)
return ret
for trial in range(3):
try:
compare(func, node, node)
return
except AssertionError as e:
print(e)
node = Variable(rand(node.shape))
raise AssertionError("Failed all attempts.")
def test_gpu_broadcast(a, b):
set_cuda_active(True)
g1 = Variable(a)
g2 = Variable(b)
assert np.allclose(a + b, (g1 + g2))
def test_copy_from_cpu():
src = Variable(rand((100, )))
dest = Variable(rand((100, )))
dest.copy_from(src)
close(src, dest)
def weight_initiallize(self, input_size):
# The first dimension is to allow different types of uncorrelated images as inputs, such as RGB information.
# After this dimension, the image data is assumed to be meaningfully correlated.
self._dims = len(input_size[1:])
if is_cuda_active():
assert self._dims < 4, "GPU Version currently only supports 2 and 3 dimensions"
if self._dims == 1 and is_cuda_active():
padding, stride, filter = self._initial_value
self._kernel = np.append(filter, 1).astype(np.int32)
self._padding = np.append(padding, 0).astype(np.int32)
self._stride = np.append(stride, 1).astype(np.int32)
def func(var):
return check_input(var, self._dims)
self._kernel, self._padding, self._stride = map(
func, [self._kernel, self._padding, self._stride])
assert all([s > 0 for s in input_size[1:]]), \
"The shape of input array {} is too small. Please give an array which size is lager than 0.".format(
input_size[1:])
f_lst = [self._channel, input_size[0]]
f_lst.extend(self._kernel)
size_f = tuple(f_lst)
size_b = tuple([1, self._channel] + [1 for _ in range(self._dims)])
self.params = {"w": Variable(self._initializer(size_f), auto_update=True)}
if not self._ignore_bias:
self.params["b"] = Variable(np.zeros(size_b), auto_update=True)
def weight_initiallize(self, input_size):
size_i = input_size[0] if isinstance(input_size, tuple) else input_size
size_o = self._output_size
self.params = {
"w":
Variable(self._initializer((size_i, size_o)), auto_update=True),
"b":
Variable(np.zeros((1, size_o)).astype(precision), auto_update=True)
}
def weight_initiallize(self, size_i):
size_i = size_i[0]
size_o = self._size_o
bias = np.zeros((1, size_o * 4), dtype=precision)
bias[:, size_o:size_o * 2] = 1
self.params = {
"w": Variable(self._initializer((size_i, size_o * 4)), auto_update=True),
"wr": Variable(self._initializer((size_o, size_o * 4)), auto_update=True),
"b": Variable(bias, auto_update=True),
}
def test_node_dump():
DEBUG_GRAPH_INIT(True)
a = Variable(np.array([1, 2, 3, 4, 5]))
b = Variable(np.array([1, 2, 3, 4, 5]))
c = a + b # NOQA
d = a + b * 2 # NOQA
DEBUG_NODE_STAT()
def test_dot(node, x, use_gpu):
node = Variable(node)
x = Variable(x)
assert_cuda_active(use_gpu)
def func(node, x):
return sum(rm.dot(node, x))
compare(func, node, node, x)
compare(func, x, node, x)
def weight_initiallize(self, input_size):
size_f = (self._channel, input_size[0], self._kernel[0],
self._kernel[1])
self.params = {
"w":
Variable(self._initializer(size_f), auto_update=True),
"b":
Variable(np.zeros((1, self._channel, 1, 1), dtype=precision),
auto_update=True)
}
def weight_initiallize(self, size_i):
size_i = size_i[0]
size_o = self._size_o
bias = np.zeros((1, size_o * 4), dtype=precision)
bias[:, size_o:size_o * 2] = 1
self.params = {
"w": Variable(self._initializer((size_i, size_o * 4)), auto_update=True, weight_decay=self._weight_decay),
"wr": Variable(self._initializer((size_o, size_o * 4)), auto_update=True, weight_decay=self._weight_decay),
"wc": Variable(self._initializer((1, size_o * 3)), auto_update=True, weight_decay=self._weight_decay)}
if not self._ignore_bias:
self.params["b"] = Variable(bias, auto_update=True)
def weight_initiallize(self, size_i):
size_i = size_i[0]
size_o = self._size_o
bias = np.zeros((1, size_o * 3), dtype=precision)
# At this point, all connected units in the same layer will use the SAME weights
self.params = {
"w": Variable(self._initializer((size_i, size_o * 3)), auto_update=True, weight_decay=self._weight_decay),
"u": Variable(self._initializer((size_o, size_o * 3)), auto_update=True, weight_decay=self._weight_decay),
}
if not self._ignore_bias:
self.params["b"] = Variable(bias, auto_update=True)
def test_node_clear():
DEBUG_GRAPH_INIT(True)
a = Variable(np.random.rand(2, 2).astype(np.float32))
b = Variable(np.random.rand(2, 2).astype(np.float32))
layer = R.Lstm(2)
c = layer(O.dot(a, b)) # NOQA
DEBUG_NODE_STAT()
def weight_initiallize(self, input_size):
size_f = (input_size[0], self._channel, self._kernel[0],
self._kernel[1])
self.params = {
"w":
Variable(self._initializer(size_f),
auto_update=True,
weight_decay=self._weight_decay)
}
if not self._ignore_bias:
self.params["b"] = Variable(np.zeros((1, self._channel, 1, 1),
dtype=precision),
auto_update=True)
def test_copy_from_gpu():
set_cuda_active(True)
src = Variable(rand((100, )))
src.to_gpu()
dest = Variable(rand((100, )))
dest.to_gpu()
dest.copy_from(src)
close(src, dest)
close(src._gpu.new_array(), dest._gpu.new_array())
def test_mean_squared_error_no_reduce(node, x, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
def func(node, x):
return sum(rm.mean_squared_error(node, x, reduce_sum=False))
for trial in range(3):
try:
compare(func, node, node, x)
return
except AssertionError:
node = Variable(rand(node.shape))
assert False
def weight_initiallize(self, input_size):
size_i = input_size[0] if isinstance(input_size, tuple) else input_size
size_o = self._output_size
self.params = {
"w":
Variable(self._initializer((size_i, size_o)),
auto_update=True,
weight_decay=self._weight_decay)
}
if not self._ignore_bias:
self.params["b"] = Variable(np.zeros(
(1, size_o)).astype(precision),
auto_update=True)
def test_copy_from_another_gpu():
set_cuda_active(True)
src = Variable(rand((100, )))
src.to_gpu()
with use_device(1):
dest = Variable(rand((100, )))
dest.to_gpu()
dest.copy_from(src)
close(src, dest)
close(src._gpu.new_array(), dest._gpu.new_array())
def weight_initiallize(self, input_size):
size_i = [
1,
]
size_i.extend(input_size)
if self._mode == BATCH_NORMALIZE_FEATUREMAP and len(size_i) > 2:
size_i[2] = 1
size_i[3] = 1
self.params = {
"w":
Variable(self._initializer(size_i).astype(precision),
auto_update=True),
"b":
Variable(np.zeros(size_i, dtype=precision), auto_update=True)
}
def test_max_pool2d(node, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
layer = MaxPool2d(filter=2, padding=1, stride=2)
def func(node):
return sum(layer(node))
for trial in range(3):
try:
compare(func, node, node)
return
except AssertionError:
node = Variable(rand(node.shape))
raise AssertionError("Failed all three attempts.")
def test_div(node, x, raise_error, use_gpu):
node = Variable(node)
x = np.array(x)
assert_cuda_active(use_gpu)
def func_div1(node, x):
return sum(x / node)
compare(func_div1, node, node, x)
def func_div2(node, x):
return sum(node / x)
compare(func_div2, node, node, x)
def func_idiv1(node, x):
node /= x
return sum(node)
try:
compare(func_idiv1, node, node, x)
assert not raise_error
except:
assert raise_error
def func_idiv2(node, x):
x /= node
return sum(node)
try:
compare(func_idiv2, node, node, x)
assert not raise_error
except:
assert raise_error
def test_softsign(node, use_gpu):
node = Variable(node)
set_cuda_active(use_gpu)
def func(node):
return sum(rm.softsign(node))
compare(func, node, node)
def test_swish_activation(node, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
def func(node):
return sum(rm.swish(node))
compare(func, node, node)
def test_softmax(node, x, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
def func(node, x):
return rm.cross_entropy(rm.softmax(node), x)
compare(func, node, node, x)
def test_leaky_relu_activation(node, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
def func(node):
return sum(rm.leaky_relu(node))
compare(func, node, node)
def test_mul(node, x, raise_error, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
def func_mul1(node, x):
return sum(x * node)
compare(func_mul1, node, node, x)
def func_mul2(node, x):
return sum(node * x)
compare(func_mul2, node, node, x)
def func_imul1(node, x):
node *= x
return sum(node)
try:
compare(func_imul1, node, node, x)
assert not raise_error
except:
assert raise_error
def func_imul2(node, x):
x *= node
return sum(node)
try:
compare(func_imul2, node, node, x)
assert not raise_error
except:
assert raise_error
def test_smooth_l1_no_reduce(node, x, delta, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
def func(node, x):
return sum(rm.smoothed_l1(node, x, delta, reduce_sum=False))
compare(func, node, node, x)
def _oper_cpu(cls, x, pz, w, u, b):
# Initialize Variables
m = w.shape[1] // 3
w_z, w_r, w_h = np.split(w, [m, m * 2, ], axis=1)
u_z, u_r, u_h = np.split(u, [m, m * 2], axis=1)
hminus = Variable(np.zeros((x.shape[0], w.shape[1] // 3),
dtype=precision)) if pz is None else pz
b_z, b_r, b_h = np.split(b, [m, m * 2], axis=1) if b is not None else (0, 0, 0)
A = dot(x, w_z) + dot(hminus, u_z) + b_z
B = dot(x, w_r) + dot(hminus, u_r) + b_r
C = dot(x, w_h) + sigmoid(B) * dot(hminus, u_h) + b_h
h = sigmoid(A) * hminus + (1 - sigmoid(A)) * tanh(C)
# Store Variables for Graph
ret = cls._create_node(h)
ret.attrs._x = x
ret.attrs._w = w
ret.attrs._w_z = w_z
ret.attrs._w_r = w_r
ret.attrs._w_h = w_h
ret.attrs._u = u
ret.attrs._u_z = u_z
ret.attrs._u_h = u_h
ret.attrs._u_r = u_r
ret.attrs._pz = hminus
ret.attrs._A = A
ret.attrs._B = B
ret.attrs._C = C
if b is not None:
ret.attrs._b = b
return ret
def test_sigmoid_activation(node, use_gpu):
node = Variable(node)
assert_cuda_active(use_gpu)
def func(node):
return sum(sigmoid(node))
compare(func, node, node)
