def test_gpu_node_deconvolution2d(a):
with use_cuda():
layer = rm.Deconv2d(channel=32)
layer.params["w"] = rm.Variable(np.random.rand(3, 32, 3, 3))
layer.params["b"] = rm.Variable(np.random.rand(1, 32, 1, 1))
g1 = Variable(a)
g2 = layer(g1)
g3 = rm.sum(g2)
g = g3.grad()
g_g1 = g.get(layer.params["w"])
g_g2 = g.get(layer.params["b"])
g_g3 = g.get(g1)
g2.to_cpu()
g3.to_cpu()
c2 = layer(g1)
c3 = rm.sum(c2)
c = c3.grad()
c_g1 = c.get(layer.params["w"])
c_g2 = c.get(layer.params["b"])
c_g3 = g.get(g1)
close(g2, c2)
close(g3, c3)
close(c_g1, g_g1)
close(c_g2, g_g2)
close(c_g3, g_g3)
def test_gpu_node_convolutionnd(a):
with use_cuda():
layer = rm.ConvNd(channel=2, filter=1, stride=1, padding=0)
#layer.params["w"] = rm.Variable(np.random.rand(32, 3, 3, 3))
#layer.params["b"] = rm.Variable(np.random.rand(1, 32, 1, 1))
g1 = Variable(a)
g2 = layer(g1)
g3 = rm.sum(g2)
g = g3.grad()
g_g1 = g.get(layer.params["w"])
g_g2 = g.get(layer.params["b"])
g_g3 = g.get(g1)
g2.to_cpu()
g3.to_cpu()
c2 = layer(g1)
c3 = rm.sum(c2)
c = c3.grad()
c_g1 = c.get(layer.params["w"])
c_g2 = c.get(layer.params["b"])
c_g3 = g.get(g1)
close(g2, c2)
close(g3, c3)
close(c_g1, g_g1)
close(c_g2, g_g2)
close(c_g3, g_g3)
def test_gpu_node_deconvolutionnd(a):
a = np.ones_like(a)
with use_cuda():
layer = rm.DeconvNd(channel=2, filter=3, stride=1, padding=0)
g1 = Variable(a)
g2 = layer(g1)
g3 = rm.sum(g2)
g = g3.grad()
g_g1 = g.get(layer.params["w"])
g_g2 = g.get(layer.params["b"])
g_g3 = g.get(g1)
g2.to_cpu()
g3.to_cpu()
c2 = layer(g1)
c3 = rm.sum(c2)
c = c3.grad()
c_g1 = c.get(layer.params["w"])
c_g2 = c.get(layer.params["b"])
c_g3 = g.get(g1)
close(g2, c2)
close(g3, c3)
close(c_g1, g_g1)
close(c_g2, g_g2)
close(c_g3, g_g3)
def test_gpu_node_dot(a, b):
set_cuda_active(True)
g1 = Variable(a)
g2 = Variable(b)
g3 = dot(g1, g2)
g4 = rm.sum(g3)
g = g4.grad()
g_g1 = g.get(g1)
g_g2 = g.get(g2)
g_g3 = g.get(g3)
g3.to_cpu()
g4.to_cpu()
set_cuda_active(False)
c3 = dot(g1, g2)
c4 = rm.sum(c3)
c = c4.grad()
c_g1 = c.get(g1)
c_g2 = c.get(g2)
c_c3 = c.get(c3)
close(g3, c3)
close(g4, c4)
close(c_g1, g_g1)
close(c_g2, g_g2)
close(c_c3, g_g3)
def test_gpu_layer_normalize(a):
set_cuda_active(True)
g1 = Variable(a)
layer = rm.LayerNormalize()
g2 = layer(g1)
g3 = rm.sum(g2)
g = g3.grad(detach_graph=False)
g_g1 = g.get(g1)
g_g2 = g.get(layer.params["gain"])
g_g3 = g.get(layer.params["bias"])
set_cuda_active(False)
c2 = layer(g1)
c3 = rm.sum(c2)
c = c3.grad(detach_graph=False)
c_c1 = c.get(g1)
c_c2 = c.get(layer.params["gain"])
c_c3 = c.get(layer.params["bias"])
close(g2, c2)
close(g3, c3)
close(g_g1, c_c1)
close(g_g2, c_c2)
close(g_g3, c_c3)
def _backward_gpu(self, context, dy, **kwargs):
norm = self.attrs._norm
if isinstance(self.attrs._x, Node):
dx = dy * norm - (rm.sum(self.attrs._x * dy, axis=1,
keepdims=True) * self.attrs._x) / norm
dx = dx / (norm * norm)
self.attrs._x._update_diff(context, get_gpu(dx * self.attrs._w), **kwargs)
if isinstance(self.attrs._w, Node):
dl = dy * (self.attrs._x / norm)
self.attrs._w._update_diff(context,
get_gpu(rm.sum(dl.as_ndarray(), axis=(0, 2, 3), keepdims=True)), **kwargs)
def forward(self, x, eps=1.e-6, noise=None):
nb = len(x)
noise_rate, noise_sigma = 0, 1
if noise:
if isinstance(noise,tuple):
noise_rate, noise_sigma = noise
else:
noise_rate = noise
noise_rate = noise_rate if noise_rate <= 1 else 1
noise_rate = noise_rate if 0 <= noise_rate else 0
noise_sigma = noise_sigma if 0 < noise_sigma else np.abs(noise_sigma)
size = (nb, self.latent_dim)
if 1:
z = np.random.randn(
np.array(size).prod()
).reshape(size).astype('float32')
else:
z = np.random.uniform(
-1, 1, np.array(size).prod()
).reshape(size).astype('float32')
self.xz = self.gen(z)
if noise_rate > 0:
noised = x.copy()
_noise_idx = np.random.permutation(nb)
_noise_idx = _noise_idx[np.random.permutation(int(nb*noise_rate))]
_noise = np.random.randn(len(_noise_idx))*noise_sigma
noised[_noise_idx] += _noise.reshape(-1, 1)
self.Disx = self.dis(noised)
else:
self.Disx = self.dis(x)
self.Disxz = self.dis(self.xz)
self.real_cost = - rm.sum(
rm.log(self.Disx + eps))/nb
self.fake_cost = - rm.sum(
rm.log(1 - self.Disxz + eps))/nb
self.GAN_loss = self.real_cost + self.fake_cost
self.real_count = (self.Disx >= 0.5).sum()/nb
self.fake_count = (self.Disxz < 0.5).sum()/nb
if self.gan_mode == 'minmax':
self.gen_loss = - self.GAN_loss
elif self.gan_mode == 'non-saturating':
self.gen_loss = - rm.sum(rm.log(self.Disxz + eps))/nb
return self.xz
def forward(self, x):
N = len(x)
h, _ = self._base(x)
D = h.shape[2] * h.shape[3]
h = rm.sum(self._last(h).reshape(N, self._num_class, -1), axis=2)
h /= D
return h
def test_indexing(node):
set_cuda_active(True)
g1 = Variable(node)
g3 = rm.sum(g1[1:2, -1])
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
g_g1.to_cpu()
set_cuda_active(False)
c3 = rm.sum(g1[1:2, -1])
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def loss(self, x, y):
if hasattr(self, 'train_class_weight'):
loss = rm.softmax_cross_entropy(x, y, reduce_sum=False) * \
np.broadcast_to(class_weight.reshape(1, -1, 1, 1), x.shape)
loss = rm.sum(loss)
else:
loss = rm.softmax_cross_entropy(x, y)
return loss / (self.imsize[0] * self.imsize[1])
def test_gpu_node_average_pooling(a):
with use_cuda():
layer = rm.AveragePool2d()
g1 = Variable(a)
g3 = rm.sum(layer(g1))
g = g3.grad()
g_g3 = g.get(g1)
g3.to_cpu()
c3 = rm.sum(layer(g1))
c3.grad()
c_g3 = g.get(g1)
close(g3, c3)
close(c_g3, g_g3)
def test_gpu_node_reshape(a):
set_cuda_active(True)
g1 = Variable(a)
g3 = rm.sum(rm.reshape(g1, shape=(-1, 1)))
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
set_cuda_active(False)
c3 = rm.sum(rm.reshape(g1, shape=(-1, 1)))
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def _oper_gpu(cls, x, w):
norm = rm.sqrt(rm.sum(x * x, axis=1, keepdims=True)) + 1e-5
z = (x / norm) * w
ret = cls._create_node(get_gpu(z))
ret.attrs._norm = norm
ret.attrs._x = x
ret.attrs._w = w
return ret
def test_batch_normalize(a):
layer = rm.Sequential([rm.BatchNormalize(momentum=0.1)])
set_cuda_active(True)
g1 = Variable(a)
g2 = layer(g1)
g3 = rm.sum(g2)
g = g3.grad()
g_g1 = g.get(g1)
g_g2 = g.get(layer.l0.params["w"])
g_g3 = g.get(layer.l0.params["b"])
layer.set_models(inference=True)
g4 = layer(g1)
layer.set_models(inference=False)
g2.to_cpu()
g3.to_cpu()
g4.to_cpu()
g_g1.to_cpu()
g_g2.to_cpu()
g_g3.to_cpu()
set_cuda_active(False)
layer.l0._mov_mean = 0
layer.l0._mov_std = 0
c2 = layer(g1)
c3 = rm.sum(c2)
c = c3.grad()
c_g1 = c.get(g1)
c_g2 = g.get(layer.l0.params["w"])
c_g3 = g.get(layer.l0.params["b"])
layer.set_models(inference=True)
c4 = layer(g1)
layer.set_models(inference=False)
close(g2, c2)
close(g3, c3)
close(g4, c4)
close(c_g1, g_g1)
close(c_g2, g_g2)
close(c_g3, g_g3)
def test_gpu_node_pow(a, b):
set_cuda_active(True)
g1 = Variable(a)
g2 = Variable(b)
g3 = rm.sum(g1**g2)
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
set_cuda_active(False)
c3 = rm.sum(g1**g2)
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def test_where(node):
# set_cuda_active(is_active)
with use_cuda():
g1 = Variable(node)
g3 = rm.sum(rm.where(g1 > 0.5, g1, 1))
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
g_g1.to_cpu()
with use_cuda():
c3 = rm.sum(rm.where(g1 > 0.5, g1, 1))
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def forward(self, x, eps=1e-3):
nb = len(x)
self.enc(x)
e = np.random.randn(nb, self.latent_dim)
self.z = self.enc.zm + rm.exp(self.enc.zlv / 2) * e
self.decd = self.dec(self.z)
self.reconE = rm.mean_squared_error(self.decd, x)
self.kl_loss = -0.5 * rm.sum(1 + self.enc.zlv - self.enc.zm**2 -
rm.exp(self.enc.zlv)) / nb
return self.decd
def test_gpu_node_spatial_dropout(a):
with use_cuda():
g1 = Variable(a)
layer = rm.SpatialDropout()
np.random.seed(1)
g3 = rm.sum(layer(g1))
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
np.random.seed(1)
c3 = rm.sum(layer(g1))
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def test_gpu_dense(a):
layer = rm.Dense(output_size=2)
set_cuda_active(True)
g1 = Variable(a)
g3 = rm.sum(layer(g1))
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
g_g1.to_cpu()
set_cuda_active(False)
c3 = rm.sum(layer(g1))
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def test_gpu_node_add(a, b):
with use_cuda():
g1 = Variable(a)
g2 = Variable(b)
g3 = rm.sum(g1 + g2)
g = g3.grad()
g_g1 = g.get(g1)
g_g2 = g.get(g2)
g3.to_cpu()
c3 = rm.sum(g1 + g2)
c = c3.grad()
c_g1 = c.get(g1)
c_g2 = c.get(g2)
close(g3, c3)
close(c_g1, g_g1)
close(c_g2, g_g2)
def test_gpu_node_dropout(a):
set_cuda_active(True)
g1 = Variable(a)
layer = rm.Dropout()
np.random.seed(1)
g3 = rm.sum(layer(g1))
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
set_cuda_active(False)
np.random.seed(1)
c3 = rm.sum(layer(g1))
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def test_lrn(node):
layer = rm.Lrn()
with use_cuda():
g1 = Variable(node)
g3 = rm.sum(layer(g1))
g = g3.grad()
g_g1 = g.get(g1)
g3.to_cpu()
g_g1.to_cpu()
set_cuda_active(False)
c3 = rm.sum(layer(g1))
c = c3.grad()
c_g1 = c.get(g1)
close(g3, c3)
close(c_g1, g_g1)
def forward(self, x):
self.z_mean, self.z_log_var = self.enc(x)
e = np.random.randn(len(x), self.latent_dim) * 1.
z_new = self.z_mean + rm.exp(self.z_log_var / 2) * e
self.decoded = self.dec(z_new)
nb, zd = self.z_log_var.shape
self.kl_loss = -0.5 * rm.sum(1 + self.z_log_var - self.z_mean**2 -
rm.exp(self.z_log_var)) / nb
#self.recon_loss = rm.sigmoid_cross_entropy(self.decoded, x)
self.recon_loss = rm.mean_squared_error(self.decoded, x)
vae_loss = self.kl_loss + self.recon_loss
return vae_loss
def test_gpu_node_max_pooling(a):
with use_cuda():
layer = rm.MaxPool2d()
g1 = Variable(a)
g2 = layer(g1)
g3 = rm.sum(g2)
g = g3.grad()
g_g3 = g.get(g1)
g2.to_cpu()
g3.to_cpu()
c2 = layer(g1)
c3 = rm.sum(c2)
c3.grad()
c_g3 = g.get(g1)
close(g2, c2)
close(g3, c3)
close(c_g3, g_g3)
def loss(self, x, y, class_weight=None):
if class_weight is not None and class_weight:
mask = np.concatenate([
np.ones((y.shape[0], 1, y.shape[2], y.shape[3])) * c
for c in class_weight
],
axis=1)
loss = rm.softmax_cross_entropy(x, y, reduce_sum=False)
loss *= mask.astype(y.dtype)
loss = rm.sum(loss) / float(len(x))
else:
loss = rm.softmax_cross_entropy(x, y)
return loss / (self.imsize[0] * self.imsize[1])
def forward(self, x, eps=1e-3):
nb = len(x)
size = (nb, self.latent_dim)
zp = np.random.randn(np.array(size).prod()).reshape(size).astype('float32')
self.xp = self.gen(zp)
self.Dis_xp = self.dis(self.xp)
self.Dis_xp_is = self.dis.raw_output
self.Dis_x = self.dis(x)
self.real_cost = - rm.sum(rm.log(self.Dis_x + eps))/nb
self.fake_cost = - rm.sum(rm.log(1 - self.Dis_xp + eps))/nb
self.GAN_loss = self.real_cost + self.fake_cost
gan_mode = 'non-saturating'
if gan_mode == 'minmax':
self.gen_loss = - self.GAN_loss
elif gan_mode == 'non-saturating':
self.gen_loss = - rm.sum(rm.log(self.Dis_xp + eps))/nb
elif gan_mode == 'max-likelihood':
self.gen_loss = - rm.sum(rm.exp(self.Dis_xp_is))/nb
return self.GAN_loss
def test_gpu_node_upconvolution2d(a):
with use_cuda():
layer = rm.Deconv2d(channel=32)
g1 = Variable(a)
g3 = rm.sum(layer(g1))
g = g3.grad()
g_g1 = g.get(layer.params["w"])
g_g2 = g.get(layer.params["b"])
g_g3 = g.get(g1)
g3.to_cpu()
c3 = rm.sum(layer(g1))
c = c3.grad()
c_g1 = c.get(layer.params["w"])
c_g2 = c.get(layer.params["b"])
c_g3 = g.get(g1)
close(g3, c3)
close(c_g1, g_g1)
close(c_g2, g_g2)
close(c_g3, g_g3)
def test_weight_decay():
set_cuda_active(False)
def add_weight_decay(weighted_model, decay):
reg = rm.sum(weighted_model.params.w * weighted_model.params.w)
return reg * (decay / 2)
test_decay = 0.25
input = np.random.rand(2, 2)
unweighted_model = rm.Dense(2, input_size=(2, ))
weighted_model = rm.Dense(2, input_size=(2, ), weight_decay=test_decay)
unweighted_model.params['w'].setflags(write=True)
unweighted_model.params['w'][...] = weighted_model.params['w'][...]
set_cuda_active(True)
with unweighted_model.train(), weighted_model.train():
unweighted_loss = rm.sum(unweighted_model(input))
weighted_loss = rm.sum(weighted_model(input))
added_loss = rm.sum(unweighted_loss +
add_weight_decay(unweighted_model, test_decay))
n_1 = unweighted_loss.grad(weight_decay=test_decay,
detach_graph=False).get(
unweighted_model.params["w"])
n_2 = weighted_loss.grad(detach_graph=False).get(
weighted_model.params["w"])
n_3 = added_loss.grad(detach_graph=False).get(unweighted_model.params["w"])
map(lambda x: x.to_cpu(), [n_1, n_2, n_3])
try:
assert np.allclose(n_1, n_2)
assert np.allclose(n_1, n_3)
except AssertionError:
print("Error in weight decay")
print(n_1)
print(n_2)
print(n_3)
assert False
def loss(self, x, y, neg_pos_ratio=3.0):
pos_samples = (y[:, :, 5] == 0)[..., None]
N = np.sum(pos_samples)
pos_Ns = np.sum(pos_samples, axis=1)
neg_Ns = np.clip(neg_pos_ratio * pos_Ns, 0, y.shape[1])
# Loc loss
loc_loss = rm.sum(
rm.smoothed_l1(x[..., :4], y[..., 1:5], reduce_sum=False) *
pos_samples)
# this is for hard negative mining.
np_x = x[..., 4:].as_ndarray()
max_np_x = np.max(np_x)
loss_c = np.log(
np.sum(np.exp(np_x.reshape(-1, self.num_class) - max_np_x),
axis=1,
keepdims=True) + 1e-8) + max_np_x
loss_c -= np_x[..., 0].reshape(-1, 1)
loss_c = loss_c.reshape(len(x), -1)
loss_c[pos_samples.astype(
np.bool)[..., 0]] = np.Inf # Cut positive samples.
sorted_index = np.argsort(-1 * loss_c,
axis=1) # Arg sort by dicending order.
index_rank = np.argsort(sorted_index, axis=1)
neg_samples = index_rank < neg_Ns
samples = (neg_samples[..., None] + pos_samples).astype(np.bool)
conf_loss = rm.sum(
rm.softmax_cross_entropy(x[..., 4:].transpose(0, 2, 1),
y[..., 5:].transpose(0, 2, 1),
reduce_sum=False).transpose(0, 2, 1) *
samples)
loss = conf_loss + loc_loss
return loss / (N / len(x))
def regularize(self):
"""
Regularization term to a loss function.
Example:
>>> x = numpu.random.rand(1, 3, 224, 224)
>>> y = numpy.random.rand(1, (5*2+20)*7*7)
>>> model = ${class}()
>>> loss = model.loss(x, y)
>>> reg_loss = loss + model.regularize() # Add weight decay term.
"""
reg = 0
for layer in self.iter_models():
if hasattr(layer, "params") and hasattr(layer.params, "w"):
reg += rm.sum(layer.params.w * layer.params.w)
return (self.decay_rate / 2) * reg