def test_clear_input_if_no_need_grad_batch_normalization(self, batch_stat):
x1 = nn.Variable([1, 1, 2], need_grad=True)
x2 = nn.Variable([1, 1, 1], need_grad=True)
x3 = nn.Variable([1, 1, 1], need_grad=True)
x4 = nn.Variable([1, 1, 1], need_grad=True)
x5 = nn.Variable([1, 1, 1], need_grad=True)
x = F.identity(x1)
beta = F.identity(x2)
gamma = F.identity(x3)
if batch_stat:
y = F.batch_normalization(
x, beta, gamma, x4, x5, batch_stat=batch_stat)
else:
mean = F.identity(x4)
var = F.identity(x5)
y = F.batch_normalization(
x, beta, gamma, mean, var, batch_stat=batch_stat)
answer = []
answer.append([False])
answer.append([False])
answer.append([False])
if not batch_stat:
answer.append([False])
answer.append([False])
answer.append([False, True, False, False, False])
y.forward(clear_no_need_grad=True)
self.check_input_data_clear_called_flags(answer)
def backtracking_line_search(grad_norm,
x,
y,
loss,
N,
val,
l2,
threshold=1e-10):
t = 10.0
beta = 0.5
params = nn.get_parameters().values()
p_data_org_list = [F.identity(p.data) for p in params]
p_grad_org_list = [F.identity(p.grad) for p in params]
while True:
for p, p_data_org, p_grad_org in zip(params, p_data_org_list,
p_grad_org_list):
p.data.copy_from(p_data_org - t * p_grad_org)
loss.forward()
if t < threshold:
print("t too small")
break
if (loss.d - val + t * grad_norm.data**2 / 2) >= 0:
t = beta * t
else:
break
return params
def test_reshape():
v = nn.Variable([2, 3, 4], need_grad=True)
grad = np.random.randn(*v.shape).astype(np.float32)
v.g = grad
v.d = np.random.randn(*v.shape)
import nnabla.functions as F
with nn.context_scope(nn.Context()), nn.auto_forward():
v2 = F.identity(v)
v2_s = v2.reshape((3, 4, 2))
v3 = F.identity(v2_s)
v3.backward(clear_buffer=False)
assert np.all(v2_s.g.flat == v2.g.flat)
assert np.all(v2_s.g == 1)
v2.d = 1
assert np.all(v2_s.d == 1)
v2.g = 1.5
assert np.all(v2_s.g == 1.5)
# Check unlink
v2_su = v2.reshape((3, 4, 2), unlink=True)
assert v2_su.need_grad
assert v2_su.parent is None
v2_su.need_grad = False
v2_su2 = v2_su.reshape((3, 4, 2), unlink=True)
assert not v2_su2.need_grad
assert v2_su2.parent is None
def graph(x1):
x1 = F.identity(x1).apply(recompute=True)
x2 = F.randn(shape=x1.shape, seed=123).apply(recompute=True)
x3 = F.rand(shape=x1.shape, seed=456).apply(recompute=True)
y = F.mul2(x1, x2).apply(recompute=True)
y = F.mul2(y, x3).apply(recompute=True)
y = F.identity(y)
return y
def network_LSTM(x, D, C, InputShape, HiddenSize, test=False):
# Input_2:x -> 687
# Delya_in:D -> 100
# Cell_in:C -> 100
# Concatenate -> 787
h = F.concatenate(D, x, axis=1)
# Affine -> 100
h1 = PF.affine(h, HiddenSize, name='Affine')
# InputGate -> 100
h2 = PF.affine(h, HiddenSize, name='InputGate')
# OutputGate -> 100
h3 = PF.affine(h, HiddenSize, name='OutputGate')
# ForgetGate -> 100
h4 = PF.affine(h, HiddenSize, name='ForgetGate')
# Sigmoid
h1 = F.sigmoid(h1)
# Sigmoid_2
h2 = F.sigmoid(h2)
# Sigmoid_3
h3 = F.sigmoid(h3)
# Sigmoid_4
h4 = F.sigmoid(h4)
# Mul2 -> 100
h1 = F.mul2(h1, h2)
# Mul2_3 -> 100
h4 = F.mul2(h4, C)
# Add2 -> 100
h1 = F.add2(h1, h4, True)
# Tanh
h5 = F.tanh(h1)
# Cell_out
h6 = F.identity(h1)
# Mul2_2 -> 100
h5 = F.mul2(h5, h3)
# Dropout
if not test:
h5 = F.dropout(h5)
# Output
h5 = F.identity(h5)
# Concatenate_2 -> 200
h5 = F.concatenate(h5, h6, axis=1)
return h5
def forward_impl(self, inputs, outputs):
x = inputs[0].data
M = inputs[1].data
y = outputs[0].data
y.copy_from(x)
if not self.training:
return
Mb = F.max(x, keepdims=True)
F.identity(self.decay * M + (1 - self.decay) * Mb, outputs=[M])
def cnn(batch_size, vocab_size, text_len, classes, features=128, train=True):
text = nn.Variable([batch_size, text_len])
with nn.parameter_scope("text_embed"):
embed = PF.embed(text, n_inputs=vocab_size, n_features=features)
print("embed", embed.shape)
embed = F.reshape(embed, (batch_size, 1, text_len, features))
print("embed", embed.shape)
combined = None
for n in range(2, 6): # 2 - 5 gram
with nn.parameter_scope(str(n) + "_gram"):
with nn.parameter_scope("conv"):
conv = PF.convolution(embed, 128, kernel=(n, features))
conv = F.relu(conv)
with nn.parameter_scope("pool"):
pool = F.max_pooling(conv, kernel=(conv.shape[2], 1))
if not combined:
combined = F.identity(pool)
else:
combined = F.concatenate(combined, pool)
if train:
combined = F.dropout(combined, 0.5)
with nn.parameter_scope("output"):
y = PF.affine(combined, classes)
t = nn.Variable([batch_size, 1])
_loss = F.softmax_cross_entropy(y, t)
loss = F.reduce_mean(_loss)
return text, y, loss, t
def test_clear_data_on_not_bwd_path(self):
a0 = nn.Variable((2, 3), need_grad=True)
a1 = F.identity(a0).apply(recompute=True)
a2 = F.sin(a1).apply(recompute=True)
# These three variables are not back-propagated.
b0 = nn.Variable((2, 3), need_grad=False)
b1 = F.identity(b0).apply(recompute=True)
b2 = F.sin(b1).apply(recompute=True)
c1 = F.add2(a2, b2).apply(recompute=True)
c2 = F.sin(c1)
# Forward
clear_called_flag_recorder.activate_clear_called_flag_recorder()
c2.forward(clear_no_need_grad=True)
# Data which will be recomputed must be cleared during forward propagation.
expected = [
[False], # a0
[True], # a1
[False], # b0
[True], # b1
[True, True], # a2, b2
[True], # c1
]
self.check_input_data_clear_called_flags(expected)
clear_called_flag_recorder.deactivate_clear_called_flag_recorder()
# Backward
clear_called_flag_recorder.activate_clear_called_flag_recorder()
c2.backward(clear_buffer=True)
# b1 is not on backward path and must be cleared during recomputation.
expected = [
# Recomputation
[False], # a0
[False], # a1
[False], # b0
[True], # b1 (not on backward path) must be cleared
[True, True], # a2, b2
[False], # c1
# Backward propagation
[True, True], # a2, b2
[False], # a1
[False], # a0
]
self.check_input_data_clear_called_flags(expected)
clear_called_flag_recorder.deactivate_clear_called_flag_recorder()
def test_rehape():
v = nn.Variable([2, 3, 4], need_grad=True)
grad = np.random.randn(*v.shape).astype(np.float32)
v.g = grad
v.d = np.random.randn(*v.shape)
import nnabla.functions as F
with nn.context_scope(nn.Context()), nn.auto_forward():
v2 = F.identity(v)
v2_s = v2.reshape((3, 4, 2))
v3 = F.identity(v2_s)
v3.backward(clear_buffer=False)
assert np.all(v2_s.g.flat == v2.g.flat)
assert np.all(v2_s.g == 1)
v2.d = 1
assert np.all(v2_s.d == 1)
v2.g = 1.5
assert np.all(v2_s.g == 1.5)
def test_unlinked():
v = nn.Variable([2, 3, 4], need_grad=True)
grad = np.random.randn(*v.shape).astype(np.float32)
v.g = grad
v.d = np.random.randn(*v.shape)
import nnabla.functions as F
with nn.context_scope(nn.Context()), nn.auto_forward():
v2 = F.identity(v)
v2_u = v2.unlinked()
v3 = F.identity(v2_u)
v2_u.grad.zero()
v2_g = v2_u.g.copy()
v3.backward(clear_buffer=False)
assert type(v2_u) == type(v2)
assert np.all(v.g == grad)
assert np.all(v2_u.g == v2.g)
assert np.all(v2_u.g == v2_g + 1)
def attention(k, q, v, div_dim=True, softmax=True):
v_shape = v.shape
k = F.identity(k)
q = F.identity(q)
k = F.reshape(k, (k.shape[0], np.prod(k.shape[1:])))
q = F.reshape(q, (q.shape[0], np.prod(q.shape[1:])))
v = q # F.reshape is inplace
cf = F.affine(q, F.transpose(k, (1, 0)))
if div_dim:
dim = np.prod(v_shape[1:])
cf /= np.sqrt(dim)
h = cf
if softmax:
h = F.softmax(h)
h = F.affine(h, v)x
h = F.reshape(h, v_shape)
return h
def test_clear_input_if_no_need_grad_convolution(self):
x1 = nn.Variable([1, 1, 2], need_grad=True)
x2 = nn.Variable([1, 1, 2], need_grad=True)
x3 = nn.Variable([1], need_grad=True)
inp = F.identity(x1)
weight = F.identity(x2)
bias = F.identity(x3)
y = F.convolution(inp, weight, bias) # (1)
answer = []
answer.append([False])
answer.append([False])
answer.append([False])
answer.append([False, False, True]) # (1) clears bias
y.forward(clear_no_need_grad=True)
self.check_input_data_clear_called_flags(answer)
def get_d_data(conf, flow_hr, gen_outputs, r_targets, rnn_length):
"""
prepare data for temporal Discriminators
"""
# 3 frames are used as one entry, the last input images%3 frames are abandoned
t_size = int(3 * (rnn_length // 3))
t_gen_output = F.reshape(
gen_outputs[:, :t_size, :, :, :],
(conf.train.batch_size * t_size, conf.train.crop_size * 4,
conf.train.crop_size * 4, 3),
inplace=False)
t_targets = F.reshape(
r_targets[:, :t_size, :, :, :],
(conf.train.batch_size * t_size, conf.train.crop_size * 4,
conf.train.crop_size * 4, 3),
inplace=False)
t_batch = conf.train.batch_size * t_size // 3
t_inputs_v_pre_batch = F.identity(
flow_hr[:, 0:t_size:3, :, :, :]) # forward motion reused,
t_inputs_v_batch = nn.Variable(t_inputs_v_pre_batch.shape)
# no motion for middle frames
t_inputs_v_batch.data.zero()
t_inputs_v_nxt_batch = F.identity(
flow_hr[:, -2:-1 - t_size:-3, :, :, :]) # backward motion
t_vel = F.stack(
*[t_inputs_v_pre_batch, t_inputs_v_batch, t_inputs_v_nxt_batch],
axis=2)
# batch, t_size/3, 3, FLAGS.crop_size*4, FLAGS.crop_size*4, 2
t_vel = F.reshape(t_vel,
(conf.train.batch_size * t_size,
conf.train.crop_size * 4, conf.train.crop_size * 4, 2),
inplace=False)
# Stop gradient to fnet from discriminator, details in TecoGAN supplemental paper
t_vel.need_grad = False
disc_data = collections.namedtuple(
'disc_data', 't_vel, t_gen_output, t_batch, t_targets, t_size')
return disc_data(t_vel=t_vel,
t_gen_output=t_gen_output,
t_batch=t_batch,
t_targets=t_targets,
t_size=t_size)
def func0(x):
assert x.recompute == False
y = F.identity(x)
assert y.recompute == f0
# First inner function call
y = func1(y)
assert y.recompute == f1
y = F.relu(y)
assert y.recompute == f0
# Second inner function call
y = func2(y)
assert y.recompute == f2
y = F.identity(y)
assert y.recompute == f0
return y
def spectral_normalization_for_conv(w, itr=1, eps=1e-12, test=False):
w_shape = w.shape
W_sn = get_parameter_or_create("W_sn", w_shape, ConstantInitializer(0),
False)
if test:
return W_sn
d0 = w.shape[0] # Out
d1 = np.prod(w.shape[1:]) # In
w = F.reshape(w, [d0, d1], inplace=False)
u0 = get_parameter_or_create("singular-vector", [d0], NormalInitializer(),
False)
u = F.reshape(u0, [1, d0])
# Power method
for _ in range(itr):
# v
v = F.affine(u, w)
v = F.div2(
v,
F.pow_scalar(F.sum(F.pow_scalar(v, 2.), keepdims=True) + eps, 0.5))
v = F.reshape(v, [d1, 1])
# u
u = F.affine(w, v)
u = F.div2(
u,
F.pow_scalar(F.sum(F.pow_scalar(u, 2.), keepdims=True) + eps, 0.5))
u = F.reshape(u, [1, d0])
# Iterate
u = F.identity(u, outputs=[u0.data])
u.persistent = True
# No grad
u.need_grad = False
v.need_grad = False
# Spectral normalization
wv = F.affine(w, v)
sigma = F.affine(u, wv)
w_sn = F.div2(w, sigma)
w_sn = F.reshape(w_sn, w_shape)
w_sn = F.identity(w_sn, outputs=[W_sn.data])
w_sn.persistent = True
return w_sn
def test_unnecessary_traverse_2(self):
def fail_with_not_cleared_data(nnabla_func):
inputs = nnabla_func.inputs
for input in inputs:
if input.parent is None:
continue
if not input.data.clear_called:
# Not cleared (recomputed) data is found.
pytest.fail()
# Prepare graph does not need any recomputation.
x1 = nn.Variable((2, 3), need_grad=True)
x1 = F.identity(x1).apply(recompute=True)
x2 = nn.Variable((2, 3), need_grad=True)
x2 = F.identity(x2).apply(recompute=True)
y = F.add2(x1, x2).apply(recompute=True)
y = F.identity(y).apply(recompute=True)
# Check unnecessary recomputation.
y.forward(clear_no_need_grad=True)
y.backward(function_pre_hook=fail_with_not_cleared_data)
def test_dropout_grad_dependency(p, seed, ctx, func_name):
from nnabla._dropout_workaround import _get_dropout_mask
# Test whether the memory clearance by grad_depends_on_inputs/outputs does
# something bad during graph execution such as the clearance values which
# is planned to be used. This test is performed by changing the
# inputs/outputs of Dropout to intermediate variables in the same manner of
# nbla_test_utils.py.
atol_f = 1e-4
with nn.context_scope(ctx):
rng = np.random.RandomState(seed)
init_x = rng.randn(2, 3, 4).astype(np.float32) * 2
init_dy_for_grad = rng.randn(*init_x.shape).astype(init_x.dtype)
init_dx = rng.randn(*init_x.shape).astype(init_x.dtype)
init_for_dx2 = rng.randn(*init_x.shape).astype(init_x.dtype)
# Graph construction
x = nn.Variable.from_numpy_array(init_x).apply(need_grad=True)
x_interm = F.identity(x)
y_interm = F.dropout(x_interm, p, seed)
y = F.identity(y_interm)
dx_interm = nn.grad(y, x, grad_outputs=[init_dy_for_grad])[0]
dx = F.identity(dx_interm)
y_dx = y + dx # replaceable with F.sink(y, dx, one_input_grad=False)
# Execution
x.g = init_dx # Accumulation
y_dx.forward(clear_no_need_grad=True)
mask = _get_dropout_mask(x_interm).d # Store mask before the clear
y_dx.backward(init_for_dx2, clear_buffer=True)
# Reference
ref_dx = ref_dropout_double_backward(init_for_dx2, mask, p) + init_dx
# Test
assert_allclose(x.g,
ref_dx,
atol=atol_f,
err_msg="Wrong output values of double backward of "
"Dropout by nn.grad.")
def test_clear_input_if_no_need_grad_branch1(self):
x1 = nn.Variable([1, 5], need_grad=True)
x2 = nn.Variable([1, 5], need_grad=True)
x3 = nn.Variable([1, 5], need_grad=True)
xx1 = F.identity(x1)
xx2 = F.identity(x2)
y1 = F.mul2(xx1, xx2) # (1)
xx3 = F.identity(x3)
y2 = F.add2(xx2, xx3) # (2)
y3 = F.add2(y1, y2) # (3)
answer = []
answer.append([False])
answer.append([False])
answer.append([False, False]) # (1)
answer.append([False])
answer.append([False, True]) # (2) use xx2 in backward
answer.append([True, True]) # (3)
y3.forward(clear_no_need_grad=True)
self.check_input_data_clear_called_flags(answer)
def test_clear_input_if_no_need_grad0(self):
x1 = nn.Variable([1, 5], need_grad=True)
xx1 = F.identity(x1)
y1 = F.add_scalar(xx1)
answer = []
answer.append([False])
answer.append([True])
y1.forward(clear_no_need_grad=True)
self.check_input_data_clear_called_flags(answer)
def test_clear_no_need_grad_during_recomputation(self):
x0 = nn.Variable((2, 3), need_grad=True)
x1 = F.identity(x0).apply(recompute=True)
# x2.data must be cleared just after recomputation because they are not need for backward propagation.
x2 = F.sin(x1).apply(recompute=True)
x3 = F.identity(x2).apply(recompute=True)
x4 = F.sin(x3)
# Forward
clear_called_flag_recorder.activate_clear_called_flag_recorder()
x4.forward(clear_no_need_grad=True)
# All intermediate data must be cleared.
expected = [
[False], # x0
[True], # x1
[True], # x2
[True], # x3
]
self.check_input_data_clear_called_flags(expected)
clear_called_flag_recorder.deactivate_clear_called_flag_recorder()
# Backward
clear_called_flag_recorder.activate_clear_called_flag_recorder()
x4.backward(clear_buffer=True)
expected = [
# Recomputation
[False], # x0
[False], # x1
[True], # x2: not need for grad calculation
# Backward propagation
[False], # x3
[True], # x2
[False], # x1
[False], # x0
]
self.check_input_data_clear_called_flags(expected)
clear_called_flag_recorder.deactivate_clear_called_flag_recorder()
def test_obsolete_inplace_option(inplace, func, num_inputs):
'''
This test confirms the construction of graph.
Since F.log_softmax requires output for backward calculation, graph cannot be constructed if it is inplaced.
'''
x0 = nn.Variable((2, 3, 4, 5), need_grad=True)
x1 = nn.Variable((2, 3, 4, 5), need_grad=True)
if num_inputs == 1:
y = F.identity(x0)
y = F.log_softmax(y)
y = func(y, inplace=inplace)
y.forward()
y.backward()
elif num_inputs == 2:
y0 = F.identity(x0)
y1 = F.identity(x1)
y0 = F.log_softmax(y0)
y1 = F.log_softmax(y1)
y = func(y0, y1, inplace=inplace)
y.forward()
y.backward()
def test_nn_grad_propagate_down_check():
register("IdentityForwardOnlyFunction",
IdentityForwardOnlyFunction_backward)
backward_func = registry["IdentityForwardOnlyFunction"]
assert backward_func is not None
x = nn.Variable.from_numpy_array(np.random.random((1, 1, 32, 32)))
y = PF.convolution(x, 1, kernel=(3, 3), pad=(1, 1), with_bias=False)
z = IdentityForwardOnlyFunction()(y)
w = F.identity(z)
# If IdentityForwardOnlyFunction_backward is called in nn.grad, an error will occur.
v = nn.grad(w, [z])
v[0].forward()
def test_leaf_indexing_access():
import nnabla.functions as F
nn.set_auto_forward(False)
shape_x = (3, 2)
dx = np.random.rand(*shape_x)
shape_y = (2, 2)
dy = np.random.rand(*shape_y)
x = nn.Variable.from_numpy_array(dx)
y = nn.Variable.from_numpy_array(dy)
x[0:2, :] = y
z = F.identity(x)
z.forward()
d1 = x.d.copy()
nn.set_auto_forward(True)
x = nn.Variable.from_numpy_array(dx)
y = nn.Variable.from_numpy_array(dy)
x[0:2, :] = y
z2 = F.identity(x)
d2 = x.d.copy()
nn.set_auto_forward(False)
x = nn.Variable.from_numpy_array(dx)
y = nn.Variable.from_numpy_array(dy)
x[0:2, :] = y
z3 = F.identity(x)
z3.forward()
d3 = x.d.copy()
d4 = z3.d.copy()
assert_allclose(d1, d2)
assert_allclose(d2, d3)
assert_allclose(d3, d4)
def network(x, d1, c1, d2, c2, test=False):
# Input:x -> 1
# OneHot -> 687
h = F.one_hot(x, (687, ))
# LSTM1 -> 200
with nn.parameter_scope('LSTM1'):
h = network_LSTM(h, d1, c1, 687, 100, test)
# Slice -> 100
h1 = F.slice(h, (0, ), (100, ), (1, ))
# h2:CellOut -> 100
h2 = F.slice(h, (100, ), (200, ), (1, ))
# LSTM2 -> 128
with nn.parameter_scope('LSTM2'):
h3 = network_LSTM(h1, d2, c2, 100, 64, test)
# h4:DelayOut
h4 = F.identity(h1)
# Slice_2 -> 64
h5 = F.slice(h3, (0, ), (64, ), (1, ))
# h6:CellOut_2 -> 64
h6 = F.slice(h3, (64, ), (128, ), (1, ))
# Affine_2 -> 687
h7 = PF.affine(h5, (687, ), name='Affine_2')
# h8:DelayOut_2
h8 = F.identity(h5)
# h7:Softmax
h7 = F.softmax(h7)
return h2, h4, h6, h8, h7
def test_clear_input_if_no_need_grad2(self):
x1 = nn.Variable([1, 5], need_grad=True)
xx1 = F.identity(x1) # (1)
y1 = F.tanh(xx1) # (2)
y2 = F.add_scalar(y1) # (3)
answer = []
answer.append([False])
answer.append([True])
answer.append([False])
# y1 must not be clear after (3) because y1 is required for backward of (2).
y2.forward(clear_no_need_grad=True)
self.check_input_data_clear_called_flags(answer)
def __call__(self, x, test=False):
'''
Input: (nb_samples, nb_channels, nb_timesteps)
or (nb_frames, nb_samples, nb_channels, nb_bins)
Outputs: Output Power/Mag Spectrogram
'''
self.test = test
if not self.input_is_spectrogram:
x = get_spectogram(*get_stft(x,
n_fft=self.n_fft,
n_hop=self.n_hop,
center=self.test),
mono=(self.nb_channels == 1))
nb_frames, nb_samples, nb_channels, nb_bins = x.shape
mix_spec = F.identity(x)
# crop
x = x[..., :self.nb_bins]
# shift and scale input to mean=0 std=1 (across all bins)
x += self.input_mean
x *= self.input_scale
# encode and normalize every instance in a batch
x = self.fc_bn(x, self.hidden_size, "fc1", activation='tanh')
# apply 3-layers of stacked LSTM
lstm_out = self.lstm(x, nb_samples, "lstm")
# lstm skip connection
x = F.concatenate(x, lstm_out)
# first dense stage + batch norm
x = self.fc_bn(x, self.hidden_size, "fc2", activation='relu')
# second dense stage + batch norm
x = self.fc_bn(x, nb_channels * nb_bins, "fc3")
# reshape back to original dim
x = F.reshape(
x, (nb_frames, nb_samples, nb_channels, self.nb_output_bins))
# apply output scaling
x *= self.output_scale
x += self.output_mean
return F.relu(x) * mix_spec
def test_clear_input_if_no_need_grad_branch0(self):
x1 = nn.Variable([1, 5], need_grad=True)
x2 = nn.Variable([1, 5], need_grad=True)
xx1 = F.identity(x1)
y1 = F.add_scalar(xx1) # (1)
y2 = F.add_scalar(xx1) # (2)
y3 = F.add2(y1, y2) # (3)
answer = []
answer.append([False])
answer.append([False]) # (1) does not clear xx1
answer.append([True]) # (2) clears xx1
answer.append([True, True])
y3.forward(clear_no_need_grad=True)
self.check_input_data_clear_called_flags(answer)
def test_clear_output_grad_inplace(self):
x1 = nn.Variable([1], need_grad=True)
xx1 = F.identity(x1)
y1 = F.add_scalar(xx1, inplace=True)
y2 = F.add_scalar(y1)
answer_grad = []
answer_grad.append([True])
answer_grad.append([True])
answer_grad.append([True])
y2.forward(clear_no_need_grad=True)
clear_called_flag_recorder.deactivate_clear_called_flag_recorder()
clear_called_flag_recorder.activate_clear_called_flag_recorder()
y2.backward(clear_buffer=True)
self.check_grad_cleared_flags(answer_grad)
def test_clearing_without_recompute_flag(self):
x0 = nn.Variable((1, 128, 128), need_grad=True)
x1 = F.sin(x0).apply(recompute=True)
x2 = F.dropout(x1)
x3 = F.sin(x2).apply(recompute=True)
x4 = F.sin(x3).apply(recompute=True)
y = F.identity(x4)
# Skip this code temporarily since it cause
# randomly crash when perform CI testing on windows 10 with nnabla-cuda-ext
pytest.skip(
'Skipped for randomly crash when perform CI testing on windows 10 with nnabla-cuda-ext')
y.forward(clear_no_need_grad=True)
x2.data.clear()
with pytest.raises(RuntimeError, match="Failed `called_setup_recompute_`"):
# x2.data cannot be recomputed correctly since `setup_recompute` is not called during forward propagation.
# Backward should raise when some intermediate variables are cleared by user.
y.backward()
def box_filter(x, szf):
"""
Box filter
"""
y = F.identity(x)
szy = list(y.shape)
b_filt = nn.Variable((szf, szf, 1, 1))
b_filt.data.fill(1.)
b_filt = b_filt / (szf**2)
# 5,5,1,1
b_filt = F.tile(b_filt, [1, 1, szy[3], 1])
b_filt = F.transpose(b_filt, (3, 2, 0, 1))
b_filt = F.reshape(b_filt, (6, 5, 5))
pp = int((szf - 1) / 2)
y = F.pad(y, (0, 0, pp, pp, pp, pp, 0, 0), mode='reflect')
y_chw = F.transpose(y, (0, 3, 1, 2))
y_chw = F.depthwise_convolution(y_chw, b_filt, multiplier=1, stride=(1, 1))
y_hwc = F.transpose(y_chw, (0, 2, 3, 1))
return y_hwc
def _build(self):
generator_fn, discriminator_fn = self._network_funcs()
# real shape
ch, w, h = self.real.shape[1:]
# inputs
self.x = nn.Variable((1, ch, w, h))
self.y = nn.Variable((1, ch, w, h))
self.rec_x = nn.Variable((1, ch, w, h))
self.rec_y = nn.Variable((1, ch, w, h))
y_real = nn.Variable.from_numpy_array(self.real)
y_real.persistent = True
# padding inputs
padded_x = _pad(self.x, self.kernel, self.num_layer)
padded_rec_x = _pad(self.rec_x, self.kernel, self.num_layer)
# generate fake image
self.fake = generator_fn(x=padded_x, y=self.y)
fake_without_grads = F.identity(self.fake)
fake_without_grads.need_grad = False
rec = generator_fn(x=padded_rec_x, y=self.rec_y)
# discriminate images
p_real = discriminator_fn(x=y_real)
p_fake = discriminator_fn(x=self.fake)
p_fake_without_grads = discriminator_fn(x=fake_without_grads)
# gradient penalty for discriminator
grad_penalty = _calc_gradient_penalty(y_real, fake_without_grads,
discriminator_fn)
# discriminator loss
self.d_real_error = -F.mean(p_real)
self.d_fake_error = F.mean(p_fake_without_grads)
self.d_error = self.d_real_error + self.d_fake_error \
+ self.lam_grad * grad_penalty
# generator loss
self.rec_error = F.mean(F.squared_error(rec, y_real))
self.g_fake_error = -F.mean(p_fake)
self.g_error = self.g_fake_error + self.alpha_recon * self.rec_error