def test_apply_gradient_directory(self):
"""get_grad correctly fetches gradient Tensor from Variable"""
w_0 = 6
sgd = nn.optimizer.SGD(learning_rate=1.0)
with nn.variable_scope(self.get_scope()):
x = nn.Input(shape=(), name='x')
w1 = nn.make_variable(
name='w',
shape=(),
initializer=nn.initializer.ConstantInitializer(w_0),
)
y1 = w1 * x
sgd.minimize(y1, w1)
dy1dw1_1 = nn.get_tensor('{}_grad'.format(w1.name))
dy1dw1_2 = nn.get_grad(w1)
self.assertIs(dy1dw1_1, dy1dw1_2)
with nn.variable_scope('{}/2'.format(self.get_scope())):
w2 = nn.make_variable(
name='w',
shape=(),
initializer=nn.initializer.ConstantInitializer(w_0),
)
y2 = w2 * x
sgd.minimize(y2, w2)
dy2dw2_1 = nn.get_tensor('{}_grad'.format(w2.name))
dy2dw2_2 = nn.get_grad(w2)
self.assertIs(dy2dw2_1, dy2dw2_2)
def test_apply_gradient_directory(self):
"""get_grad correctly fetches gradient Tensor from Variable"""
w_0 = 6
sgd = nn.optimizer.SGD(learning_rate=1.0)
with nn.variable_scope(self.get_scope()):
x = nn.Input(shape=(), name='x')
w1 = nn.make_variable(
name='w', shape=(),
initializer=nn.initializer.ConstantInitializer(w_0),
)
y1 = w1 * x
sgd.minimize(y1, w1)
dy1dw1_1 = nn.get_tensor('{}_grad'.format(w1.name))
dy1dw1_2 = nn.get_grad(w1)
self.assertIs(dy1dw1_1, dy1dw1_2)
with nn.variable_scope('{}/2'.format(self.get_scope())):
w2 = nn.make_variable(
name='w', shape=(),
initializer=nn.initializer.ConstantInitializer(w_0),
)
y2 = w2 * x
sgd.minimize(y2, w2)
dy2dw2_1 = nn.get_tensor('{}_grad'.format(w2.name))
dy2dw2_2 = nn.get_grad(w2)
self.assertIs(dy2dw2_1, dy2dw2_2)
def test_fetch_output(self):
"""Output of Anonymous layer is fetched"""
base_scope, scope = self.get_scope(), 'anon'
with nn.variable_scope(base_scope):
layer = nn.layer.Anonymous(exp='x', scope=scope)
output_var = layer(nn.Input(shape=(3, 4)))
_tensor = nn.get_tensor('{}/output'.format(scope))
self.assertIs(output_var, _tensor)
_tensor = nn.get_tensor('{}/{}/output'.format(base_scope, scope))
self.assertIs(output_var, _tensor)
def test_get_tensor_from_current_scope(self):
"""get_tensor retrieve existing tensor"""
scope, name = self.get_scope(), 'foo'
with nn.variable_scope(scope):
tensor = fixture.create_ones_tensor([3, 1], 'float32', name=name)
self.assertIs(tensor, nn.get_tensor(name))
self.assertIs(tensor, nn.get_tensor('{}/{}'.format(scope, name)))
with self.assertRaises(ValueError):
nn.get_tensor(name)
def test_get_tensor_from_current_scope(self):
"""get_tensor retrieve existing tensor"""
scope, name = self.get_scope(), 'foo'
with nn.variable_scope(scope):
tensor = fixture.create_ones_tensor([3, 1], 'float32', name=name)
self.assertIs(tensor, nn.get_tensor(name))
self.assertIs(tensor, nn.get_tensor('{}/{}'.format(scope, name)))
with self.assertRaises(ValueError):
nn.get_tensor(name)
def test_conv2dtranspose(self):
"""Compnents consisting Conv2DTranspose layer are retrieved"""
scope = self.get_scope()
with nn.variable_scope(scope) as vs:
input_ = nn.Input(shape=(32, 4, 8, 8), name='input')
layer = nn.get_layer('Conv2D')(filter_height=4,
filter_width=4,
n_filters=4,
strides=1,
with_bias=True,
name='Conv2D')
output = layer(input_)
layer = nn.get_layer('Conv2DTranspose')(filter_height=4,
filter_width=4,
n_filters=4,
strides=1,
with_bias=True,
output_shape=input_.shape,
name='Conv2DT')
output = layer(output)
filters = layer.get_parameter_variable('filter')
bias = layer.get_parameter_variable('bias')
with nn.variable_scope(vs, reuse=True):
self.assertIs(filters, nn.get_variable('Conv2DT/filter'))
self.assertIs(bias, nn.get_variable('Conv2DT/bias'))
self.assertIs(output, nn.get_tensor('Conv2DT/output'))
self.assertIs(input_, nn.get_input('input'))
def test_get_gradients(self):
"""gradients can be retrieved with get_tensor"""
scope = self.get_scope()
with nn.variable_scope(scope):
xs = [nn.make_variable(
name='x_{}'.format(i), shape=(), trainable=True,
) for i in range(5)]
y = xs[0] + xs[1] + xs[2] + xs[3] + xs[4]
grads_and_vars = nn.ops.compute_gradient(loss=y, wrt=xs)
for i in range(5):
grad = nn.get_tensor('{}_grad'.format(xs[i].name))
self.assertIs(grads_and_vars[i][0], grad)
for i in range(5):
grad = nn.get_tensor('{}/{}_grad'.format(scope, xs[i].name))
self.assertIs(grads_and_vars[i][0], grad)
def test_concat(self):
"""Compnents consisting Concat layer are retrieved"""
with nn.variable_scope(self.get_scope()):
layer = nn.fetch_layer('Concat')(axis=1, scope='Concat')
output = layer([
nn.Input(shape=(32, 4), name='input_1'),
nn.Input(shape=(32, 5), name='input_2'),
])
self.assertIs(output, nn.get_tensor('Concat/output'))
def test_mean(self):
"""Compnents consisting Mean layer are retrieved"""
scope = self.get_scope()
with nn.variable_scope(scope):
input_ = nn.Input(shape=(32, 4, 8, 8), name='input')
layer = nn.get_layer('Mean')(axis=[1, 2], name='Mean')
output = layer(input_)
self.assertIs(output, nn.get_tensor('Mean/output'))
self.assertIs(input_, nn.get_input('input'))
def test_tile(self):
"""Compnents consisting Tile layer are retrieved"""
scope = self.get_scope()
with nn.variable_scope(scope):
input_ = nn.Input(shape=(32, ), name='input')
layer = nn.get_layer('Tile')(pattern=(1, 2), name='Tile')
output = layer(input_)
self.assertIs(output, nn.get_tensor('Tile/output'))
self.assertIs(input_, nn.get_input('input'))
def test_true_div(self):
"""Compnents consisting truediv layer are retrieved"""
scope = self.get_scope()
with nn.variable_scope(scope):
input_ = nn.Input(shape=(32, 4, 8, 8), name='input')
layer = nn.fetch_layer('TrueDiv')(denom=1.0, scope='TrueDiv')
output = layer(input_)
self.assertIs(output, nn.get_tensor('TrueDiv/output'))
self.assertIs(input_, nn.get_input('input'))
def _test_layer_io(self, layer_name, input_shape):
scope = '{}/{}'.format(self.get_scope(), layer_name)
with nn.variable_scope(scope) as vs:
input_ = nn.Input(shape=input_shape, name='input')
layer = nn.fetch_layer(layer_name)(scope=layer_name)
output = layer(input_)
with nn.variable_scope(vs, reuse=True):
output_tensor_name = '{}/output'.format(layer_name)
self.assertIs(input_, nn.get_input('input'))
self.assertIs(output, nn.get_tensor(output_tensor_name))
def test_concat(self):
"""Compnents consisting Concat layer are retrieved"""
scope = self.get_scope()
with nn.variable_scope(scope):
input_ = [
nn.Input(shape=(32, 4), name='input'),
nn.Input(shape=(32, 5), name='input'),
]
layer = nn.get_layer('Concat')(axis=1, name='Concat')
output = layer(input_)
self.assertIs(output, nn.get_tensor('Concat/output'))
def test_sub(self):
"""Compnents consisting Sub layer are retrieved"""
scope = self.get_scope()
with nn.variable_scope(scope):
input_ = [
nn.Input(shape=(32, 4), name='input'),
nn.Input(shape=(32, 4), name='input'),
]
layer = nn.get_layer('Sub')(name='Sub')
output = layer(input_)
self.assertIs(output, nn.get_tensor('Sub/output'))
def test_dense(self):
"""Compnents consisting Dense layer are retrieved"""
with nn.variable_scope(self.get_scope()) as vs:
input_ = nn.Input(shape=(32, 5), name='input')
layer = nn.fetch_layer('Dense')(
n_nodes=4, with_bias=True, scope='Dense')
output = layer(input_)
weight = layer.get_parameter_variable('weight')
bias = layer.get_parameter_variable('bias')
with nn.variable_scope(vs, reuse=True):
self.assertIs(weight, nn.get_variable('Dense/weight'))
self.assertIs(bias, nn.get_variable('Dense/bias'))
self.assertIs(output, nn.get_tensor('Dense/output'))
self.assertIs(input_, nn.get_input('input'))
def test_conv2d(self):
"""Compnents consisting Conv2D layer are retrieved"""
scope = self.get_scope()
with nn.variable_scope(scope) as vs:
input_ = nn.Input(shape=(32, 4, 8, 8), name='input')
layer = nn.fetch_layer('Conv2D')(
filter_height=4, filter_width=4, n_filters=4,
strides=1, with_bias=True, name='Conv2D')
output = layer(input_)
filters = layer.get_parameter_variable('filter')
bias = layer.get_parameter_variable('bias')
with nn.variable_scope(vs, reuse=True):
self.assertIs(filters, nn.get_variable('Conv2D/filter'))
self.assertIs(bias, nn.get_variable('Conv2D/bias'))
self.assertIs(output, nn.get_tensor('Conv2D/output'))
self.assertIs(input_, nn.get_input('input'))
def test_bn(self):
"""Compnents consisting BatchNormalization layer are retrieved"""
base_scope, scope = self.get_scope(), 'BN'
with nn.variable_scope(base_scope) as vs:
input_ = nn.Input(shape=(32, 4), name='input')
layer = nn.fetch_layer('BatchNormalization')(scope=scope)
output = layer(input_)
mean = layer.get_parameter_variable('mean')
var = layer.get_parameter_variable('var')
scale = layer.get_parameter_variable('scale')
offset = layer.get_parameter_variable('offset')
updates = layer.get_update_operations()
with nn.variable_scope(vs, reuse=True):
self.assertIs(mean, nn.get_variable('BN/mean'))
self.assertIs(var, nn.get_variable('BN/var'))
self.assertIs(scale, nn.get_variable('BN/scale'))
self.assertIs(offset, nn.get_variable('BN/offset'))
self.assertIs(output, nn.get_tensor('BN/output'))
self.assertIs(updates[0], nn.get_operation('BN/update_mean'))
self.assertIs(updates[1], nn.get_operation('BN/update_var'))
def test_check_optimizer_slot(self):
"""Slot variables are updated when applying gradient directly"""
name, b1_0, b2_0 = 'Adam', 0.5, 0.4
opt = nn.optimizer.Adam(
learning_rate=1.0, name=name, beta1=b1_0, beta2=b2_0)
with nn.variable_scope(self.get_scope()) as vs:
x = nn.Input(shape=(), name='x')
w = nn.get_variable(shape=(), name='w')
update_op = opt.minimize(w * x, w)
vs.reuse_variables()
dw = nn.get_tensor('{}_grad'.format(w.name))
b1 = nn.get_variable('{}/beta1_power'.format(name))
b2 = nn.get_variable('{}/beta2_power'.format(name))
session = nn.Session()
session.initialize()
for i in range(10):
b1_val, b2_val = session.run(outputs=[b1, b2])
np.testing.assert_almost_equal(b1_val, b1_0 ** (i + 1))
np.testing.assert_almost_equal(b2_val, b2_0 ** (i + 1))
session.run(updates=update_op, givens={dw: 1.0})
def test_apply_gradient_directory(self):
"""Variables can be updated by appyling gradient directly"""
w_0 = 6
with nn.variable_scope(self.get_scope()):
x = nn.Input(shape=(), name='x')
w = nn.get_variable(
name='w', shape=(),
initializer=nn.initializer.ConstantInitializer(w_0),
)
y = w * x
sgd = nn.optimizer.SGD(learning_rate=1.0)
update_op = sgd.minimize(y, w)
dw = nn.get_tensor('{}_grad'.format(w.name))
session = nn.Session()
session.initialize()
val0 = 3.
session.run(updates=update_op, givens={dw: val0})
val_w = session.run(outputs=w)
np.testing.assert_almost_equal(val_w, w_0 - val0)