def test_paramter_reuse_conv2d(self):
"""Conv2D layer is built using existing Variables"""
shape = (10, 11, 12, 13)
with nn.variable_scope(self.get_scope()):
layer1 = nn.layer.Conv2D(filter_width=5,
filter_height=3,
n_filters=4,
strides=1,
padding='VALID')
layer2 = nn.layer.Conv2D(filter_width=5,
filter_height=3,
n_filters=4,
strides=1,
padding='VALID')
tensor = nn.Input(shape=shape)
out1 = layer1(tensor)
layer2.set_parameter_variables(
filter=layer1.get_parameter_variable('filter'),
bias=layer1.get_parameter_variable('bias'))
out2 = layer2(tensor)
for key in ['filter', 'bias']:
var1 = layer1.get_parameter_variable(key)
var2 = layer2.get_parameter_variable(key)
self.assertIs(var1, var2)
session = nn.Session()
session.initialize()
input_val = np.random.rand(*shape)
out1, out2 = session.run(outputs=[out1, out2],
inputs={tensor: input_val})
np.testing.assert_almost_equal(out1, out2)
def test_paramter_reuse_dense(self):
"""Dense layer is built using existing Variables"""
shape = (3, 5)
with nn.variable_scope(self.get_scope()):
layer1 = nn.layer.Dense(n_nodes=5)
layer2 = nn.layer.Dense(n_nodes=5)
tensor = nn.Input(shape=shape)
out1 = layer1(tensor)
layer2.set_parameter_variables(
weight=layer1.get_parameter_variable('weight'),
bias=layer1.get_parameter_variable('bias'),
)
out2 = layer2(tensor)
for key in ['weight', 'bias']:
var1 = layer1.get_parameter_variable(key)
var2 = layer2.get_parameter_variable(key)
self.assertIs(var1, var2)
session = nn.Session()
session.initialize()
input_val = np.random.rand(*shape)
out1, out2 = session.run(outputs=[out1, out2],
inputs={tensor: input_val})
np.testing.assert_almost_equal(out1, out2)
def test_concate_2d_axis_1_3(self):
"""Concatenate 3 2D tensors"""
axis, shape1, shape2, shape3 = 1, (2, 5), (2, 3), (2, 4)
with nn.variable_scope(self.get_scope(), reuse=False):
var1 = nn.get_variable(name='var1', shape=shape1)
var2 = nn.get_variable(name='var2', shape=shape2)
var3 = nn.get_variable(name='var3', shape=shape3)
conc_var = nn.layer.Concat(axis=axis).build([var1, var2, var3])
session = nn.Session()
val1, val2 = np.random.rand(*shape1), np.random.rand(*shape2)
val3 = np.random.rand(*shape3)
conc_val = session.run(outputs=conc_var,
givens={
var1: val1,
var2: val2,
var3: val3
})
expected = conc_val.shape
found = conc_var.shape
self.assertEqual(found, expected)
expected = np.concatenate((val1, val2, val3), axis=axis)
found = conc_val
np.testing.assert_almost_equal(found, expected)
def test_clip_variable_by_norm(self):
"""Test clip_by_norm with Variable"""
shape, clip_norm = (3, 4), np.asarray(15, dtype='float32')
with nn.variable_scope(self.get_scope()):
input_ = nn.Input(shape, dtype='float32')
clip_var = nn.Input(shape=[], dtype='float32')
output = nn.ops.clip_by_norm(input_, clip_norm=clip_var)
session = nn.Session()
in_val = np.random.rand(*shape).astype('float32')
out_val = session.run(
outputs=output,
givens={input_: in_val, clip_var: clip_norm}
)
np.testing.assert_almost_equal(out_val, in_val)
in_val += 10.0
out_val = session.run(
outputs=output,
givens={input_: in_val, clip_var: clip_norm}
)
l2_norm = np.sqrt(np.sum(in_val ** 2))
np.testing.assert_almost_equal(
out_val, clip_norm * in_val / l2_norm, decimal=3)
def test_dynamic_initializer(self):
"""Initializers are correctly selected"""
n_in, n_nodes, weight_val, bias_val = 4, 5, 13, 7
with nn.variable_scope(self.get_scope()):
dense = nn.layer.Dense(n_nodes=5,
initializers={
'weight': {
'typename': 'ConstantInitializer',
'args': {
'value': weight_val,
},
},
'bias': {
'typename': 'ConstantInitializer',
'args': {
'value': bias_val,
}
}
})
dense(nn.Input(shape=(3, n_in)))
session = nn.Session()
session.initialize()
weight, bias = session.run(outputs=[
dense.get_parameter_variable('weight'),
dense.get_parameter_variable('bias'),
])
np.testing.assert_almost_equal(weight,
weight_val * np.ones((n_in, n_nodes)))
np.testing.assert_almost_equal(bias, bias_val * np.ones((n_nodes, )))
def test_clip_number_by_norm_with_axes(self):
"""Test clip_by_norm with axis"""
shape, clip_norm, axis = (3, 4), 15.0, 1
with nn.variable_scope(self.get_scope()):
input_ = nn.Input(shape, dtype='float32')
output = nn.ops.clip_by_norm(
input_, clip_norm=clip_norm, axes=axis)
session = nn.Session()
in_val = np.random.rand(*shape).astype('float32')
out_val = session.run(
outputs=output,
givens={input_: in_val}
)
np.testing.assert_almost_equal(out_val, in_val)
in_val += 10.0
out_val = session.run(
outputs=output,
givens={input_: in_val}
)
l2_norm = np.sqrt(np.sum(in_val ** 2, axis=axis, keepdims=True))
np.testing.assert_almost_equal(
out_val, clip_norm * in_val / l2_norm, decimal=3)
def test_sync_with_tau(self):
"""sync op copies weighted sum of source and target variables"""
tau = 0.1
with nn.variable_scope(self.get_scope()):
source_var, target_var = _create_variables()
sync_op = nn.ops.build_sync_op(
[source_var], [target_var], tau=tau)
session = nn.Session()
session.initialize()
src_val, tgt_val = session.run([source_var, target_var])
self.assertTrue((src_val == 1).all())
self.assertTrue((tgt_val == 0).all())
for _ in range(10):
expected = tau * src_val + (1 - tau) * tgt_val
session.run(updates=sync_op)
src_val, found = session.run([source_var, target_var])
self.assertTrue((src_val == 1).all())
self.assertTrue(
np.square(expected - found).sum() < 1e-10,
'\nExpected: \n{}\nFound: \n{}'.format(expected, found)
)
tgt_val = found
def _main():
args = _parase_command_line_args()
_initialize_logger(args.debug)
data_format = luchador.get_nn_conv_format()
batch_size = 32
input_shape = (
[batch_size, 28, 28, 1] if data_format == 'NHWC' else
[batch_size, 1, 28, 28]
)
classifier = _build_model(args.model, input_shape, batch_size)
dataset = _load_data(args.mnist, data_format)
session = nn.Session()
session.initialize()
summary = nn.SummaryWriter(output_dir='tmp')
if session.graph:
summary.add_graph(session.graph)
try:
_train(session, classifier, dataset['train'], batch_size)
_test(session, classifier, dataset['test'], batch_size)
except KeyboardInterrupt:
pass
def test_clip_gradients(self):
"""Gradients are clipped"""
sgd = nn.optimizer.SGD(learning_rate=1.0)
shape = (32, 1)
with nn.variable_scope(self.get_scope()):
initializer = nn.get_initializer('UniformInitializer')(minval=-3,
maxval=3)
x = nn.get_variable(name='x', shape=shape, initializer=initializer)
y = x * x / 2
grads_and_vars = [
(nn.clip_by_value(grad, max_value=1.0, min_value=-1.0), var)
for grad, var in sgd.compute_gradients(y.sum(), wrt=x)
]
op = sgd.apply_gradients(grads_and_vars)
session = nn.Session()
session.initialize()
val_0 = session.run(outputs=x)
session.run(updates=op)
val_1_be = session.run(outputs=x)
val_1_np = np.zeros(shape)
val_1_np[val_0 > 1] = val_0[val_0 > 1] - 1
val_1_np[val_0 < -1] = val_0[val_0 < -1] + 1
np.testing.assert_almost_equal(val_1_be, val_1_np)
def test_dict(self):
"""Anonymous layer can handle dict inputs"""
shape, dtype = (3, 4), 'float32'
exp = ('x["0"] + x["1"]')
input_vars = {
'0': nn.Input(shape=shape, dtype=dtype, name='input1'),
'1': nn.Input(shape=shape, dtype=dtype, name='input2')
}
input_vals = {
'0': np.random.rand(3, 4).astype(dtype),
'1': np.random.rand(3, 4).astype(dtype),
}
output_val = sum(input_vals.values())
with nn.variable_scope(self.get_scope()):
layer = nn.layer.Anonymous(exp)
output_var = layer(**input_vars)
session = nn.Session()
output_val_ = session.run(outputs=output_var,
inputs={
input_vars['0']: input_vals['0'],
input_vars['1']: input_vals['1']
})
np.testing.assert_almost_equal(output_val, output_val_)
def test_list(self):
"""Anonymous layer can handle list inputs"""
shape, dtype = (3, 4), 'float32'
exp = ('x[0] + x[1]')
input_vars = [
nn.Input(shape=shape, dtype=dtype, name='input1'),
nn.Input(shape=shape, dtype=dtype, name='input2')
]
input_vals = [
np.random.rand(3, 4).astype(dtype),
np.random.rand(3, 4).astype(dtype),
]
output_val = sum(input_vals)
with nn.variable_scope(self.get_scope()):
layer = nn.layer.Anonymous(exp)
output_var = layer(*input_vars)
session = nn.Session()
output_val_ = session.run(outputs=output_var,
inputs={
input_vars[0]: input_vals[0],
input_vars[1]: input_vals[1]
})
np.testing.assert_almost_equal(output_val, output_val_)
def _exe(exp, input_val, scope):
input_var = nn.Input(shape=input_val.shape, dtype=input_val.dtype)
with nn.variable_scope(scope):
layer = nn.layer.Anonymous(exp)
output_var = layer(input_var)
session = nn.Session()
return session.run(outputs=output_var, inputs={input_var: input_val})
def test_beta_power_update(self):
"""Beta parameter is updated every time update is evaluated"""
beta1, beta2, x_init_val = 0.9, 0.999, 3.0
adamax = nn.optimizer.Adamax(learning_rate=0.01, beta1=beta1)
x_tensor, y_tensor = _get_y_equals_x_squared(scope=self.get_scope(),
x_init=x_init_val)
minimize_op = adamax.minimize(loss=y_tensor, wrt=x_tensor)
beta1_pow_tensor = _get_slot_var(adamax, 'beta1_power')
m_tensor = _get_slot_var(adamax, 'm', var_name=x_tensor.name)
u_tensor = _get_slot_var(adamax, 'u', var_name=x_tensor.name)
session = nn.Session()
session.initialize()
x_val_prev, m_val_prev, u_val_prev = x_init_val, 0, 0
for i in range(1, 10):
session.run(updates=minimize_op, name='optimization')
beta1_pow_val = session.run(outputs=beta1_pow_tensor,
name='fetch1')
expected = beta1**(i + 1)
found = beta1_pow_val
diff = abs(expected - found)
self.assertTrue(
diff < 0.01, 'Beta1 is not correctly updated. '
'Expected: {}, Found: {}'.format(expected, found))
m_val, u_val = session.run(outputs=[m_tensor, u_tensor],
name='fetch3')
grad = 2 * x_val_prev
expected = m_val_prev + (1.0 - beta1) * (grad - m_val_prev)
found = m_val
diff = abs(expected - found)
self.assertTrue(
diff < 0.01, 'The value of `m` is not correctly updated. '
'Expected: {}, Fround: {}'.format(expected, found))
m_val_prev = m_val
expected = max(u_val_prev * beta2, abs(grad))
found = u_val
diff = abs(expected - found)
self.assertTrue(
diff < 0.01, 'The value of `u` is not correctly updated. '
'Expected: {}, Fround: {}'.format(expected, found))
u_val_prev = u_val
x_val = session.run(outputs=x_tensor, name='fetch2')
self.assertTrue(
0 <= x_val < x_val_prev,
'The value of `x` must regress to zero at each update. '
'Previous value: {}, current value: {}'.format(
x_val_prev, x_val))
x_val_prev = x_val
def test_neg(self):
"""-Tensor is correct elementwise"""
shape = (3, 5)
with nn.variable_scope(self.get_scope()):
tensor1 = fixture.create_ones_tensor(shape, dtype='int32')
tensor2 = -tensor1
session = nn.Session()
val1, val2 = session.run(outputs=[tensor1, tensor2], )
np.testing.assert_equal(-val1, val2)
def _test_max(self, axis, shape, keep_dims):
with nn.variable_scope(self.get_scope()):
tensor0 = fixture.create_ones_tensor(shape, dtype='float32')
tensor1 = tensor0.max(axis=axis, keep_dims=keep_dims)
session = nn.Session()
val0, val1 = session.run(outputs=[tensor0, tensor1], )
expected = val0.max(axis=axis, keepdims=keep_dims)
np.testing.assert_equal(val1, expected)
def test_floordiv_tensor(self):
"""Tensor // Tensor is correct elementwise"""
shape = (3, 5)
with nn.variable_scope(self.get_scope()):
tensor1 = fixture.create_ones_tensor(shape, dtype='float32')
tensor2 = tensor1 // tensor1
session = nn.Session()
val1, val2 = session.run(outputs=[tensor1, tensor2], )
np.testing.assert_equal(np.floor_divide(val1, val1), val2)
def _check(self, input_var, output_var):
session = nn.Session()
session.initialize()
input_val = np.random.randn(*input_var.shape)
output_val = session.run(outputs=output_var,
inputs={input_var: input_val})
self.assertEqual(output_var.shape, input_var.shape)
self.assertEqual(output_var.dtype, input_var.dtype)
self.assertEqual(output_var.shape, output_val.shape)
self.assertEqual(output_var.dtype, output_val.dtype)
def _convert(layer, shape):
input_tensor = nn.Input(shape=shape)
input_value = np.random.randn(*shape) - 100
session = nn.Session()
output_tensor = layer(input_tensor)
output_value = session.run(
outputs=output_tensor,
inputs={input_tensor: input_value},
)
return output_value, output_tensor
def test_floordiv_numbers(self):
"""Tensor // number is correct elementwise"""
constant, shape = 10., (3, 5)
with nn.variable_scope(self.get_scope()):
tensor1 = fixture.create_ones_tensor(shape, dtype='float32')
tensor2 = tensor1 // constant
tensor3 = constant // tensor1
session = nn.Session()
val1, val2, val3 = session.run(outputs=[tensor1, tensor2, tensor3], )
np.testing.assert_equal(np.floor_divide(val1, constant), val2)
np.testing.assert_equal(np.floor_divide(constant, val1), val3)
def _test_tile(self, pattern, input_shape, input_value):
scope = self.get_scope()
with nn.variable_scope(scope, reuse=False):
input_tensor = nn.Input(shape=input_shape)
tile = nn.layer.Tile(pattern=pattern)
output_tensor = tile(input_tensor)
session = nn.Session()
output_value = session.run(outputs=output_tensor,
inputs={input_tensor: input_value})
expected = np.tile(input_value, pattern)
np.testing.assert_almost_equal(expected, output_value)
def test_sub_numbers(self):
"""Tensor - number is correct elementwise"""
constant, shape = 10, (3, 5)
with nn.variable_scope(self.get_scope()):
tensor1 = fixture.create_ones_tensor(shape, dtype='int32')
tensor2 = tensor1 - constant
tensor3 = constant - tensor1
session = nn.Session()
val1, val2, val3 = session.run(outputs=[tensor1, tensor2, tensor3], )
np.testing.assert_equal(val1 - constant, val2)
np.testing.assert_equal(constant - val1, val3)
def _test_transpose_tensor(self, shape, axes):
with nn.variable_scope(self.get_scope()):
in_var = nn.Input(shape=shape)
out_var = in_var.transpose(axes=axes)
in_val = np.random.random(size=shape)
session = nn.Session()
out_val = session.run(
outputs=out_var,
inputs={in_var: in_val},
)
np.testing.assert_almost_equal(
np.transpose(in_val, axes=axes), out_val)
def _test_load_dataset(self, dtype1, dtype2):
name = 'test_load_dataset_{}_{}'.format(dtype1, dtype2)
shape = (3, 3)
target_value = 10
variable = nn.get_variable(name=name, shape=shape, dtype=dtype1)
value = target_value * np.ones(shape, dtype=dtype2)
session = nn.Session()
session.load_dataset({name: value}, cast=not dtype1 == dtype2)
updated_value = session.run(outputs=variable)
self.assertTrue(np.all(target_value == updated_value))
def _optimize(optimizer, loss, wrt, n_ite):
minimize_op = optimizer.minimize(loss=loss, wrt=wrt)
sess = nn.Session()
sess.initialize()
result = []
for _ in range(n_ite + 1):
output = sess.run(outputs=[loss, wrt], name='output')
result.append({
'loss': output[0],
'wrt': output[1],
})
sess.run(updates=minimize_op, name='minimize')
return result
def _run_forward_prop(layer, input_value, parameter_file, iteration=1):
sess = nn.Session()
if parameter_file:
_LG.info('Loading parameter values from %s', parameter_file)
sess.load_from_file(parameter_file, strict=False)
_LG.info('Running forward path for %s times', iteration)
for _ in range(iteration):
ret = sess.run(
outputs=layer.output, updates=layer.get_update_operations(),
inputs={layer.input: input_value.astype(layer.input.dtype)},
)
_LG.info('Run forward path. Output shape: %s', ret.shape)
return ret
def _validate(self, in_var, in_val, mean, std, out_var):
session = nn.Session()
out_val = session.run(outputs=out_var, givens={in_var: in_val})
self.assertEqual(in_var.shape, out_var.shape)
self.assertEqual(in_var.dtype, out_var.dtype)
self.assertEqual(in_var.shape, out_val.shape)
self.assertEqual(in_var.dtype, out_val.dtype)
mean_diff = abs(np.mean(out_val) - mean)
std_diff = abs(np.std(out_val) - std)
self.assertLess(mean_diff, 0.3)
self.assertLess(std_diff, 0.3)
def test_relu(self):
"""Test Sigmoid output """
shape = (3, 4)
with nn.variable_scope(self.get_scope()):
in_var = nn.Input(shape=shape)
layer = nn.layer.Sigmoid()
out_var = layer(in_var)
in_val = np.random.rand(*shape)
session = nn.Session()
expected = 1 / (1 + np.exp(-in_val))
out_val = session.run(outputs=out_var, inputs={in_var: in_val})
self.assertEqual(out_var.shape, out_val.shape)
np.testing.assert_almost_equal(out_val, expected)
def _compute_cost(cost, target, logit):
target_tensor = nn.Input(shape=target.shape)
logit_tensor = nn.Input(shape=logit.shape)
output_tensor = cost.build(target_tensor, logit_tensor)
session = nn.Session()
output_value = session.run(
outputs=output_tensor,
inputs={
logit_tensor: logit,
target_tensor: target,
},
)
return output_value
def test_sub_input(self):
"""Tensor - Input is correct elementwise"""
shape = (3, 5)
with nn.variable_scope(self.get_scope()):
input_ = nn.Input(shape=[], dtype='int32')
tensor1 = fixture.create_ones_tensor(shape, dtype='int32')
tensor2 = tensor1 - input_
tensor3 = input_ - tensor1
session = nn.Session()
val0 = np.array(10, dtype='int32')
val1, val2, val3 = session.run(outputs=[tensor1, tensor2, tensor3],
givens={input_: val0})
np.testing.assert_equal(val1 - val0, val2)
np.testing.assert_equal(val0 - val1, val3)
def test_abs(self):
"""abs(Tensor) is correct elementwise"""
shape = (3, 5)
with nn.variable_scope(self.get_scope()):
input_ = nn.Input(shape, dtype='float64')
output = abs(input_)
input_val = np.random.randn(*shape)
session = nn.Session()
out_val = session.run(
outputs=output,
inputs={input_: input_val},
)
np.testing.assert_almost_equal(out_val, abs(input_val))
