def test_paramter_reuse_conv2d(self): """Conv2D layer is built using existing Variables""" shape = (10, 11, 12, 13) with nn.variable_scope(self.get_scope()): layer1 = nn.layer.Conv2D(filter_width=5, filter_height=3, n_filters=4, strides=1, padding='VALID') layer2 = nn.layer.Conv2D(filter_width=5, filter_height=3, n_filters=4, strides=1, padding='VALID') tensor = nn.Input(shape=shape) out1 = layer1(tensor) layer2.set_parameter_variables( filter=layer1.get_parameter_variable('filter'), bias=layer1.get_parameter_variable('bias')) out2 = layer2(tensor) for key in ['filter', 'bias']: var1 = layer1.get_parameter_variable(key) var2 = layer2.get_parameter_variable(key) self.assertIs(var1, var2) session = nn.Session() session.initialize() input_val = np.random.rand(*shape) out1, out2 = session.run(outputs=[out1, out2], inputs={tensor: input_val}) np.testing.assert_almost_equal(out1, out2)
def test_paramter_reuse_dense(self): """Dense layer is built using existing Variables""" shape = (3, 5) with nn.variable_scope(self.get_scope()): layer1 = nn.layer.Dense(n_nodes=5) layer2 = nn.layer.Dense(n_nodes=5) tensor = nn.Input(shape=shape) out1 = layer1(tensor) layer2.set_parameter_variables( weight=layer1.get_parameter_variable('weight'), bias=layer1.get_parameter_variable('bias'), ) out2 = layer2(tensor) for key in ['weight', 'bias']: var1 = layer1.get_parameter_variable(key) var2 = layer2.get_parameter_variable(key) self.assertIs(var1, var2) session = nn.Session() session.initialize() input_val = np.random.rand(*shape) out1, out2 = session.run(outputs=[out1, out2], inputs={tensor: input_val}) np.testing.assert_almost_equal(out1, out2)
def test_concate_2d_axis_1_3(self): """Concatenate 3 2D tensors""" axis, shape1, shape2, shape3 = 1, (2, 5), (2, 3), (2, 4) with nn.variable_scope(self.get_scope(), reuse=False): var1 = nn.get_variable(name='var1', shape=shape1) var2 = nn.get_variable(name='var2', shape=shape2) var3 = nn.get_variable(name='var3', shape=shape3) conc_var = nn.layer.Concat(axis=axis).build([var1, var2, var3]) session = nn.Session() val1, val2 = np.random.rand(*shape1), np.random.rand(*shape2) val3 = np.random.rand(*shape3) conc_val = session.run(outputs=conc_var, givens={ var1: val1, var2: val2, var3: val3 }) expected = conc_val.shape found = conc_var.shape self.assertEqual(found, expected) expected = np.concatenate((val1, val2, val3), axis=axis) found = conc_val np.testing.assert_almost_equal(found, expected)
def test_clip_variable_by_norm(self): """Test clip_by_norm with Variable""" shape, clip_norm = (3, 4), np.asarray(15, dtype='float32') with nn.variable_scope(self.get_scope()): input_ = nn.Input(shape, dtype='float32') clip_var = nn.Input(shape=[], dtype='float32') output = nn.ops.clip_by_norm(input_, clip_norm=clip_var) session = nn.Session() in_val = np.random.rand(*shape).astype('float32') out_val = session.run( outputs=output, givens={input_: in_val, clip_var: clip_norm} ) np.testing.assert_almost_equal(out_val, in_val) in_val += 10.0 out_val = session.run( outputs=output, givens={input_: in_val, clip_var: clip_norm} ) l2_norm = np.sqrt(np.sum(in_val ** 2)) np.testing.assert_almost_equal( out_val, clip_norm * in_val / l2_norm, decimal=3)
def test_dynamic_initializer(self): """Initializers are correctly selected""" n_in, n_nodes, weight_val, bias_val = 4, 5, 13, 7 with nn.variable_scope(self.get_scope()): dense = nn.layer.Dense(n_nodes=5, initializers={ 'weight': { 'typename': 'ConstantInitializer', 'args': { 'value': weight_val, }, }, 'bias': { 'typename': 'ConstantInitializer', 'args': { 'value': bias_val, } } }) dense(nn.Input(shape=(3, n_in))) session = nn.Session() session.initialize() weight, bias = session.run(outputs=[ dense.get_parameter_variable('weight'), dense.get_parameter_variable('bias'), ]) np.testing.assert_almost_equal(weight, weight_val * np.ones((n_in, n_nodes))) np.testing.assert_almost_equal(bias, bias_val * np.ones((n_nodes, )))
def test_clip_number_by_norm_with_axes(self): """Test clip_by_norm with axis""" shape, clip_norm, axis = (3, 4), 15.0, 1 with nn.variable_scope(self.get_scope()): input_ = nn.Input(shape, dtype='float32') output = nn.ops.clip_by_norm( input_, clip_norm=clip_norm, axes=axis) session = nn.Session() in_val = np.random.rand(*shape).astype('float32') out_val = session.run( outputs=output, givens={input_: in_val} ) np.testing.assert_almost_equal(out_val, in_val) in_val += 10.0 out_val = session.run( outputs=output, givens={input_: in_val} ) l2_norm = np.sqrt(np.sum(in_val ** 2, axis=axis, keepdims=True)) np.testing.assert_almost_equal( out_val, clip_norm * in_val / l2_norm, decimal=3)
def test_sync_with_tau(self): """sync op copies weighted sum of source and target variables""" tau = 0.1 with nn.variable_scope(self.get_scope()): source_var, target_var = _create_variables() sync_op = nn.ops.build_sync_op( [source_var], [target_var], tau=tau) session = nn.Session() session.initialize() src_val, tgt_val = session.run([source_var, target_var]) self.assertTrue((src_val == 1).all()) self.assertTrue((tgt_val == 0).all()) for _ in range(10): expected = tau * src_val + (1 - tau) * tgt_val session.run(updates=sync_op) src_val, found = session.run([source_var, target_var]) self.assertTrue((src_val == 1).all()) self.assertTrue( np.square(expected - found).sum() < 1e-10, '\nExpected: \n{}\nFound: \n{}'.format(expected, found) ) tgt_val = found
def _main(): args = _parase_command_line_args() _initialize_logger(args.debug) data_format = luchador.get_nn_conv_format() batch_size = 32 input_shape = ( [batch_size, 28, 28, 1] if data_format == 'NHWC' else [batch_size, 1, 28, 28] ) classifier = _build_model(args.model, input_shape, batch_size) dataset = _load_data(args.mnist, data_format) session = nn.Session() session.initialize() summary = nn.SummaryWriter(output_dir='tmp') if session.graph: summary.add_graph(session.graph) try: _train(session, classifier, dataset['train'], batch_size) _test(session, classifier, dataset['test'], batch_size) except KeyboardInterrupt: pass
def test_clip_gradients(self): """Gradients are clipped""" sgd = nn.optimizer.SGD(learning_rate=1.0) shape = (32, 1) with nn.variable_scope(self.get_scope()): initializer = nn.get_initializer('UniformInitializer')(minval=-3, maxval=3) x = nn.get_variable(name='x', shape=shape, initializer=initializer) y = x * x / 2 grads_and_vars = [ (nn.clip_by_value(grad, max_value=1.0, min_value=-1.0), var) for grad, var in sgd.compute_gradients(y.sum(), wrt=x) ] op = sgd.apply_gradients(grads_and_vars) session = nn.Session() session.initialize() val_0 = session.run(outputs=x) session.run(updates=op) val_1_be = session.run(outputs=x) val_1_np = np.zeros(shape) val_1_np[val_0 > 1] = val_0[val_0 > 1] - 1 val_1_np[val_0 < -1] = val_0[val_0 < -1] + 1 np.testing.assert_almost_equal(val_1_be, val_1_np)
def test_dict(self): """Anonymous layer can handle dict inputs""" shape, dtype = (3, 4), 'float32' exp = ('x["0"] + x["1"]') input_vars = { '0': nn.Input(shape=shape, dtype=dtype, name='input1'), '1': nn.Input(shape=shape, dtype=dtype, name='input2') } input_vals = { '0': np.random.rand(3, 4).astype(dtype), '1': np.random.rand(3, 4).astype(dtype), } output_val = sum(input_vals.values()) with nn.variable_scope(self.get_scope()): layer = nn.layer.Anonymous(exp) output_var = layer(**input_vars) session = nn.Session() output_val_ = session.run(outputs=output_var, inputs={ input_vars['0']: input_vals['0'], input_vars['1']: input_vals['1'] }) np.testing.assert_almost_equal(output_val, output_val_)
def test_list(self): """Anonymous layer can handle list inputs""" shape, dtype = (3, 4), 'float32' exp = ('x[0] + x[1]') input_vars = [ nn.Input(shape=shape, dtype=dtype, name='input1'), nn.Input(shape=shape, dtype=dtype, name='input2') ] input_vals = [ np.random.rand(3, 4).astype(dtype), np.random.rand(3, 4).astype(dtype), ] output_val = sum(input_vals) with nn.variable_scope(self.get_scope()): layer = nn.layer.Anonymous(exp) output_var = layer(*input_vars) session = nn.Session() output_val_ = session.run(outputs=output_var, inputs={ input_vars[0]: input_vals[0], input_vars[1]: input_vals[1] }) np.testing.assert_almost_equal(output_val, output_val_)
def _exe(exp, input_val, scope): input_var = nn.Input(shape=input_val.shape, dtype=input_val.dtype) with nn.variable_scope(scope): layer = nn.layer.Anonymous(exp) output_var = layer(input_var) session = nn.Session() return session.run(outputs=output_var, inputs={input_var: input_val})
def test_beta_power_update(self): """Beta parameter is updated every time update is evaluated""" beta1, beta2, x_init_val = 0.9, 0.999, 3.0 adamax = nn.optimizer.Adamax(learning_rate=0.01, beta1=beta1) x_tensor, y_tensor = _get_y_equals_x_squared(scope=self.get_scope(), x_init=x_init_val) minimize_op = adamax.minimize(loss=y_tensor, wrt=x_tensor) beta1_pow_tensor = _get_slot_var(adamax, 'beta1_power') m_tensor = _get_slot_var(adamax, 'm', var_name=x_tensor.name) u_tensor = _get_slot_var(adamax, 'u', var_name=x_tensor.name) session = nn.Session() session.initialize() x_val_prev, m_val_prev, u_val_prev = x_init_val, 0, 0 for i in range(1, 10): session.run(updates=minimize_op, name='optimization') beta1_pow_val = session.run(outputs=beta1_pow_tensor, name='fetch1') expected = beta1**(i + 1) found = beta1_pow_val diff = abs(expected - found) self.assertTrue( diff < 0.01, 'Beta1 is not correctly updated. ' 'Expected: {}, Found: {}'.format(expected, found)) m_val, u_val = session.run(outputs=[m_tensor, u_tensor], name='fetch3') grad = 2 * x_val_prev expected = m_val_prev + (1.0 - beta1) * (grad - m_val_prev) found = m_val diff = abs(expected - found) self.assertTrue( diff < 0.01, 'The value of `m` is not correctly updated. ' 'Expected: {}, Fround: {}'.format(expected, found)) m_val_prev = m_val expected = max(u_val_prev * beta2, abs(grad)) found = u_val diff = abs(expected - found) self.assertTrue( diff < 0.01, 'The value of `u` is not correctly updated. ' 'Expected: {}, Fround: {}'.format(expected, found)) u_val_prev = u_val x_val = session.run(outputs=x_tensor, name='fetch2') self.assertTrue( 0 <= x_val < x_val_prev, 'The value of `x` must regress to zero at each update. ' 'Previous value: {}, current value: {}'.format( x_val_prev, x_val)) x_val_prev = x_val
def test_neg(self): """-Tensor is correct elementwise""" shape = (3, 5) with nn.variable_scope(self.get_scope()): tensor1 = fixture.create_ones_tensor(shape, dtype='int32') tensor2 = -tensor1 session = nn.Session() val1, val2 = session.run(outputs=[tensor1, tensor2], ) np.testing.assert_equal(-val1, val2)
def _test_max(self, axis, shape, keep_dims): with nn.variable_scope(self.get_scope()): tensor0 = fixture.create_ones_tensor(shape, dtype='float32') tensor1 = tensor0.max(axis=axis, keep_dims=keep_dims) session = nn.Session() val0, val1 = session.run(outputs=[tensor0, tensor1], ) expected = val0.max(axis=axis, keepdims=keep_dims) np.testing.assert_equal(val1, expected)
def test_floordiv_tensor(self): """Tensor // Tensor is correct elementwise""" shape = (3, 5) with nn.variable_scope(self.get_scope()): tensor1 = fixture.create_ones_tensor(shape, dtype='float32') tensor2 = tensor1 // tensor1 session = nn.Session() val1, val2 = session.run(outputs=[tensor1, tensor2], ) np.testing.assert_equal(np.floor_divide(val1, val1), val2)
def _check(self, input_var, output_var): session = nn.Session() session.initialize() input_val = np.random.randn(*input_var.shape) output_val = session.run(outputs=output_var, inputs={input_var: input_val}) self.assertEqual(output_var.shape, input_var.shape) self.assertEqual(output_var.dtype, input_var.dtype) self.assertEqual(output_var.shape, output_val.shape) self.assertEqual(output_var.dtype, output_val.dtype)
def _convert(layer, shape): input_tensor = nn.Input(shape=shape) input_value = np.random.randn(*shape) - 100 session = nn.Session() output_tensor = layer(input_tensor) output_value = session.run( outputs=output_tensor, inputs={input_tensor: input_value}, ) return output_value, output_tensor
def test_floordiv_numbers(self): """Tensor // number is correct elementwise""" constant, shape = 10., (3, 5) with nn.variable_scope(self.get_scope()): tensor1 = fixture.create_ones_tensor(shape, dtype='float32') tensor2 = tensor1 // constant tensor3 = constant // tensor1 session = nn.Session() val1, val2, val3 = session.run(outputs=[tensor1, tensor2, tensor3], ) np.testing.assert_equal(np.floor_divide(val1, constant), val2) np.testing.assert_equal(np.floor_divide(constant, val1), val3)
def _test_tile(self, pattern, input_shape, input_value): scope = self.get_scope() with nn.variable_scope(scope, reuse=False): input_tensor = nn.Input(shape=input_shape) tile = nn.layer.Tile(pattern=pattern) output_tensor = tile(input_tensor) session = nn.Session() output_value = session.run(outputs=output_tensor, inputs={input_tensor: input_value}) expected = np.tile(input_value, pattern) np.testing.assert_almost_equal(expected, output_value)
def test_sub_numbers(self): """Tensor - number is correct elementwise""" constant, shape = 10, (3, 5) with nn.variable_scope(self.get_scope()): tensor1 = fixture.create_ones_tensor(shape, dtype='int32') tensor2 = tensor1 - constant tensor3 = constant - tensor1 session = nn.Session() val1, val2, val3 = session.run(outputs=[tensor1, tensor2, tensor3], ) np.testing.assert_equal(val1 - constant, val2) np.testing.assert_equal(constant - val1, val3)
def _test_transpose_tensor(self, shape, axes): with nn.variable_scope(self.get_scope()): in_var = nn.Input(shape=shape) out_var = in_var.transpose(axes=axes) in_val = np.random.random(size=shape) session = nn.Session() out_val = session.run( outputs=out_var, inputs={in_var: in_val}, ) np.testing.assert_almost_equal( np.transpose(in_val, axes=axes), out_val)
def _test_load_dataset(self, dtype1, dtype2): name = 'test_load_dataset_{}_{}'.format(dtype1, dtype2) shape = (3, 3) target_value = 10 variable = nn.get_variable(name=name, shape=shape, dtype=dtype1) value = target_value * np.ones(shape, dtype=dtype2) session = nn.Session() session.load_dataset({name: value}, cast=not dtype1 == dtype2) updated_value = session.run(outputs=variable) self.assertTrue(np.all(target_value == updated_value))
def _optimize(optimizer, loss, wrt, n_ite): minimize_op = optimizer.minimize(loss=loss, wrt=wrt) sess = nn.Session() sess.initialize() result = [] for _ in range(n_ite + 1): output = sess.run(outputs=[loss, wrt], name='output') result.append({ 'loss': output[0], 'wrt': output[1], }) sess.run(updates=minimize_op, name='minimize') return result
def _run_forward_prop(layer, input_value, parameter_file, iteration=1): sess = nn.Session() if parameter_file: _LG.info('Loading parameter values from %s', parameter_file) sess.load_from_file(parameter_file, strict=False) _LG.info('Running forward path for %s times', iteration) for _ in range(iteration): ret = sess.run( outputs=layer.output, updates=layer.get_update_operations(), inputs={layer.input: input_value.astype(layer.input.dtype)}, ) _LG.info('Run forward path. Output shape: %s', ret.shape) return ret
def _validate(self, in_var, in_val, mean, std, out_var): session = nn.Session() out_val = session.run(outputs=out_var, givens={in_var: in_val}) self.assertEqual(in_var.shape, out_var.shape) self.assertEqual(in_var.dtype, out_var.dtype) self.assertEqual(in_var.shape, out_val.shape) self.assertEqual(in_var.dtype, out_val.dtype) mean_diff = abs(np.mean(out_val) - mean) std_diff = abs(np.std(out_val) - std) self.assertLess(mean_diff, 0.3) self.assertLess(std_diff, 0.3)
def test_relu(self): """Test Sigmoid output """ shape = (3, 4) with nn.variable_scope(self.get_scope()): in_var = nn.Input(shape=shape) layer = nn.layer.Sigmoid() out_var = layer(in_var) in_val = np.random.rand(*shape) session = nn.Session() expected = 1 / (1 + np.exp(-in_val)) out_val = session.run(outputs=out_var, inputs={in_var: in_val}) self.assertEqual(out_var.shape, out_val.shape) np.testing.assert_almost_equal(out_val, expected)
def _compute_cost(cost, target, logit): target_tensor = nn.Input(shape=target.shape) logit_tensor = nn.Input(shape=logit.shape) output_tensor = cost.build(target_tensor, logit_tensor) session = nn.Session() output_value = session.run( outputs=output_tensor, inputs={ logit_tensor: logit, target_tensor: target, }, ) return output_value
def test_sub_input(self): """Tensor - Input is correct elementwise""" shape = (3, 5) with nn.variable_scope(self.get_scope()): input_ = nn.Input(shape=[], dtype='int32') tensor1 = fixture.create_ones_tensor(shape, dtype='int32') tensor2 = tensor1 - input_ tensor3 = input_ - tensor1 session = nn.Session() val0 = np.array(10, dtype='int32') val1, val2, val3 = session.run(outputs=[tensor1, tensor2, tensor3], givens={input_: val0}) np.testing.assert_equal(val1 - val0, val2) np.testing.assert_equal(val0 - val1, val3)
def test_abs(self): """abs(Tensor) is correct elementwise""" shape = (3, 5) with nn.variable_scope(self.get_scope()): input_ = nn.Input(shape, dtype='float64') output = abs(input_) input_val = np.random.randn(*shape) session = nn.Session() out_val = session.run( outputs=output, inputs={input_: input_val}, ) np.testing.assert_almost_equal(out_val, abs(input_val))