def test_json_storage(self):
connection_1 = layers.join(
layers.Input(10),
[
layers.Sigmoid(5),
layers.Relu(5),
],
layers.Elementwise(),
)
predict_1 = connection_1.compile()
connection_2 = layers.join(
layers.Input(10),
[
layers.Sigmoid(5),
layers.Relu(5),
],
layers.Elementwise(),
)
predict_2 = connection_2.compile()
random_input = asfloat(np.random.random((13, 10)))
random_output_1 = predict_1(random_input)
random_output_2_1 = predict_2(random_input)
# Outputs has to be different
self.assertFalse(np.any(random_output_1 == random_output_2_1))
with tempfile.NamedTemporaryFile() as temp:
storage.save_json(connection_1, temp.name)
storage.load_json(connection_2, temp.name)
random_output_2_2 = predict_2(random_input)
np.testing.assert_array_almost_equal(random_output_1,
random_output_2_2)
def test_simple_storage_hdf5(self):
network_1 = layers.join(
layers.Input(10),
layers.parallel(
layers.Sigmoid(5),
layers.Relu(5),
),
layers.Elementwise(),
)
network_2 = layers.join(
layers.Input(10),
layers.parallel(
layers.Sigmoid(5),
layers.Relu(5),
),
layers.Elementwise(),
)
random_input = asfloat(np.random.random((13, 10)))
random_output_1 = self.eval(network_1.output(random_input))
random_output_2_1 = self.eval(network_2.output(random_input))
# Outputs has to be different
self.assertFalse(np.any(random_output_1 == random_output_2_1))
with tempfile.NamedTemporaryFile() as temp:
storage.save_hdf5(network_1, temp.name)
storage.load_hdf5(network_2, temp.name)
random_output_2_2 = self.eval(network_2.output(random_input))
np.testing.assert_array_almost_equal(random_output_1,
random_output_2_2)
def test_many_to_many_graph(self):
l0 = layers.Input(1)
l11 = layers.Sigmoid(10)
le = layers.Elementwise()
l3 = layers.Sigmoid(30)
l4 = layers.Sigmoid(40)
l5 = layers.Sigmoid(50)
l6 = layers.Sigmoid(60)
l12 = layers.Sigmoid(70)
# Graph Structure:
# [l0, l12] -> [l6, l4]
#
# l0 - l11 - l5 - l6
# \
# l12 - le - l4
# \
# -- l3
graph = LayerGraph()
# Connection #1
graph.connect_layers(l0, l11)
graph.connect_layers(l11, l5)
graph.connect_layers(l5, l6)
# Connection #2
graph.connect_layers(l11, le)
graph.connect_layers(le, l4)
# Connection #3
graph.connect_layers(le, l3)
# Connection #4
graph.connect_layers(l12, le)
def test_elementwise_in_connections(self):
input_layer = layers.Input(2)
hidden_layer_1 = layers.Relu(1,
weight=init.Constant(1),
bias=init.Constant(0))
hidden_layer_2 = layers.Relu(1,
weight=init.Constant(2),
bias=init.Constant(0))
elem_layer = layers.Elementwise(merge_function=tf.add)
connection = layers.join(input_layer, hidden_layer_1, elem_layer)
connection = layers.join(input_layer, hidden_layer_2, elem_layer)
connection.initialize()
self.assertEqual(elem_layer.output_shape, (1, ))
test_input = asfloat(np.array([
[0, 1],
[-1, -1],
]))
actual_output = self.eval(connection.output(test_input))
expected_output = np.array([
[3],
[0],
])
np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_compilation_multiple_inputs(self):
input_matrix = asfloat(np.ones((7, 10)))
expected_output = np.ones((7, 5))
network = layers.join([[
layers.Input(10),
], [
layers.Input(10),
]], layers.Elementwise(),
layers.Linear(5,
weight=init.Constant(0.1),
bias=None))
# Generated input variables
predict = network.compile()
actual_output = predict(input_matrix * 0.7, input_matrix * 0.3)
np.testing.assert_array_almost_equal(actual_output, expected_output)
# Pre-defined input variables
input_variable_1 = T.matrix('x1')
input_variable_2 = T.matrix('x2')
predict = network.compile(input_variable_1, input_variable_2)
actual_output = predict(input_matrix * 0.7, input_matrix * 0.3)
np.testing.assert_array_almost_equal(actual_output, expected_output)
def test_elementwise_init_error(self):
input_layer_1 = layers.Input(10)
input_layer_2 = layers.Input(20)
elem_layer = layers.Elementwise()
layers.join(input_layer_1, elem_layer)
with self.assertRaises(LayerConnectionError):
layers.join(input_layer_2, elem_layer)
def test_elementwise_basic(self):
elem_layer = layers.Elementwise(merge_function=tf.add)
x1_matrix = asfloat(np.random.random((10, 2)))
x2_matrix = asfloat(np.random.random((10, 2)))
expected_output = x1_matrix + x2_matrix
actual_output = self.eval(elem_layer.output(x1_matrix, x2_matrix))
np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_elementwise_exceptions(self):
with self.assertRaises(ValueError):
not_callable_object = (1, 2, 3)
layers.Elementwise(merge_function=not_callable_object)
with self.assertRaises(ValueError):
layers.Elementwise(merge_function='wrong-func-name')
message = "expected multiple inputs"
with self.assertRaisesRegexp(LayerConnectionError, message):
layers.join(layers.Input(5), layers.Elementwise('multiply'))
inputs = layers.parallel(
layers.Input(2),
layers.Input(1),
)
error_message = "layer have incompatible shapes"
with self.assertRaisesRegexp(LayerConnectionError, error_message):
layers.join(inputs, layers.Elementwise('add'))
def create_VIN(input_image_shape=(8, 8, 2), n_hidden_filters=150,
n_state_filters=10, k=10):
SamePadConvolution = partial(layers.Convolution, padding='SAME', bias=None)
R = layers.join(
layers.Input(input_image_shape, name='grid-input'),
layers.Convolution((3, 3, n_hidden_filters),
padding='SAME',
weight=init.Normal(),
bias=init.Normal()),
SamePadConvolution((1, 1, 1), weight=init.Normal()),
)
# Create shared weights
q_weight = random_weight((3, 3, 1, n_state_filters))
fb_weight = random_weight((3, 3, 1, n_state_filters))
Q = R > SamePadConvolution((3, 3, n_state_filters), weight=q_weight)
for i in range(k):
V = Q > ChannelGlobalMaxPooling()
Q = layers.join(
# Convolve R and V separately and then add outputs together with
# the Elementwise layer. This part of the code looks different
# from the one that was used in the original VIN repo, but
# it does the same operation.
#
# conv(x, w) == (conv(x1, w1) + conv(x2, w2))
# where, x = concat(x1, x2)
# w = concat(w1, w2)
#
# See code sample from Github Gist: https://bit.ly/2zm3ntN
[[
R,
SamePadConvolution((3, 3, n_state_filters), weight=q_weight)
], [
V,
SamePadConvolution((3, 3, n_state_filters), weight=fb_weight)
]],
layers.Elementwise(merge_function=tf.add),
)
input_state_1 = layers.Input(UNKNOWN, name='state-1-input')
input_state_2 = layers.Input(UNKNOWN, name='state-2-input')
# Select the conv-net channels at the state position (S1, S2)
VIN = [Q, input_state_1, input_state_2] > SelectValueAtStatePosition()
# Set up softmax layer that predicts actions base on (S1, S2)
# position. Each action encodes specific direction:
# N, S, E, W, NE, NW, SE, SW (in the same order)
VIN = VIN > layers.Softmax(8, bias=None, weight=init.Normal())
return VIN
def test_one_to_one_graph(self):
l0 = layers.Input(1)
l1 = layers.Sigmoid(10)
l2 = layers.Sigmoid(20)
l3 = layers.Sigmoid(30)
l41 = layers.Sigmoid(40)
l42 = layers.Sigmoid(40)
le = layers.Elementwise()
# Graph Structure:
# l0 -> le
#
# l0 - l1 - l41 -- le
# \ /
# l2 - l42
# \
# -- l3
graph = LayerGraph()
# Connection #1
graph.connect_layers(l0, l1)
graph.connect_layers(l1, l41)
graph.connect_layers(l41, le)
graph.connect_layers(l1, l2)
graph.connect_layers(l2, l42)
graph.connect_layers(l42, le)
# Connection #2
graph.connect_layers(l2, l3)
for layer in graph.forward_graph:
layer.initialize()
subgraph = graph.subgraph_for_output(le)
self.assertIsNot(l3, subgraph.forward_graph)
self.assertEqual(6, len(subgraph))
# Input layers
self.assertEqual(1, len(subgraph.input_layers))
self.assertEqual([l0], subgraph.input_layers)
# Output layers
self.assertEqual(1, len(subgraph.output_layers))
self.assertEqual([le], subgraph.output_layers)
x = T.matrix()
y = subgraph.propagate_forward(x)
test_input = asfloat(np.array([[1]]))
output = y.eval({x: test_input})
self.assertEqual((1, 40), output.shape)
def test_elementwise_basic(self):
elem_layer = layers.Elementwise(merge_function=T.add)
x1 = T.matrix()
x2 = T.matrix()
y = theano.function([x1, x2], elem_layer.output(x1, x2))
x1_matrix = asfloat(np.random.random((10, 2)))
x2_matrix = asfloat(np.random.random((10, 2)))
expected_output = x1_matrix + x2_matrix
actual_output = y(x1_matrix, x2_matrix)
np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_elementwise_in_network(self):
network = layers.join(
layers.Input(2),
layers.parallel(
layers.Relu(1, weight=1, bias=0),
layers.Relu(1, weight=2, bias=0),
),
layers.Elementwise('add'),
)
self.assertShapesEqual(network.input_shape, (None, 2))
self.assertShapesEqual(network.output_shape, (None, 1))
test_input = asfloat(np.array([[0, 1], [-1, -1]]))
actual_output = self.eval(network.output(test_input))
expected_output = np.array([[3, 0]]).T
np.testing.assert_array_almost_equal(expected_output, actual_output)
def create_VIN(input_image_shape=(2, 8, 8), n_hidden_filters=150,
n_state_filters=10, k=10):
HalfPaddingConv = partial(layers.Convolution, padding='half', bias=None)
R = layers.join(
layers.Input(input_image_shape, name='grid-input'),
layers.Convolution((n_hidden_filters, 3, 3),
padding='half',
weight=init.Normal(),
bias=init.Normal()),
HalfPaddingConv((1, 1, 1), weight=init.Normal()),
)
# Create shared weights
q_weight = random_weight((n_state_filters, 1, 3, 3))
fb_weight = random_weight((n_state_filters, 1, 3, 3))
Q = R > HalfPaddingConv((n_state_filters, 3, 3), weight=q_weight)
for i in range(k):
V = Q > GlobalMaxPooling()
Q = layers.join(
# Convolve R and V separately and then add
# outputs together with the Elementwise layer
[[
R,
HalfPaddingConv((n_state_filters, 3, 3), weight=q_weight)
], [
V,
HalfPaddingConv((n_state_filters, 3, 3), weight=fb_weight)
]],
layers.Elementwise(merge_function=T.add),
)
input_state_1 = layers.Input(10, name='state-1-input')
input_state_2 = layers.Input(10, name='state-2-input')
# Select the conv-net channels at the state position (S1, S2)
VIN = [Q, input_state_1, input_state_2] > SelectValueAtStatePosition()
# Set up softmax layer that predicts actions base on (S1, S2)
# position. Each action encodes specific direction:
# N, S, E, W, NE, NW, SE, SW (in the same order)
VIN = VIN > layers.Softmax(8, bias=None, weight=init.Normal())
return VIN
def test_many_to_one_graph(self):
l0 = layers.Input(1)
l11 = layers.Sigmoid(10)
le = layers.Elementwise()
l3 = layers.Sigmoid(30)
l4 = layers.Sigmoid(40)
l5 = layers.Input(50)
l6 = layers.Sigmoid(60)
l12 = layers.Sigmoid(10)
# Graph Structure:
# [l0, l12] -> l6
#
# l0 - l11 - le - l6
# /
# l5 - l12 - l4
# \
# -- l3
graph = LayerGraph()
# Connection #1
graph.connect_layers(l0, l11)
graph.connect_layers(l11, le)
graph.connect_layers(le, l6)
graph.connect_layers(l5, l12)
graph.connect_layers(l12, le)
# Connection #2
graph.connect_layers(l12, l4)
# Connection #3
graph.connect_layers(l12, l3)
subgraph = graph.subgraph_for_output(l6)
self.assertIsNot(l4, subgraph.forward_graph)
self.assertIsNot(l3, subgraph.forward_graph)
self.assertEqual(6, len(subgraph))
# Input layers
self.assertEqual(2, len(subgraph.input_layers))
self.assertIn(l0, subgraph.input_layers)
self.assertIn(l5, subgraph.input_layers)
# Output layers
self.assertEqual(1, len(subgraph.output_layers))
self.assertEqual([l6], subgraph.output_layers)
def test_inline_connections_after_exception(self):
# One possibility to solve it is to reset all states in
# connections/inline.py and when we assing new shape
# in connections/graph.py:connect_layers catch error if happens
# and destroy connection between layers
input_layer = layers.Input(2)
with self.assertRaises(LayerConnectionError):
# it suppose to fail because layers in parallel connections
# specified with different output shapes.
input_layer > [layers.Sigmoid(20),
layers.Sigmoid(10)] > layers.Elementwise()
# Issue #181. Bug presented in NeuPy versions <= 0.7.2
network = input_layer > layers.Softmax(5)
self.assertEqual(network.input_shape, (2, ))
self.assertEqual(network.output_shape, (5, ))
def test_elementwise_custom_function(self):
def weighted_sum(a, b):
return 0.2 * a + 0.8 * b
network = layers.join(
layers.Input(2),
layers.parallel(
layers.Relu(1, weight=1, bias=0),
layers.Relu(1, weight=2, bias=0),
),
layers.Elementwise(weighted_sum),
)
self.assertShapesEqual(network.input_shape, (None, 2))
self.assertShapesEqual(network.output_shape, (None, 1))
test_input = asfloat(np.array([[0, 1], [-1, -1]]))
actual_output = self.eval(network.output(test_input))
expected_output = np.array([[1.8, 0]]).T
np.testing.assert_array_almost_equal(expected_output, actual_output)
def ResidualUnit(n_in_filters, n_out_filters, stride, has_branch=False):
main_branch = layers.join(
layers.Convolution((n_in_filters, 1, 1), stride=stride, bias=None),
layers.BatchNorm(),
layers.Relu(),
layers.Convolution((n_in_filters, 3, 3), padding=1, bias=None),
layers.BatchNorm(),
layers.Relu(),
layers.Convolution((n_out_filters, 1, 1), bias=None),
layers.BatchNorm(),
)
residual_branch = []
if has_branch:
residual_branch = layers.join(
layers.Convolution((n_out_filters, 1, 1), stride=stride,
bias=None),
layers.BatchNorm(),
)
return layers.join(
[main_branch, residual_branch],
layers.Elementwise() > layers.Relu(),
)
def test_elementwise_output_shape_no_connection(self):
elem_layer = layers.Elementwise()
self.assertEqual(elem_layer.output_shape, None)
def test_elementwise_not_function(self):
with self.assertRaises(ValueError):
not_callable_object = (1, 2, 3)
layers.Elementwise(merge_function=not_callable_object)
def ResidualUnit(n_input_filters, stride=1, rate=1, has_branch=False,
name=None):
def bn_name(index):
return 'bn' + name + '_branch' + index
def conv_name(index):
return 'res' + name + '_branch' + index
n_output_filters = 4 * n_input_filters
main_branch = layers.join(
# The main purpose of this 1x1 convolution layer is to
# reduce number of filters. For instance, for the tensor with
# 256 filters it can be reduced to 64. This trick allows to
# reduce computation by factor of 4.
layers.Convolution(
size=(1, 1, n_input_filters),
stride=stride,
bias=None,
name=conv_name('2a'),
),
layers.BatchNorm(name=bn_name('2a')),
layers.Relu(),
# This convolution layer applies 3x3 filter in order to
# extract features.
layers.Convolution(
(3, 3, n_input_filters),
padding='same',
dilation=rate,
bias=None,
name=conv_name('2b'),
),
layers.BatchNorm(name=bn_name('2b')),
layers.Relu(),
# Last layer reverses operations of the first layer. In this
# case we increase number of filters. For instance, from previously
# obtained 64 filters we can increase it back to the 256 filters
layers.Convolution(
(1, 1, n_output_filters),
bias=None,
name=conv_name('2c')
),
layers.BatchNorm(name=bn_name('2c')),
)
if has_branch:
residual_branch = layers.join(
layers.Convolution(
(1, 1, n_output_filters),
stride=stride,
bias=None,
name=conv_name('1'),
),
layers.BatchNorm(name=bn_name('1')),
)
else:
# Empty list defines residual connection, meaning that
# output from this branch would be equal to its input
residual_branch = layers.Identity('residual-' + name)
return layers.join(
# For the output from two branches we just combine results
# with simple elementwise sum operation. The main purpose of
# the residual connection is to build shortcuts for the
# gradient during backpropagation.
(main_branch | residual_branch),
layers.Elementwise(),
layers.Relu(),
)
def test_elementwise_single_input(self):
elem_layer = layers.Elementwise()
output = elem_layer.output(None)
self.assertEqual(output, None)
def test_elementwise_initialize(self):
# Suppose not to fail if you initialize
# it without connection
elem_layer = layers.Elementwise()
elem_layer.initialize()
