def test_forward(self, b, w, h, c, filters, size1, size2, stride1, stride2,
pad):
if pad:
keras_pad = 'same'
else:
keras_pad = 'valid'
keras_activs = ['relu', 'sigmoid', 'tanh', 'linear']
numpynet_activs = [Relu, Logistic, Tanh, Linear]
size = (size1, size2)
stride = (stride1, stride2)
inpt = np.random.uniform(low=-1., high=1., size=(b, w, h, c))
weights = np.random.uniform(low=-1.,
high=1.,
size=(size1, size2) + (c, filters))
bias = np.random.uniform(low=-1., high=1., size=(filters, ))
for numpynet_activ, keras_activ in zip(numpynet_activs, keras_activs):
layer = Convolutional_layer(filters=filters,
input_shape=inpt.shape,
weights=weights,
bias=bias,
activation=numpynet_activ,
size=(size1, size2),
stride=stride,
pad=pad)
# Tensorflow layer
model = tf.keras.layers.Conv2D(
filters=filters,
kernel_initializer=lambda shape, dtype=None: weights,
bias_initializer=lambda shape, dtype=None: bias,
kernel_size=size,
strides=stride,
padding=keras_pad,
activation=keras_activ,
data_format='channels_last',
use_bias=True,
dilation_rate=1)
# FORWARD
forward_out_keras = model(inpt)
layer.forward(inpt, copy=False)
forward_out_numpynet = layer.output
assert forward_out_keras.shape == forward_out_numpynet.shape
np.testing.assert_allclose(forward_out_keras,
forward_out_numpynet,
atol=1e-4,
rtol=1e-3)
def test_printer(self, b, w, h, c, filters, size, stride, pad):
activ = Linear
layer = Convolutional_layer(filters=filters,
size=size,
stride=stride,
pad=pad,
input_shape=(b, w, h, c),
activations=activ)
print(layer)
############################################
n_train = X_train.shape[0]
n_test = X_test.shape[0]
# transform y to array of dimension 10 and in 4 dimension
y_train = to_categorical(y_train).reshape(n_train, 1, 1, -1)
y_test = to_categorical(y_test).reshape(n_test, 1, 1, -1)
# Create the model and training
model = Network(batch=batch, input_shape=X_train.shape[1:])
model.add(
Convolutional_layer(size=3,
filters=32,
stride=1,
pad=True,
activation='Relu'))
model.add(BatchNorm_layer())
model.add(Maxpool_layer(size=2, stride=1, padding=True))
model.add(Connected_layer(outputs=100, activation='Relu'))
model.add(BatchNorm_layer())
model.add(Connected_layer(outputs=num_classes, activation='Linear'))
model.add(Softmax_layer(spatial=True, groups=1, temperature=1.))
# model.add(Cost_layer(cost_type=cost_type.mse))
def test_constructor(self, b, w, h, c, filters, size1, size2, stride, pad):
numpynet_activ = [Relu, Logistic, Tanh, Linear]
input_shape = (b, w, h, c)
size = choice([(size1, size2), size1, size2])
stride = choice([stride, None])
if filters <= 0:
with pytest.raises(ValueError):
layer = Convolutional_layer(filters=filters, size=1)
filters += 10
if hasattr(size, '__iter__'):
if size[0] < 0 or size[1] < 0:
with pytest.raises(LayerError):
layer = Convolutional_layer(filters=filters, size=size)
size[0] += 10
size[1] += 10
weights_choice = [
np.random.uniform(low=-1,
high=1.,
size=(size[0], size[1], c, filters)), None
]
else:
if size <= 0:
with pytest.raises(LayerError):
layer = Convolutional_layer(filters=filters, size=size)
size += 10
weights_choice = [
np.random.uniform(low=-1,
high=1.,
size=(size, size, c, filters)), None
]
if stride:
if stride < 0:
with pytest.raises(LayerError):
layer = Convolutional_layer(filters=filters,
size=size,
stride=stride)
stride += 10
bias_choice = [
np.random.uniform(low=-1, high=1., size=(filters, )), None
]
weights = choice(weights_choice)
bias = choice(bias_choice)
for activ in numpynet_activ:
layer = Convolutional_layer(filters=filters,
size=size,
stride=stride,
pad=pad,
weights=weights,
bias=bias,
input_shape=input_shape,
activations=activ)
if weights is not None:
assert np.allclose(layer.weights, weights)
else:
assert (layer.weights is not None)
if bias is None:
assert np.allclose(layer.bias, np.zeros(shape=(filters, )))
else:
assert np.allclose(layer.bias, bias)
assert layer.delta == None
assert layer.output == None
assert layer.weights_update == None
assert layer.bias_update == None
assert layer.optimizer == None
assert layer.pad == pad
def test_backward(self, b, w, h, c, filters, size1, size2, stride1,
stride2, pad):
if pad:
keras_pad = 'same'
else:
keras_pad = 'valid'
keras_activs = ['relu', 'sigmoid', 'tanh', 'linear']
numpynet_activs = [Relu, Logistic, Tanh, Linear]
size = (size1, size2)
stride = (stride1, stride2)
inpt = np.random.uniform(low=-1., high=1., size=(b, w, h, c))
tf_input = tf.Variable(inpt.astype('float32'))
weights = np.random.uniform(low=-1.,
high=1.,
size=(size1, size2) + (c, filters))
bias = np.random.uniform(low=-1., high=1., size=(filters, ))
for numpynet_activ, keras_activ in zip(numpynet_activs, keras_activs):
layer = Convolutional_layer(filters=filters,
input_shape=inpt.shape,
weights=weights,
bias=bias,
activation=numpynet_activ,
size=(size1, size2),
stride=stride,
pad=pad)
# Tensorflow layer
model = tf.keras.layers.Conv2D(
filters=filters,
kernel_initializer=lambda shape, dtype=None: weights,
bias_initializer=lambda shape, dtype=None: bias,
kernel_size=size,
strides=stride,
padding=keras_pad,
activation=keras_activ,
data_format='channels_last',
use_bias=True,
dilation_rate=1)
# Try backward:
with pytest.raises(NotFittedError):
delta = np.empty(shape=inpt.shape)
layer.backward(inpt, delta)
# FORWARD
with tf.GradientTape(persistent=True) as tape:
preds = model(tf_input)
grad1 = tape.gradient(preds, tf_input)
grad2 = tape.gradient(preds, model.trainable_weights)
forward_out_keras = preds.numpy()
delta_keras = grad1.numpy()
updates = grad2
layer.forward(inpt, copy=False)
forward_out_numpynet = layer.output
assert forward_out_keras.shape == forward_out_numpynet.shape
np.testing.assert_allclose(forward_out_keras,
forward_out_numpynet,
atol=1e-4,
rtol=1e-3)
# BACKWARD
weights_updates_keras = updates[0]
bias_updates_keras = updates[1]
delta_numpynet = np.zeros(shape=inpt.shape, dtype=float)
layer.delta = np.ones(shape=layer.out_shape, dtype=float)
layer.backward(delta_numpynet, copy=False)
np.testing.assert_allclose(delta_numpynet,
delta_keras,
atol=1e-3,
rtol=1e-2)
np.testing.assert_allclose(layer.weights_update,
weights_updates_keras,
atol=1e-3,
rtol=1e-3)
np.testing.assert_allclose(layer.bias_update,
bias_updates_keras,
atol=1e-5,
rtol=1e-3)
n_train = X_train.shape[0]
n_test = X_test.shape[0]
# transform y to array of dimension 10 and in 4 dimension
y_train = to_categorical(y_train).reshape(n_train, 1, 1, -1)
y_test = to_categorical(y_test).reshape(n_test, 1, 1, -1)
# Create the modeland training
model = Network(batch=batch, input_shape=X_train.shape[1:])
# model.add(Input_layer(input_shape=(batch, 32, 32, 3))) # not necessary if input_shape is given to Network
model.add(
Convolutional_layer(input_shape=(batch, 8, 8, 3),
size=3,
filters=32,
stride=1,
pad=True,
activation='Relu'))
model.add(
Convolutional_layer(input_shape=(batch, 8, 8, 32),
size=3,
filters=64,
stride=1,
pad=True,
activation='Relu'))
model.add(Maxpool_layer(size=2, stride=1, padding=True))
model.add(Dropout_layer(prob=0.25))
model.add(
Connected_layer(input_shape=(batch, 8, 8, 64),
outputs=128,
activation='Relu'))
def test_convolutional_layer():
'''
Tests:
if the convolutional layer forward is consistent with keras
if the convolutional layer backward is consistent with keras
to be:
update
'''
from NumPyNet.activations import Relu, Logistic, Linear, Tanh
from NumPyNet.layers.convolutional_layer import Convolutional_layer
from keras.layers import Conv2D
keras_activations = ['relu', 'sigmoid', 'tanh','linear']
numpynet_activations = [Relu, Logistic, Tanh, Linear]
channels_in = [1,3,5]
channels_out= [5,10,20]
sizes = [(3,3),(20,20),(30,30)]
strides = [(2,2), (10,10),(20,20)]
padding = [False, True]
for c_in in channels_in:
for c_out in channels_out:
for keras_activ, numpynet_activ in zip(keras_activations,numpynet_activations):
for size,stride in zip(sizes,strides):
for pad in padding:
batch = np.random.randint(low=1, high=10)
batch = 1
if pad:
keras_pad = 'same'
else :
keras_pad = 'valid'
inpt = np.random.uniform(-1., 1., size = (batch, 100, 100, c_in))
b, w, h, c = inpt.shape
# Shape (size1,size2,c_in, c_out), reshape inside numpynet.forward.
filters = np.random.uniform(-1., 1., size = size + (c,c_out))
bias = np.random.uniform(-1., 1., size = (c_out,))
# Numpy_net model
numpynet = Convolutional_layer(channels_out=c_out , inputs=inpt.shape,
weights=filters, bias=bias,
activation = numpynet_activ,
size = size, stride = stride,
padding = pad)
# Keras model
inp = Input(shape = inpt.shape[1:], batch_shape = inpt.shape)
Conv2d = Conv2D(filters=c_out,
kernel_size=size, strides=stride,
padding=keras_pad,
activation=keras_activ,
data_format='channels_last',
use_bias=True , bias_initializer='zeros',
dilation_rate=1)(inp) # dilation rate = 1 is no dilation (I think)
model = Model(inputs=[inp], outputs=[Conv2d])
model.set_weights([filters, bias])
# FORWARD
print(c_in, c_out, keras_activ, size, stride, pad, keras_pad, '\n', sep = '\n')
global forward_out_keras, forward_out_numpynet
forward_out_keras = model.predict(inpt)
numpynet.forward(inpt)
forward_out_numpynet = numpynet.output
assert forward_out_keras.shape == forward_out_numpynet.shape
assert np.allclose(forward_out_keras, forward_out_numpynet, atol = 1e-04, rtol = 1e-3)
# BACKWARD
grad1 = K.gradients(model.output, [model.input])
grad2 = K.gradients(model.output, model.trainable_weights)
func1 = K.function(model.inputs + model.outputs, grad1 )
func2 = K.function(model.inputs + model.trainable_weights + model.outputs, grad2)
delta_keras = func1([inpt])[0]
updates = func2([inpt]) # it does something at least
delta_numpynet = np.zeros(shape = inpt.shape)
numpynet.backward(delta_numpynet, inpt)
assert np.allclose(delta_numpynet, delta_keras)
assert np.allclose(numpynet.weights_updates, updates[0], atol = 1e-4, rtol = 1e-3) # for a lot of operations, atol is lower
assert np.allclose(numpynet.bias_updates, updates[1])