def test_forward(self, b, w, h, c, filters, size1, size2, stride1, stride2, pad): if pad: keras_pad = 'same' else: keras_pad = 'valid' keras_activs = ['relu', 'sigmoid', 'tanh', 'linear'] numpynet_activs = [Relu, Logistic, Tanh, Linear] size = (size1, size2) stride = (stride1, stride2) inpt = np.random.uniform(low=-1., high=1., size=(b, w, h, c)) weights = np.random.uniform(low=-1., high=1., size=(size1, size2) + (c, filters)) bias = np.random.uniform(low=-1., high=1., size=(filters, )) for numpynet_activ, keras_activ in zip(numpynet_activs, keras_activs): layer = Convolutional_layer(filters=filters, input_shape=inpt.shape, weights=weights, bias=bias, activation=numpynet_activ, size=(size1, size2), stride=stride, pad=pad) # Tensorflow layer model = tf.keras.layers.Conv2D( filters=filters, kernel_initializer=lambda shape, dtype=None: weights, bias_initializer=lambda shape, dtype=None: bias, kernel_size=size, strides=stride, padding=keras_pad, activation=keras_activ, data_format='channels_last', use_bias=True, dilation_rate=1) # FORWARD forward_out_keras = model(inpt) layer.forward(inpt, copy=False) forward_out_numpynet = layer.output assert forward_out_keras.shape == forward_out_numpynet.shape np.testing.assert_allclose(forward_out_keras, forward_out_numpynet, atol=1e-4, rtol=1e-3)
def test_printer(self, b, w, h, c, filters, size, stride, pad): activ = Linear layer = Convolutional_layer(filters=filters, size=size, stride=stride, pad=pad, input_shape=(b, w, h, c), activations=activ) print(layer)
############################################ n_train = X_train.shape[0] n_test = X_test.shape[0] # transform y to array of dimension 10 and in 4 dimension y_train = to_categorical(y_train).reshape(n_train, 1, 1, -1) y_test = to_categorical(y_test).reshape(n_test, 1, 1, -1) # Create the model and training model = Network(batch=batch, input_shape=X_train.shape[1:]) model.add( Convolutional_layer(size=3, filters=32, stride=1, pad=True, activation='Relu')) model.add(BatchNorm_layer()) model.add(Maxpool_layer(size=2, stride=1, padding=True)) model.add(Connected_layer(outputs=100, activation='Relu')) model.add(BatchNorm_layer()) model.add(Connected_layer(outputs=num_classes, activation='Linear')) model.add(Softmax_layer(spatial=True, groups=1, temperature=1.)) # model.add(Cost_layer(cost_type=cost_type.mse))
def test_constructor(self, b, w, h, c, filters, size1, size2, stride, pad): numpynet_activ = [Relu, Logistic, Tanh, Linear] input_shape = (b, w, h, c) size = choice([(size1, size2), size1, size2]) stride = choice([stride, None]) if filters <= 0: with pytest.raises(ValueError): layer = Convolutional_layer(filters=filters, size=1) filters += 10 if hasattr(size, '__iter__'): if size[0] < 0 or size[1] < 0: with pytest.raises(LayerError): layer = Convolutional_layer(filters=filters, size=size) size[0] += 10 size[1] += 10 weights_choice = [ np.random.uniform(low=-1, high=1., size=(size[0], size[1], c, filters)), None ] else: if size <= 0: with pytest.raises(LayerError): layer = Convolutional_layer(filters=filters, size=size) size += 10 weights_choice = [ np.random.uniform(low=-1, high=1., size=(size, size, c, filters)), None ] if stride: if stride < 0: with pytest.raises(LayerError): layer = Convolutional_layer(filters=filters, size=size, stride=stride) stride += 10 bias_choice = [ np.random.uniform(low=-1, high=1., size=(filters, )), None ] weights = choice(weights_choice) bias = choice(bias_choice) for activ in numpynet_activ: layer = Convolutional_layer(filters=filters, size=size, stride=stride, pad=pad, weights=weights, bias=bias, input_shape=input_shape, activations=activ) if weights is not None: assert np.allclose(layer.weights, weights) else: assert (layer.weights is not None) if bias is None: assert np.allclose(layer.bias, np.zeros(shape=(filters, ))) else: assert np.allclose(layer.bias, bias) assert layer.delta == None assert layer.output == None assert layer.weights_update == None assert layer.bias_update == None assert layer.optimizer == None assert layer.pad == pad
def test_backward(self, b, w, h, c, filters, size1, size2, stride1, stride2, pad): if pad: keras_pad = 'same' else: keras_pad = 'valid' keras_activs = ['relu', 'sigmoid', 'tanh', 'linear'] numpynet_activs = [Relu, Logistic, Tanh, Linear] size = (size1, size2) stride = (stride1, stride2) inpt = np.random.uniform(low=-1., high=1., size=(b, w, h, c)) tf_input = tf.Variable(inpt.astype('float32')) weights = np.random.uniform(low=-1., high=1., size=(size1, size2) + (c, filters)) bias = np.random.uniform(low=-1., high=1., size=(filters, )) for numpynet_activ, keras_activ in zip(numpynet_activs, keras_activs): layer = Convolutional_layer(filters=filters, input_shape=inpt.shape, weights=weights, bias=bias, activation=numpynet_activ, size=(size1, size2), stride=stride, pad=pad) # Tensorflow layer model = tf.keras.layers.Conv2D( filters=filters, kernel_initializer=lambda shape, dtype=None: weights, bias_initializer=lambda shape, dtype=None: bias, kernel_size=size, strides=stride, padding=keras_pad, activation=keras_activ, data_format='channels_last', use_bias=True, dilation_rate=1) # Try backward: with pytest.raises(NotFittedError): delta = np.empty(shape=inpt.shape) layer.backward(inpt, delta) # FORWARD with tf.GradientTape(persistent=True) as tape: preds = model(tf_input) grad1 = tape.gradient(preds, tf_input) grad2 = tape.gradient(preds, model.trainable_weights) forward_out_keras = preds.numpy() delta_keras = grad1.numpy() updates = grad2 layer.forward(inpt, copy=False) forward_out_numpynet = layer.output assert forward_out_keras.shape == forward_out_numpynet.shape np.testing.assert_allclose(forward_out_keras, forward_out_numpynet, atol=1e-4, rtol=1e-3) # BACKWARD weights_updates_keras = updates[0] bias_updates_keras = updates[1] delta_numpynet = np.zeros(shape=inpt.shape, dtype=float) layer.delta = np.ones(shape=layer.out_shape, dtype=float) layer.backward(delta_numpynet, copy=False) np.testing.assert_allclose(delta_numpynet, delta_keras, atol=1e-3, rtol=1e-2) np.testing.assert_allclose(layer.weights_update, weights_updates_keras, atol=1e-3, rtol=1e-3) np.testing.assert_allclose(layer.bias_update, bias_updates_keras, atol=1e-5, rtol=1e-3)
n_train = X_train.shape[0] n_test = X_test.shape[0] # transform y to array of dimension 10 and in 4 dimension y_train = to_categorical(y_train).reshape(n_train, 1, 1, -1) y_test = to_categorical(y_test).reshape(n_test, 1, 1, -1) # Create the modeland training model = Network(batch=batch, input_shape=X_train.shape[1:]) # model.add(Input_layer(input_shape=(batch, 32, 32, 3))) # not necessary if input_shape is given to Network model.add( Convolutional_layer(input_shape=(batch, 8, 8, 3), size=3, filters=32, stride=1, pad=True, activation='Relu')) model.add( Convolutional_layer(input_shape=(batch, 8, 8, 32), size=3, filters=64, stride=1, pad=True, activation='Relu')) model.add(Maxpool_layer(size=2, stride=1, padding=True)) model.add(Dropout_layer(prob=0.25)) model.add( Connected_layer(input_shape=(batch, 8, 8, 64), outputs=128, activation='Relu'))
def test_convolutional_layer(): ''' Tests: if the convolutional layer forward is consistent with keras if the convolutional layer backward is consistent with keras to be: update ''' from NumPyNet.activations import Relu, Logistic, Linear, Tanh from NumPyNet.layers.convolutional_layer import Convolutional_layer from keras.layers import Conv2D keras_activations = ['relu', 'sigmoid', 'tanh','linear'] numpynet_activations = [Relu, Logistic, Tanh, Linear] channels_in = [1,3,5] channels_out= [5,10,20] sizes = [(3,3),(20,20),(30,30)] strides = [(2,2), (10,10),(20,20)] padding = [False, True] for c_in in channels_in: for c_out in channels_out: for keras_activ, numpynet_activ in zip(keras_activations,numpynet_activations): for size,stride in zip(sizes,strides): for pad in padding: batch = np.random.randint(low=1, high=10) batch = 1 if pad: keras_pad = 'same' else : keras_pad = 'valid' inpt = np.random.uniform(-1., 1., size = (batch, 100, 100, c_in)) b, w, h, c = inpt.shape # Shape (size1,size2,c_in, c_out), reshape inside numpynet.forward. filters = np.random.uniform(-1., 1., size = size + (c,c_out)) bias = np.random.uniform(-1., 1., size = (c_out,)) # Numpy_net model numpynet = Convolutional_layer(channels_out=c_out , inputs=inpt.shape, weights=filters, bias=bias, activation = numpynet_activ, size = size, stride = stride, padding = pad) # Keras model inp = Input(shape = inpt.shape[1:], batch_shape = inpt.shape) Conv2d = Conv2D(filters=c_out, kernel_size=size, strides=stride, padding=keras_pad, activation=keras_activ, data_format='channels_last', use_bias=True , bias_initializer='zeros', dilation_rate=1)(inp) # dilation rate = 1 is no dilation (I think) model = Model(inputs=[inp], outputs=[Conv2d]) model.set_weights([filters, bias]) # FORWARD print(c_in, c_out, keras_activ, size, stride, pad, keras_pad, '\n', sep = '\n') global forward_out_keras, forward_out_numpynet forward_out_keras = model.predict(inpt) numpynet.forward(inpt) forward_out_numpynet = numpynet.output assert forward_out_keras.shape == forward_out_numpynet.shape assert np.allclose(forward_out_keras, forward_out_numpynet, atol = 1e-04, rtol = 1e-3) # BACKWARD grad1 = K.gradients(model.output, [model.input]) grad2 = K.gradients(model.output, model.trainable_weights) func1 = K.function(model.inputs + model.outputs, grad1 ) func2 = K.function(model.inputs + model.trainable_weights + model.outputs, grad2) delta_keras = func1([inpt])[0] updates = func2([inpt]) # it does something at least delta_numpynet = np.zeros(shape = inpt.shape) numpynet.backward(delta_numpynet, inpt) assert np.allclose(delta_numpynet, delta_keras) assert np.allclose(numpynet.weights_updates, updates[0], atol = 1e-4, rtol = 1e-3) # for a lot of operations, atol is lower assert np.allclose(numpynet.bias_updates, updates[1])