def conv(self, inputs, filters, kernel_size, strides=1): inputs = batch_normalization()(inputs) inputs = relu()(inputs) inputs = conv2d(filters=filters, kernel_size=kernel_size, strides=strides)(inputs) return inputs
def separable_conv_relu_block(inputs, filters, kernel_size=(3, 3), strides=1): inputs = separable_conv_block(inputs, filters=filters, kernel_size=kernel_size, strides=strides) inputs = relu()(inputs) return inputs
def activate(self, param): inp = self.result with tf.name_scope('activation_' + str(self.layernum)): if param == 0: res = L.relu(inp, name='relu_' + str(self.layernum)) elif param == 1: res = L.lrelu(inp, name='lrelu_' + str(self.layernum)) elif param == 2: res = L.elu(inp, name='elu_' + str(self.layernum)) elif param == 3: res = L.tanh(inp, name='tanh_' + str(self.layernum)) elif param == 4: self.inpsize[-1] = self.inpsize[-1] // 2 res = L.MFM(inp, self.inpsize[-1], name='mfm_' + str(self.layernum)) elif param == 5: self.inpsize[-1] = self.inpsize[-1] // 2 res = L.MFMfc(inp, self.inpsize[-1], name='mfm_' + str(self.layernum)) elif param == 6: res = L.sigmoid(inp, name='sigmoid_' + str(self.layernum)) else: res = inp self.result = res return self.result
def __init__(self): self.lr = 0.01 # conv net self.c1 = conv(1, 6, kernel=5, learning_rate=self.lr) self.relu1 = relu() self.s2 = max_pool(kernel=2, stride=2) self.c3 = conv(6, 16, kernel=5, learning_rate=self.lr) self.relu3 = relu() self.s4 = max_pool(kernel=2, stride=2) self.c5 = conv(16, 120, kernel=4, learning_rate=self.lr) self.relu5 = relu() # fc net self.f6 = fc(120, 84, learning_rate=self.lr) self.relu6 = relu() self.f7 = fc(84, 10) self.sig7 = softmax() # record the shape between the conv net and fc net self.conv_out_shape = None
def classifier_aux(self, inputs, classes): filters = 128 outputs = average_pooling2d(pool_size=(5, 5), strides=3, padding='valid')(inputs) outputs = conv2d(filters=filters, kernel_size=(1, 1), strides=1, padding='same')(outputs) outputs = flatten()(outputs) outputs = relu()(outputs) outputs = dense(1024)(outputs) outputs = relu()(outputs) outputs = dropout(0.7)(outputs) outputs = dense(classes)(outputs) outputs = softmax()(outputs) return outputs
def deploy(self,input_layer): with tf.variable_scope('RPN_',reuse=self.reuse): shared_feature = L.conv2D(input_layer,3,512,stride=self.anchor_stride,name='share_conv') shared_feature = L.relu(shared_feature,'share_relu') rpn_bf_logits = L.conv2D(shared_feature,1,2*self.anchors_per_loc,'bf') rpn_bf_logits = tf.reshape(rpn_bf_logits,[tf.shape(rpn_bf_logits)[0],-1,2]) rpn_bf_prob = tf.nn.softmax(rpn_bf_logits) rpn_bbox = L.conv2D(shared_feature,1,4*self.anchors_per_loc,'bbox') rpn_bbox = tf.reshape(rpn_bbox,[tf.shape(rpn_bbox[0],-1,2)]) self.reuse = True return rpn_bf_logits, rpn_bf_prob, rpn_bbox
def __init__(self, learning_rate, input_shape, BS): #input_shape example: [BS,1,28,28] self.lr = learning_rate self.conv2d_1 = ly.conv2d(input_shape, [5, 5, 1, 6], [1, 1], 'VALID') self.relu_1 = ly.relu() self.conv2d_2 = ly.conv2d([BS, 6, 12, 12], [3, 3, 6, 10], [2, 2], 'VALID') self.relu_2 = ly.relu() self.flatter = ly.flatter() self.full_connect_1 = ly.full_connect(250, 84) self.relu_3 = ly.relu() self.dropout = ly.dropout(lenth=84) self.full_connect_2 = ly.full_connect(84, 10) self.loss_func = ly.softmax_cross_entropy_error()
def __init__(self, input_shape, learning_rate=0.001): BS = input_shape[0] self.lr = learning_rate #conv1 : 1*28*28->6*12*12 self.conv1 = ly.conv2d(input_shape, [5, 5, 1, 6], [1, 1], 'same') self.conv1_relu = ly.relu() self.pool1 = ly.max_pooling(self.conv1.out_shape, [3, 3], [2, 2], 'valid') # conv2 : 6*12*12 - > 10*5*5 self.conv2 = ly.conv2d(self.pool1.out_shape, [3, 3, 6, 10], [1, 1], 'same') self.conv2_relu = ly.relu() self.pool2 = ly.max_pooling(self.conv2.out_shape, [3, 3], [2, 2], 'valid') self.conv_fc = ly.conv_fc() self.fc1 = ly.full_connect(360, 84) self.fc1_relu = ly.relu() self.fc2 = ly.full_connect(84, 10) self.loss = ly.softmax_cross_with_entropy()
def transition_block(self, inputs): filters = keras.backend.int_shape(inputs)[ self.batch_normalization_axis] if self.compression: filters = int(filters * (1 - self.reduction_rate)) outputs = batch_normalization()(inputs) outputs = relu()(outputs) outputs = conv2d(filters=filters, kernel_size=(1, 1))(outputs) outputs = average_pooling2d(pool_size=(2, 2), strides=2)(outputs) return outputs
def discriminator(self, name, input, is_training=True): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): input = tf.pad(input, [[0, 0], [1, 1], [1, 1], [0, 0]]) input = ly.conv2d(input, 64, strides=2, kernal_size=4, padding='VALID', name='d_conv2d_0') input = ly.relu(input, alpha=0.2) input = tf.pad(input, [[0, 0], [1, 1], [1, 1], [0, 0]]) input = ly.conv2d(input, 128, strides=2, name='d_conv2d_1') input = ly.batch_normal(input, name='d_bn1', is_training=is_training) input = ly.relu(input, alpha=0.2) ### 31 | 15 input = tf.pad(input, [[0, 0], [1, 1], [1, 1], [0, 0]]) input = ly.conv2d(input, 256, strides=1, name='d_conv2d_2', padding='VALID') input = ly.batch_normal(input, name='d_bn2', is_training=is_training) input = ly.relu(input, alpha=0.2) ### 30 | 14 input = tf.pad(input, [[0, 0], [1, 1], [1, 1], [0, 0]]) input = ly.conv2d(input, 1, strides=1, name='d_conv2d_3', padding='VALID') return tf.nn.sigmoid(input)
def test_relu_forward(this): # Test the relu forward method. x = np.random.randint(low=-10, high=10, size=(30, 1)) layer = layers.relu() y = layer.forward(x) y_exp = x y_exp[x < 0] = 0 nptest.assert_array_equal( y, y_exp, err_msg="relu layer forward incorrect for \n x = \n {} \n".format( x))
def net(self, X, reuse=None): with tf.variable_scope('EyeNet', reuse=reuse): conv1 = conv2d(X,output_dims=20,k_h=5,k_w=5,s_h=1,s_w=1,padding='VALID',name='conv1') pool1 = max_pool(conv1,k_h=2,k_w=2,s_h=2,s_w=2,padding='SAME',name='pool1') conv2 = conv2d(pool1,output_dims=50,k_h=5,k_w=5,s_h=1,s_w=1,padding='VALID',name='conv2') pool2 = max_pool(conv2,k_h=2,k_w=2,s_h=2,s_w=2,padding='SAME',name='pool2') flatten = tf.reshape(pool2,[-1, pool2.get_shape().as_list()[1] *pool2.get_shape().as_list()[2] *pool2.get_shape().as_list()[3]], name='conv_reshape') fc1 = fc(flatten, output_dims=500, name='fc1') relu1 = relu(fc1, name='relu1') out = fc(relu1, output_dims=2, name='output') return out
def test_relu_x_gradient(this): # Test the x gradient of relu. x = np.random.randint(low=-10, high=10, size=(30, 1)) layer = layers.relu() dydx = layer.x_gradient(x) idx = (layer.forward(x) != x) dydx_exp = np.eye(30) dydx_exp[idx[:, 0], idx[:, 0]] = 0 nptest.assert_array_equal( dydx, dydx_exp, err_msg="relu layer x gradient incorrect for \n x = \n {} \n". format(x))
def net(name,image,output0,trainable = True,reuse = None): with tf.variable_scope(name,reuse=reuse): params = [] #conv bias 42->40 conv1,k1 = layers.conv(name + "conv1",image,3,3,1,1,"VALID",1,16,trainable) bias1,b1 = layers.bias(name + "bias1",conv1,16,trainable) relu1 = layers.relu(name + "relu1",bias1) params += [k1,b1] #pool 40->20 pool1 = layers.pooling(name + "pool1",relu1,2,2,2,2) #conv bias 20->20 conv2,k2 = layers.conv(name + "conv2",pool1,3,3,1,1, "SAME",16,32,trainable) bias2,b2 = layers.bias(name + "bias2",conv2,32,trainable) relu2 = layers.relu(name + "relu2",bias2) params += [k2,b2] #conv bias 20->20 conv2_,k2_ = layers.conv(name + "conv2_",relu2,3,3,1,1, "SAME",32,32,trainable) bias2_,b2_ = layers.bias(name + "bias2_",conv2_,32,trainable) relu2_ = layers.relu(name + "relu2_",bias2_) params += [k2_,b2_] #pool 20->10 pool2 = layers.pooling(name + "pool2",relu2_,2,2,2,2) #conv bias 10->10 conv3,k3 = layers.conv(name + "conv3",pool2,3,3,1,1,"SAME",32,64,trainable) bias3,b3 = layers.bias(name + "bias3",conv3,64,trainable) relu3 = layers.relu(name + "relu3",bias3) params += [k3,b3] #conv bias 10->10 conv3_,k3_ = layers.conv(name + "conv3_",relu3,3,3,1,1,"SAME",64,64,trainable) bias3_,b3_ = layers.bias(name + "bias3_",conv3_,64,trainable) relu3_ = layers.relu(name + "relu3_",bias3_) params += [k3_,b3_] #pool 10->5 pool3 = layers.pooling(name + "pool3",relu3_,2,2,2,2) #conv4 5->3 conv4,k4 = layers.conv(name + "conv4",pool3,3,3,1,1,"VALID",64,128,trainable) bias4,b4 = layers.bias(name + "bias4",conv4,128,trainable) relu4 = layers.relu(name + "relu4",bias4) params += [k4,b4] #conv5 3->1 conv5,k5 = layers.conv(name + "conv5",relu4,3,3,1,1,"VALID",128,128,trainable) bias5,b5 = layers.bias(name + "bias5",conv5,128,trainable) relu5 = layers.relu(name + "relu5",bias5) params += [k5,b5] #fcn feature0,dim0 = layers.reshapeToLine(relu5) fcn1,k6 = layers.fcn(name + "fcn1",feature0,dim0,output0,trainable) fcn1_bias,b6 = layers.bias(name + "fcn1_bias",fcn1,output0,trainable) params += [k6,b6] return fcn1_bias,params
def forward(self, x, weights=None, prefix=''): ''' Runs the net forward; if weights are None it uses 'self' layers, otherwise keeps the structure and uses 'weights' instead. ''' if weights is None: x = self.features(x) x = self.fa(x) else: for i in range(self.num_layers): x = linear(x, weights[prefix + 'fc' + str(i) + '.weight'], weights[prefix + 'fc' + str(i) + '.bias']) if i < self.num_layers - 1: x = relu(x) x = self.fa(x) return x
def __init__( self, blocks, filters, bottleneck=False, input_layers=[ batch_normalization(), relu(), conv2d(filters=64, kernel_size=(7, 7), strides=2), ], output_layers=[average_pooling2d(pool_size=(2, 2)), flatten()]): self.blocks = blocks self.filters = filters self.bottleneck = bottleneck self.bn_axis = -1 if keras.backend.image_data_format( ) == 'channels_last' else 1 self.input_layers = input_layers self.output_layer = output_layers
def forward_propagate(x, y, _weights, debug=True): activation_caches = {} m = x.shape[0] activation_caches["conv1"] = conv_fast(x, _weights["W1"], _weights["B1"], 2, 1) activation_caches["A1"] = relu(activation_caches["conv1"]) activation_caches["pool1"] = max_pooling(activation_caches["A1"], 2, 2) # Sanity check to make sure that our convolution vectorization is correct if debug: # Conv kconv, kcache = karpathy_conv_forward_naive(x, _weights["W1"], _weights["B1"], { 'stride': 1, 'pad': 2 }) assert np.mean(np.isclose(activation_caches["conv1"], kconv)) == 1.0 conv1_verify = conv_forward_naive(x, _weights["W1"], _weights["B1"], 2, 1) assert np.mean(np.isclose(activation_caches["conv1"], conv1_verify)) == 1.0 kpool1, kcache1 = karpathy_max_pool_forward_naive( activation_caches["A1"], { 'pool_height': 2, 'pool_width': 2, 'stride': 2 }) assert np.mean(np.isclose(activation_caches["pool1"], kpool1)) == 1.0 activation_caches["conv2"] = conv_fast(activation_caches["pool1"], _weights["W2"], _weights["B2"], 2, 1) activation_caches["A2"] = relu(activation_caches["conv2"]) activation_caches["pool2"] = max_pooling(activation_caches["A2"], 2, 2) activation_caches["Ar2"] = activation_caches["pool2"].reshape( (m, activation_caches["pool2"].shape[1] * activation_caches["pool2"].shape[2] * activation_caches["pool2"].shape[3])) if debug: conv2_verify = conv_forward_naive(activation_caches["pool1"], _weights["W2"], _weights["B2"], 2, 1) assert np.mean(np.isclose(activation_caches["conv2"], conv2_verify)) == 1.0 activation_caches["Z3"] = fully_connected(activation_caches["Ar2"], _weights["W3"], _weights["B3"]) activation_caches["A3"] = relu(activation_caches["Z3"]) activation_caches["Z4"] = fully_connected(activation_caches["A3"], _weights["W4"], _weights["B4"]) activation_caches["A4"] = softmax(activation_caches["Z4"]) cost = np.mean(softmax_cost(y, activation_caches["A4"], m)) return activation_caches, cost
def build(self, input_shape, classes): inputs = keras.Input(shape=input_shape) # Entry flow filters = 32 outputs = conv_relu_block(inputs, filters=filters, strides=(2, 2)) filters = 64 outputs = conv_relu_block(inputs, filters=filters) residual = outputs filters = 128 outputs = separable_conv_block(outputs, filters=filters) outputs = relu_separable_conv_block(outputs, filters=filters) outputs = max_pooling2d(pool_size=(3, 3), strides=(2, 2))(outputs) residual = conv_block(residual, filters=filters, kernel_size=(1, 1), strides=(2, 2)) outputs = add()([outputs, residual]) filters = 256 outputs = relu_separable_conv_block(outputs, filters=filters) outputs = relu_separable_conv_block(outputs, filters=filters) outputs = max_pooling2d(pool_size=(3, 3), strides=(2, 2))(outputs) residual = conv_block(residual, filters=filters, kernel_size=(1, 1), strides=(2, 2)) outputs = add()([outputs, residual]) filters = 728 outputs = relu_separable_conv_block(outputs, filters=filters) outputs = relu_separable_conv_block(outputs, filters=filters) outputs = max_pooling2d(pool_size=(3, 3), strides=(2, 2))(outputs) residual = conv_block(residual, filters=filters, kernel_size=(1, 1), strides=(2, 2)) outputs = add()([outputs, residual]) # Middle flow filters = 728 for _ in range(8): residual = outputs outputs = relu_separable_conv_block(outputs, filters=filters) outputs = relu_separable_conv_block(outputs, filters=filters) outputs = relu_separable_conv_block(outputs, filters=filters) outputs = add()([outputs, residual]) # Exit flow residual = outputs outputs = relu_separable_conv_block(outputs, filters=filters) filters = 1024 outputs = relu_separable_conv_block(outputs, filters=filters) outputs = add()([outputs, residual]) filters = 1536 outputs = separable_conv_relu_block(outputs, filters=filters) filters = 2048 outputs = separable_conv_relu_block(outputs, filters=filters) outputs = global_average_pooling2d()(outputs) outputs = dense(filters)(outputs) outputs = relu()(outputs) outputs = dense(1000)(outputs) outputs = softmax()(outputs) model = keras.Model(inputs, outputs) model.summary() return model
# Number of classes K = 10 # Dimensions of input num_examples, Xh, Xw = X_train.shape Xd = 1 ########################### # Multilayer Perceptron classifier ########################### # Initialise parameters cnn1 = cnn2d(input_shape=(Xh,Xw,Xd), filter_shape=(f1_field, f1_field), num_filters=h1_units) def flatten(x, n_examples): return np.reshape(x, (n_examples,-1)) relu1 = relu() fc2 = fc2d(cnn1.yw*cnn1.yh*Xd*h1_units, K) data_loss_fn = softmax_loss(y_train) for i in range(n_epochs): # Forward pass conv1 = cnn1.forward(X_train) print("conv1:", conv1.shape) flatten1 = flatten(conv1, num_examples) h1 = relu1.forward(flatten1) scores = fc2.forward(h1) data_loss = data_loss_fn.forward(scores) reg_loss = 0.5*reg*(np.sum(cnn1.W * cnn1.W) + np.sum(fc2.W*fc2.W)) loss = data_loss + reg_loss if i % 1 == 0: print("Epoch: %d, Loss: %f" % (i, loss))
########################### # Multilayer Perceptron classifier ########################### # Initialise parameters cnn1 = cnn2d(input_shape=(Xh, Xw, Xd), filter_shape=(f1_field, f1_field), num_filters=h1_units) def flatten(x, n_examples): return np.reshape(x, (n_examples, -1)) relu1 = relu() fc2 = fc2d(cnn1.yw * cnn1.yh * Xd * h1_units, K) data_loss_fn = softmax_loss(y_train) for i in range(n_epochs): # Forward pass conv1 = cnn1.forward(X_train) print("conv1:", conv1.shape) flatten1 = flatten(conv1, num_examples) h1 = relu1.forward(flatten1) scores = fc2.forward(h1) data_loss = data_loss_fn.forward(scores) reg_loss = 0.5 * reg * (np.sum(cnn1.W * cnn1.W) + np.sum(fc2.W * fc2.W)) loss = data_loss + reg_loss if i % 1 == 0: print("Epoch: %d, Loss: %f" % (i, loss))
def conv_block(self, inputs, filters, blocks): for _ in range(blocks): inputs = conv2d(filters=filters, kernel_size=(3, 3))(inputs) inputs = relu()(inputs) return inputs
def test_get_training_parameters(this): # Test the get_training_parameters method. layer = layers.relu() params = layer.get_training_parameters() this.assertDictEqual(params, {})
def generator(self, name, input, is_training=True): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): ### 3 -> 64 64 e0 = ly.conv2d(input, 64, strides=2, name='g_conv2d_0') e0 = ly.batch_normal(e0, name='g_bn_0', is_training=is_training) e0 = ly.relu(e0, alpha=0.2) ### 64 -> 128 32 e1 = ly.conv2d(e0, 128, strides=2, name='g_conv2d_1') e1 = ly.batch_normal(e1, name='g_bn_1', is_training=is_training) e1 = ly.relu(e1, alpha=0.2) ### 128 -> 256 16 e2 = ly.conv2d(e1, 256, strides=2, name='g_conv2d_2') e2 = ly.batch_normal(e2, name='g_bn_2', is_training=is_training) e2 = ly.relu(e2, alpha=0.2) ### 256 -> 512 8 e3 = ly.conv2d(e2, 512, strides=2, name='g_conv2d_3') e3 = ly.batch_normal(e3, name='g_bn_3', is_training=is_training) e3 = ly.relu(e3, alpha=0.2) ### 512 -> 512 4 e4 = ly.conv2d(e3, 512, strides=2, name='g_conv2d_4') e4 = ly.batch_normal(e4, name='g_bn_4', is_training=is_training) e4 = ly.relu(e4, alpha=0.2) ### 512 -> 512 2 e5 = ly.conv2d(e4, 512, strides=2, name='g_conv2d_5') e5 = ly.batch_normal(e5, name='g_bn_5', is_training=is_training) e5 = ly.relu(e5, alpha=0.2) ### 512 -> 512 4 d1 = ly.deconv2d(e5, 512, strides=2, name='g_deconv2d_1') d1 = ly.batch_normal(d1, name='g_bn_6', is_training=is_training) d1 = tf.nn.dropout(d1, keep_prob=0.5) d1 = tf.concat([d1, e4], axis=3) d1 = ly.relu(d1, alpha=0.2) ### 512 -> 512 8 d2 = ly.deconv2d(d1, 512, strides=2, name='g_deconv2d_2') d2 = ly.batch_normal(d2, name='g_bn_7', is_training=is_training) d2 = tf.nn.dropout(d2, keep_prob=0.5) d2 = ly.relu(d2, alpha=0.2) d2 = tf.concat([d2, e3], axis=3) ### 512 -> 256 16 d3 = ly.deconv2d(d2, 256, strides=2, name='g_deconv2d_3') d3 = ly.batch_normal(d3, name='g_bn_8', is_training=is_training) d3 = ly.relu(d3, alpha=0.2) d3 = tf.concat([d3, e2], axis=3) ### 256 -> 128 32 d4 = ly.deconv2d(d3, 128, strides=2, name='g_deconv2d_4') d4 = ly.batch_normal(d4, name='g_bn_9', is_training=is_training) d4 = ly.relu(d4, alpha=0.2) d4 = tf.concat([d4, e1], axis=3) ### 128 -> 64 64 d5 = ly.deconv2d(d4, 64, strides=2, name='g_deconv2d_5') d5 = ly.batch_normal(d5, name='g_bn_10', is_training=is_training) d5 = ly.relu(d5, alpha=0.2) d5 = tf.concat([d5, e0], axis=3) ### 64 -> 3 128 d6 = ly.deconv2d(d5, 3, strides=2, name='g_deconv2d_6') d6 = ly.batch_normal(d6, name='g_bn_11', is_training=is_training) d6 = ly.relu(d6, alpha=0.2) return tf.nn.tanh(d6)
import unittest
def test_constructor(this): # Test the constructor layer = layers.relu() this.assertIsInstance(layer, layers.relu)