def __init__(self, lossfunc, optimizer, batch_size): super().__init__(lossfunc, optimizer, batch_size) self.conv0 = L.Convolution_(n_filter=8, filter_size=(3, 3), stride=1) self.conv1 = L.Convolution_(n_filter=16, filter_size=(3, 3), stride=1) self.fc0 = L.Linear_(output_size=1024) self.fc1 = L.Linear_(output_size=10) self.bn0 = L.BatchNormalization_() self.bn1 = L.BatchNormalization_() self.bn4 = L.BatchNormalization_() self.acti0 = L.ELU() self.acti1 = L.ELU() self.acti4 = L.ELU() self.pool0 = L.MaxPooling(7, 7) self.pool1 = L.MaxPooling(5, 5) self.flat = L.Flatten() self.drop0 = L.Dropout(0.5) self.drop1 = L.Dropout(0.5) self.layers = [ self.conv0, self.acti0, self.pool0, self.bn0, #self.drop0, self.conv1, self.acti1, self.pool1, self.bn1, #self.drop1, self.flat, self.fc0, self.acti4, self.bn4, self.fc1, ]
test_x = ds_test.data.reshape([-1, 1, 28, 28]).numpy().astype(np.float) / 255 test_y = ds_test.targets.numpy() train_mean = train_x.mean() train_x, valid_x, test_x = (x - train_mean for x in (train_x, valid_x, test_x)) train_y, valid_y, test_y = (dense_to_one_hot(y, 10) for y in (train_y, valid_y, test_y)) weight_decay = config['weight_decay'] net = [] regularizers = [] inputs = np.random.randn(config['batch_size'], 1, 28, 28) net += [layers.Convolution(inputs, 16, 5, "conv1")] regularizers += [ layers.L2Regularizer(net[-1].weights, weight_decay, 'conv1_l2reg') ] net += [layers.MaxPooling(net[-1], "pool1")] net += [layers.ReLU(net[-1], "relu1")] net += [layers.Convolution(net[-1], 32, 5, "conv2")] regularizers += [ layers.L2Regularizer(net[-1].weights, weight_decay, 'conv2_l2reg') ] net += [layers.MaxPooling(net[-1], "pool2")] net += [layers.ReLU(net[-1], "relu2")] ## 7x7 net += [layers.Flatten(net[-1], "flatten3")] net += [layers.FC(net[-1], 512, "fc3")] regularizers += [ layers.L2Regularizer(net[-1].weights, weight_decay, 'fc3_l2reg') ] net += [layers.ReLU(net[-1], "relu3")] net += [layers.FC(net[-1], 10, "logits")]
print("Error norm = ", np.linalg.norm(grad_b_num - grad_b)) print("Convolution") x = np.random.randn(4, 3, 5, 5) grad_out = np.random.randn(4, 2, 5, 5) conv = layers.Convolution(x, 2, 3, "conv1") print("Check grad wrt input") check_grad_inputs(conv, x, grad_out) print("Check grad wrt params") check_grad_params(conv, x, conv.weights, conv.bias, grad_out) print("\nMaxPooling") x = np.random.randn(5, 4, 8, 8) grad_out = np.random.randn(5, 4, 4, 4) pool = layers.MaxPooling(x, "pool", 2, 2) print("Check grad wrt input") check_grad_inputs(pool, x, grad_out) print("\nReLU") x = np.random.randn(4, 3, 5, 5) grad_out = np.random.randn(4, 3, 5, 5) relu = layers.ReLU(x, "relu") print("Check grad wrt input") check_grad_inputs(relu, x, grad_out) print("\nFC") x = np.random.randn(20, 40) grad_out = np.random.randn(20, 30) fc = layers.FC(x, 30, "fc") print("Check grad wrt input")
loss_function="cross-entropy", lr=1e-4, reg=1e-5) graph.add_layer( layers.Convolution("conv1", kernel_size=11, kernel_channels=3, n_kernels=96, stride=4, same_padding=False, logging=False)) graph.add_layer(layers.Activation(activation="relu", logging=False)) graph.add_layer( layers.MaxPooling(window_size=3, window_channels=96, stride=2, logging=False)) graph.add_layer( layers.Convolution("conv2", kernel_size=5, kernel_channels=96, n_kernels=256, stride=1, same_padding=True, logging=False)) graph.add_layer(layers.Activation(activation="relu", logging=False)) graph.add_layer( layers.MaxPooling(window_size=3, window_channels=256, stride=2,
valid_y = dataset.validation.labels test_x = dataset.test.images test_x = test_x.reshape([-1, 1, 28, 28]) test_y = dataset.test.labels train_mean = train_x.mean() train_x -= train_mean valid_x -= train_mean test_x -= train_mean net = [] inputs = np.random.randn( config['batch_size'], 1, 28, 28 ) #cemu normalna razdioba, neki placeholder za stvarne podatke? samo zbog shapea? net += [layers.Convolution(inputs, 16, 5, "conv1")] #num filters = 16, kernel size= 5 net += [layers.MaxPooling(net[-1], "pool1")] #ulazni parametar je ulaz u layer net += [layers.ReLU(net[-1], "relu1")] net += [layers.Convolution(net[-1], 32, 5, "conv2")] net += [layers.MaxPooling(net[-1], "pool2")] net += [layers.ReLU(net[-1], "relu2")] # out = 7x7 net += [layers.Flatten(net[-1], "flatten3")] net += [layers.FC(net[-1], 512, "fc3")] net += [layers.ReLU(net[-1], "relu3")] net += [layers.FC(net[-1], 10, "logits")] loss = layers.SoftmaxCrossEntropyWithLogits() nn.train(train_x, train_y, valid_x, valid_y, net, loss, config) nn.evaluate("Test", test_x, test_y, net, loss, config)
kernel_channels=3, n_kernels=10, stride=1, same_padding=True) X = conv.forward(X) print("[+] CONVOLVED_shape:", X.shape) print() activation = layers.Activation(activation="relu") X = activation.forward(X) print("[+] ACTIVATION_shape:", X.shape) dActivation = activation.backward(np.ones_like(X)) print("dActivation_SHAPE:", dActivation.shape) print() maxpool = layers.MaxPooling(window_size=2, window_channels=10, stride=2) X = maxpool.forward(X) print("[+] MAXPOOLED_shape:", X.shape) dMaxpool = maxpool.backward(np.ones_like(X)) print("dMaxpool_SHAPE:", dMaxpool.shape) N, C, H, W = X.shape flatten = layers.Flatten((C * H * W, N)) X = flatten.forward(X) print(X.shape) I, N = X.shape fc1 = layers.FullyConnected("fc1", I, 1000, activation="relu") X = fc1.forward(X)
def __init__(self, rng, batch_size, input): conv1_1 = layers.ConvLayer(rng, input=input, image_shape=(batch_size, 3, 480, 320), filter_shape=(32, 3, 3, 3), bias=True, padding='valid') conv1_2 = layers.ConvLayer(rng, input=K.spatial_2d_padding(conv1_1.output), image_shape=(batch_size, 32, 480, 320), filter_shape=(64, 32, 3, 3), padding='valid') maxp1 = layers.MaxPooling(input=K.spatial_2d_padding(conv1_2.output), poolsize=(2, 2)) conv2_1 = layers.ConvLayer(rng, input=maxp1.output, image_shape=(batch_size, 64, 240, 160), filter_shape=(64, 64, 3, 3), padding='valid') conv2_2 = layers.ConvLayer(rng, input=K.spatial_2d_padding(conv2_1.output), image_shape=(batch_size, 64, 240, 160), filter_shape=(64, 64, 3, 3), padding='valid') # maxp2 = layers.MaxPooling( # input=K.spatial_2d_padding(conv2_2.output), # poolsize=(2, 2) # ) # # conv3_1 = layers.ConvLayer( # rng, # input=maxp2.output, # image_shape=(batch_size, 10, 120, 80), # filter_shape=(10, 10, 3, 3), # padding='valid' # ) # # conv3_2 = layers.ConvLayer( # rng, # input=K.spatial_2d_padding(conv3_1.output), # image_shape=(batch_size, 10, 120, 80), # filter_shape=(10, 10, 3, 3), # padding='valid' # ) # # maxp3 = layers.MaxPooling( # input=conv3_2.output, # poolsize=(2, 2) # ) # # conv4_1 = layers.ConvLayer( # rng, # input=maxp3.output, # image_shape=(batch_size, 10, 60, 40), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # conv4_2 = layers.ConvLayer( # rng, # input=conv4_1.output, # image_shape=(batch_size, 10, 60, 40), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # maxp4 = layers.MaxPooling( # input=conv4_2.output, # poolsize=(2, 2) # ) # # conv5_1 = layers.ConvLayer( # rng, # input=maxp4.output, # image_shape=(batch_size, 10, 30, 20), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # conv5_2 = layers.ConvLayer( # rng, # input=conv5_1.output, # image_shape=(batch_size, 10, 30, 20), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # maxp5 = layers.MaxPooling( # input=conv5_2.output, # poolsize=(2, 2) # ) # # flat1_input = maxp5.output.flatten(2) # flat1_shape = 10*15*10 # # flat1 = layers.HiddenLayer( # rng=rng, # input=flat1_input, # n_in=flat1_shape, # n_out=flat1_shape, # activation=T.nnet.relu # ) # # flat2 = layers.HiddenLayer( # rng=rng, # input=flat1.output, # n_in=flat1_shape, # n_out=flat1_shape, # activation=T.nnet.relu # ) # # remax5_input = flat2.output.reshape((batch_size,10,15,10)) # # remax5 = layers.ReverseMaxPooling( # input=remax5_input, # mask=maxp5.mask, # poolsize=(2, 2) # ) # # deconv5_1 = layers.DeConvLayer( # rng, # #input=remax5.output, # input=conv5_2.output, # W=conv5_2.W, # image_shape=(batch_size, 10, 30, 20), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # deconv5_2 = layers.DeConvLayer( # rng, # input=deconv5_1.output, # W=conv5_1.W, # image_shape=(batch_size, 10, 30, 20), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # remax4 = layers.ReverseMaxPooling( # input=deconv5_2.output, # mask=maxp4.mask, # poolsize=(2, 2) # ) # # deconv4_1 = layers.DeConvLayer( # rng, # input=remax4.output, # W=conv4_2.W, # image_shape=(batch_size, 10, 60, 40), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # deconv4_2 = layers.DeConvLayer( # rng, # input=deconv4_1.output, # W=conv4_1.W, # image_shape=(batch_size, 10, 60, 40), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # remax3 = layers.ReverseMaxPooling( # input=deconv4_2.output, # mask=maxp3.mask, # poolsize=(2, 2) # ) # # deconv3_1 = layers.DeConvLayer( # rng, # #input=remax3.output, # input=conv3_2.output, # W=conv3_2.W, # image_shape=(batch_size, 10, 118, 78), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # deconv3_2 = layers.DeConvLayer( # rng, # input=K.spatial_2d_padding(deconv3_1.output), # W=conv3_1.W, # image_shape=(batch_size, 10, 120, 80), # filter_shape=(10, 10, 3, 3), # padding='same' # ) # # remax2 = layers.ReverseMaxPooling( # input=deconv3_2.output, # mask=maxp2.mask, # poolsize=(2, 2) # ) # deconv2_1 = layers.DeConvLayer( rng, #input=remax2.output, input=K.spatial_2d_padding(conv2_2.output), W=conv2_2.W, image_shape=(batch_size, 64, 240, 160), filter_shape=(64, 64, 3, 3), padding='same') deconv2_2 = layers.DeConvLayer(rng, input=deconv2_1.output, W=conv2_1.W, image_shape=(batch_size, 64, 240, 160), filter_shape=(64, 64, 3, 3), padding='same') remax1 = layers.ReverseMaxPooling(input=deconv2_2.output, mask=maxp1.mask, poolsize=(2, 2)) deconv1_1 = layers.DeConvLayer( rng, input=remax1.output, #input=K.spatial_2d_padding(conv1_2.output), W=conv1_2.W, image_shape=(batch_size, 64, 480, 320), filter_shape=(32, 64, 3, 3), padding='same', ) deconv1_2 = layers.ConvLayer(rng, input=deconv1_1.output, image_shape=(batch_size, 32, 480, 320), filter_shape=(1, 32, 3, 3), bias=True, padding='same', activation=T.nnet.sigmoid) self.y_pred = T.clip(deconv1_2.output, 0.001, 0.999) # self.params=conv1_1.params+conv1_2.params+conv2_1.params+conv2_2.params+\ # conv3_1.params+conv3_2.params+conv4_1.params+conv4_2.params+conv5_1.params+conv5_2.params+\ # deconv5_1.params+deconv5_2.params+deconv4_1.params+deconv4_2.params+deconv3_1.params+\ # deconv3_2.params+deconv2_1.params+deconv2_2.params+deconv1_1.params+deconv1_2.params self.params = conv1_1.params + conv1_2.params + conv2_1.params + conv2_2.params + deconv1_2.params self.input = input
def __init__( self, lossfunc, optimizer, batch_size=32, ): self.lossfunc = lossfunc self.optimizer = optimizer self.batch_size = batch_size input_size = 64 hidden_size = 3136 output_size = 10 # self.lr = 0.001 # self.alpha = 0.9 self.l1 = 1e-4 self.l2 = 1e-4 self.optimizer = optimizers.Adam(l1=self.l1, l2=self.l2) self.conv0 = L.Convolution_(n_filter=8, filter_size=(3, 3), stride=1) self.conv1 = L.Convolution_(n_filter=16, filter_size=(3, 3), stride=1) self.conv2 = L.Convolution_(n_filter=32, filter_size=(5, 5), stride=1) self.conv3 = L.Convolution_(n_filter=64, filter_size=(5, 5), stride=1) self.fc0 = L.Linear_(output_size=1024) self.fc1 = L.Linear_(output_size=10) self.bn0 = L.BatchNormalization_() self.bn1 = L.BatchNormalization_() self.bn2 = L.BatchNormalization_() self.bn3 = L.BatchNormalization_() self.bn4 = L.BatchNormalization_() self.acti0 = L.ELU() self.acti1 = L.ELU() self.acti2 = L.ELU() self.acti3 = L.ELU() self.acti4 = L.ELU() self.pool0 = L.MaxPooling(7, 7) self.pool1 = L.MaxPooling(5, 5) self.pool2 = L.MaxPooling(3, 3) self.pool3 = L.MaxPooling(3, 3) self.flat = L.Flatten() self.drop0 = L.Dropout(0.5) self.drop1 = L.Dropout(0.5) self.drop2 = L.Dropout(0.5) self.drop3 = L.Dropout(0.25) self.layers = [ self.conv0, self.acti0, self.pool0, self.bn0, #self.drop0, self.conv1, self.acti1, self.pool1, self.bn1, #self.drop1, #self.conv2, #self.acti2, #self.pool2, #self.bn2, #self.drop2, #self.conv3, #self.acti3, #self.pool3, #self.bn3, #self.drop3, self.flat, self.fc0, self.acti4, self.bn4, self.fc1, ]
def upperBlock(x, conv_size=[128, 256, 512, 256, 128], n_channels=3): x = L.GaussianNoise(x) x = L.Conv2D(x, filter_size=3, n_channels=n_channels, n_filters=conv_size[0], padding='SAME', name='1a') x = L.LeakyReLU(x) x = L.Conv2D(x, filter_size=3, n_channels=conv_size[0], n_filters=conv_size[0], name='1b') x = L.LeakyReLU(x) x = L.Conv2D(x, filter_size=3, n_channels=conv_size[0], n_filters=conv_size[0], name='1c') x = L.MaxPooling(x, ksize=2, stride_length=2) x = L.Dropout(x, probability=0.5) x = L.Conv2D(x, filter_size=3, n_channels=conv_size[0], n_filters=conv_size[1], name='2a') x = L.LeakyReLU(x) x = L.Conv2D(x, filter_size=3, n_channels=conv_size[1], n_filters=conv_size[1], name='2b') x = L.LeakyReLU(x) x = L.Conv2D(x, filter_size=3, n_channels=conv_size[1], n_filters=conv_size[1], name='2c') x = L.LeakyReLU(x) x = L.MaxPooling(x, ksize=2, stride_length=2) x = L.Conv2D(x, filter_size=3, n_channels=conv_size[1], n_filters=conv_size[2], padding='VALID', name='3a') x = L.LeakyReLU(x) x = L.Conv2D(x, filter_size=1, n_channels=conv_size[2], n_filters=conv_size[3], name='3b') x = L.LeakyReLU(x) x = L.Conv2D(x, filter_size=1, n_channels=conv_size[3], n_filters=conv_size[4], name='3c') x = L.LeakyReLU(x) x = L.GlobalAveragePooling(x) # x = L.Dropout(x, probability=0.5) # x = L.Dense(x, conv_size[4], 10) # x = L.SoftMax(x) return x