def __init__(self, lossfunc, optimizer, batch_size):
super().__init__(lossfunc, optimizer, batch_size)
self.conv0 = L.Convolution_(n_filter=8, filter_size=(3, 3), stride=1)
self.conv1 = L.Convolution_(n_filter=16, filter_size=(3, 3), stride=1)
self.fc0 = L.Linear_(output_size=1024)
self.fc1 = L.Linear_(output_size=10)
self.bn0 = L.BatchNormalization_()
self.bn1 = L.BatchNormalization_()
self.bn4 = L.BatchNormalization_()
self.acti0 = L.ELU()
self.acti1 = L.ELU()
self.acti4 = L.ELU()
self.pool0 = L.MaxPooling(7, 7)
self.pool1 = L.MaxPooling(5, 5)
self.flat = L.Flatten()
self.drop0 = L.Dropout(0.5)
self.drop1 = L.Dropout(0.5)
self.layers = [
self.conv0,
self.acti0,
self.pool0,
self.bn0,
#self.drop0,
self.conv1,
self.acti1,
self.pool1,
self.bn1,
#self.drop1,
self.flat,
self.fc0,
self.acti4,
self.bn4,
self.fc1,
]
test_x = ds_test.data.reshape([-1, 1, 28, 28]).numpy().astype(np.float) / 255
test_y = ds_test.targets.numpy()
train_mean = train_x.mean()
train_x, valid_x, test_x = (x - train_mean for x in (train_x, valid_x, test_x))
train_y, valid_y, test_y = (dense_to_one_hot(y, 10)
for y in (train_y, valid_y, test_y))
weight_decay = config['weight_decay']
net = []
regularizers = []
inputs = np.random.randn(config['batch_size'], 1, 28, 28)
net += [layers.Convolution(inputs, 16, 5, "conv1")]
regularizers += [
layers.L2Regularizer(net[-1].weights, weight_decay, 'conv1_l2reg')
]
net += [layers.MaxPooling(net[-1], "pool1")]
net += [layers.ReLU(net[-1], "relu1")]
net += [layers.Convolution(net[-1], 32, 5, "conv2")]
regularizers += [
layers.L2Regularizer(net[-1].weights, weight_decay, 'conv2_l2reg')
]
net += [layers.MaxPooling(net[-1], "pool2")]
net += [layers.ReLU(net[-1], "relu2")]
## 7x7
net += [layers.Flatten(net[-1], "flatten3")]
net += [layers.FC(net[-1], 512, "fc3")]
regularizers += [
layers.L2Regularizer(net[-1].weights, weight_decay, 'fc3_l2reg')
]
net += [layers.ReLU(net[-1], "relu3")]
net += [layers.FC(net[-1], 10, "logits")]
print("Error norm = ", np.linalg.norm(grad_b_num - grad_b))
print("Convolution")
x = np.random.randn(4, 3, 5, 5)
grad_out = np.random.randn(4, 2, 5, 5)
conv = layers.Convolution(x, 2, 3, "conv1")
print("Check grad wrt input")
check_grad_inputs(conv, x, grad_out)
print("Check grad wrt params")
check_grad_params(conv, x, conv.weights, conv.bias, grad_out)
print("\nMaxPooling")
x = np.random.randn(5, 4, 8, 8)
grad_out = np.random.randn(5, 4, 4, 4)
pool = layers.MaxPooling(x, "pool", 2, 2)
print("Check grad wrt input")
check_grad_inputs(pool, x, grad_out)
print("\nReLU")
x = np.random.randn(4, 3, 5, 5)
grad_out = np.random.randn(4, 3, 5, 5)
relu = layers.ReLU(x, "relu")
print("Check grad wrt input")
check_grad_inputs(relu, x, grad_out)
print("\nFC")
x = np.random.randn(20, 40)
grad_out = np.random.randn(20, 30)
fc = layers.FC(x, 30, "fc")
print("Check grad wrt input")
loss_function="cross-entropy",
lr=1e-4,
reg=1e-5)
graph.add_layer(
layers.Convolution("conv1",
kernel_size=11,
kernel_channels=3,
n_kernels=96,
stride=4,
same_padding=False,
logging=False))
graph.add_layer(layers.Activation(activation="relu", logging=False))
graph.add_layer(
layers.MaxPooling(window_size=3,
window_channels=96,
stride=2,
logging=False))
graph.add_layer(
layers.Convolution("conv2",
kernel_size=5,
kernel_channels=96,
n_kernels=256,
stride=1,
same_padding=True,
logging=False))
graph.add_layer(layers.Activation(activation="relu", logging=False))
graph.add_layer(
layers.MaxPooling(window_size=3,
window_channels=256,
stride=2,
valid_y = dataset.validation.labels
test_x = dataset.test.images
test_x = test_x.reshape([-1, 1, 28, 28])
test_y = dataset.test.labels
train_mean = train_x.mean()
train_x -= train_mean
valid_x -= train_mean
test_x -= train_mean
net = []
inputs = np.random.randn(
config['batch_size'], 1, 28, 28
) #cemu normalna razdioba, neki placeholder za stvarne podatke? samo zbog shapea?
net += [layers.Convolution(inputs, 16, 5,
"conv1")] #num filters = 16, kernel size= 5
net += [layers.MaxPooling(net[-1], "pool1")] #ulazni parametar je ulaz u layer
net += [layers.ReLU(net[-1], "relu1")]
net += [layers.Convolution(net[-1], 32, 5, "conv2")]
net += [layers.MaxPooling(net[-1], "pool2")]
net += [layers.ReLU(net[-1], "relu2")]
# out = 7x7
net += [layers.Flatten(net[-1], "flatten3")]
net += [layers.FC(net[-1], 512, "fc3")]
net += [layers.ReLU(net[-1], "relu3")]
net += [layers.FC(net[-1], 10, "logits")]
loss = layers.SoftmaxCrossEntropyWithLogits()
nn.train(train_x, train_y, valid_x, valid_y, net, loss, config)
nn.evaluate("Test", test_x, test_y, net, loss, config)
kernel_channels=3,
n_kernels=10,
stride=1,
same_padding=True)
X = conv.forward(X)
print("[+] CONVOLVED_shape:", X.shape)
print()
activation = layers.Activation(activation="relu")
X = activation.forward(X)
print("[+] ACTIVATION_shape:", X.shape)
dActivation = activation.backward(np.ones_like(X))
print("dActivation_SHAPE:", dActivation.shape)
print()
maxpool = layers.MaxPooling(window_size=2, window_channels=10, stride=2)
X = maxpool.forward(X)
print("[+] MAXPOOLED_shape:", X.shape)
dMaxpool = maxpool.backward(np.ones_like(X))
print("dMaxpool_SHAPE:", dMaxpool.shape)
N, C, H, W = X.shape
flatten = layers.Flatten((C * H * W, N))
X = flatten.forward(X)
print(X.shape)
I, N = X.shape
fc1 = layers.FullyConnected("fc1", I, 1000, activation="relu")
X = fc1.forward(X)
def __init__(self, rng, batch_size, input):
conv1_1 = layers.ConvLayer(rng,
input=input,
image_shape=(batch_size, 3, 480, 320),
filter_shape=(32, 3, 3, 3),
bias=True,
padding='valid')
conv1_2 = layers.ConvLayer(rng,
input=K.spatial_2d_padding(conv1_1.output),
image_shape=(batch_size, 32, 480, 320),
filter_shape=(64, 32, 3, 3),
padding='valid')
maxp1 = layers.MaxPooling(input=K.spatial_2d_padding(conv1_2.output),
poolsize=(2, 2))
conv2_1 = layers.ConvLayer(rng,
input=maxp1.output,
image_shape=(batch_size, 64, 240, 160),
filter_shape=(64, 64, 3, 3),
padding='valid')
conv2_2 = layers.ConvLayer(rng,
input=K.spatial_2d_padding(conv2_1.output),
image_shape=(batch_size, 64, 240, 160),
filter_shape=(64, 64, 3, 3),
padding='valid')
# maxp2 = layers.MaxPooling(
# input=K.spatial_2d_padding(conv2_2.output),
# poolsize=(2, 2)
# )
#
# conv3_1 = layers.ConvLayer(
# rng,
# input=maxp2.output,
# image_shape=(batch_size, 10, 120, 80),
# filter_shape=(10, 10, 3, 3),
# padding='valid'
# )
#
# conv3_2 = layers.ConvLayer(
# rng,
# input=K.spatial_2d_padding(conv3_1.output),
# image_shape=(batch_size, 10, 120, 80),
# filter_shape=(10, 10, 3, 3),
# padding='valid'
# )
#
# maxp3 = layers.MaxPooling(
# input=conv3_2.output,
# poolsize=(2, 2)
# )
#
# conv4_1 = layers.ConvLayer(
# rng,
# input=maxp3.output,
# image_shape=(batch_size, 10, 60, 40),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# conv4_2 = layers.ConvLayer(
# rng,
# input=conv4_1.output,
# image_shape=(batch_size, 10, 60, 40),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# maxp4 = layers.MaxPooling(
# input=conv4_2.output,
# poolsize=(2, 2)
# )
#
# conv5_1 = layers.ConvLayer(
# rng,
# input=maxp4.output,
# image_shape=(batch_size, 10, 30, 20),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# conv5_2 = layers.ConvLayer(
# rng,
# input=conv5_1.output,
# image_shape=(batch_size, 10, 30, 20),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# maxp5 = layers.MaxPooling(
# input=conv5_2.output,
# poolsize=(2, 2)
# )
#
# flat1_input = maxp5.output.flatten(2)
# flat1_shape = 10*15*10
#
# flat1 = layers.HiddenLayer(
# rng=rng,
# input=flat1_input,
# n_in=flat1_shape,
# n_out=flat1_shape,
# activation=T.nnet.relu
# )
#
# flat2 = layers.HiddenLayer(
# rng=rng,
# input=flat1.output,
# n_in=flat1_shape,
# n_out=flat1_shape,
# activation=T.nnet.relu
# )
#
# remax5_input = flat2.output.reshape((batch_size,10,15,10))
#
# remax5 = layers.ReverseMaxPooling(
# input=remax5_input,
# mask=maxp5.mask,
# poolsize=(2, 2)
# )
#
# deconv5_1 = layers.DeConvLayer(
# rng,
# #input=remax5.output,
# input=conv5_2.output,
# W=conv5_2.W,
# image_shape=(batch_size, 10, 30, 20),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# deconv5_2 = layers.DeConvLayer(
# rng,
# input=deconv5_1.output,
# W=conv5_1.W,
# image_shape=(batch_size, 10, 30, 20),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# remax4 = layers.ReverseMaxPooling(
# input=deconv5_2.output,
# mask=maxp4.mask,
# poolsize=(2, 2)
# )
#
# deconv4_1 = layers.DeConvLayer(
# rng,
# input=remax4.output,
# W=conv4_2.W,
# image_shape=(batch_size, 10, 60, 40),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# deconv4_2 = layers.DeConvLayer(
# rng,
# input=deconv4_1.output,
# W=conv4_1.W,
# image_shape=(batch_size, 10, 60, 40),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# remax3 = layers.ReverseMaxPooling(
# input=deconv4_2.output,
# mask=maxp3.mask,
# poolsize=(2, 2)
# )
#
# deconv3_1 = layers.DeConvLayer(
# rng,
# #input=remax3.output,
# input=conv3_2.output,
# W=conv3_2.W,
# image_shape=(batch_size, 10, 118, 78),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# deconv3_2 = layers.DeConvLayer(
# rng,
# input=K.spatial_2d_padding(deconv3_1.output),
# W=conv3_1.W,
# image_shape=(batch_size, 10, 120, 80),
# filter_shape=(10, 10, 3, 3),
# padding='same'
# )
#
# remax2 = layers.ReverseMaxPooling(
# input=deconv3_2.output,
# mask=maxp2.mask,
# poolsize=(2, 2)
# )
#
deconv2_1 = layers.DeConvLayer(
rng,
#input=remax2.output,
input=K.spatial_2d_padding(conv2_2.output),
W=conv2_2.W,
image_shape=(batch_size, 64, 240, 160),
filter_shape=(64, 64, 3, 3),
padding='same')
deconv2_2 = layers.DeConvLayer(rng,
input=deconv2_1.output,
W=conv2_1.W,
image_shape=(batch_size, 64, 240, 160),
filter_shape=(64, 64, 3, 3),
padding='same')
remax1 = layers.ReverseMaxPooling(input=deconv2_2.output,
mask=maxp1.mask,
poolsize=(2, 2))
deconv1_1 = layers.DeConvLayer(
rng,
input=remax1.output,
#input=K.spatial_2d_padding(conv1_2.output),
W=conv1_2.W,
image_shape=(batch_size, 64, 480, 320),
filter_shape=(32, 64, 3, 3),
padding='same',
)
deconv1_2 = layers.ConvLayer(rng,
input=deconv1_1.output,
image_shape=(batch_size, 32, 480, 320),
filter_shape=(1, 32, 3, 3),
bias=True,
padding='same',
activation=T.nnet.sigmoid)
self.y_pred = T.clip(deconv1_2.output, 0.001, 0.999)
# self.params=conv1_1.params+conv1_2.params+conv2_1.params+conv2_2.params+\
# conv3_1.params+conv3_2.params+conv4_1.params+conv4_2.params+conv5_1.params+conv5_2.params+\
# deconv5_1.params+deconv5_2.params+deconv4_1.params+deconv4_2.params+deconv3_1.params+\
# deconv3_2.params+deconv2_1.params+deconv2_2.params+deconv1_1.params+deconv1_2.params
self.params = conv1_1.params + conv1_2.params + conv2_1.params + conv2_2.params + deconv1_2.params
self.input = input
def __init__(
self,
lossfunc,
optimizer,
batch_size=32,
):
self.lossfunc = lossfunc
self.optimizer = optimizer
self.batch_size = batch_size
input_size = 64
hidden_size = 3136
output_size = 10
# self.lr = 0.001
# self.alpha = 0.9
self.l1 = 1e-4
self.l2 = 1e-4
self.optimizer = optimizers.Adam(l1=self.l1, l2=self.l2)
self.conv0 = L.Convolution_(n_filter=8, filter_size=(3, 3), stride=1)
self.conv1 = L.Convolution_(n_filter=16, filter_size=(3, 3), stride=1)
self.conv2 = L.Convolution_(n_filter=32, filter_size=(5, 5), stride=1)
self.conv3 = L.Convolution_(n_filter=64, filter_size=(5, 5), stride=1)
self.fc0 = L.Linear_(output_size=1024)
self.fc1 = L.Linear_(output_size=10)
self.bn0 = L.BatchNormalization_()
self.bn1 = L.BatchNormalization_()
self.bn2 = L.BatchNormalization_()
self.bn3 = L.BatchNormalization_()
self.bn4 = L.BatchNormalization_()
self.acti0 = L.ELU()
self.acti1 = L.ELU()
self.acti2 = L.ELU()
self.acti3 = L.ELU()
self.acti4 = L.ELU()
self.pool0 = L.MaxPooling(7, 7)
self.pool1 = L.MaxPooling(5, 5)
self.pool2 = L.MaxPooling(3, 3)
self.pool3 = L.MaxPooling(3, 3)
self.flat = L.Flatten()
self.drop0 = L.Dropout(0.5)
self.drop1 = L.Dropout(0.5)
self.drop2 = L.Dropout(0.5)
self.drop3 = L.Dropout(0.25)
self.layers = [
self.conv0,
self.acti0,
self.pool0,
self.bn0,
#self.drop0,
self.conv1,
self.acti1,
self.pool1,
self.bn1,
#self.drop1,
#self.conv2,
#self.acti2,
#self.pool2,
#self.bn2,
#self.drop2,
#self.conv3,
#self.acti3,
#self.pool3,
#self.bn3,
#self.drop3,
self.flat,
self.fc0,
self.acti4,
self.bn4,
self.fc1,
]
def upperBlock(x, conv_size=[128, 256, 512, 256, 128], n_channels=3):
x = L.GaussianNoise(x)
x = L.Conv2D(x,
filter_size=3,
n_channels=n_channels,
n_filters=conv_size[0],
padding='SAME',
name='1a')
x = L.LeakyReLU(x)
x = L.Conv2D(x,
filter_size=3,
n_channels=conv_size[0],
n_filters=conv_size[0],
name='1b')
x = L.LeakyReLU(x)
x = L.Conv2D(x,
filter_size=3,
n_channels=conv_size[0],
n_filters=conv_size[0],
name='1c')
x = L.MaxPooling(x, ksize=2, stride_length=2)
x = L.Dropout(x, probability=0.5)
x = L.Conv2D(x,
filter_size=3,
n_channels=conv_size[0],
n_filters=conv_size[1],
name='2a')
x = L.LeakyReLU(x)
x = L.Conv2D(x,
filter_size=3,
n_channels=conv_size[1],
n_filters=conv_size[1],
name='2b')
x = L.LeakyReLU(x)
x = L.Conv2D(x,
filter_size=3,
n_channels=conv_size[1],
n_filters=conv_size[1],
name='2c')
x = L.LeakyReLU(x)
x = L.MaxPooling(x, ksize=2, stride_length=2)
x = L.Conv2D(x,
filter_size=3,
n_channels=conv_size[1],
n_filters=conv_size[2],
padding='VALID',
name='3a')
x = L.LeakyReLU(x)
x = L.Conv2D(x,
filter_size=1,
n_channels=conv_size[2],
n_filters=conv_size[3],
name='3b')
x = L.LeakyReLU(x)
x = L.Conv2D(x,
filter_size=1,
n_channels=conv_size[3],
n_filters=conv_size[4],
name='3c')
x = L.LeakyReLU(x)
x = L.GlobalAveragePooling(x)
# x = L.Dropout(x, probability=0.5)
# x = L.Dense(x, conv_size[4], 10)
# x = L.SoftMax(x)
return x
