def __init__(self,
network,
filter_width,
filter_height,
filter_number,
zero_padding,
stride,
activator,
learning_rate,
momentum_rate=0.0,
decay_rate=0.0):
upstream_layer = network.layers[-1]
input_width = upstream_layer.get_output().shape[1]
input_height = upstream_layer.get_output().shape[2]
channel_number = upstream_layer.get_output().shape[0]
ConvLayer.__init__(self,
network,
input_width,
input_height,
channel_number,
filter_width,
filter_height,
filter_number,
zero_padding,
stride,
activator,
learning_rate,
momentum_rate=momentum_rate,
decay_rate=decay_rate)
def __init__(self, num_conv_in_channel, num_conv_out_channel,
num_primary_unit, primary_unit_size, num_classes,
output_unit_size, num_routing, use_reconstruction_loss,
regularization_scale, input_width, input_height,
cuda_enabled):
"""
In the constructor we instantiate one ConvLayer module and two CapsuleLayer modules
and assign them as member variables.
"""
super(capsulenet, self).__init__()
self.cuda_enabled = cuda_enabled
# Configurations used for image reconstruction.
self.use_reconstruction_loss = use_reconstruction_loss
# Input image size and number of channel.
# By default, for MNIST, the image width and height is 28x28
# and 1 channel for black/white.
self.image_width = input_width
self.image_height = input_height
self.image_channel = num_conv_in_channel
# Also known as lambda reconstruction. Default value is 0.0005.
# We use sum of squared errors (SSE) similar to paper.
self.regularization_scale = regularization_scale
# Layer 1: Conventional Conv2d layer.
self.conv1 = ConvLayer(in_channel=3, out_channel=256, kernel_size=9)
self.conv2 = ConvLayer(in_channel=256, out_channel=256, kernel_size=9)
# PrimaryCaps
# Layer 2: Conv2D layer with `squash` activation.
self.primary = CapsuleLayer(
in_unit=0,
in_channel=num_conv_out_channel,
num_unit=num_primary_unit,
unit_size=primary_unit_size, # capsule outputs
use_routing=False,
num_routing=num_routing,
cuda_enabled=cuda_enabled)
# DigitCaps
# Final layer: Capsule layer where the routing algorithm is.
self.digits = CapsuleLayer(
in_unit=num_primary_unit,
in_channel=primary_unit_size,
num_unit=num_classes,
unit_size=output_unit_size, # 16D capsule per digit class
use_routing=True,
num_routing=num_routing,
cuda_enabled=cuda_enabled)
# Reconstruction network
if use_reconstruction_loss:
self.decoder = Decoder(num_classes, output_unit_size, input_width,
input_height, num_conv_in_channel,
cuda_enabled)
def setUp(self):
self.layer = ConvLayer(2, (2, 3), 1, padding_mode=False)
self.layer.set_input_shape((2, 8))
self.layer._filter_weights = np.ones((2, 2, 3), dtype=np.double)
self.layer._filter_weights[1, :, :] /= 2.0
self.input = np.array([[2, 2, 2, 2, 2, 2, 2, 4],
[4, 4, 4, 4, 4, 4, 4, 8]], dtype=np.double)
def __init__(self, maps_input, fm_sizes, stride=2):
self.session = None
self.f = tf.nn.relu
self.stride = stride
self.conv0 = ConvLayer(1, maps_input, fm_sizes[0], stride)
self.btn0 = BatchNormLayer(fm_sizes[0])
self.conv_block1 = ConvBlock(fm_sizes[0], [fm_sizes[0], 64, 256],
stride=1,
padding='SAME')
self.btn_block1 = BatchNormLayer(256)
self.layers = [self.conv0, self.btn0]
self.input_ = tf.placeholder(tf.float32,
shape=(1, 224, 224, maps_input))
self.output = self.forward(self.input_)
# TODO
pass
def parse_layers(layer_str):
layers = layer_str.split('-')
layer_arr = []
for i, layer in enumerate(layers):
if 'c' in layer:
conv_channels = int(re.findall(r'\((.*?)\)', layer)[0])
filter_size = int(layer[-2])
cl = ConvLayer(filter_size, conv_channels, None)
layer_arr.append(cl)
elif 'p' in layer:
pool_size = int(layer.replace('p', ''))
layer_arr[-1].pool_size = pool_size
return layer_arr
def __init__(self, layer_types, layer_shapes, conv_layer_types=None, layers=None):
self.layer_types = layer_types
self.layer_shapes = layer_shapes
self.num_genes = 0
self.conv_layer_types = conv_layer_types
if layers is not None:
self.layers = layers
for typ, shpe in zip(layer_types, layer_shapes):
if typ == "conv":
self.num_genes += shpe[1][0]
elif typ == "dense":
self.num_genes += shpe[0][0]
elif typ == "soft":
self.num_genes += shpe[0][0]
else:
self.layers = []
cntr = 0
n_conv_layer_types = -1
if conv_layer_types is not None:
n_conv_layer_types = len(conv_layer_types)
for typ, shpe in zip(layer_types, layer_shapes):
if typ == "conv":
if cntr <= n_conv_layer_types:
self.layers.append(ConvLayer(image_shape=shpe[0],
filter_shape=shpe[1],
filter_method=conv_layer_types[cntr][0],
zero_padding=conv_layer_types[cntr][1]))
cntr += 1
else:
self.layers.append(ConvLayer(image_shape=shpe[0],
filter_shape=shpe[1]))
self.num_genes += shpe[1][0]
elif typ == "dense":
self.layers.append(DenseLayer(layer_shape=shpe[0]))
self.num_genes += shpe[0][0]
elif typ == "soft":
self.layers.append(SoftmaxLayer(layer_shape=shpe[0]))
self.num_genes += shpe[0][0]
def __init__(self, layer_types, layer_shapes, layers=None, cost_func=QuadCost):
self.layer_types = layer_types
self.layer_shapes = layer_shapes
self.num_layers = len(layer_types)
self.cost_func = cost_func
if layers is not None:
self.layers = layers
else:
self.layers = []
for lt, ls in zip(layer_types, layer_shapes):
if lt == "conv":
self.layers.append(ConvLayer(image_shape=ls[0], kernel_shape=ls[1]))
elif lt == "dense":
self.layers.append(DenseLayer(layer_shape=ls[0]))
def __init__(self, input_size, n_class=1):
# input (1,8,8)
conv1 = ConvLayer(
input_size=input_size,
input_dim=1,
zero_padding=2,
stride=1,
kernel_size=np.array([5, 5]),
n_kernels=3,
activator=ReluActivator())
# output (3,8,8)
self.conv1 = conv1
pool1 = PoolingLayer(
input_size=conv1.output_size,
input_dim=conv1.n_kernels,
kernel_size=2,
stride=2,
mode='max')
# output (3,4,4)
self.pool1 = pool1
stack = StackingLayer(
input_size=pool1.output_size,
input_dim=pool1.input_dim)
# output(3*4*4,1)
self.stack = stack
fc1 = FcLayer(
input_size=stack.output_size,
output_size=16,
activator=ReluActivator())
# output (16,1)
self.fc1 = fc1
fc2 = FcLayer(
input_size=fc1.output_size,
output_size=n_class,
activator=SoftmaxActivator())
# output (10,1)
self.fc2 = fc2
self.layers = [conv1, pool1, stack, fc1]
self.output_layer = fc2
def __init__(self, img_width, img_height, img_channel,
num_conv_input_channels, num_conv_output_channels,
num_prim_units, prim_unit_size, num_classes, output_unit_size,
num_routing, CUDA, conv_kernel_size, prim_kernel_size,
prim_output_channels):
"""
Constructor for CapsuleNetwork class
:param img_width: Width of image (ex. MNIST=28)
:param img_height: Height of image (ex. MNIST=28)
:param img_channel: Number Channels per image (ex. MNIST=1)
:param num_conv_input_channels: Number of input channels for ConvLayer (ex. MNIST=1)
:param num_conv_output_channels: Number of output channels from ConvLayer (ex. MNIST=256)
:param num_prim_units: Number of primary unit (ex. MNIST=8)
:param prim_unit_size: Number of input channels of DigitCapsLayer (ex. MNIST=1152)
:param num_classes: Number of classifications (ex. MNIST=10)
:param output_unit_size: Size of the output unit from DigitCapsLayer (ex. MNIST=16)
:param num_routing: Number of iterations for the CapsNet routing mechanism
:param CUDA: True if running on GPU, False otherwise
:param conv_kernel_size: Kernel length/width for ConvLayer (ex. MNIST=9)
:param prim_kernel_size: Kernel length/width for PrimaryCapsLayer (ex. MNIST=9)
:param prim_output_channels: Number of output channels of PrimaryCapsLayer (ex. MNIST=32)
"""
super(CapsuleNetwork, self).__init__()
self.CUDA = CUDA
self.img_width = img_width
self.img_height = img_height
self.img_channel = img_channel
self.num_classes = num_classes
self.output_unit_size = output_unit_size
self.nn = torch.nn.Sequential(
ConvLayer(input_channels=num_conv_input_channels,
output_channels=num_conv_output_channels,
kernel_size=conv_kernel_size),
PrimaryCapsLayer(input_channels=num_conv_output_channels,
num_unit=num_prim_units,
kernel_size=prim_kernel_size,
output_channels=prim_output_channels),
DigitCapsLayer(input_unit=num_prim_units,
input_channels=prim_unit_size,
num_unit=num_classes,
unit_size=output_unit_size,
num_routes=num_routing,
CUDA=CUDA))
self.decoder = None
class ResNet:
def __init__(self, maps_input, fm_sizes, stride=2):
self.session = None
self.f = tf.nn.relu
self.stride = stride
self.conv0 = ConvLayer(1, maps_input, fm_sizes[0], stride)
self.btn0 = BatchNormLayer(fm_sizes[0])
self.conv_block1 = ConvBlock(fm_sizes[0], [fm_sizes[0], 64, 256],
stride=1,
padding='SAME')
self.btn_block1 = BatchNormLayer(256)
self.layers = [self.conv0, self.btn0]
self.input_ = tf.placeholder(tf.float32,
shape=(1, 224, 224, maps_input))
self.output = self.forward(self.input_)
# TODO
pass
def forward(self, X):
FX = self.conv0.forward(X)
FX = self.btn0.forward(FX)
FX = self.f(FX)
FX = tf.nn.max_pool2d(FX,
strides=(self.stride, self.stride),
ksize=[2, 2],
padding='VALID')
FX = self.conv_block1.forward(FX)
FX = self.btn_block1.forward(FX)
return FX
def predict(self, X):
assert (self.session is not None)
return (self.session.run(self.output, feed_dict={self.input_: X}))
pass
def set_session(self, session):
self.session = session
self.conv0.session = session
self.conv_block1.session = session
self.btn0.session = session
def __init__(self,
layers,
l2_decay=0.001,
debug=False,
learning_rate=0.001):
super(Network, self).__init__()
mapping = {
"input": lambda x: InputLayer(x),
"fc": lambda x: FullyConnectedLayer(x),
"convolution": lambda x: ConvLayer(x),
"pool": lambda x: MaxPoolLayer(x),
"squaredloss": lambda x: SquaredLossLayer(x),
"softmax": lambda x: SoftmaxLayer(x),
"relu": lambda x: ReLULayer(x),
"dropout": lambda x: DropoutLayer(x)
}
self.layers = []
self.l2_decay = l2_decay
self.debug = debug
self.learning_rate = learning_rate
def crossover(father, mother, alpha=0.5):
layers = []
for lt, ls, fl, ml in zip(father.get_layer_types(),
father.get_layer_shapes(), father.get_layers(),
mother.get_layers()):
if lt == "conv":
lyr = ConvLayer(image_shape=ls[0], filter_shape=ls[1])
# Loop for each filter
for i in range(ls[1][0]):
parent = ml
# Randomly pick either mother or father gene based on alpha
if random.random() < alpha:
parent = fl
lyr.set_filter(index=i, filtr=deepcopy(parent.get_filter(i)))
layers.append(lyr)
elif lt == "dense" or lt == "soft":
weights = []
biases = []
lyr = DenseLayer(layer_shape=ls[0])
if lt == "soft":
lyr = SoftmaxLayer(layer_shape=ls[0])
# Loop for each neuron
for i in range(ls[0][0]):
parent = ml
# Randomly pick either mother or father gene based on alpha
if random.random() < alpha:
parent = fl
weights.append(deepcopy(parent.get_weights(i)))
biases.append(deepcopy(parent.get_biases(i)))
lyr.set_weights_biases(weights, biases)
layers.append(lyr)
child = Individual(deepcopy(father.get_layer_types()),
deepcopy(father.get_layer_shapes()),
deepcopy(father.get_conv_layer_types()), layers)
child = mutate_individual(child)
return child
def __init__(self, CUDA):
"""
Constructor for CapsuleNetwork class
"""
super(CIFAR10nn, self).__init__()
self.img_width = 32
self.img_height = 32
self.img_channel = 3
self.CUDA = CUDA
self.nn = nn.Sequential(
ConvLayer(input_channels=3, output_channels=256, kernel_size=9),
PrimaryCapsLayer(input_channels=256,
num_unit=8,
kernel_size=9,
output_channels=32),
DigitCapsLayer(input_unit=8,
input_channels=2048,
num_unit=10,
unit_size=36,
num_routes=3,
CUDA=CUDA))
self.decoder = Decoder_Color(CUDA=self.CUDA)
def __init__(self, n_channel=3, n_classes=2, image_size=64):
self.images = tf.placeholder(
dtype=tf.float32,
shape=[None, image_size, image_size, n_channel],
name='images')
self.labels = tf.placeholder(dtype=tf.int64,
shape=[None],
name='labels')
self.keep_prob = tf.placeholder(dtype=tf.float32, name='keep_prob')
self.global_step = tf.Variable(0, dtype=tf.int32, name='global_step')
#cnn网络
conv_layer1 = ConvLayer(
input_shape=(None, image_size, image_size, n_channel),
n_size=3, #卷积核3*3
n_filter=64,
stride=1,
activation='relu',
batch_normal=True,
weight_decay=1e-4,
name='conv1')
pool_layer1 = PoolLayer(n_size=2,
stride=2,
mode='max',
resp_normal=True,
name='pool1')
conv_layer2 = ConvLayer(input_shape=(None, int(image_size / 2),
int(image_size / 2), 64),
n_size=3,
n_filter=128,
stride=1,
activation='relu',
batch_normal=True,
weight_decay=1e-4,
name='conv2')
pool_layer2 = PoolLayer(n_size=2,
stride=2,
mode='max',
resp_normal=True,
name='pool2')
conv_layer3 = ConvLayer(input_shape=(None, int(image_size / 4),
int(image_size / 4), 128),
n_size=3,
n_filter=256,
stride=1,
activation='relu',
batch_normal=True,
weight_decay=1e-4,
name='conv3')
pool_layer3 = PoolLayer(n_size=2,
stride=2,
mode='max',
resp_normal=True,
name='pool3')
dense_layer1 = DenseLayer(input_shape=(None, int(image_size / 8) *
int(image_size / 8) * 256),
hidden_dim=1024,
activation='relu',
dropout=True,
keep_prob=self.keep_prob,
batch_normal=True,
weight_decay=1e-4,
name='dense1')
dense_layer2 = DenseLayer(input_shape=(None, 1024),
hidden_dim=n_classes,
activation='none',
dropout=False,
keep_prob=None,
batch_normal=False,
weight_decay=1e-4,
name='dense2')
#数据流
hidden_conv1 = conv_layer1.get_output(input=self.images)
hidden_pool1 = pool_layer1.get_output(input=hidden_conv1)
hidden_conv2 = conv_layer2.get_output(input=hidden_pool1)
hidden_pool2 = pool_layer2.get_output(input=hidden_conv2)
hidden_conv3 = conv_layer3.get_output(input=hidden_pool2)
hidden_pool3 = pool_layer3.get_output(input=hidden_conv3)
input_dense1 = tf.reshape(
hidden_pool3,
[-1, int(image_size / 8) * int(image_size / 8) * 256])
output_dense1 = dense_layer1.get_output(input=input_dense1)
logits = dense_layer2.get_output(input=output_dense1)
logits = tf.multiply(logits, 1, name='logits')
# 目标/损失函数
self.objective = tf.reduce_sum(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=self.labels))
tf.add_to_collection('losses', self.objective)
self.avg_loss = tf.add_n(tf.get_collection('losses'))
# 优化器
lr = tf.cond(
tf.less(self.global_step, 500), lambda: tf.constant(0.01),
lambda: tf.cond(tf.less(self.global_step, 1000), lambda: tf.
constant(0.001), lambda: tf.constant(0.0001)))
self.optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(
self.avg_loss, global_step=self.global_step)
# 观察值
correct_prediction = tf.equal(self.labels, tf.argmax(logits, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
def __init__(self, master):
self.master = master
master.title("RuneScape Bot")
self.gamefield = (29, 0, 510, 340) # (x_min, y_min, x_max, y_max)
self.mapfield = (595, 9, 740 - 595, 160 - 9)
self.win_info = wt.get_wininfo()
self.raw_im = Image.open("data/train1.png")
self.ar_im = np.array([])
self.time = 0
self.x_offset = 0 # image offset in canvas
self.y_offset = 0
self.player_pos = (285, 180) # absolute coordinates (x, y)
self.nearest_ore = self.player_pos
# image and buttons
self.canvas = tk.Canvas(master, width=900, height=800)
self.canvas.pack()
self.button_update = tk.Button(master,
text='Update',
command=self.update)
self.button_update.pack()
self.img = ImageTk.PhotoImage(self.raw_im)
self.canvas.create_image(self.x_offset,
self.y_offset,
anchor=tk.NW,
image=self.img)
self.ar_gamefield = np.array(self.raw_im.crop(self.gamefield))
self.mark_iron_ores(self.ar_gamefield)
self.player = self.canvas.create_oval(
self.player_pos[0] - 5 + self.x_offset,
self.player_pos[1] - 5 + self.y_offset,
self.player_pos[0] + 5 + self.x_offset,
self.player_pos[1] + 5 + self.y_offset,
fill='blue')
in_shape = (1, 340, 510, 3)
lamb = 0.001
# l1 = ConvLayer(in_shape, 2, 10, 1, 1, activation='lrelu')
# l2 = PoolLayer(l1.dim_out, 2, 2, mode='max')
# l3 = ConvLayer(l2.dim_out, 4, 20, 1, 2, activation='lrelu')
# l4 = PoolLayer(l3.dim_out, 4, 4, mode='max')
# l5 = ConvLayer(l4.dim_out, 3, 20, 1, 0, activation='lrelu')
# l6 = PoolLayer(l5.dim_out, 4, 4, mode='max')
# l7 = ConvLayer(l6.dim_out, 1, 5, 1, 0, activation='lrelu')
# self.net = nn.Network((l1, l2, l3, l4, l5, l6, l7), lamb=lamb)
l1 = ConvLayer(in_shape, 2, 10, 1, 1, activation='lrelu')
l2 = PoolLayer(l1.dim_out, 2, 2, mode='max')
l3 = ConvLayer(l2.dim_out, 4, 20, 1, 2, activation='lrelu')
l4 = PoolLayer(l3.dim_out, 2, 2, mode='max')
l5 = ConvLayer(l4.dim_out, 2, 20, 1, 0, activation='lrelu')
l6 = PoolLayer(l5.dim_out, 4, 4, mode='max')
l7 = ConvLayer(l6.dim_out, 2, 20, 1, 0, activation='lrelu')
l8 = PoolLayer(l7.dim_out, 2, 2, mode='max')
#fully connected
l9 = ConvLayer(l8.dim_out, 1, 5, 1, 0, activation='lrelu')
self.net = nn.Network((l1, l2, l3, l4, l5, l6, l7, l8, l9), lamb=lamb)
self.net.load_network("model_9L")
def add_conv_layer(self, n_filters, kernel_size, eta=0.001):
self.layers.append(ConvLayer(n_filters, kernel_size, eta))
import mnist
import numpy as np
from conv_layer import ConvLayer
from max_pool_layer import MaxPoolLayer
from softmax_layer import SoftmaxLayer
train_images = mnist.train_images()[:1000]
train_labels = mnist.train_labels()[:1000]
test_images = mnist.test_images()[:1000]
test_labels = mnist.test_labels()[:1000]
conv = ConvLayer(8)
pool = MaxPoolLayer()
softmax = SoftmaxLayer(13 * 13 * 8, 10)
def check(image):
out = conv.forward((image / 255) - 0.5)
out = pool.forward(out)
out = softmax.forward(out)
return np.argmax(out)
def forward(image, label):
'''
Completes a forward pass of the CNN and calculates the accuracy and
cross-entropy loss.
- image is a 2d numpy array
- label is a digit
'''
X_data = np.array(X) / 255
Y_data = np.array(Y)
f.close()
return X_data, Y_data
X_data, Y_data = load_data("data/copper_data.h5")
X_test, Y_test = load_data("data/copper_test_data.h5")
X_test = X_test[:6]
Y_test = Y_test[:6]
mb_size = 5
in_shape = (mb_size, *X_data.shape[1:])
lamb = 0.001
l1 = ConvLayer(in_shape, 2, 10, 1, 1, activation='lrelu')
l2 = PoolLayer(l1.dim_out, 2, 2, mode='max')
l3 = ConvLayer(l2.dim_out, 4, 20, 1, 2, activation='lrelu')
l4 = PoolLayer(l3.dim_out, 2, 2, mode='max')
l5 = ConvLayer(l4.dim_out, 2, 20, 1, 0, activation='lrelu')
l6 = PoolLayer(l5.dim_out, 4, 4, mode='max')
l7 = ConvLayer(l6.dim_out, 2, 20, 1, 0, activation='lrelu')
l8 = PoolLayer(l7.dim_out, 2, 2, mode='max')
#fully connected
l9 = ConvLayer(l8.dim_out, 1, 5, 1, 0, activation='lrelu')
net = nn.Network((l1, l2, l3, l4, l5, l6, l7, l8, l9), lamb=lamb)
#net.load_network("model2")
cost = net.train_network(X_data, Y_data, (X_test, Y_test), 200, 0.002, mb_size)
def __init__(self, input_size, n_class=1):
# input (1,28,28)
conv1 = ConvLayer(input_size=input_size,
input_dim=1,
zero_padding=2,
stride=1,
kernel_size=np.array([5, 5]),
n_kernels=32,
activator=ReluActivator())
# output (32,28,28)
self.conv1 = conv1
pool1 = PoolingLayer(input_size=conv1.output_size,
input_dim=conv1.n_kernels,
kernel_size=2,
stride=2,
mode='max')
# output (32,14,14)
self.pool1 = pool1
conv2 = ConvLayer(input_size=pool1.output_size,
input_dim=pool1.input_dim,
zero_padding=2,
stride=1,
kernel_size=np.array([5, 5]),
n_kernels=64,
activator=ReluActivator())
# output (64,14,14)
self.conv2 = conv2
pool2 = PoolingLayer(input_size=conv2.output_size,
input_dim=conv2.n_kernels,
kernel_size=2,
stride=2,
mode='max')
# output (64,7,7)
self.pool2 = pool2
stack = StackingLayer(input_size=pool2.output_size,
input_dim=pool2.input_dim)
# output(64*7*7,1)
self.stack = stack
fc1 = FcLayer(input_size=stack.output_size,
output_size=1024,
activator=ReluActivator())
# output (1024,1)
self.fc1 = fc1
fc2 = FcLayer(input_size=fc1.output_size,
output_size=n_class,
activator=SoftmaxActivator())
# output (10,1)
self.fc2 = fc2
self.layers = [conv1, pool1, conv2, pool2, stack, fc1]
self.output_layer = fc2
import cupy as np from conv_layer import ConvLayer m = 10 X = np.random.rand(m, 700, 400, 3) l = ConvLayer(X.shape, 3, 6, 1, 1)
def __init__(self, num_conv_in_channel, num_conv_out_channel,
num_primary_unit, primary_unit_size, num_classes,
output_unit_size, num_routing, use_reconstruction_loss,
regularization_scale, cuda_enabled):
"""
In the constructor we instantiate one ConvLayer module and two CapsuleLayer modules
and assign them as member variables.
"""
print 'num_conv_in_channel,', num_conv_in_channel
print 'num_conv_out_channel,', num_conv_out_channel
print 'num_primary_unit,', num_primary_unit
print 'primary_unit_size,', primary_unit_size
print 'num_classes,', num_classes
print 'output_unit_size,', output_unit_size
print 'num_routing,', num_routing
print 'use_reconstruction_loss,', use_reconstruction_loss
print 'regularization_scale,', regularization_scale
print 'cuda_enabled,', cuda_enabled
super(Net, self).__init__()
self.cuda_enabled = cuda_enabled
# Configurations used for image reconstruction.
self.use_reconstruction_loss = use_reconstruction_loss
self.image_width = 28 # MNIST digit image width
self.image_height = 28 # MNIST digit image height
self.image_channel = 1 # MNIST digit image channel
self.regularization_scale = regularization_scale
# Layer 1: Conventional Conv2d layer
self.conv1 = ConvLayer(in_channel=num_conv_in_channel,
out_channel=num_conv_out_channel,
kernel_size=9)
# PrimaryCaps
# Layer 2: Conv2D layer with `squash` activation
print('n_channel=', num_conv_out_channel)
print('num_unit=', num_primary_unit)
print('unit_size=', primary_unit_size)
print('use_routing=', False)
print('num_routing=', num_routing)
print('cuda_enabled=', cuda_enabled)
self.primary = CapsuleLayer(
in_unit=0,
in_channel=num_conv_out_channel,
num_unit=num_primary_unit,
unit_size=primary_unit_size, # capsule outputs
use_routing=False,
num_routing=num_routing,
cuda_enabled=cuda_enabled)
# DigitCaps
# Final layer: Capsule layer where the routing algorithm is.
print('SECOND')
print('in_unit=', num_primary_unit)
print('in_channel=', primary_unit_size)
print('num_unit=', num_classes)
print('unit_size=', output_unit_size)
print('use_routing=', True)
print('num_routing=', num_routing)
print('cuda_enabled=', cuda_enabled)
self.digits = CapsuleLayer(
in_unit=num_primary_unit,
in_channel=primary_unit_size,
num_unit=num_classes,
unit_size=output_unit_size, # 16D capsule per digit class
use_routing=True,
num_routing=num_routing,
cuda_enabled=cuda_enabled)
# Reconstruction network
if use_reconstruction_loss:
# The output of the digit capsule is fed into a decoder consisting of
# 3 fully connected layers that model the pixel intensities.
fc1_output_size = 512
fc2_output_size = 1024
self.fc1 = nn.Linear(num_classes * output_unit_size,
fc1_output_size)
self.fc2 = nn.Linear(fc1_output_size, fc2_output_size)
self.fc3 = nn.Linear(fc2_output_size, 784)
# Activation functions
self.relu = nn.ReLU(inplace=True)
self.sigmoid = nn.Sigmoid()
class TestConvLayer(unittest.TestCase):
def setUp(self):
self.layer = ConvLayer(2, (2, 3), 1, padding_mode=False)
self.layer.set_input_shape((2, 8))
self.layer._filter_weights = np.ones((2, 2, 3), dtype=np.double)
self.layer._filter_weights[1, :, :] /= 2.0
self.input = np.array([[2, 2, 2, 2, 2, 2, 2, 4],
[4, 4, 4, 4, 4, 4, 4, 8]], dtype=np.double)
def test_forward_prop(self):
expected_output = np.array([[18, 18, 18, 18, 18, 24],
[9, 9, 9, 9, 9, 12]], dtype=np.double)
output = self.layer.forward_prop(self.input)
numpy.testing.assert_array_equal(output, expected_output)
def test_back_prop(self):
self.layer.forward_prop(self.input)
out_grad = np.ones((2, 6), dtype=np.double)
expected_in_grad = np.array([[1.5, 3, 4.5, 4.5, 4.5, 4.5, 3, 1.5],
[1.5, 3, 4.5, 4.5, 4.5, 4.5, 3, 1.5]], dtype=np.double)
in_grad = self.layer.back_prop(out_grad)
numpy.testing.assert_array_equal(in_grad, expected_in_grad)
def test_get_output_shape(self):
self.assertEqual(self.layer.get_output_shape(), (2, 6))
def test_update_parameters(self):
self.layer.forward_prop(self.input)
out_grad = np.ones((2, 6), dtype=np.double)
self.layer.back_prop(out_grad)
self.layer.update_parameters(0.1)
expected_output = np.array([[-20.6, -20.6, -20.6, -20.6, -20.6, -28.6],
[-29.6, -29.6, -29.6, -29.6, -29.6, -40.6]], dtype=np.double)
output = self.layer.forward_prop(self.input)
numpy.testing.assert_array_almost_equal(output, expected_output)
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_train = x_train.astype('float32')
x_train /= 255
# encode output which is a number in range [0,9] into a vector of size 10
# e.g. number 3 will become [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]
y_train = np_utils.to_categorical(y_train)
# same for test data : 10000 samples
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
x_test = x_test.astype('float32')
x_test /= 255
y_test = np_utils.to_categorical(y_test)
# Network
net = Network()
net.add(ConvLayer((28, 28, 1), (3, 3),
1)) # input_shape=(28, 28, 1) ; output_shape=(26, 26, 1)
net.add(ActivationLayer(tanh, tanh_prime))
net.add(
FlattenLayer()) # input_shape=(26, 26, 1) ; output_shape=(1, 26*26*1)
net.add(FCLayer(26 * 26 * 1,
100)) # input_shape=(1, 26*26*1) ; output_shape=(1, 100)
net.add(ActivationLayer(tanh, tanh_prime))
net.add(FCLayer(100, 10)) # input_shape=(1, 100) ; output_shape=(1, 10)
net.add(ActivationLayer(tanh, tanh_prime))
# train on 1000 samples
# as we didn't implemented mini-batch GD, training will be pretty slow if we update at each iteration on 60000 samples...
net.use(mse, mse_prime)
net.fit(x_train[0:1000], y_train[0:1000], epochs=100, learning_rate=0.1)
# test on 3 samples
import numpy as np
from network import Network
from conv_layer import ConvLayer
from activation_layer import ActivationLayer
from activations import tanh, tanh_prime
from losses import mse, mse_prime
# training data
x_train = [np.random.rand(10,10,1)]
y_train = [np.random.rand(4,4,2)]
# network
net = Network()
net.add(ConvLayer((10,10,1), (3,3), 1))
net.add(ActivationLayer(tanh, tanh_prime))
net.add(ConvLayer((8,8,1), (3,3), 1))
net.add(ActivationLayer(tanh, tanh_prime))
net.add(ConvLayer((6,6,1), (3,3), 2))
net.add(ActivationLayer(tanh, tanh_prime))
# train
net.use(mse, mse_prime)
net.fit(x_train, y_train, epochs=1000, learning_rate=0.3)
# test
out = net.predict(x_train)
print("predicted = ", out)
print("expected = ", y_train)
def __init__(self,
corpus,
n_emb,
n_hidden,
batch_size,
conv_size,
pooling,
rng=None,
th_rng=None,
load_from=None,
gensim_w2v=None):
'''
n_hidden: output conv stack size
conv_size: filter height size
'''
self.corpus = corpus
self.n_emb = n_emb
self.n_hidden = n_hidden
self.batch_size = batch_size
self.conv_size = conv_size
self.pooling = pooling
assert pooling in ('mean', 'max')
if rng is None:
rng = np.random.RandomState(1226)
if th_rng is None:
th_rng = RandomStreams(1226)
# x/mask: (batch size, nsteps)
x = T.matrix('x', dtype='int32')
mask = T.matrix('mask', dtype=theano.config.floatX)
y = T.vector('y', dtype='int32')
batch_idx_seq = T.vector('index', dtype='int32')
use_noise = theano.shared(th_floatX(0.))
self.x, self.mask, self.y, self.batch_idx_seq, self.use_noise = x, mask, y, batch_idx_seq, use_noise
# No need for transpose of x/mask in CNN
n_samples, n_steps = x.shape
# transpose mask-matrix to be consistent with pooling-layer-inputs
trans_mask = mask.T
# truncate mask-matrix to be consistent with conv-outputs
trunc_mask = trans_mask[(conv_size - 1):]
# list of model layers
model_layers = []
model_layers.append(
EmbLayer(x,
load_from=load_from,
rand_init_params=(rng, (corpus.dic.size, n_emb)),
gensim_w2v=gensim_w2v,
dic=corpus.dic))
# emb-out: (batch size, n_words/steps, emb_dim)
# conv-in: (batch size, 1(input stack size), n_words/steps, emb_dim)
# conv-out: (batch size, n_hidden(output stack size), output feature map height, 1(output feature map width))
# pooling-in: (output feature map height, batch size, output stack size)
conv_in = model_layers[-1].outputs[:, None, :, :]
model_layers.append(
ConvLayer(conv_in,
image_shape=(batch_size, 1, corpus.maxlen, n_emb),
load_from=load_from,
rand_init_params=(rng, (n_hidden, 1, conv_size, n_emb))))
pooling_in = T.transpose(model_layers[-1].outputs.flatten(3),
axes=(2, 0, 1))
if pooling == 'mean':
model_layers.append(MeanPoolingLayer(pooling_in, trunc_mask))
else:
model_layers.append(MaxPoolingLayer(pooling_in, trunc_mask))
model_layers.append(
DropOutLayer(model_layers[-1].outputs, use_noise, th_rng))
model_layers.append(
HiddenLayer(model_layers[-1].outputs,
activation=T.nnet.softmax,
load_from=load_from,
rand_init_params=(rng, (n_hidden, corpus.n_type))))
self.model_layers = model_layers
model_params = []
for layer in model_layers:
model_params += layer.params
self.pred_prob = model_layers[-1].outputs
self.pred = T.argmax(self.pred_prob, axis=1)
off = 1e-8
self.cost = -T.mean(
T.log(self.pred_prob[T.arange(n_samples), y] + off))
# attributes with `func` suffix is compiled function
self.predict_func = theano.function(inputs=[x, mask],
outputs=self.pred)
self.predict_prob_func = theano.function(inputs=[x, mask],
outputs=self.pred_prob)
grads = T.grad(self.cost, model_params)
self.gr_updates, self.gr_sqr_updates, self.dp_sqr_updates, self.param_updates = ada_updates(
model_params, grads)
def load_population(file_name):
counter = 0
file = open(file_name, 'r')
#read line by line
arr = file.read().splitlines()
pop_size = int(arr[counter])
counter += 1
initial_pop = []
clt = []
for i in range(pop_size):
layer_types = []
layer_shapes = []
layers = []
conv_layer_types = []
num_layers = int(arr[counter])
counter += 1
for j in range(num_layers):
type = arr[counter]
layer_types.append(type)
counter += 1
if type == "conv":
filter_method = arr[counter]
counter += 1
zero_padding = int(arr[counter])
counter += 1
conv_layer_types.append((filter_method, zero_padding))
image_shape = (int(arr[counter]), int(arr[counter + 1]),
int(arr[counter + 2]))
counter += 3
filter_shape = (int(arr[counter]), int(arr[counter + 1]),
int(arr[counter + 2]), int(arr[counter + 3]))
counter += 4
layer_shapes.append([image_shape, filter_shape])
filters = []
for i in range(filter_shape[0]):
weights = np.zeros(filter_shape[1:])
bias = 0
for i in range(filter_shape[1]):
for j in range(filter_shape[2]):
for k in range(filter_shape[3]):
weights[i][j][k] = float(arr[counter])
counter += 1
bias = float(arr[counter])
counter += 1
filters.append(Filter(filter_shape[1:], weights, bias))
clt = conv_layer_types
layers.append(
ConvLayer(image_shape, filter_shape, filter_method,
zero_padding, filters))
elif type == "dense" or type == "soft":
shpe = (int(arr[counter]), int(arr[counter + 1]))
layer_shapes.append([shpe])
counter += 2
weights = np.zeros(shpe)
biases = np.zeros(shpe[0])
for cl in range(shpe[0]):
for pl in range(shpe[1]):
weights[cl][pl] = float(arr[counter])
counter += 1
for cl in range(shpe[0]):
biases[cl] = float(arr[counter])
counter += 1
if type == "dense":
layers.append(DenseLayer(shpe, weights, biases))
elif type == "soft":
layers.append(SoftmaxLayer(shpe, weights, biases))
initial_pop.append(Individual(layer_types, layer_shapes, clt, layers))
population = Population(pop_size, initial_pop[0].get_layer_types(),
initial_pop[0].get_layer_shapes(), clt,
initial_pop)
return population