def _init_params(self, X=None):
self._dv = {}
self.conv1 = Conv2D(
pad=self.pad1,
init=self.init,
act_fn=self.act_fn,
out_ch=self.out_ch1,
stride=self.stride1,
optimizer=self.optimizer,
kernel_shape=self.kernel_shape1,
)
self.conv2 = Conv2D(
pad=self.pad2,
init=self.init,
out_ch=self.out_ch2,
stride=self.stride2,
optimizer=self.optimizer,
kernel_shape=self.kernel_shape2,
act_fn=Affine(slope=1, intercept=0),
)
# we can't initialize `conv_skip` without X's dimensions; see `forward`
# for further details
self.batchnorm1 = BatchNorm2D(epsilon=self.epsilon,
momentum=self.momentum)
self.batchnorm2 = BatchNorm2D(epsilon=self.epsilon,
momentum=self.momentum)
self.batchnorm_skip = BatchNorm2D(epsilon=self.epsilon,
momentum=self.momentum)
self.add3 = Add(self.act_fn)
def basicConv2Layer():
model = Network()
model.add(Conv2D('conv1', 1, 4, 3, 1, 1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 4, 4, 3, 1, 1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 4 x 7 x 7
model.add(Reshape('flatten', (-1, 196)))
model.add(Linear('fc3', 196, 10, 0.1))
loss = SoftmaxCrossEntropyLoss(name='loss')
return model, loss
def _build_encoder(self):
"""
CNN encoder
Conv1 -> ReLU -> MaxPool1 -> Conv2 -> ReLU -> MaxPool2 ->
Flatten -> FC1 -> ReLU -> FC2
"""
self.encoder = OrderedDict()
self.encoder["Conv1"] = Conv2D(
act_fn=ReLU(),
init=self.init,
pad=self.enc_conv1_pad,
optimizer=self.optimizer,
out_ch=self.enc_conv1_out_ch,
stride=self.enc_conv1_stride,
kernel_shape=self.enc_conv1_kernel_shape,
)
self.encoder["Pool1"] = Pool2D(
mode="max",
optimizer=self.optimizer,
stride=self.enc_pool1_stride,
kernel_shape=self.enc_pool1_kernel_shape,
)
self.encoder["Conv2"] = Conv2D(
act_fn=ReLU(),
init=self.init,
pad=self.enc_conv2_pad,
optimizer=self.optimizer,
out_ch=self.enc_conv2_out_ch,
stride=self.enc_conv2_stride,
kernel_shape=self.enc_conv2_kernel_shape,
)
self.encoder["Pool2"] = Pool2D(
mode="max",
optimizer=self.optimizer,
stride=self.enc_pool2_stride,
kernel_shape=self.enc_pool2_kernel_shape,
)
self.encoder["Flatten3"] = Flatten(optimizer=self.optimizer)
self.encoder["FC4"] = FullyConnected(
n_out=self.latent_dim, act_fn=ReLU(), optimizer=self.optimizer
)
self.encoder["FC5"] = FullyConnected(
n_out=self.T * 2,
optimizer=self.optimizer,
act_fn=Affine(slope=1, intercept=0),
init=self.init,
)
def TEST_conv():
import ipdb
ipdb.set_trace()
x = T.tensor4("x")
x.tag.test_value = np.random.randn(1, 3, 32, 32)
conv2d = Conv2D(x, 1, 3, 3, (1, 1), 1, "conv1")
def LeNet():
model = Network()
model.add(Conv2D('conv1', 1, 6, 5, 2, 1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 6 x 14 x 14
model.add(Conv2D('conv2', 6, 16, 5, 0, 1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 16 x 5 x 5
model.add(Reshape('flatten', (-1, 400)))
model.add(Linear('fc1', 400, 120, 0.1))
model.add(Relu('relu3'))
model.add(Linear('fc2', 120, 84, 0.1))
model.add(Relu('relu4'))
model.add(Linear('fc3', 84, 10, 0.1))
loss = SoftmaxCrossEntropyLoss(name='loss')
return model, loss
def build_policy_and_value_network(self):
data_format = 'NHWC'
#init = tf.truncated_normal_initializer(0, 0.02)
self.state = tf.placeholder(
'float', [None, self.width, self.height, self.hist_len], 'state')
print self.state.get_shape()
l = Conv2D('conv0',
self.state,
out_channel=32,
kernel_shape=8,
stride=4)
l = Conv2D('conv1', l, out_channel=64, kernel_shape=4, stride=2)
l = Conv2D('conv2', l, out_channel=64, kernel_shape=3, stride=1)
l = Linear('l1', l, out_dim=512, nl=tf.nn.relu)
self.logits = tf.nn.softmax(
Linear('fc-pi', l, out_dim=self.num_actions, nl=tf.identity))
self.value = Linear('fc-v', l, 1, nl=tf.identity)
def _init_conv2(self):
self.conv2 = Conv2D(
pad="same",
init=self.init,
out_ch=self.in_ch,
stride=self.stride2,
optimizer=self.optimizer,
kernel_shape=self.kernel_shape2,
act_fn=Affine(slope=1, intercept=0),
)
def _init_conv_skip(self, X):
self._calc_skip_padding(X)
self.conv_skip = Conv2D(
init=self.init,
pad=self.pad_skip,
out_ch=self.out_ch2,
stride=self.stride_skip,
kernel_shape=self.kernel_shape_skip,
act_fn=Affine(slope=1, intercept=0),
optimizer=self.optimizer,
)
def vgg_bn():
return [
Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(32),
Activation(tf.nn.relu),
Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(32),
Activation(tf.nn.relu),
Conv2D([3, 3], 64, [1, 2, 2, 1]),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
Conv2D([3, 3], 128, [1, 2, 2, 1]),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
Conv2D([3, 3], 128, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
Flatten(),
Dense(128),
Activation(tf.sigmoid),
Dropout(0.5),
Dense(10),
Activation(tf.nn.softmax),
]
def mnist_feature_learner(name, images, DIM, dropout=None, is_training=None):
initializer = 'glorot_uniform'
normalizer = 'BN'
output = Conv2D('{}.feature.Conv.1'.format(name), images, DIM, initializer=initializer)
output = tf.nn.max_pool(output, (1,1,2,2), (1,1,2,2), 'VALID', data_format='NCHW')
output = Normalize('{}.feature.NORM.1'.format(name), output, method=normalizer, bn_is_training=is_training)
output = tf.nn.relu(output)
output = Conv2D('{}.feature.Conv.2'.format(name), output, 2*DIM, initializer=initializer)
output = tf.nn.max_pool(output, (1,1,2,2), (1,1,2,2), 'VALID', data_format='NCHW')
output = Normalize('{}.feature.NORM.2'.format(name), output, method=normalizer, bn_is_training=is_training)
output = tf.nn.relu(output)
output = Conv2D('{}.feature.Conv.3'.format(name), output, 4*DIM, initializer=initializer)
output = tf.nn.max_pool(output, (1,1,2,2), (1,1,2,2), 'VALID', data_format='NCHW')
output = Normalize('{}.feature.NORM.3'.format(name), output, method=normalizer, bn_is_training=is_training)
output = tf.nn.relu(output)
output = tf.reduce_mean(output, [2,3])
return output
def add(self, layer_type, **kwargs):
layer = None
# send the layer the output shape of the last layer, or the input shape if it's the first
kwargs['input_shape'] = self._layers[-1].output_shape if len(self._layers) else self._input_shape
if layer_type == 'conv':
layer = Conv2D(**kwargs)
elif layer_type == 'dense':
layer = Dense(**kwargs)
elif layer_type == 'pool':
layer = MaxPooling2D(**kwargs)
elif layer_type == 'flat':
layer = Flatten(**kwargs)
self._layers.append(layer)
def vgg_bn():
return [
#1
Conv2D([7, 7], 64, [1, 3, 3, 1]),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
MaxPool([1,4,4,1],[1,1,1,1]),
#2
Convolutional_block(f = 3, filters = [64,64,256],s = 1),
MaxPool([1,5,5,1],[1,1,1,1]),
Dropout(0.5),
Identity_block(f = 3, filters=[64,64,256]),
Dropout(0.5),
Identity_block(f = 3, filters=[64,64,256]),
Dropout(0.5),
MaxPool([1,2,2,1],[1,1,1,1]),
#3
Convolutional_block(f = 3, filters = [128,128,512],s = 2),
Dropout(0.5),
Identity_block(f = 3, filters=[128,128,512]),
Dropout(0.5),
Identity_block(f = 3, filters=[128,128,512]),
Dropout(0.5),
MaxPool([1,2,2,1],[1,1,1,1]),
#4
Convolutional_block(f = 3, filters = [256,256,1024],s = 2),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Flatten(),
Dense(128),
Activation(tf.sigmoid),
Dropout(0.5),
Dense(10),
#Fully_connected(),
Activation(tf.nn.softmax),
]
def _init_params(self):
self._dv = {}
self.conv1 = Conv2D(
pad="same",
init=self.init,
out_ch=self.out_ch,
act_fn=self.act_fn,
stride=self.stride1,
optimizer=self.optimizer,
kernel_shape=self.kernel_shape1,
)
# we can't initialize `conv2` without X's dimensions; see `forward`
# for further details
self.batchnorm1 = BatchNorm2D(epsilon=self.epsilon,
momentum=self.momentum)
self.batchnorm2 = BatchNorm2D(epsilon=self.epsilon,
momentum=self.momentum)
self.add3 = Add(self.act_fn)
def convert(keras_model, class_map, description="Neural Network Model"):
"""
Convert a keras model to PMML
@model. The keras model object
@class_map. A map in the form {class_id: class_name}
@description. A short description of the model
Returns a DeepNeuralNetwork object which can be exported to PMML
"""
pmml = DeepNetwork(description=description, class_map=class_map)
pmml.keras_model = keras_model
pmml.model_name = keras_model.name
config = keras_model.get_config()
for layer in config['layers']:
layer_class = layer['class_name']
layer_config = layer['config']
layer_inbound_nodes = layer['inbound_nodes']
# Input
if layer_class is "InputLayer":
pmml._append_layer(InputLayer(
name=layer_config['name'],
input_size=layer_config['batch_input_shape'][1:]
))
# Conv2D
elif layer_class is "Conv2D":
pmml._append_layer(Conv2D(
name=layer_config['name'],
channels=layer_config['filters'],
kernel_size=layer_config['kernel_size'],
dilation_rate=layer_config['dilation_rate'],
use_bias=layer_config['use_bias'],
activation=layer_config['activation'],
strides=layer_config['strides'],
padding=layer_config['padding'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# DepthwiseConv2D
elif layer_class is "DepthwiseConv2D":
pmml._append_layer(DepthwiseConv2D(
name=layer_config['name'],
kernel_size=layer_config['kernel_size'],
depth_multiplier=layer_config['depth_multiplier'],
use_bias=layer_config['use_bias'],
activation=layer_config['activation'],
strides=layer_config['strides'],
padding=layer_config['padding'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# MaxPooling
elif layer_class is "MaxPooling2D":
pmml._append_layer(MaxPooling2D(
name=layer_config['name'],
pool_size=layer_config['pool_size'],
strides=layer_config['strides'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "AveragePooling2D":
pmml._append_layer(AveragePooling2D(
name=layer_config['name'],
pool_size=layer_config['pool_size'],
strides=layer_config['strides'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "GlobalAveragePooling2D":
pmml._append_layer(GlobalAveragePooling2D(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Flatten
elif layer_class is "Flatten":
pmml._append_layer(Flatten(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Dense
elif layer_class is "Dense":
pmml._append_layer(Dense(
name=layer_config['name'],
channels=layer_config['units'],
use_bias=layer_config['use_bias'],
activation=layer_config['activation'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Zero padding layer
elif layer_class is "ZeroPadding2D":
pmml._append_layer(ZeroPadding2D(
name=layer_config['name'],
padding=layer_config['padding'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Reshape layer
elif layer_class is "Reshape":
pmml._append_layer(Reshape(
name=layer_config['name'],
target_shape=layer_config['target_shape'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "Dropout":
pmml._append_layer(Dropout(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Batch Normalization
elif layer_class is "BatchNormalization":
pmml._append_layer(BatchNormalization(
name=layer_config['name'],
axis=layer_config['axis'],
momentum=layer_config['momentum'],
epsilon=layer_config['epsilon'],
center=layer_config['center'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "Add":
pmml._append_layer(Merge(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Subtract":
pmml._append_layer(Merge(
name=layer_config['name'],
operator='subtract',
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Dot":
pmml._append_layer(Merge(
name=layer_config['name'],
operator='dot',
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Concatenate":
pmml._append_layer(Merge(
name=layer_config['name'],
axis=layer_config['axis'],
operator='concatenate',
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Activation":
pmml._append_layer(Activation(
name=layer_config['name'],
activation=layer_config['activation'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "ReLU":
pmml._append_layer(Activation(
name=layer_config['name'],
activation='relu',
threshold = layer_config['threshold'],
max_value = layer_config['max_value'],
negative_slope = layer_config['negative_slope'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Unknown layer
else:
raise ValueError("Unknown layer type:",layer_class)
return pmml
from network import Network
from layers import Relu, Linear, Conv2D, AvgPool2D, Reshape
from utils import LOG_INFO
from loss import EuclideanLoss, SoftmaxCrossEntropyLoss
from solve_net import train_net, test_net, get_feature_map
from load_data import load_mnist_4d
import matplotlib.pyplot as plt
train_data, test_data, train_label, test_label = load_mnist_4d('data')
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Conv2D('conv1', 1, 8, 3, 1, 0.01))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 8, 16, 3, 1, 0.01))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 4 x 7 x 7
model.add(Reshape('flatten', (-1, 784)))
model.add(Linear('fc3', 784, 256, 0.01))
model.add(Relu('relu3'))
model.add(Linear('fc4', 256, 10, 0.01))
loss = SoftmaxCrossEntropyLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
plt.imshow(data)
plt.axis('off')
train_data, test_data, train_label, test_label = load_mnist_4d('data')
if len(sys.argv) >= 2:
filename = sys.argv[1]
else:
filename = "test"
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Conv2D('conv1', 1, 4, 3, 1, 1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 4, 4, 3, 1, 1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 4 x 7 x 7
model.add(Reshape('flatten', (-1, 196)))
model.add(Linear('fc3', 196, 10, 0.1))
loss = SoftmaxCrossEntropyLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
from losses import mse, cross_entry
from compute import accuracy, to_categorical, shuffle
from optimizers import SGD, Adam, RMSprpo
x, y = load_mnist('mnist')
x = x / 255.
y = to_categorical(y, 10)
x, y = shuffle(x, y)
x = x[:100]
y = y[:100]
x = np.reshape(x, (-1, 1, 28, 28))
model = Model()
n = 32
act = activation.ReLU
model.addlayer(Conv2D(3, x.shape[1], n, 1, activate=act))
model.addlayer(Pool2D(2))
model.addlayer(Conv2D(3, n, n, 1, activate=act))
model.addlayer(Pool2D(2))
model.addlayer(Conv2D(3, n, n, 2, activate=act))
model.addlayer(Conv2D(3, n, 1, activate=act))
model.addlayer(Flatten())
model.addlayer(Dense(4**2, y.shape[1], activate=activation.Softmax))
# model.addlayer(Dense(x.shape[1], n,activate=act))
# model.addlayer(Dense(n, n,activate=act))
# model.addlayer(Dense(n, n,activate=act))
# model.addlayer(Dense(n, y.shape[1], activate=activation.Softmax))
model.compile(opt=SGD, lr=1e-2, loss=cross_entry)
model.fit(x, y, epoch=50, btsize=10, verbose=2)
# plt.imshow(data)
# plt.axis('off')
train_data, test_data, train_label, test_label = load_mnist_4d('data')
# Your model defintion here
# You should explore different model architecture
# Batch = N x 28 x 28
# Origin CNN
model = Network()
model.add(
Conv2D('conv1',
in_channel=1,
out_channel=4,
kernel_size=3,
pad=1,
init_std=0.01))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', kernel_size=2,
pad=0)) # output shape: N x 4 x 14 x 14
model.add(
Conv2D('conv2',
in_channel=4,
out_channel=4,
kernel_size=3,
pad=1,
init_std=0.01))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', kernel_size=2,
pad=0)) # output shape: N x 4 x 7 x 7
def main():
train_x, train_y, valid_x, valid_y, test_x, test_y = get_cifar10(
'./cifar-10-batches-py/')
labels = unpickle('./cifar-10-batches-py/batches.meta')['label_names']
train_x = train_x.astype(np.float32) / 255.0
valid_x = valid_x.astype(np.float32) / 255.0
test_x = test_x.astype(np.float32) / 255.0
num_epochs = args.epochs
eta = args.lr
batch_size = args.batch_size
# input
x = T.tensor4("x")
y = T.ivector("y")
drop_switch = T.scalar("drop_switch")
# test values
# x.tag.test_value = np.random.randn(6, 3, 32, 32).astype(np.float32)
# y.tag.test_value = np.array([1,2,1,4,5]).astype(np.int32)
# x.tag.test_value = x.tag.test_value / x.tag.test_value.max()
# drop_switch.tag.test_value = 1.0
# import ipdb; ipdb.set_trace()
# network definition
conv1 = Conv2D(input=x,
num_filters=50,
input_channels=3,
size=3,
strides=(1, 1),
padding=1,
name="conv1")
act1 = Activation(input=conv1.output, activation="relu", name="act1")
pool1 = Pool2D(input=act1.output, stride=(2, 2), name="pool1")
conv2 = Conv2D(input=pool1.output,
num_filters=100,
input_channels=50,
size=3,
strides=(1, 1),
padding=1,
name="conv2")
act2 = Activation(input=conv2.output, activation="relu", name="act2")
pool2 = Pool2D(input=act2.output, stride=(2, 2), name="pool2")
conv3 = Conv2D(input=pool2.output,
num_filters=200,
input_channels=100,
size=3,
strides=(1, 1),
padding=1,
name="conv3")
act3 = Activation(input=conv3.output, activation="relu", name="act3")
pool3 = Pool2D(input=act3.output, stride=(2, 2), name="pool3")
flat = Flatten(input=pool3.output)
drop4 = Dropout(input=flat.output, p=0.5, drop_switch=drop_switch)
fc1 = Dense(input=drop4.output, n_in=200 * 4 * 4, n_out=500, name="fc1")
act4 = Activation(input=fc1.output, activation="relu", name="act4")
drop5 = Dropout(input=act4.output, p=0.5, drop_switch=drop_switch)
fc2 = Dense(input=drop5.output, n_in=500, n_out=10, name="fc2")
softmax = Activation(input=fc2.output,
activation="softmax",
name="softmax")
# loss
xent = T.nnet.nnet.categorical_crossentropy(softmax.output, y)
cost = xent.mean()
# errors
y_pred = T.argmax(softmax.output, axis=1)
errors = T.mean(T.neq(y, y_pred))
# updates
params = conv1.params + conv2.params + conv3.params + fc1.params + fc2.params
grads = [T.grad(cost, param) for param in params]
updates = []
for p, g in zip(params, grads):
updates.append((p, p - eta * g) #sgd
)
# compiling train, predict and test fxns
train = theano.function(inputs=[x, y, drop_switch],
outputs=cost,
updates=updates)
predict = theano.function(inputs=[x, drop_switch], outputs=y_pred)
test = theano.function(inputs=[x, y, drop_switch], outputs=errors)
# train
checkpoint = ModelCheckpoint(folder="snapshots")
logger = Logger("logs/{}".format(time()))
for epoch in range(num_epochs):
print "Epoch: ", epoch
print "LR: ", eta
epoch_hist = {"loss": []}
t = tqdm(range(0, len(train_x), batch_size))
for lower in t:
upper = min(len(train_x), lower + batch_size)
loss = train(train_x[lower:upper],
train_y[lower:upper].astype(np.int32), 1.0) # drop
t.set_postfix(loss="{:.2f}".format(float(loss)))
epoch_hist["loss"].append(loss.astype(np.float32))
# epoch loss
average_loss = sum(epoch_hist["loss"]) / len(epoch_hist["loss"])
t.set_postfix(loss="{:.2f}".format(float(average_loss)))
logger.log_scalar(tag="Training Loss", value=average_loss, step=epoch)
# validation accuracy
val_acc = 1.0 - test(valid_x, valid_y.astype(np.int32), 0.0) # nodrop
print "Validation Accuracy: ", val_acc
logger.log_scalar(tag="Validation Accuracy", value=val_acc, step=epoch)
checkpoint.check(val_acc, params)
# Report Results on test set (w/ best val acc file)
best_val_acc_filename = checkpoint.best_val_acc_filename
print "Using ", best_val_acc_filename, " to calculate best test acc."
load_model(path=best_val_acc_filename, params=params)
test_acc = 1.0 - test(test_x, test_y.astype(np.int32), 0.0) # no drop
print "Test accuracy: ", test_acc
def __init__(self,
sizes,
batch_size,
epoch_num,
use_trained_params=False,
filename=None,
img_dim=(3, 32, 32),
conv_param={
'filter_num': 32,
'filter_size': 3,
'padding': 1,
'stride': 1
},
optimizer='Adam',
activation='ReLU',
use_dropout=True,
dropout_p=0.2,
use_bn=True):
self.num_layers = len(sizes)
self.sizes = sizes
self.batch_size = batch_size
self.epoch_num = epoch_num
# self.learning_rate = learning_rate
self.activation = activation
self.use_dropout = use_dropout
self.dropout_p = dropout_p
self.use_bn = use_bn
self.filter_num = conv_param['filter_num']
self.filter_size = conv_param['filter_size']
self.filter_padding = conv_param['padding']
self.filter_stride = conv_param['stride']
self.img_c = img_dim[0]
self.img_wh = img_dim[1]
self.conv_output_size = int(
(img_dim[1] - self.filter_size + 2 * self.filter_padding) /
self.filter_stride) + 1
self.pool_output_size = int(self.filter_num *
(self.conv_output_size / 2) *
(self.conv_output_size / 2))
self.opt = optimizer
optimizers = {
'SGD': SGD,
'Momentum_SGD': Momentum_SGD,
'AdaGrad': AdaGrad,
'RMSProp': RMSProp,
'AdaDelta': AdaDelta,
'Adam': Adam
}
self.optimizer = optimizers[self.opt]()
if use_trained_params:
path = os.path.dirname(os.path.abspath(__file__))
loaded_params = np.load(os.path.join(path, filename))
self.W1 = loaded_params['W1']
self.b1 = loaded_params['b1']
self.W2 = loaded_params['W2']
self.b2 = loaded_params['b2']
self.W3 = loaded_params['W3']
self.b3 = loaded_params['b3']
self.gamma = loaded_params['gamma']
self.beta = loaded_params['beta']
if use_bn:
self.running_mean = loaded_params['running_mean']
self.running_var = loaded_params['running_var']
else:
np.random.seed(12)
# Conv層重み
self.W1 = np.sqrt(1 / sizes[0]) * np.random.randn(
self.filter_num, img_dim[0], self.filter_size,
self.filter_size)
self.b1 = np.sqrt(1 / sizes[0]) * np.random.randn(self.filter_num)
# BatchNorm層
self.gamma = np.ones(self.filter_num * self.conv_output_size *
self.conv_output_size)
self.beta = np.zeros(self.filter_num * self.conv_output_size *
self.conv_output_size)
# FullyConnected層重み（中間層)
self.W2 = np.sqrt(1 / self.pool_output_size) * np.random.randn(
self.pool_output_size, self.sizes[1]) #(pool,100)
self.b2 = np.sqrt(1 / self.pool_output_size) * np.random.randn(
self.sizes[1])
# Fullyconnected層重み(出力層)
self.W3 = np.sqrt(1 / sizes[1]) * np.random.randn(
self.sizes[1], self.sizes[2])
self.b3 = np.sqrt(1 / sizes[1]) * np.random.randn(self.sizes[2])
# layers of network
activation_function = {'Sigmoid': Sigmoid, 'ReLU': ReLU}
self.layers = {}
self.layers['Conv'] = Conv2D(self.W1, self.b1, self.filter_stride,
self.filter_padding)
if self.use_bn:
if use_trained_params:
self.layers['BatchNorm'] = BatchNorm(self.gamma, self.beta,\
running_mean=self.running_mean,running_var=self.running_var)
else:
self.layers['BatchNorm'] = BatchNorm(self.gamma, self.beta)
self.layers['Activation'] = activation_function[self.activation]()
if self.use_dropout:
self.layers['Dropout'] = Dropout(self.dropout_p)
self.layers['Pool'] = MaxPool(pool_h=2, pool_w=2, stride=2)
self.layers['FullyConnected1'] = FullyConnected(self.W2, self.b2)
self.layers['Activation2'] = activation_function[self.activation]()
self.layers['FullyConnected2'] = FullyConnected(self.W3, self.b3)
self.lastLayer = SoftmaxLoss()
def feature_learner(name, images, DIM, dropout=None, is_training=None):
output = images
output = Conv2D('{}.conv.Init'.format(name),
output,
DIM,
initializer='glorot_uniform')
output = ResidualLayer('{}.conv.1.1'.format(name),
output,
DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.1.2'.format(name),
output,
DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.1.3'.format(name),
output,
DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.1.4'.format(name),
output,
2 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.1'.format(name),
output,
2 * DIM,
stride=2,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.2'.format(name),
output,
2 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.3'.format(name),
output,
2 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.4'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.1'.format(name),
output,
4 * DIM,
stride=2,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.2'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.3'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.4'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.1'.format(name),
output,
4 * DIM,
stride=2,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.2'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.3'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.4'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = Normalize('{}.convout.3.NORM'.format(name),
output,
bn_is_training=is_training)
output = tf.nn.relu(output)
output = tf.reduce_mean(output, [2, 3])
return output
from network import Network
from layers import Relu, Linear, Conv2D, AvgPool2D, Reshape
from utils import LOG_INFO
from loss import EuclideanLoss, SoftmaxCrossEntropyLoss
from solve_net import train_net, test_net
from load_data import load_mnist_4d
from plot import show
from solve_net import show4category
train_data, test_data, train_label, test_label = load_mnist_4d('data')
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Conv2D('conv1', 1, 4, 3, 1, 0.01))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 4, 8, 3, 1, 0.01))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 8 x 7 x 7
model.add(Reshape('flatten', (-1, 392)))
model.add(Linear('fc3', 392, 10, 0.01))
loss = SoftmaxCrossEntropyLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
import time
import pickle
import numpy as np
from layers import Dense, Flatten, Conv2D, MaxPooling2D
from loss import binary_crossentropy
from model import Linear
from activation import Sigmoid
from optimizer import gradient_descent
start_time = time.time()
image = np.random.rand(64, 64)
model = Linear()
model.add(Conv2D(64, input_shape=(64, 64)))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(128))
model.add(MaxPooling2D((2, 2)))
model.add(Flatten())
model.add(Dense(128))
model.add(Dense(64))
model.add(Dense(1, activation=Sigmoid, normalize_signal=False))
model.summary()
model.eval(image)
input_data = open("input_data.pickle", "rb")
input_data = np.array(pickle.load(input_data)) / 255.0
label = open("label.pickle", "rb")
label = np.expand_dims(np.array(pickle.load(label)), axis=1)
model.compile(optimizer=gradient_descent, loss=binary_crossentropy)
model.fit(input_data, label, epochs=20, batch_size=16)
def vgg_bn():
return [
#1
Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
#2
Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
#3
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#4
Conv2D([3, 3], 128, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
#5
Conv2D([3, 3], 128, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
#6
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#7
Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(256),
Activation(tf.nn.relu),
#8
Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(256),
Activation(tf.nn.relu),
#9
Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(256),
Activation(tf.nn.relu),
#10
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#11
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#12
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#13
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#14
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#15
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#16
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#17
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#18
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
Flatten(),
Dense(4096),
Activation(tf.nn.relu),
Dropout(0.5),
Dense(4096),
Activation(tf.nn.relu),
Dense(1000),
Activation(tf.nn.relu),
Dense(10),
Activation(tf.nn.softmax),
]
data = data.reshape((n * data.shape[1], n * data.shape[3]) +
data.shape[4:])
cv2.imwrite('try%d.png' % id, data * 255)
# print(data.shape)
# print(data)
# plt.savefig('try%d.pdf'%id)
# plt.imshow(data)
# plt.axis('off')
train_data, test_data, train_label, test_label = load_mnist_4d('data')
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Conv2D('conv1', 1, 12, 3, 1, 1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 12, 10, 3, 1, 1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 4 x 7 x 7
model.add(Reshape('flatten', (-1, 49 * 10)))
model.add(Linear('fc3', 49 * 10, 10, 0.1))
# loss = EuclideanLoss(name='loss')
loss = SoftmaxCrossEntropyLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
from cnn import ConvolutionalNeuralNetwork
from layers import Conv2D, Dense, MaxPooling2D
import numpy as np
x = np.array([[1, 2, 3, 4],
[2, 1, 6, 6],
[1, 5, 4, 3],
[5, 4, 8, 2]])
f = np.array([[2, 3],
[1, 1]])
cnn = ConvolutionalNeuralNetwork()
cnn.add(Conv2D(f, stride=2, activation='relu'))
cnn.fit(x)
def main():
test_id = 1
if test_id == 1:
#----------
# Conv Net
#----------
optimizer = Adam()
data = datasets.load_digits()
X = data.data # （1797, 64)
y = data.target
# Convert to one-hot encoding
y = to_categorical(y.astype("int")) # (n_sample, n_class)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.4,
seed=1)
# Reshape X to (n_samples, channels, height, width)
X_train = X_train.reshape((-1, 1, 8, 8))
X_test = X_test.reshape((-1, 1, 8, 8))
clf = NeuralNetwork(optimizer=optimizer,
loss=CrossEntropy,
validation_data=(X_test, y_test))
clf.add(
Conv2D(n_filters=16,
filter_shape=(3, 3),
stride=1,
input_shape=(1, 8, 8),
padding='same'))
clf.add(Activation('relu'))
clf.add(Dropout(0.25))
clf.add(BatchNormalization())
clf.add(
Conv2D(n_filters=32, filter_shape=(3, 3), stride=1,
padding='same'))
clf.add(Activation('relu'))
clf.add(Dropout(0.25))
clf.add(BatchNormalization())
clf.add(Flatten()) # 展平层
clf.add(Dense(256)) # 全连接层
clf.add(Activation('relu'))
clf.add(Dropout(0.4))
clf.add(BatchNormalization())
clf.add(Dense(10))
clf.add(Activation('softmax'))
print()
clf.summary(name="ConvNet")
train_err, val_err = clf.fit(X_train,
y_train,
n_epochs=50,
batch_size=64)
# Training and validation error plot
n = len(train_err)
training, = plt.plot(range(n), train_err, label="Training Error")
validation, = plt.plot(range(n), val_err, label="Validation Error")
plt.legend(handles=[training, validation])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
_, accuracy = clf.test_on_batch(X_test, y_test)
print("Accuracy:", accuracy)
y_pred = np.argmax(clf.predict(X_test), axis=1)
X_test = X_test.reshape(-1, 8 * 8)
# Reduce dimension to 2D using PCA and plot the results
Plot().plot_in_2d(X_test,
y_pred,
title="Convolutional Neural Network",
accuracy=accuracy,
legend_labels=range(10))
if test_id == 2:
dataset = MultiClassDataset(n_samples=300,
centers=3,
n_features=2,
center_box=(-10.0, 10.0),
cluster_std=1.0,
norm=True,
one_hot=True)
X = dataset.datas
y = dataset.labels
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
seed=1)
clf = NeuralNetwork(optimizer=optimizer,
loss=CrossEntropy,
validation_data=(X_test, y_test))
clf.add(Dense(3))
clf.add(Activation('softmax'))
clf.summary(name="SoftmaxReg")
train_err, val_err = clf.fit(X_train,
y_train,
n_epochs=50,
batch_size=256)
import time
from layers import Dense, Flatten, Conv2D
from model import Linear
import numpy as np
start_time = time.time()
image = np.random.rand(64, 64)
model = Linear()
model.add(Conv2D(64))
model.add(Flatten())
model.add(Dense(512))
model.add(Dense(256))
model.add(Dense(64))
model.summary()
print(model.eval(image))
end_time = time.time()
print(f"Total execution time:: {end_time - start_time}")
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = (x_train.astype('float32') / 255).reshape(-1, 1, 28, 28)
y_train = np_utils.to_categorical(y_train.astype('int32'), 10)
x_test = (x_test.astype('float32') / 255).reshape(-1, 1, 28, 28)
y_test = np_utils.to_categorical(y_test.astype('int32'), 10)
threshold = 1000
x_train = x_train[:threshold]
y_train = y_train[:threshold]
x_test = x_test[:threshold]
y_test = y_test[:threshold]
seed = 15
model = Sequential(seed=seed)
model.add(Conv2D(32, (5, 5), activation="relu",
inputs_shape=x_train.shape[1:]))
model.add(Pooling((2, 2)))
model.add(Conv2D(16, (3, 3), activation="relu"))
model.add(Pooling((2, 2)))
model.add(Dense(10, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation="softmax"))
model.compile(loss="categorical_crossentropy",
optimizer=Adam(),
metric="accuracy")
model.fit(x_train=x_train,
t_train=y_train,
x_test=x_test,
t_test=y_test,
batch_size=128,
# tile the filters into an image
data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3) + tuple(range(4, data.ndim + 1)))
data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])
cv2.imwrite('try%d.png'%id, data*255)
# print(data.shape)
# print(data)
# plt.savefig('try%d.pdf'%id)
# plt.imshow(data)
# plt.axis('off')
train_data, test_data, train_label, test_label = load_mnist_4d('data')
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Conv2D('conv1', 1, 24, 5, 2, 1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 24, 10, 5, 2, 1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 4 x 7 x 7
model.add(Reshape('flatten', (-1, 49*10)))
model.add(Linear('fc3', 49*10, 10, 0.1))
# loss = EuclideanLoss(name='loss')
loss = SoftmaxCrossEntropyLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.