def vgg_bn():
return [
Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(32),
Activation(tf.nn.relu),
Conv2D([3, 3], 32, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(32),
Activation(tf.nn.relu),
Conv2D([3, 3], 64, [1, 2, 2, 1]),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
Conv2D([3, 3], 128, [1, 2, 2, 1]),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
Conv2D([3, 3], 128, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
Flatten(),
Dense(128),
Activation(tf.sigmoid),
Dropout(0.5),
Dense(10),
Activation(tf.nn.softmax),
]
def model_builder(n_inputs, n_outputs):
model = NeuralNetwork(optimizer=Adam(), loss=CrossEntropy)
model.add(Dense(64, input_shape=(n_inputs, )))
model.add(Activation('relu'))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('linear'))
return model
def vgg_bn():
return [
#1
Conv2D([7, 7], 64, [1, 3, 3, 1]),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
MaxPool([1,4,4,1],[1,1,1,1]),
#2
Convolutional_block(f = 3, filters = [64,64,256],s = 1),
MaxPool([1,5,5,1],[1,1,1,1]),
Dropout(0.5),
Identity_block(f = 3, filters=[64,64,256]),
Dropout(0.5),
Identity_block(f = 3, filters=[64,64,256]),
Dropout(0.5),
MaxPool([1,2,2,1],[1,1,1,1]),
#3
Convolutional_block(f = 3, filters = [128,128,512],s = 2),
Dropout(0.5),
Identity_block(f = 3, filters=[128,128,512]),
Dropout(0.5),
Identity_block(f = 3, filters=[128,128,512]),
Dropout(0.5),
MaxPool([1,2,2,1],[1,1,1,1]),
#4
Convolutional_block(f = 3, filters = [256,256,1024],s = 2),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Identity_block(f = 3, filters=[256,256,1024]),
Flatten(),
Dense(128),
Activation(tf.sigmoid),
Dropout(0.5),
Dense(10),
#Fully_connected(),
Activation(tf.nn.softmax),
]
def __init__(self, device=torch.device('cuda:0')):
super().__init__(device)
self.conv1 = Conv(1, 6, kernel_size=5, noise_std=1e-0, act='ReLU', device=self.device)
self.act1 = Activation('ReLU')
self.pool1 = Pool(2, device=self.device)
self.conv2 = Conv(6, 16, kernel_size=5, noise_std=1e-0, act='ReLU', device=self.device)
self.act2 = Activation('ReLU')
self.pool2 = Pool(2, device=self.device)
self.fc1 = Linear(256, 120, noise_std=1e-0, act='ReLU', device=self.device)
self.act3 = Activation('ReLU')
self.fc2 = Linear(120, 84, noise_std=1e-0, act='ReLU', device=self.device)
self.act4 = Activation('ReLU')
self.fc3 = Linear(84, 10, noise_std=1e-0, act='ReLU', device=self.device)
self.softmax = Activation('Softmax')
self.layers = [self.conv1, self.conv2, self.fc1, self.fc2, self.fc3]
def convModule(flow, filters, dropout, strides=(1, 1), s=(3, 3)): # {{{
if mirroring:
flow = CReLU()(flow)
flow = Convolution(filters, s=s, initialisation=init, initKWArgs=initKWArgs, strides=strides, regFunction=regF, reg=regP)(flow)
if observing and not resNet:
flow = Observation()(flow)
if not mirroring:
flow = Activation('relu')(flow)
if doDropout:
flow = Dropout(dropout)(flow)
return flow # }}}
class DenseNet(Net):
def __init__(self, device=torch.device('cuda:0')):
super().__init__(device)
self.fc1 = Linear(28*28, 50, noise_std=1e-0, device=self.device)
self.act1 = Activation('TanH')
self.fc2 = Linear(50, 10, noise_std=1e-0, device=self.device)
self.softmax = Activation('Softmax')
self.layers = [self.fc1, self.fc2]
def forward(self, input):
input = input.reshape(len(input), -1)
fc_out_1 = self.fc1.forward(input)
act_out_1 = self.act1.forward(fc_out_1)
fc_out_2 = self.fc2.forward(act_out_1)
output = self.softmax.forward(fc_out_2)
return output
def fcModule(flow, w, dropout, regf=regF, reg=regP): # {{{
if mirroring:
flow = CReLU()(flow)
flow = FC(w, initialisation=init, initKWArgs=initKWArgs, reg=regP, regFunction=regF)(flow)
if observing and not resNet:
flow = Observation()(flow)
if not mirroring:
flow = Activation('relu')(flow)
if doDropout:
flow = Dropout(dropout)(flow)
return flow # }}}
class LeNet5(Net):
def __init__(self, device=torch.device('cuda:0')):
super().__init__(device)
self.conv1 = Conv(1, 6, kernel_size=5, noise_std=1e-0, act='ReLU', device=self.device)
self.act1 = Activation('ReLU')
self.pool1 = Pool(2, device=self.device)
self.conv2 = Conv(6, 16, kernel_size=5, noise_std=1e-0, act='ReLU', device=self.device)
self.act2 = Activation('ReLU')
self.pool2 = Pool(2, device=self.device)
self.fc1 = Linear(256, 120, noise_std=1e-0, act='ReLU', device=self.device)
self.act3 = Activation('ReLU')
self.fc2 = Linear(120, 84, noise_std=1e-0, act='ReLU', device=self.device)
self.act4 = Activation('ReLU')
self.fc3 = Linear(84, 10, noise_std=1e-0, act='ReLU', device=self.device)
self.softmax = Activation('Softmax')
self.layers = [self.conv1, self.conv2, self.fc1, self.fc2, self.fc3]
def forward(self, input):
conv_out_1 = self.conv1.forward(input)
act_out_1 = self.act1.forward(conv_out_1)
pool_out_1 = self.pool1.forward(act_out_1)
conv_out_2 = self.conv2.forward(pool_out_1)
act_out_2 = self.act2.forward(conv_out_2)
pool_out_2 = self.pool2.forward(act_out_2)
pool_out_2 = pool_out_2.reshape(len(pool_out_2), -1)
fc_out_1 = self.fc1.forward(pool_out_2)
act_out_3 = self.act3.forward(fc_out_1)
fc_out_2 = self.fc2.forward(act_out_3)
act_out_4 = self.act4.forward(fc_out_2)
fc_out_3 = self.fc3.forward(act_out_4)
output = self.softmax.forward(fc_out_3)
return output
class DenseNet_CNN(Net):
def __init__(self, device=torch.device('cuda:0')):
super().__init__(device)
# self.fc1 = Linear(28*28, 25, noise_std=1e-0, device=self.device)
self.fc1 = Conv(1, 25, kernel_size=25, noise_std=1e-0, act='TanH', device=self.device)
self.act1 = Activation('TanH')
self.fc2 = Linear(16*25, 10, noise_std=1e-0, device=self.device)
self.softmax = Activation('Softmax')
self.layers = [self.fc1, self.fc2]
def forward(self, input):
#input = input.reshape(len(input), -1)
fc_out_1 = self.fc1.forward(input)
act_out_1 = self.act1.forward(fc_out_1)
act_out_1 = act_out_1.reshape(len(act_out_1), -1)
fc_out_2 = self.fc2.forward(act_out_1)
output = self.softmax.forward(fc_out_2)
return output
def __init__(self, device=torch.device('cuda:0')):
super().__init__(device)
self.conv1 = Conv(1, 6, kernel_size=5, noise_std=1e-0, act='TanH', device=self.device)
self.act1 = Activation('TanH')
self.pool1 = Pool(2, device=self.device)
self.fc1 = Linear(6*12*12, 100, noise_std=1e-0, act='TanH', device=self.device)
self.act2 = Activation('TanH')
self.fc2 = Linear(100, 10, noise_std=1e-0, act='TanH', device=self.device)
self.softmax = Activation('Softmax')
self.layers = [self.conv1, self.fc1, self.fc2]
def _deserialize(self, params):
layers_attrs = ['layers']
if params['best_layers_']:
layers_attrs.append('best_layers_')
for layers_attr in layers_attrs:
for i, layer_dict in enumerate(params[layers_attr]):
if layer_dict['layer'] == 'activation':
params[layers_attr][i] = Activation(**layer_dict)
if layer_dict['layer'] == 'dropout':
params[layers_attr][i] = Dropout(**layer_dict)
if layer_dict['layer'] == 'fully_connected':
fc = FullyConnected(**layer_dict)
fc.W = np.asarray(layer_dict['W'])
fc.b = np.asarray(layer_dict['b'])
fc.dW = np.asarray(layer_dict['dW'])
fc.db = np.asarray(layer_dict['db'])
params[layers_attr][i] = fc
return params
def main():
test_id = 1
if test_id == 1:
#----------
# Conv Net
#----------
optimizer = Adam()
data = datasets.load_digits()
X = data.data # （1797, 64)
y = data.target
# Convert to one-hot encoding
y = to_categorical(y.astype("int")) # (n_sample, n_class)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.4,
seed=1)
# Reshape X to (n_samples, channels, height, width)
X_train = X_train.reshape((-1, 1, 8, 8))
X_test = X_test.reshape((-1, 1, 8, 8))
clf = NeuralNetwork(optimizer=optimizer,
loss=CrossEntropy,
validation_data=(X_test, y_test))
clf.add(
Conv2D(n_filters=16,
filter_shape=(3, 3),
stride=1,
input_shape=(1, 8, 8),
padding='same'))
clf.add(Activation('relu'))
clf.add(Dropout(0.25))
clf.add(BatchNormalization())
clf.add(
Conv2D(n_filters=32, filter_shape=(3, 3), stride=1,
padding='same'))
clf.add(Activation('relu'))
clf.add(Dropout(0.25))
clf.add(BatchNormalization())
clf.add(Flatten()) # 展平层
clf.add(Dense(256)) # 全连接层
clf.add(Activation('relu'))
clf.add(Dropout(0.4))
clf.add(BatchNormalization())
clf.add(Dense(10))
clf.add(Activation('softmax'))
print()
clf.summary(name="ConvNet")
train_err, val_err = clf.fit(X_train,
y_train,
n_epochs=50,
batch_size=64)
# Training and validation error plot
n = len(train_err)
training, = plt.plot(range(n), train_err, label="Training Error")
validation, = plt.plot(range(n), val_err, label="Validation Error")
plt.legend(handles=[training, validation])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
_, accuracy = clf.test_on_batch(X_test, y_test)
print("Accuracy:", accuracy)
y_pred = np.argmax(clf.predict(X_test), axis=1)
X_test = X_test.reshape(-1, 8 * 8)
# Reduce dimension to 2D using PCA and plot the results
Plot().plot_in_2d(X_test,
y_pred,
title="Convolutional Neural Network",
accuracy=accuracy,
legend_labels=range(10))
if test_id == 2:
dataset = MultiClassDataset(n_samples=300,
centers=3,
n_features=2,
center_box=(-10.0, 10.0),
cluster_std=1.0,
norm=True,
one_hot=True)
X = dataset.datas
y = dataset.labels
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
seed=1)
clf = NeuralNetwork(optimizer=optimizer,
loss=CrossEntropy,
validation_data=(X_test, y_test))
clf.add(Dense(3))
clf.add(Activation('softmax'))
clf.summary(name="SoftmaxReg")
train_err, val_err = clf.fit(X_train,
y_train,
n_epochs=50,
batch_size=256)
def convert(keras_model, class_map, description="Neural Network Model"):
"""
Convert a keras model to PMML
@model. The keras model object
@class_map. A map in the form {class_id: class_name}
@description. A short description of the model
Returns a DeepNeuralNetwork object which can be exported to PMML
"""
pmml = DeepNetwork(description=description, class_map=class_map)
pmml.keras_model = keras_model
pmml.model_name = keras_model.name
config = keras_model.get_config()
for layer in config['layers']:
layer_class = layer['class_name']
layer_config = layer['config']
layer_inbound_nodes = layer['inbound_nodes']
# Input
if layer_class is "InputLayer":
pmml._append_layer(InputLayer(
name=layer_config['name'],
input_size=layer_config['batch_input_shape'][1:]
))
# Conv2D
elif layer_class is "Conv2D":
pmml._append_layer(Conv2D(
name=layer_config['name'],
channels=layer_config['filters'],
kernel_size=layer_config['kernel_size'],
dilation_rate=layer_config['dilation_rate'],
use_bias=layer_config['use_bias'],
activation=layer_config['activation'],
strides=layer_config['strides'],
padding=layer_config['padding'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# DepthwiseConv2D
elif layer_class is "DepthwiseConv2D":
pmml._append_layer(DepthwiseConv2D(
name=layer_config['name'],
kernel_size=layer_config['kernel_size'],
depth_multiplier=layer_config['depth_multiplier'],
use_bias=layer_config['use_bias'],
activation=layer_config['activation'],
strides=layer_config['strides'],
padding=layer_config['padding'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# MaxPooling
elif layer_class is "MaxPooling2D":
pmml._append_layer(MaxPooling2D(
name=layer_config['name'],
pool_size=layer_config['pool_size'],
strides=layer_config['strides'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "AveragePooling2D":
pmml._append_layer(AveragePooling2D(
name=layer_config['name'],
pool_size=layer_config['pool_size'],
strides=layer_config['strides'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "GlobalAveragePooling2D":
pmml._append_layer(GlobalAveragePooling2D(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Flatten
elif layer_class is "Flatten":
pmml._append_layer(Flatten(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Dense
elif layer_class is "Dense":
pmml._append_layer(Dense(
name=layer_config['name'],
channels=layer_config['units'],
use_bias=layer_config['use_bias'],
activation=layer_config['activation'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Zero padding layer
elif layer_class is "ZeroPadding2D":
pmml._append_layer(ZeroPadding2D(
name=layer_config['name'],
padding=layer_config['padding'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Reshape layer
elif layer_class is "Reshape":
pmml._append_layer(Reshape(
name=layer_config['name'],
target_shape=layer_config['target_shape'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "Dropout":
pmml._append_layer(Dropout(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Batch Normalization
elif layer_class is "BatchNormalization":
pmml._append_layer(BatchNormalization(
name=layer_config['name'],
axis=layer_config['axis'],
momentum=layer_config['momentum'],
epsilon=layer_config['epsilon'],
center=layer_config['center'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "Add":
pmml._append_layer(Merge(
name=layer_config['name'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Subtract":
pmml._append_layer(Merge(
name=layer_config['name'],
operator='subtract',
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Dot":
pmml._append_layer(Merge(
name=layer_config['name'],
operator='dot',
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Concatenate":
pmml._append_layer(Merge(
name=layer_config['name'],
axis=layer_config['axis'],
operator='concatenate',
inbound_nodes=get_inbound_nodes(layer_inbound_nodes)
))
elif layer_class is "Activation":
pmml._append_layer(Activation(
name=layer_config['name'],
activation=layer_config['activation'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
elif layer_class is "ReLU":
pmml._append_layer(Activation(
name=layer_config['name'],
activation='relu',
threshold = layer_config['threshold'],
max_value = layer_config['max_value'],
negative_slope = layer_config['negative_slope'],
inbound_nodes=get_inbound_nodes(layer_inbound_nodes),
))
# Unknown layer
else:
raise ValueError("Unknown layer type:",layer_class)
return pmml
W1 = np.array([[.15, .20], [.25, .30]])
W2 = np.array([[.40, .45], [.50, .55]])
b1 = .35
b2 = 0.60
y_true = np.array([[.01, .99]])
#Layers Generation
dense = Dense(2, W1, b1)
dense2 = Dense(2, W2, b2)
activation1 = Sigmoid()
# activation2=Sigmoid()
activation2 = Activation("sigmoid")
loss_func = MSE()
#Forward Pass
# Dense -> Activation -> Dense -> Activation -> y_pred
z1 = dense.forward(x)
a1 = activation1.forward(z1)
print("Activation Value:", a1)
z2 = dense2.forward(a1)
a2 = activation2.forward(z2)
y_pred = a2
loss = loss_func.loss(y_true, y_pred)
X_train = X_train[:256]
y_train = y_train[:256]
X_train = X_train.reshape((-1, 1, 8, 8))
X_test = X_test.reshape((-1, 1, 8, 8))
X_test = X_train[:256]
y_test = y_train[:256]
# Model
model = NeuralNetwork(SquareLoss(), (X_test, y_test))
model.add(
Conv2D(16,
filter_shape=(3, 3),
stride=1,
input_shape=(1, 8, 8),
padding='same'))
model.add(Activation('relu'))
model.add(Dropout(p=0.2))
model.add(Conv2D(n_filters=32, filter_shape=(3, 3), stride=1, padding='same'))
model.add(Activation('relu'))
model.add(Dropout(p=0.2))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(Dense(10))
model.add(Activation('softmax'))
train_err = model.fit(X_train, y_train, n_epochs=5, batch_size=256)
# print(model.layers[-1])
y = np.zeros([nums, 10, 20], dtype=float)
for i in range(nums):
start = np.random.randint(0, 10)
num_seq = np.arange(start, start + 10)
X[i] = to_categorical(num_seq, n_col=20)
y[i] = np.roll(X[i], -1, axis=0)
y[:, -1, 1] = 1 # Mark endpoint as 1
return X, y
if __name__ == '__main__':
X, y = gen_mult_ser(3000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy)
clf.add(RNN(10, activation="tanh", bptt_trunc=5, input_shape=(10, 61)))
clf.add(Activation('softmax'))
tmp_X = np.argmax(X_train[0], axis=1)
tmp_y = np.argmax(y_train[0], axis=1)
print("Number Series Problem:")
print("X = [" + " ".join(tmp_X.astype("str")) + "]")
print("y = [" + " ".join(tmp_y.astype("str")) + "]")
print()
train_err, _ = clf.fit(X_train, y_train, n_epochs=500, batch_size=512)
y_pred = np.argmax(clf.predict(X_test), axis=2)
y_test = np.argmax(y_test, axis=2)
accuracy = np.mean(accuracy_score(y_test, y_pred))
print(accuracy)
print()
print("Results:")
# Output padding
sys.stdout.write(' ' * 20)
# Allow progress bar to persist if it's complete
if current == total:
sys.stdout.write('\n')
# Flush to standard out
sys.stdout.flush()
# Return the time of the progress update
return time.time()
model = Sequential()
model.add(Input(2))
model.add(Dense(25))
model.add(Activation("relu"))
model.add(Dense(50))
model.add(Activation("relu"))
model.add(Dense(50))
model.add(Activation("relu"))
model.add(Dense(25))
model.add(Activation("relu"))
model.add(Dense(1))
model.add(Activation("sigmoid"))
def initialise_layer_parameters(seed=2):
# random seed initiation
np.random.seed(seed)
# Iterate over the layers of the neural network
def main():
train_x, train_y, valid_x, valid_y, test_x, test_y = get_cifar10('./cifar-10-batches-py/')
labels = unpickle('./cifar-10-batches-py/batches.meta')['label_names']
train_x = train_x.astype(np.float32) / 255.0
valid_x = valid_x.astype(np.float32) / 255.0
test_x = test_x.astype(np.float32) / 255.0
num_epochs = args.epochs
eta = args.lr
batch_size = args.batch_size
# input
x = T.tensor4("x")
y = T.ivector("y")
# test values
# x.tag.test_value = np.random.randn(6, 3, 32, 32).astype(np.float32)
# y.tag.test_value = np.array([1,2,1,4,5]).astype(np.int32)
# x.tag.test_value = x.tag.test_value / x.tag.test_value.max()
# import ipdb; ipdb.set_trace()
# network definition
conv1 = BinaryConv2D(input=x, num_filters=50, input_channels=3, size=3, strides=(1,1), padding=1, name="conv1")
act1 = Activation(input=conv1.output, activation="relu", name="act1")
pool1 = Pool2D(input=act1.output, stride=(2,2), name="pool1")
conv2 = BinaryConv2D(input=pool1.output, num_filters=100, input_channels=50, size=3, strides=(1,1), padding=1, name="conv2")
act2 = Activation(input=conv2.output, activation="relu", name="act2")
pool2 = Pool2D(input=act2.output, stride=(2,2), name="pool2")
conv3 = BinaryConv2D(input=pool2.output, num_filters=200, input_channels=100, size=3, strides=(1,1), padding=1, name="conv3")
act3 = Activation(input=conv3.output, activation="relu", name="act3")
pool3 = Pool2D(input=act3.output, stride=(2,2), name="pool3")
flat = Flatten(input=pool3.output)
fc1 = BinaryDense(input=flat.output, n_in=200*4*4, n_out=500, name="fc1")
act4 = Activation(input=fc1.output, activation="relu", name="act4")
fc2 = BinaryDense(input=act4.output, n_in=500, n_out=10, name="fc2")
softmax = Activation(input=fc2.output, activation="softmax", name="softmax")
# loss
xent = T.nnet.nnet.categorical_crossentropy(softmax.output, y)
cost = xent.mean()
# errors
y_pred = T.argmax(softmax.output, axis=1)
errors = T.mean(T.neq(y, y_pred))
# updates + clipping (+-1)
params = conv1.params + conv2.params + conv3.params + fc1.params + fc2.params
params_bin = conv1.params_bin + conv2.params_bin + conv3.params_bin + fc1.params_bin + fc2.params_bin
grads = [T.grad(cost, param) for param in params_bin] # calculate grad w.r.t binary parameters
updates = []
for p,g in zip(params, grads):
updates.append(
(p, clip_weights(p - eta*g)) #sgd + clipping update
)
# compiling train, predict and test fxns
train = theano.function(
inputs = [x,y],
outputs = cost,
updates = updates
)
predict = theano.function(
inputs = [x],
outputs = y_pred
)
test = theano.function(
inputs = [x,y],
outputs = errors
)
# train
checkpoint = ModelCheckpoint(folder="snapshots")
logger = Logger("logs/{}".format(time()))
for epoch in range(num_epochs):
print "Epoch: ", epoch
print "LR: ", eta
epoch_hist = {"loss": []}
t = tqdm(range(0, len(train_x), batch_size))
for lower in t:
upper = min(len(train_x), lower + batch_size)
loss = train(train_x[lower:upper], train_y[lower:upper].astype(np.int32))
t.set_postfix(loss="{:.2f}".format(float(loss)))
epoch_hist["loss"].append(loss.astype(np.float32))
# epoch loss
average_loss = sum(epoch_hist["loss"])/len(epoch_hist["loss"])
t.set_postfix(loss="{:.2f}".format(float(average_loss)))
logger.log_scalar(
tag="Training Loss",
value= average_loss,
step=epoch
)
# validation accuracy
val_acc = 1.0 - test(valid_x, valid_y.astype(np.int32))
print "Validation Accuracy: ", val_acc
logger.log_scalar(
tag="Validation Accuracy",
value= val_acc,
step=epoch
)
checkpoint.check(val_acc, params)
# Report Results on test set (w/ best val acc file)
best_val_acc_filename = checkpoint.best_val_acc_filename
print "Using ", best_val_acc_filename, " to calculate best test acc."
load_model(path=best_val_acc_filename, params=params)
test_acc = 1.0 - test(test_x, test_y.astype(np.int32))
print "Test accuracy: ",test_acc
def vgg_bn():
return [
#1
Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
#2
Conv2D([3, 3], 64, [1, 1, 1, 1], padding='SAME'),
Conv2DBatchNorm(64),
Activation(tf.nn.relu),
#3
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#4
Conv2D([3, 3], 128, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
#5
Conv2D([3, 3], 128, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(128),
Activation(tf.nn.relu),
#6
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#7
Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(256),
Activation(tf.nn.relu),
#8
Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(256),
Activation(tf.nn.relu),
#9
Conv2D([3, 3], 256, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(256),
Activation(tf.nn.relu),
#10
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#11
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#12
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#13
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#14
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
#15
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#16
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#17
Conv2D([3, 3], 512, [1, 1, 1, 1],padding='SAME'),
Conv2DBatchNorm(512),
Activation(tf.nn.relu),
#18
MaxPool([1,2,2,1],[1,2,2,1],padding='SAME'),
Flatten(),
Dense(4096),
Activation(tf.nn.relu),
Dropout(0.5),
Dense(4096),
Activation(tf.nn.relu),
Dense(1000),
Activation(tf.nn.relu),
Dense(10),
Activation(tf.nn.softmax),
]
def get_brats_nets(input_shape, filters_list, kernel_size_list, dense_size,
nlabels):
inputs = Input(shape=input_shape)
conv = inputs
for filters, kernel_size in zip(filters_list, kernel_size_list):
conv = Conv(filters,
kernel_size=(kernel_size, ) * 3,
activation='relu',
data_format='channels_first')(conv)
full = Conv(dense_size,
kernel_size=(1, 1, 1),
data_format='channels_first',
name='fc_dense',
activation='relu')(conv)
full_roi = Conv(nlabels[0],
kernel_size=(1, 1, 1),
data_format='channels_first',
name='fc_roi')(full)
full_sub = Conv(nlabels[1],
kernel_size=(1, 1, 1),
data_format='channels_first',
name='fc_sub')(full)
rf_roi = Concatenate(axis=1)([conv, full_roi])
rf_sub = Concatenate(axis=1)([conv, full_sub])
rf_num = 1
while np.product(rf_roi.shape[2:]) > 1:
rf_roi = Conv(dense_size,
kernel_size=(3, 3, 3),
data_format='channels_first',
name='rf_roi%d' % rf_num)(rf_roi)
rf_sub = Conv(dense_size,
kernel_size=(3, 3, 3),
data_format='channels_first',
name='rf_sub%d' % rf_num)(rf_sub)
rf_num += 1
full_roi = Reshape((nlabels[0], -1))(full_roi)
full_sub = Reshape((nlabels[1], -1))(full_sub)
full_roi = Permute((2, 1))(full_roi)
full_sub = Permute((2, 1))(full_sub)
full_roi_out = Activation('softmax', name='fc_roi_out')(full_roi)
full_sub_out = Activation('softmax', name='fc_sub_out')(full_sub)
combo_roi = Concatenate(axis=1)([Flatten()(conv), Flatten()(rf_roi)])
combo_sub = Concatenate(axis=1)([Flatten()(conv), Flatten()(rf_sub)])
tumor_roi = Dense(nlabels[0], activation='softmax',
name='tumor_roi')(combo_roi)
tumor_sub = Dense(nlabels[1], activation='softmax',
name='tumor_sub')(combo_sub)
outputs_roi = [tumor_roi, full_roi_out]
net_roi = Model(inputs=inputs,
outputs=outputs_roi,
optimizer='adadelta',
loss='categorical_cross_entropy',
metrics='accuracy')
outputs_sub = [tumor_sub, full_sub_out]
net_sub = Model(inputs=inputs,
outputs=outputs_sub,
optimizer='adadelta',
loss='categorical_cross_entropy',
metrics='accuracy')
return net_roi, net_sub
def main():
train_x, train_y, valid_x, valid_y, test_x, test_y = get_mnist()
num_epochs = args.epochs
eta = args.lr
batch_size = args.batch_size
# input
x = T.matrix("x")
y = T.ivector("y")
#x.tag.test_value = np.random.randn(3, 784).astype("float32")
#y.tag.test_value = np.array([1,2,3])
#drop_switch.tag.test_value = 0
#import ipdb; ipdb.set_trace()
hidden_1 = BinaryDense(input=x, n_in=784, n_out=2048, name="hidden_1")
act_1 = Activation(input=hidden_1.output, activation="relu", name="act_1")
hidden_2 = BinaryDense(input=act_1.output,
n_in=2048,
n_out=2048,
name="hidden_2")
act_2 = Activation(input=hidden_2.output, activation="relu", name="act_2")
hidden_3 = BinaryDense(input=act_2.output,
n_in=2048,
n_out=2048,
name="hidden_3")
act_3 = Activation(input=hidden_3.output, activation="relu", name="act_3")
output = BinaryDense(input=act_3.output,
n_in=2048,
n_out=10,
name="output")
softmax = Activation(input=output.output,
activation="softmax",
name="softmax")
# loss
xent = T.nnet.nnet.categorical_crossentropy(softmax.output, y)
cost = xent.mean()
# errors
y_pred = T.argmax(softmax.output, axis=1)
errors = T.mean(T.neq(y, y_pred))
# updates + clipping (+-1)
params_bin = hidden_1.params_bin + hidden_2.params_bin + hidden_3.params_bin
params = hidden_1.params + hidden_2.params + hidden_3.params
grads = [T.grad(cost, param)
for param in params_bin] # calculate grad w.r.t binary parameters
updates = []
for p, g in zip(
params, grads
): # gradient update on full precision weights (NOT binarized wts)
updates.append((p, clip_weights(p - eta * g)) #sgd + clipping update
)
# compiling train, predict and test fxns
train = theano.function(inputs=[x, y], outputs=cost, updates=updates)
predict = theano.function(inputs=[x], outputs=y_pred)
test = theano.function(inputs=[x, y], outputs=errors)
# train
checkpoint = ModelCheckpoint(folder="snapshots")
logger = Logger("logs/{}".format(time()))
for epoch in range(num_epochs):
print "Epoch: ", epoch
print "LR: ", eta
epoch_hist = {"loss": []}
t = tqdm(range(0, len(train_x), batch_size))
for lower in t:
upper = min(len(train_x), lower + batch_size)
loss = train(train_x[lower:upper],
train_y[lower:upper].astype(np.int32))
t.set_postfix(loss="{:.2f}".format(float(loss)))
epoch_hist["loss"].append(loss.astype(np.float32))
# epoch loss
average_loss = sum(epoch_hist["loss"]) / len(epoch_hist["loss"])
t.set_postfix(loss="{:.2f}".format(float(average_loss)))
logger.log_scalar(tag="Training Loss", value=average_loss, step=epoch)
# validation accuracy
val_acc = 1.0 - test(valid_x, valid_y.astype(np.int32))
print "Validation Accuracy: ", val_acc
logger.log_scalar(tag="Validation Accuracy", value=val_acc, step=epoch)
checkpoint.check(val_acc, params)
# Report Results on test set
best_val_acc_filename = checkpoint.best_val_acc_filename
print "Using ", best_val_acc_filename, " to calculate best test acc."
load_model(path=best_val_acc_filename, params=params)
test_acc = 1.0 - test(test_x, test_y.astype(np.int32))
print "Test accuracy: ", test_acc
def call(self, inputs, *args, **kwargs):
x = inputs
x = super().call(x)
if self.activation:
x = Activation(self.activation)(x)
return tf.nn.depth_to_space(x, self.rate)