def _inference(self, X, keep_prob, is_train):
dropout_rate = [0.9, 0.8, 0.7, 0.6, 0.5]
layers = [64, 128, 256, 512, 512]
iters = [2, 2, 3, 3]
h = X
# VGG Network Layer
for i in range(4):
for j in range(iters[i]):
with tf.variable_scope('layers%s_%s' % (i, j)) as scope:
h = F.conv(h, layers[i])
h = F.batch_norm(h, is_train)
h = F.activation(h)
h = F.dropout(h, dropout_rate[i], is_train)
h = F.max_pool(h)
# Fully Connected Layer
with tf.variable_scope('fully_connected_layer') as scope:
h = F.dense(h, layers[i + 1])
h = F.batch_norm(h, is_train)
h = F.activation(h)
h = F.dropout(h, dropout_rate[i + 1], is_train)
# Softmax Layer
with tf.variable_scope('softmax_layer') as scope:
h = F.dense(h, self._num_classes)
return h
def _inference(self, CC, MLO, keep_prob, is_train):
layers = [3, 16, 32, 64, 64]
cc = CC
mlo = MLO
for i in range(4):
with tf.variable_scope('CC_layers_%s' % i) as scope:
cc = F.conv(cc, layers[i])
cc = F.batch_norm(cc, is_train)
cc = F.activation(cc)
cc = F.max_pool(cc)
with tf.variable_scope('CC_features') as scope:
cc = F.dense(cc, layers[i + 1])
cc = F.batch_norm(cc, is_train)
cc = F.activation(cc)
for j in range(4):
with tf.variable_scope('MLO_layers_%s' % j) as scope:
mlo = F.conv(mlo, layers[j])
mlo = F.batch_norm(mlo, is_train)
mlo = F.activation(mlo)
mlo = F.max_pool(mlo)
with tf.variable_scope('MLO_features') as scope:
mlo = F.dense(mlo, layers[j + 1])
mlo = F.batch_norm(mlo, is_train)
mlo = F.activation(mlo)
with tf.variable_scope('softmax') as scope:
concat = tf.concat(1, [cc, mlo])
h = F.dense(concat, self._num_classes)
return h
def _inference(self, X, keep_prob):
h = F.max_pool(F.activation(F.conv(X, 64)))
h = F.max_pool(F.activation(F.conv(h, 128)))
h = F.max_pool(F.activation(F.conv(h, 256)))
h = F.activation(F.dense(F.flatten(h), 1024))
h = F.dense(h, self._num_classes)
return tf.nn.softmax(h)
def _inference(self, X, keep_prob):
h = F.max_pool(F.activation(F.conv(X, 64)))
h = F.max_pool(F.activation(F.conv(h, 128)))
h = F.max_pool(F.activation(F.conv(h, 256)))
h = F.activation(F.dense(F.flatten(h), 1024))
h = F.dense(h, self._num_classes)
return h
def _residual(self, h, channels, strides, keep_prob, is_train):
h0 = h
h1 = F.conv(F.activation(F.batch_norm(self, 'bn1', h0, is_train)), channels, strides)
h1 = F.dropout(h1, keep_prob, is_train)
h2 = F.conv(F.activation(F.batch_norm(self, 'bn2', h1, is_train)), channels)
if F.volume(h0) == F.volume(h2):
h = h0 + h2
else :
h4 = F.conv(h0, channels, strides)
h = h2 + h4
return h
def _residual(self, h, channels, strides, keep_prob, is_train):
h0 = h
with tf.variable_scope('residual_first'):
h1 = F.conv(F.activation(F.batch_norm(h0, is_train)), channels, strides)
h1 = F.dropout(h1, keep_prob, is_train)
with tf.variable_scope('residual_second'):
h2 = F.conv(F.activation(F.batch_norm(h1, is_train)), channels)
if F.volume(h0) == F.volume(h2):
h = h0 + h2
else :
h4 = F.conv(h0, channels, strides)
h = h2 + h4
return h
def _residual(self, h, channels, strides, keep_prob):
h0 = h
h1 = F.dropout(
F.conv(F.activation(F.batch_normalization(h0)), channels, strides),
keep_prob)
h2 = F.conv(F.activation(F.batch_normalization(h1)), channels)
# c.f. http://gitxiv.com/comments/7rffyqcPLirEEsmpX
if F.volume(h0) == F.volume(h2):
h = h2 + h0
else:
h4 = F.conv(h0, channels, strides)
h = h2 + h4
return h
def _residual(self, h, channels, strides):
h0 = h
h1 = F.batch_normalization(
F.conv(F.activation(h0), channels, strides, bias_term=False))
h2 = F.batch_normalization(
F.conv(F.activation(h1), channels, bias_term=False))
if F.volume(h0) == F.volume(h2):
h = h2 + h0
else:
h3 = F.avg_pool(h0)
h4 = tf.pad(h3,
[[0, 0], [0, 0], [0, 0], [channels / 4, channels / 4]])
h = h2 + h4
return h
def _residual(self, h, channels, strides):
h0 = h
h1 = F.activation(
F.batch_normalization(
F.conv(h0, channels, strides, bias_term=False)))
h2 = F.batch_normalization(F.conv(h1, channels, bias_term=False))
# c.f. http://gitxiv.com/comments/7rffyqcPLirEEsmpX
if F.volume(h0) == F.volume(h2):
h = h2 + h0
else:
h3 = F.avg_pool(h0)
h4 = tf.pad(h3,
[[0, 0], [0, 0], [0, 0], [channels / 4, channels / 4]])
h = h2 + h4
return F.activation(h)
def forward(conf, X_batch, params, is_training):
"""
Forward propagation through fully connected network.
X_batch:
(batch_size, channels * height * width)
"""
n = conf["layer_dimensions"]
L = len(n) - 1
# Saves the input
A = X_batch
features = {}
features["A_0"] = A
# Loop over each layer in network
for l in range(1, L + 1):
A_prev = A.copy()
Z = np.dot(params["W_" + str(l)].T, A_prev) + params["b_" + str(l)]
# Calculates activation (Relu, or softmax for output)
if l < L:
A = activation(Z.copy(), "relu")
else:
A = softmax(Z.copy())
if is_training:
# Save activations if training
features["Z_" + str(l)] = Z.copy()
features["A_" + str(l)] = A.copy()
# Y_proposed is the probabilities returned by passing
# activations through the softmax function.
Y_proposed = A
return Y_proposed, features
def forward(conf, input_layer, params, is_training=False):
"""
Forward propagation through the Convolutional layer.
input_layer:
(batch_size, channels_x, height_x, width_x)
"""
# Get weights an parameters
weight = params["W_1"]
bias = params["b_1"]
# Get parameters
stride = conf["stride"]
pad_size = conf["pad_size"]
# Padding width and height
(batch_size, channels_x, height_x, width_x) = input_layer.shape
input_padded = np.pad(input_layer,
((0, ), (0, ), (pad_size, ), (pad_size, )),
mode="constant")
(num_filters, channels_w, height_w, width_w) = weight.shape
# Calculate dimensions of output layer and initialize
height_y = 1 + (height_x + 2 * pad_size - height_w) // stride
width_y = 1 + (width_x + 2 * pad_size - width_w) // stride
output_layer = np.zeros((batch_size, num_filters, height_y, width_y))
# Save input layer
A = input_layer
features = {}
features["A_0"] = A
# Forward pass loop in numba
output_layer = forwardloop(
batch_size,
num_filters,
output_layer,
weight,
bias,
input_padded,
channels_x,
width_y,
height_y,
width_w,
height_w,
stride,
)
# Save output to Z
Z = output_layer.copy()
A = functions.activation(Z.copy(), "relu")
if is_training:
# If training, save outputs
features["Z_1"] = Z.copy()
features["A_1"] = A.copy()
return A, features
def _feedforward(self, x):
x_in = x
for layer in self.net:
x_out = []
for node in layer:
active = activation(node['func'])
node["a"] = active(dotprod(node['weights'], x_in))
x_out.append(node["a"])
x_in = x_out # set output as next input
return x_in
def _inference(self, X, keep_prob):
h = X
h = F.activation(F.batch_normalization(F.conv(h, 16, bias_term=False)))
for i in range(self._layers):
h = self._residual(h, channels=16, strides=1)
for channels in [32, 64]:
for i in range(self._layers):
strides = 2 if i == 0 else 1
h = self._residual(h, channels, strides)
h = tf.reduce_mean(h, reduction_indices=[1,
2]) # Global Average Pooling
h = F.dense(h, self._num_classes)
return h
def _inference(self, X, keep_prob, is_train):
h = F.conv(X, 16)
for i in range(self._layers):
with tf.variable_scope(str(16*self._k)+'_layers_%s' %i):
h = self._residual(h, channels=16*self._k, strides=1, keep_prob=keep_prob, is_train=is_train)
for channels in [32*self._k, 64*self._k]:
for i in range(self._layers):
with tf.variable_scope(str(channels)+'_layers_%s' %i):
strides = 2 if i == 0 else 1
h = self._residual(h, channels, strides, keep_prob, is_train)
h = F.activation(F.batch_norm(self, 'bn', h, is_train))
h = tf.reduce_mean(h, reduction_indices=[1,2])
h = F.dense(h, self._num_classes)
return h
def _inference(self, X, keep_prob, is_train):
# Conv_layer 1
conv = F.conv(X, 192)
batch_norm = F._batch_norm(self, 'bn1', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.9, is_train)
conv = F.conv(dropout, 192)
batch_norm = F._batch_norm(self, 'bn2', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.9, is_train)
max_pool = F.max_pool(dropout) # 16 x 16
# Conv_layer 2
conv = F.conv(max_pool, 192)
batch_norm = F._batch_norm(self, 'bn3', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.8, is_train)
conv = F.conv(dropout, 192)
batch_norm = F._batch_norm(self, 'bn4', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.8, is_train)
max_pool = F.max_pool(dropout) # 8 x 8
# Conv_layer 3
conv = F.conv(max_pool, 256)
batch_norm = F._batch_norm(self, 'bn5', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.7, is_train)
conv = F.conv(dropout, 256)
batch_norm = F._batch_norm(self, 'bn6', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.7, is_train)
conv = F.conv(dropout, 256)
batch_norm = F._batch_norm(self, 'bn7', conv, is_train)
dropout = F.dropout(relu, 0.7, is_train)
max_pool = F.max_pool(dropout) # 4 x 4
# Conv_layer 4
conv = F.conv(max_pool, 512)
batch_norm = F._batch_norm(self, 'bn8', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.6, is_train)
conv = F.conv(dropout, 512)
batch_norm = F._batch_norm(self, 'bn9', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.6, is_train)
conv = F.conv(max_pool, 512)
batch_norm = F._batch_norm(self, 'bn10', conv, is_train)
relu = F.activation(batch_norm)
dropout = F.dropout(relu, 0.6, is_train)
max_pool = F.max_pool(dropout) # 2 x 2
# Fully Connected Layer
h = tf.reduce_mean(max_pool, reduction_indices=[1,2])
h = F.dropout(h, 0.5, is_train)
h = F.dense(h, 512)
h = F._batch_norm(self, 'bn11', h, is_train)
h = F.activation(h)
h = F.dropout(h, 0.5, is_train)
h = F.dense(h, self._num_classes)
return h
weights += [random.uniform(0, 1)]
# Initialize arrays to contain historical loss and accuracy for plotting
train_loss = np.zeros(args.numepoch + 1)
valid_loss = np.zeros(args.numepoch + 1)
train_acc = np.zeros(args.numepoch + 1)
valid_acc = np.zeros(args.numepoch + 1)
# Gradient descent method with specified # epochs
for e in range(args.numepoch):
# Get list of sum (Z) values for training and validation data
sums = F.sum_function(weights, train_data, bias)
valid_sums = F.sum_function(weights, valid_data, bias)
# Run summations (Z) through activation function to get array Y
train_act = F.activation(args.actfunction, sums)
valid_act = F.activation(args.actfunction, valid_sums)
# Record loss and accuracy at end of each epoch on training and validation data
train_loss[e] = F.loss(train_act, train_label)
valid_loss[e] = F.loss(valid_act, valid_label)
train_acc[e] = F.accuracy(train_label, train_act)
valid_acc[e] = F.accuracy(valid_label, valid_act)
# Calculate weights gradient for each of 9 weights
weight_grad = []
for w in range(0, 9, 1):
weight_grad += F.weights_gradient(args.actfunction, train_act,
train_data, train_label, w)
