def predict_with_network(input_data, weights):
# Calculate node 0 in the first hidden layer
node_0_0_input = (input_data * weights['node_0_0']).sum()
node_0_0_output = relu(node_0_0_input)
# Calculate node 1 in the first hidden layer
node_0_1_input = (input_data * weights['node_0_1']).sum()
node_0_1_output = relu(node_0_1_input)
# Put node values into array: hidden_0_outputs
hidden_0_outputs = np.array([node_0_0_output, node_0_1_output])
# Calculate node 0 in the second hidden layer
node_1_0_input = (hidden_0_outputs * weights['node_1_0']).sum()
node_1_0_output = relu(node_1_0_input)
# Calculate node 1 in the second hidden layer
node_1_1_input = (hidden_0_outputs * weights['node_1_1']).sum()
node_1_1_output = relu(node_1_1_input)
# Put node values into array: hidden_1_outputs
hidden_1_outputs = np.array([node_1_0_output, node_1_1_output])
# Calculate model output: model_output
model_output = (hidden_1_outputs * weights['output']).sum()
# Return model_output
return (model_output)
def __forward__(self, x):
"""
this method is an implementation of forward propagation with one sample at a time.
Parameters:
x : numpy array (contains one sample of features)
Returns:
zs : list (contains numpy arrays, each array coresponds to sum(xW+b) of respective layer)
activations: list (contains numpy arrays, each array coresponds to output of respective layer)
"""
# demo shapes
l0 = x.T # [1, 784]
z1 = np.dot(l0, self.weights['l1'].T) + self.biases['l1'] # [1, 300] = [1, 784] .* [784, 300] + [1, 300]
l1 = util.relu(z1) # [1, 300]
z2 = np.dot(l1, self.weights['l2'].T) + self.biases['l2'] # [1, 90] = [1, 300] .* [300, 90] + [1, 90]
l2 = util.relu(z2) # [1, 90]
z3 = np.dot(l2, self.weights['l3'].T) + self.biases['l3'] # [1, 10] = [1, 90] .* [90, 10] + [1, 10]
l3 = util.softmax(z3) # [1, 10]
zs = [z1, z2, z3]
activations = [l0, l1, l2, l3]
return zs, activations
def predict(self,input):
activation = [input]
for i in range(self.Nlayers):
act = np.dot(activation[i],self.weights[i]) + self.biases[i]
activation.append(relu(act))
out = np.dot(activation[-1],self.weights[-1]) + self.biases[-1]
activation.append(relu(out))
return activation
def predict(self, input):
activation = [input]
for i in range(self.Nlayers):
act = np.dot(activation[i], self.weights[i]) + self.biases[i]
activation.append(relu(act))
out = np.dot(activation[-1], self.weights[-1]) + self.biases[-1]
activation.append(relu(out))
return activation
def cnn(x):
""" CNN model to detect lung cancer
Args:
x: tensor of shape [batch_size, width, height, channels]
Returns:
pool2: tensor with all convolutions, pooling applied
"""
with tf.name_scope('cnn') as scope:
with tf.name_scope('conv1') as inner_scope:
wcnn1 = tu.weight([3, 3, 1, 64], name='wcnn1')
bcnn1 = tu.bias(1.0, [64], name='bcnn1')
conv1 = tf.add(tu.conv2d(x, wcnn1, stride=(1, 1), padding='SAME'), bcnn1)
conv1 = tu.relu(conv1)
# (?, 192, 192, 64)
with tf.name_scope('conv2') as inner_scope:
wcnn2 = tu.weight([3, 3, 64, 64], name='wcnn2')
bcnn2 = tu.bias(1.0, [64], name='bcnn2')
conv2 = tf.add(tu.conv2d(conv1, wcnn2, stride=(1, 1), padding='SAME'), bcnn2)
conv2 = tu.relu(conv2)
#(?, 192, 192, 64)
with tf.name_scope('max_pool') as inner_scope:
pool1 = tu.max_pool2d(conv2, kernel=[1, 2, 2, 1], stride=[1, 2, 2, 1], padding='SAME')
# (?, 96, 96, 64)
with tf.name_scope('conv3') as inner_scope:
wcnn3 = tu.weight([3, 3, 64, 64], name='wcnn3')
bcnn3 = tu.bias(1.0, [64], name='bcnn3')
conv3 = tf.add(tu.conv2d(pool1, wcnn3, stride=(1, 1), padding='SAME'), bcnn3)
conv3 = tu.relu(conv3)
# (?, 96, 96, 64)
with tf.name_scope('conv4') as inner_scope:
wcnn4 = tu.weight([3, 3, 64, 64], name='wcnn4')
bcnn4 = tu.bias(1.0, [64], name='bcnn4')
conv4 = tf.add(tu.conv2d(conv3, wcnn4, stride=(1, 1), padding='SAME'), bcnn4)
conv4 = tu.relu(conv4)
# (?, 96, 96, 64)
with tf.name_scope('conv5') as inner_scope:
wcnn5 = tu.weight([3, 3, 64, 64], name='wcnn5')
bcnn5 = tu.bias(1.0, [64], name='bcnn5')
conv5 = tf.add(tu.conv2d(conv4, wcnn5, stride=(1, 1), padding='SAME'), bcnn5)
conv5 = tu.relu(conv5)
# (?, 96, 96, 64)
with tf.name_scope('max_pool') as inner_scope:
pool2 = tu.max_pool2d(conv5, kernel=[1, 2, 2, 1], stride=[1, 2, 2, 1], padding='SAME')
# (?, 48, 48, 64)
return pool2
def predict(self,input):
L = np.shape(input)[0]
#output = np.zeros((L,self.O))
a1 = relu(np.dot(input,self.W1) + self.b1)
a2 = np.zeros((L,self.H))
a2prev = np.zeros((1,self.H))
for i in range(L):
a2[i,:] = relu(np.dot(a1[i,:],self.W2) + np.dot(a2prev,self.WF) + self.b2)
a2prev = a2[i,:]
out = np.exp(np.dot(a2,self.W3) + self.b3)
output = out.T / (np.sum(out,1)+ 3.5e-15)
return output.T
def predict(self, input):
L = np.shape(input)[0]
#output = np.zeros((L,self.O))
a1 = relu(np.dot(input, self.W1) + self.b1)
a2 = np.zeros((L, self.H))
a2prev = np.zeros((1, self.H))
for i in range(L):
a2[i, :] = relu(
np.dot(a1[i, :], self.W2) + np.dot(a2prev, self.WF) + self.b2)
a2prev = a2[i, :]
out = np.exp(np.dot(a2, self.W3) + self.b3)
output = out.T / (np.sum(out, 1) + 3.5e-15)
return output.T
def linear_activation_forward(A_prev, W, b, activation):
"""
Implement the forward propagation for the LINEAR->ACTIVATION layer
Arguments:
A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)
W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)
b -- bias vector, numpy array of shape (size of the current layer, 1)
activation -- the activation to be used in this layer, stored as a text string: "sigmoid" or "relu"
Returns:
A -- the output of the activation function, also called the post-activation value
cache -- a python dictionary containing "linear_cache" and "activation_cache";
stored for computing the backward pass efficiently
"""
if activation == "sigmoid":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = sigmoid(Z)
elif activation == "relu":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = relu(Z)
assert (A.shape == (W.shape[0], A_prev.shape[1]))
cache = (linear_cache, activation_cache)
return A, cache
def forward(self):
temp = np.vstack((np.ones((1, self._input.shape[1])), self._input))
self._forward_cache_acted = [temp]
self._forward_cache_raw = [temp]
if not self._constructed:
print("use the build method before forwarding.")
assert 0
times = len(self._layers) - 1
for i in range(times):
# temp = np.vstack((np.ones((1, self._input.shape[1])), temp))
temp = np.dot(self._weights[i], temp)
self._forward_cache_raw.append(temp)
if not self._activations[i]:
pass
elif self._activations[i].lower() == 'sigmoid':
temp = util.sigmoid(temp)
elif self._activations[i].lower() == 'tanh':
temp = util.tanh(temp)
elif self._activations[i].lower() == 'relu':
temp = util.relu(temp)
else:
print(
"Activation function should be None, 'sigmoid', 'tanh' or 'relu'."
)
assert 0
self._forward_cache_acted.append(temp)
self._predictions = temp
return temp
def konvolucija(self, slika: np.ndarray,
konvolucijski_filteri: np.ndarray) -> np.ndarray:
'''
Odrađuje proces konvolucije i primjenjuje aktivacijsku funkciju nad slikom.
Kreira n broj filtera ovisno o broju konvolucijskih filtera koje dobiva kao argument.
'''
mapa_znacajki = np.zeros(
(slika.shape[0] - konvolucijski_filteri.shape[1] + 1,
slika.shape[1] - konvolucijski_filteri.shape[1] + 1,
konvolucijski_filteri.shape[0]))
for broj_filtera in range(konvolucijski_filteri.shape[0]):
trenutni_filter = konvolucijski_filteri[
broj_filtera, :] # dohvačanje filtera
# primjene se filteri nad svakom sliku u podatkovnom skupu
if len(slika.shape) > 2:
konvolucijska_mapa = self.trazenje_znacajki(
slika[:, :, 0], trenutni_filter)
for broj_kanala in range(1, slika.shape[-1]):
konvolucijska_mapa = konvolucijska_mapa + self.trazenje_znacajki(
slika[:, :, broj_kanala], trenutni_filter)
else:
konvolucijska_mapa = self.trazenje_znacajki(
slika, trenutni_filter)
mapa_znacajki[:, :, broj_filtera] = konvolucijska_mapa
# primjena aktivacijske funkcije
return relu(mapa_znacajki)
def forward(self, X):
Z = X.dot(self.W1) + self.b1
Z = relu(Z)
A = Z.dot(self.W2) + self.b2
A = sigmoid(A)
return A, Z
def predict_with_network(input_data_row, weights): # Calculate node 0 value
node_0_input = (input_data_row * weights['node_0']).sum()
node_0_output = relu(node_0_input)
# Calculate node 1 value
node_1_input = (input_data_row * weights['node_1']).sum()
node_1_output = relu(node_1_input)
# Put node values into array: hidden_layer_outputs
hidden_layer_outputs = np.array([node_0_output, node_1_output])
# Calculate model output
input_to_final_layer = (hidden_layer_outputs * weights['output']).sum()
model_output = relu(input_to_final_layer)
# Return model output
return (model_output)
def classifier(x, dropout):
"""
AlexNet fully connected layers definition
Args:
x: tensor of shape [batch_size, width, height, channels]
dropout: probability of non dropping out units
Returns:
fc3: 1000 linear tensor taken just before applying the softmax operation
it is needed to feed it to tf.softmax_cross_entropy_with_logits()
softmax: 1000 linear tensor representing the output probabilities of the image to classify
"""
pool5 = alexnet(x)
dim = pool5.get_shape().as_list()
flat_dim = dim[1] * dim[2] * dim[3] # 6 * 6 * 256
flat = tf.reshape(pool5, [-1, flat_dim])
with tf.name_scope('classifier') as scope:
with tf.name_scope('fullyconected1') as inner_scope:
wfc1 = tu.weight([flat_dim, 4096], name='wfc1')
bfc1 = tu.bias(0.0, [4096], name='bfc1')
fc1 = tf.add(tf.matmul(flat, wfc1), bfc1)
#fc1 = tu.batch_norm(fc1)
fc1 = tu.relu(fc1)
fc1 = tf.nn.dropout(fc1, dropout)
with tf.name_scope('fullyconected2') as inner_scope:
wfc2 = tu.weight([4096, 4096], name='wfc2')
bfc2 = tu.bias(0.0, [4096], name='bfc2')
fc2 = tf.add(tf.matmul(fc1, wfc2), bfc2)
#fc2 = tu.batch_norm(fc2)
fc2 = tu.relu(fc2)
fc2 = tf.nn.dropout(fc2, dropout)
with tf.name_scope('classifier_output') as inner_scope:
wfc3 = tu.weight([4096, 1000], name='wfc3')
bfc3 = tu.bias(0.0, [1000], name='bfc3')
fc3 = tf.add(tf.matmul(fc2, wfc3), bfc3)
softmax = tf.nn.softmax(fc3)
return fc3, softmax
def classifier(x, dropout):
"""cnn fully connected layers definition
Args:
x: tensor of shape [batch_size, width, height, channels]
dropout: probability of non dropping out units
Returns:
fc3: 2 linear tensor taken just before applying the softmax operation
it is needed to feed it to tf.softmax_cross_entropy_with_logits()
softmax: 2 linear tensor representing the output probabilities of the image to classify
"""
pool2 = cnn(x)
dim = pool2.get_shape().as_list()
flat_dim = dim[1] * dim[2] * dim[3] # 48 * 48 * 64
flat = tf.reshape(pool2, [-1, flat_dim])
with tf.name_scope('classifier') as scope:
with tf.name_scope('fullyconected1') as inner_scope:
wfc1 = tu.weight([flat_dim, 500], name='wfc1')
bfc1 = tu.bias(1.0, [500], name='bfc1')
fc1 = tf.add(tf.matmul(flat, wfc1), bfc1)
fc1 = tu.relu(fc1)
fc1 = tf.nn.dropout(fc1, dropout)
with tf.name_scope('fullyconected2') as inner_scope:
wfc2 = tu.weight([500, 100], name='wfc2')
bfc2 = tu.bias(1.0, [100], name='bfc2')
fc2 = tf.add(tf.matmul(fc1, wfc2), bfc2)
fc2 = tu.relu(fc2)
fc2 = tf.nn.dropout(fc2, dropout)
with tf.name_scope('classifier_output') as inner_scope:
wfc3 = tu.weight([100, 2], name='wfc3')
bfc3 = tu.bias(1.0, [2], name='bfc3')
fc3 = tf.add(tf.matmul(fc2, wfc3), bfc3)
softmax = tf.nn.softmax(fc3)
return fc3, softmax
def __backward__(self, zs, activations, y):
"""
this method is an implementation of backpropagation with one sample at a time.
Parameters:
zs : list (contains numpy arrays, each array coresponds to sum(xW+b) of respective layer)
activations: list (contains numpy arrays, each array coresponds to output of respective layer)
y : numpy array (contains one sample of target values)
Returns:
l3_error : numpy array (contains error of last layer 3 (or) network error)
new_weights: numpy array (contains numpy arrays, each array coresponds to new weights of respective layer)
new_biases : numpy array (contains numpy arrays, each array coresponds to new biases of respective layer)
"""
l0, l1, l2, l3 = activations[0], activations[1], activations[2], activations[3]
z1, z2 = zs[0], zs[1]
# calculating loss of network (or) layer 3 # demo shapes
l3_error = l3 - y.T # [1, 10] = [1, 10] - [1, 10]
# calculating layer3 weights and biases
l3_new_biases = l3_error # [1, 10] = [1, 10] * [1, 10]
l3_new_weights = l3_error.T.dot(l2) # [10, 90] = [10, 1] * [1,90]
# calculating layer2 weights and biases
l2_error = l3_error.dot(self.weights['l3']) # [1, 90] = [1, 10] * [10, 90]
l2_error = np.multiply(l2_error, util.relu(z2, derivative=True)) # [1, 90] = [1, 90] * [1,90]
l2_new_biases = l2_error
l2_new_weights = l2_error.T.dot(l1) # [90, 300] = [90, 1] * [1, 300]
# calculating layer1 weights and biases
l1_error = l2_error.dot(self.weights['l2']) # [1, 300] = [1, 90] * [90, 300]
l1_error = np.multiply(l1_error, util.relu(z1, derivative=True)) # [1, 300] = [1, 300] * [1, 300]
l1_new_biases = l1_error
l1_new_weights = l1_error.T.dot(l0) # [300, 784] = [300, 1] * [1, 784]
new_weights = [l3_new_weights, l2_new_weights, l1_new_weights]
new_biases = [l3_new_biases, l2_new_biases, l1_new_biases]
return l3_error, new_weights, new_biases
def the_rectified_activation_function():
weights = {
'node_0': np.array([2, 4]),
'node_1': np.array([4, -5]),
'output': np.array([2, 7])
}
input_data = np.array([3, 5])
# Calculate node 0 value: node_0_output
node_0_input = (input_data * weights['node_0']).sum()
node_0_output = relu(node_0_input)
# Calculate node 1 value: node_1_output
node_1_input = (input_data * weights['node_1']).sum()
node_1_output = relu(node_1_input)
# Put node values into array: hidden_layer_outputs
hidden_layer_outputs = np.array([node_0_output, node_1_output])
# Calculate model output (do not apply relu)
model_output = (hidden_layer_outputs * weights['output']).sum()
# Print model output
print(model_output)
def feedforward(self, img):
N, k1_height, _ = self.k1.shape
C1_height = int(
(img.shape[0] - k1_height + 2 * self.padding) / self.stride) + 1
# Initialize C
C1 = np.zeros((N, C1_height, C1_height), dtype=np.float64)
dC1S1 = np.zeros(C1.shape, dtype=np.float64)
for n in range(N):
for i in range(C1_height):
for j in range(C1_height):
region = img[i:(i + k1_height), j:(j + k1_height)]
S1_nij = np.sum(region * self.k1[n]) + self.b1[n]
C1[n, i, j] = relu(S1_nij)
dC1S1[n, i, j] = 1 if S1_nij > 0 else 0
return C1, dC1S1
def feedforward(self, P1):
M, N, k2_height, _ = self.k2.shape
C2_height = int(
(P1.shape[1] - k2_height + 2 * self.padding) / self.stride) + 1
C2 = np.zeros((M, C2_height, C2_height), dtype=np.float64)
dC2S2 = np.zeros(C2.shape, dtype=np.float64)
dS2P1 = np.zeros(P1.shape + C2.shape, dtype=np.float64)
for m in range(M):
for u in range(C2_height):
for v in range(C2_height):
region = P1[0:N, u:(u + k2_height), v:(v + k2_height)]
S2_muv = np.sum(region * self.k2[m]) + self.b2[m]
C2[m, u, v] = relu(S2_muv)
dC2S2[m, u, v] = 1 if S2_muv > 0 else 0
dS2P1[0:N, u:(u + k2_height), v:(v + k2_height), m, u,
v] = self.k2[m]
return C2, dC2S2, dS2P1
def guranje_naprijed(
self, podaci: np.ndarray, oznaka: np.ndarray,
velicina_skupa: int) -> Tuple[np.ndarray, np.ndarray, float]:
'''
Generira težinske faktore i pristranost. Nad izravnatim slojem odrađuje guranje prema naprijed.
'''
if self.tf_ss is None:
self.generiraj_tezinske_faktore(podaci.shape[0])
skriveni_sloj: np.ndarray = relu(
np.dot(self.tf_ss, podaci) + self.o_ss)
izlazni_sloj: np.ndarray = softmax(
np.dot(self.tf_is, skriveni_sloj) + self.o_is)
gubitak: float = self.krizna_entropija(izlazni_sloj, oznaka,
velicina_skupa)
print()
print(f'Predviđena vrijednost: ')
print(izlazni_sloj)
print(f'Prava vrijednost: ')
print(oznaka)
print(f'Gubitak: {gubitak}')
return skriveni_sloj, izlazni_sloj, gubitak
def forward(self, X):
return relu(X.dot(self.W) + self.b)
def predict1step(self,input,a2prev):
a1 = relu(np.dot(input,self.W1) + self.b1)
a2 = relu(np.dot(a1,self.W2) + np.dot(a2prev,self.WF) + self.b2)
out = np.exp(np.dot(a2,self.W3) + self.b3)
output = out.T / (np.sum(out,1)+ 3.5e-15)
return output.T, a2
def main():
#Get train and test data
XTrain, YTrain = get_digit_train_data()
YTrain_ind = y2indicator(YTrain)
XTest = get_digit_test_data()
N,K = YTrain_ind.shape
M=300
lr = np.float32(0.0001)
reg = np.float32(0.01)
mu = np.float32(0.99)
poolsize = (2,2)
batch_sz = 500
no_batches = int(N/batch_sz)
#Initial random weight values
W1_shape = (20, 1, 5, 5)
W1_init = init_filter(W1_shape, poolsize)
b1_init = np.zeros([W1_shape[0]])
W2_shape = (50, 20, 5, 5)
W2_init = init_filter(W2_shape, poolsize)
b2_init = np.zeros([W2_shape[0]])
W3_init = np.random.randn(W2_shape[0]*4*4, M)/np.sqrt(W2_shape[0]*4*4 + M)
b3_init = np.zeros([M])
W4_init = np.random.randn(M,K)/np.sqrt(M+K)
b4_init = np.zeros([K])
#Create theano variables
X = T.tensor4('X', dtype='float32') #inputs
Y = T.matrix('Y')
W1 = theano.shared(W1_init.astype(np.float32), 'W1') #Weights
b1 = theano.shared(b1_init.astype(np.float32), 'b1')
W2 = theano.shared(W2_init.astype(np.float32), 'W2')
b2 = theano.shared(b2_init.astype(np.float32), 'b2')
W3 = theano.shared(W3_init.astype(np.float32), 'W3')
b3 = theano.shared(b3_init.astype(np.float32), 'b3')
W4 = theano.shared(W4_init.astype(np.float32), 'W4')
b4 = theano.shared(b4_init.astype(np.float32), 'b4')
# dW1 = theano.shared(np.zeros(W1_init.shape, dtype=np.float32)) #Momentum variables
# db1 = theano.shared(np.zeros(b1_init.shape, dtype=np.float32))
# dW2 = theano.shared(np.zeros(W2_init.shape, dtype=np.float32))
# db2 = theano.shared(np.zeros(b2_init.shape, dtype=np.float32))
# dW3 = theano.shared(np.zeros(W3_init.shape, dtype=np.float32))
# db3 = theano.shared(np.zeros(b3_init.shape, dtype=np.float32))
# dW4 = theano.shared(np.zeros(W4_init.shape, dtype=np.float32))
# db4 = theano.shared(np.zeros(b4_init.shape, dtype=np.float32))
#Forward prop equations
Z1 = convpool(X, W1, b1) #2 Conv-pool layer
Z2 = convpool(Z1, W2, b2)
Z3 = relu(Z2.flatten(ndim=2).dot(W3) + b3) #Fully connected NN
P = T.nnet.softmax(Z3.dot(W4) + b4)
#Cost and prediction equations
# params = (W1, b1, W2, b2, W3, b3, W4, b4)
# reg_cost = reg*np.sum([(param*param).sum() for param in params])
cost = (Y * T.log(P)).sum() #+ reg_cost
pred = T.argmax(P, axis=1)
#Update Weights
W1_update = W1 + lr*T.grad(cost, W1)
b1_update = b1 + lr*T.grad(cost,b1)
W2_update = W2 + lr*T.grad(cost, W2)
b2_update = b2 + lr*T.grad(cost,b2)
W3_update = W3 + lr*T.grad(cost, W3)
b3_update = b3 + lr*T.grad(cost,b3)
W4_update = W4 + lr*T.grad(cost, W4)
b4_update = b4 + lr*T.grad(cost,b4)
#Gradient updates for momentum
# dW1_update = mu*dW1 + lr*T.grad(cost, W1)
# db1_update = mu*db1 + lr*T.grad(cost, b1)
# dW2_update = mu*dW2 + lr*T.grad(cost, W2)
# db2_update = mu*db2 + lr*T.grad(cost, b2)
# dW3_update = mu*dW3 + lr*T.grad(cost, W3)
# db3_update = mu*db3 + lr*T.grad(cost, b3)
# dW4_update = mu*dW4 + lr*T.grad(cost, W4)
# db4_update = mu*db4 + lr*T.grad(cost, b4)
#Train function
train = theano.function(
inputs=[X,Y],
updates=[ (W1, W1_update),
(b1, b1_update),
(W2, W2_update),
(b2, b2_update),
(W3, W3_update),
(b3, b3_update),
(W4, W4_update),
(b4, b4_update),
# (dW1, dW1_update),
# (db1, db1_update),
# (dW2, dW2_update),
# (db2, db2_update),
# (dW3, dW3_update),
# (db3, db3_update),
# (dW4, dW4_update),
# (db4, db4_update),
])
#Get cost and prediction function
get_res = theano.function(
inputs=[X,Y],
outputs=[cost,pred])
get_prediction = theano.function(
inputs=[X],
outputs=[pred])
#Run batch gradient descent
costs = []
for i in range(210):
for n in range(no_batches):
#get current batches
XBatch = XTrain[n*batch_sz:(n*batch_sz + batch_sz), :]
YBatch_ind = YTrain_ind[n*batch_sz:(n*batch_sz + batch_sz), :]
#Forward prop
train(XBatch, YBatch_ind)
if(n%200 == 0):
YBatch = YTrain[n*batch_sz:(n*batch_sz + batch_sz)]
c, P = get_res(XBatch, YBatch_ind)
er = error_rate(P, YBatch)
print("Iteration: ", i, "Cost: ", c, "Error rate: ", er)
#Write test result to csv file
pY = get_prediction(XTest)
N = XTest.shape[0]
f = open("Result.csv","w")
f.write("ImageId,Label\n")
for n in range(N):
f.write(str(n+1) + "," + str(pY[0][n]) + "\n")
f.close()
def convpool(X, W, b, poolsz=(2,2)):
conv = conv2d(X,W) #Convolution
max_pool = pool.pool_2d(conv, ws=poolsz, ignore_border=True) #Max-pooling
return relu(max_pool +b.dimshuffle('x', 0, 'x', 'x')) #Non-linearity
def alexnet(x):
"""
AlexNet conv layers definition
Args:
x: tensor of shape[batch_size,width,height,channels]
Returns:
pool5: tensor with all convolutions ,pooling and lrn operations applied
"""
with tf.name_scope('alexnetwork') as scope:
with tf.name_scope('conv1') as inner_scope:
wcnn1 = tu.weight([11, 11, 3, 96], name='wcnn1')
bcnn1 = tu.bias(0.0, [96], name='bcnn1')
conv1 = tf.add(tu.conv2d(x, wcnn1, stride=(4, 4), padding='SAME'),
bcnn1)
#conv1 = tu.batch_norm(conv1)
conv1 = tu.relu(conv1)
norm1 = tu.lrn(conv1,
depth_radius=5,
bias=1.0,
alpha=1e-04,
beta=0.75)
pool1 = tu.max_pool2d(norm1,
kernel=[1, 3, 3, 1],
stride=[1, 2, 2, 1],
padding='VALID')
with tf.name_scope('conv2') as inner_scope:
wcnn2 = tu.weights([5, 5, 96, 256], name='wcnn2')
bcnn2 = tu.bias(1.0, [256], name='bcnn2')
conv2 = tf.add(
tu.conv2d(pool1, wcnn2, stride=(1, 1), padding='SAME'), bcnn2)
#conv2 = tu.batch_norm(conv2)
conv2 = tu.relu(conv2)
norm2 = tu.lrn(conv2,
depth_radius=5,
bias=1.0,
alpha=1e-04,
beta=0.75)
pool2 = tu.max_pool2d(norm2,
kernel=[1, 3, 3, 1],
stride=[1, 2, 2, 1],
padding='VALID')
with tf.name_scope('conv3') as inner_scope:
wcnn3 = tu.weights([3, 3, 256, 384], name='wcnn3')
bcnn3 = tu.bias(0.0, [384], name='bcnn3')
conv3 = tf.add(
tu.conv2d(pool2, wcnn3, stride=(1, 1), padding='SAME'), bcnn3)
#conv3 = tu.batch_norm(conv3)
conv3 = tu.relu(conv3)
with tf.name_scope('conv4') as inner_scope:
wcnn4 = tu.weight([3, 3, 384, 384], name='wcnn4')
bcnn4 = tu.bias(1.0, [384], name='bcnn4')
conv4 = tf.add(
tu.conv2d(conv3, wcnn5, stride=(1, 1), padding='SAME'), bcnn5)
#conv5 = tu.batch_norm(conv5)
conv5 = tu.relu(conv5)
pool5 = tu.max_pool2d(conv5,
kernel=[1, 3, 3, 1],
stride=[1, 2, 2, 1],
padding='VALID')
return pool5
def forward(self, X):
if self.activation_func == 'relu':
Z = relu(X.dot(self.W1) + self.b1)
elif self.activation_func == 'tanh':
Z = np.tanh(X.dot(self.W1) + self.b1)
return sigmoid(Z.dot(self.W2)), Z
def predict1step(self, input, a2prev):
a1 = relu(np.dot(input, self.W1) + self.b1)
a2 = relu(np.dot(a1, self.W2) + np.dot(a2prev, self.WF) + self.b2)
out = np.exp(np.dot(a2, self.W3) + self.b3)
output = out.T / (np.sum(out, 1) + 3.5e-15)
return output.T, a2
def cnn(x):
""" CNN model to detect lung cancer
Args:
x: tensor of shape [batch_size, width, height, channels]
Returns:
pool2: tensor with all convolutions, pooling applied
"""
with tf.name_scope('cnn') as scope:
with tf.name_scope('conv1') as inner_scope:
wcnn1 = tu.weight([3, 3, 1, 64], name='wcnn1')
bcnn1 = tu.bias(1.0, [64], name='bcnn1')
conv1 = tf.add(tu.conv2d(x, wcnn1, stride=(1, 1), padding='SAME'),
bcnn1)
conv1 = tu.relu(conv1)
# (?, 192, 192, 64)
with tf.name_scope('conv2') as inner_scope:
wcnn2 = tu.weight([3, 3, 64, 64], name='wcnn2')
bcnn2 = tu.bias(1.0, [64], name='bcnn2')
conv2 = tf.add(
tu.conv2d(conv1, wcnn2, stride=(1, 1), padding='SAME'), bcnn2)
conv2 = tu.relu(conv2)
#(?, 192, 192, 64)
with tf.name_scope('max_pool') as inner_scope:
pool1 = tu.max_pool2d(conv2,
kernel=[1, 2, 2, 1],
stride=[1, 2, 2, 1],
padding='SAME')
# (?, 96, 96, 64)
with tf.name_scope('conv3') as inner_scope:
wcnn3 = tu.weight([3, 3, 64, 64], name='wcnn3')
bcnn3 = tu.bias(1.0, [64], name='bcnn3')
conv3 = tf.add(
tu.conv2d(pool1, wcnn3, stride=(1, 1), padding='SAME'), bcnn3)
conv3 = tu.relu(conv3)
# (?, 96, 96, 64)
with tf.name_scope('conv4') as inner_scope:
wcnn4 = tu.weight([3, 3, 64, 64], name='wcnn4')
bcnn4 = tu.bias(1.0, [64], name='bcnn4')
conv4 = tf.add(
tu.conv2d(conv3, wcnn4, stride=(1, 1), padding='SAME'), bcnn4)
conv4 = tu.relu(conv4)
# (?, 96, 96, 64)
with tf.name_scope('conv5') as inner_scope:
wcnn5 = tu.weight([3, 3, 64, 64], name='wcnn5')
bcnn5 = tu.bias(1.0, [64], name='bcnn5')
conv5 = tf.add(
tu.conv2d(conv4, wcnn5, stride=(1, 1), padding='SAME'), bcnn5)
conv5 = tu.relu(conv5)
# (?, 96, 96, 64)
with tf.name_scope('max_pool') as inner_scope:
pool2 = tu.max_pool2d(conv5,
kernel=[1, 2, 2, 1],
stride=[1, 2, 2, 1],
padding='SAME')
# (?, 48, 48, 64)
return pool2
def forward(self,X):
Z = relu(X.dot(self.W1) + self.b1)
return sigmoid(Z.dot(self.W2) + self.b2),Z
def forward(self, input_layer): self.A = util.relu(input_layer) return self.A
def forward(self, X):
return relu(X.dot(self.W) + self.b)
