def train(training_data,
labels,
n_iter,
n_classes,
n_filter,
learn_rate,
print_acc=True):
input_dim = int((((training_data[0].shape[0] - 3 + 1) / 2)**2) * n_filter)
np.random.seed(seed=30)
own_filter_conv = np.random.randn(n_filter, 3, 3) / 9
np.random.seed(seed=30)
own_weight_soft = (np.random.randn(input_dim, n_classes) / input_dim)
own_bias_soft = np.random.randn(n_classes)
num_correct = 0
for i in range(n_iter):
image = training_data[i] / 255 - 0.5
label = labels[i]
own_feature_map, own_filter_conv = fun.convolute(
image=image, filter_matrix=own_filter_conv)
own_maxpool_map = fun.maxpool(feature_map=own_feature_map)
own_probs, own_inter_soft = fun.softmax(own_maxpool_map,
weight_matrix=own_weight_soft,
bias_vector=own_bias_soft)
own_weight_soft, own_bias_soft, own_gradient_soft = fun.backprop_softmax(
inter_soft=own_inter_soft,
probabilities=own_probs,
label=label,
learn_rate=learn_rate)
own_gradient_max = fun.backprop_maxpool(feature_map=own_feature_map,
gradient=own_gradient_soft)
own_filter_conv = fun.backprop_conv(image=image,
filter_conv=own_filter_conv,
gradient=own_gradient_max,
learn_rate=learn_rate)
prediction = np.argmax(own_probs)
acc = 1 if prediction == label else 0
num_correct += acc
if i % 100 == 0 and i != 0 and print_acc:
accuracy = num_correct / i
print(f"accuracy for the first {i} samples: {accuracy}")
print(f"{num_correct} predictions for {i} samples were correct")
return None
test_image = np.array([
list(range(1, 7)),
list(range(6, 12)),
list(range(11, 17)),
list(range(16, 22)),
list(range(21, 27)),
list(range(26, 32))
])
test_image.shape
test_filter1 = np.array([[0, 0, 0], [1, 2, 1], [0, 0, 0]])
test_filter2 = test_filter1.T
test_filter = np.array([test_filter1, test_filter2])
test_conv, filter_mat, inter_conv = fun.convolute(image=test_image,
filter_matrix=test_filter)
out_maxpool, index_maxpool = fun.max_pool(test_conv)
ind = np.where(
index_maxpool[0] ==
True) # for these indices we need gradients with respect to inmage?
test_conv[index_maxpool] # diese indices wollen wir mit deltaL * image[]
test_conv[0, ind[0], ind[1]] # if this works for image, wooooow!
test_conv.shape
ind[0]
ind[1]
os.listdir(path) import functions as fun np.random.seed(seed=666); image = np.random.randn(6, 6) * 3 image = np.round(image) label = 1 num_filters = 2 np.random.seed(seed=666); filter_conv = np.random.randn(num_filters, 3, 3) / 9 ## after transpsosing input to softmax, feedforward agrees, # but maybe it would agree in backprop, because I put the num filters first, # he put them last out_ownconv, filter_conv, inter = fun.convolute((image / 255) - 0.5, filter_conv) out_ownconv.T out_maxown, index_maxown = fun.max_pool(out_ownconv) np.random.seed(seed=666); weight_soft = np.random.randn(8, 2) / 8 np.random.seed(seed=666); bias_soft = np.zeros(2) probabilities, intermediates = fun.softmax(out_maxown.T, weight_soft, bias_soft) out_maxown[0] out_maxown.T[:, :, 0] ################################## backprop ###################### weight_soft.shape
def debug_cnn(n_iter, version, learn_rate):
# from importlib import reload
path = "/home/konstantin/Documents/master_arbeit/"
sys.path.append(path)
import functions as fun
#import os
if version == "changed":
print("changed blog version is being used")
path_blog_changed = "/home/konstantin/Documents/master_arbeit/cnn_python/cnn-from-scratch-changed/"
sys.path.append(path_blog_changed)
#os.listdir(path_blog_changed)
from conv import Conv3x3
from maxpool import MaxPool2
from softmax import Softmax
if version == "original":
print("original version is being used")
path_blog_original = "/home/konstantin/Documents/master_arbeit/cnn_python/original_blog/cnn-from-scratch"
sys.path.append(path_blog_original)
from conv import Conv3x3
from maxpool import MaxPool2
from softmax import Softmax
# Conv3x3 = reload(Conv3x3)
# MaxPool2 = reload(MaxPool2)
# Softmax = reload(Softmax)
num_filters = 8
np.random.seed(seed=444); own_filter_conv = np.random.randn(num_filters, 3, 3) / 9
own_filter_conv = np.round(own_filter_conv)
dim_maxpool = 13 * 13 *8
np.random.seed(seed=666); own_weight_soft = (np.random.randn(dim_maxpool, 10) / dim_maxpool)
own_bias_soft = np.zeros(10)
conv = Conv3x3(8) # 28x28x1 -> 26x26x8
pool = MaxPool2() # 26x26x8 -> 13x13x8
dim_maxpool = np.prod(13 * 13 * 8)
softmax = Softmax(dim_maxpool, 10)
for i in range(n_iter):
image = test_images[i] / 255 - 0.5
label = test_labels[i]
own_feature_map, own_filter_conv = fun.convolute(image=image, filter_matrix=own_filter_conv)
own_maxpool_map = fun.maxpool(feature_map=own_feature_map)
own_probs, own_inter_soft = fun.softmax(own_maxpool_map,
weight_matrix=own_weight_soft,
bias_vector=own_bias_soft)
own_weight_soft, own_bias_soft, own_gradient_soft = fun.backprop_softmax(inter_soft=own_inter_soft,
probabilities=own_probs,
label = label,
learn_rate=learn_rate)
own_gradient_max = fun.backprop_maxpool(feature_map=own_feature_map,
gradient=own_gradient_soft)
own_filter_conv = fun.backprop_conv(image=image, filter_conv=own_filter_conv,
gradient=own_gradient_max, learn_rate=learn_rate)
# run model from blog with same data
blog_out_conv = conv.forward(image)
#print(out_conv)
blog_out_max = pool.forward(blog_out_conv)
blog_out_soft = softmax.forward(blog_out_max) #
#print(blog_out_soft)
gradient_L = np.zeros(10)
gradient_L[label] = -1 / blog_out_soft[label]
blog_gradient_soft = softmax.backprop(
gradient_L, learn_rate)
blog_gradient_max = pool.backprop(blog_gradient_soft)
conv.backprop(blog_gradient_max, learn_rate)
################## compare feedforward ####################################
###########################################################################
print("This is iteration", i)
if np.sum(own_feature_map == blog_out_conv) == np.prod(own_feature_map.shape):
print("YEAAAH! FeatureMaps are the same")
else:
print("NOOOO! featuremaps are not the same")
# conv.filters == filter_conv #
# after first iteration these are not the same anymore,
# since they get updated
if np.sum(own_maxpool_map == blog_out_max) == np.prod(blog_out_max.shape):
print("YEAHHH! maxpool is the same")
else:
print("NOOOO! maxpool is not the same")
if np.sum(own_probs == blog_out_soft) == np.prod(blog_out_soft.shape):
print("YEAAAH! predicted probabilities are the same")
else:
print("NOOOO! predicted probabilities are not the same")
print("Own probabilities")
print(own_probs)
print("Blog probabilities")
print(blog_out_soft)
# break
######################### compare backprop #################################
############################################################################
######## softmax: gradients:
if np.sum(own_gradient_soft == blog_gradient_soft) == np.prod(blog_gradient_soft.shape):
print("YEAHHHH! gradients softmax are the same")
else:
print("NOOOO! gradients softmax are not the same")
## weight updates weight matrix softmax layer
# if np.sum(own_weight_soft == blog_weights_updated) == np.prod(own_weight_soft.shape):
# print("Yeaaah! updated weightmatrix softmax is the same")
# else:
# print("updated weightmatrix softmax is not the same")
# ## weight updates bias vector
# if np.sum(own_bias_soft == blog_biases_updated) == np.prod(blog_biases_updated.shape):
# print("Yeaaah! Updated bias vector softmax is the same")
# else:
# print("updated bias vector is not the same")
#### maxpool
if np.sum(own_gradient_max== blog_gradient_max) == np.prod(blog_gradient_max.shape):
print("YEAHHHH! gradients maxpool layer are the same")
else:
print("NOOOO! updated gradients maxpool are not the same")
## conv
# if np.sum(own_filter_conv == blog_filter_update) == np.prod(own_filter_conv.shape):
# print("YEAAAHHH! updated filter convlayer are the same")
# else:
# print("NOOOOO! updated filter conv layer is not the same")
#
# So! After two runs the predicted probabilities are already different, why?
return None
softmax = Softmax(dim_maxpool, 10) # 13x13x8 -> 10
num_filters = 8
np.random.seed(seed=666)
filter_conv = np.random.randn(num_filters, 3, 3) * 3 # / 9
filter_conv = np.round(filter_conv)
###############################################################################
## pass in net from blog and own net
###############################################################################
########################### feedforward
# feedforward conv layer
out_conv = conv.forward(image)
out_convown, filter_conv, inter = fun.convolute(image, filter_conv)
if np.sum(out_conv == out_convown) == np.prod(out_convown.shape):
print("Yeaaaah!")
# feedforward maxpool layer
out_max = pool.forward(out_conv)
out_maxown = fun.max_pool(out_convown)
if np.sum(out_max == out_maxown) == np.prod(out_maxown.shape):
print("Yeaaaah!")
# feedforward softmax layer
out_soft, weights, summe = softmax.forward(out_max) #
np.random.seed(seed=666)
weight_soft = (np.random.randn(dim_maxpool, 10) / dim_maxpool) * 10