def feedforward_backprop(data, label, weights):
# feedforward hidden layer and relu
fully1_out = fullyconnect_feedforward(data, weights['fully1_weight'],
weights['fully1_bias'])
#print('fully1_out', fully1_out)
relu1_out = relu_feedforward(fully1_out)
#print("relu1_out", relu1_out)
# softmax loss (probs = e^(w*x+b) / sum(e^(w*x+b))) is implemented in two parts for convenience.
# first part: y = w * x + b is a fullyconnect.
fully2_out = fullyconnect_feedforward(relu1_out, weights['fully2_weight'],
weights['fully2_bias'])
#print("fully2_out", fully2_out)
# second part: probs = e^y / sum(e^y) is the so-called softmax_loss here.
loss, accuracy, fully2_sensitivity = softmax_loss(fully2_out, label)
#print("fully2_sensitivity", fully2_sensitivity)
gradients = {}
gradients['fully2_weight_grad'], gradients[
'fully2_bias_grad'], relu1_sensitivity = fullyconnect_backprop(
fully2_sensitivity, relu1_out, weights['fully2_weight'])
# backprop of relu and then hidden layer
fully1_sensitivity = relu_backprop(relu1_sensitivity, fully1_out)
gradients['fully1_weight_grad'], gradients[
'fully1_bias_grad'], _ = fullyconnect_backprop(
fully1_sensitivity, data, weights['fully1_weight'])
return loss, accuracy, gradients
def relu_backprop(in_sensitivity, in_):
'''
The backpropagation process of relu
input paramter:
in_sensitivity : the sensitivity from the upper layer, shape:
: [number of images, number of outputs in feedforward]
in_ : the input in feedforward process, shape: same as in_sensitivity
output paramter:
out_sensitivity : the sensitivity to the lower layer, shape: same as in_sensitivity
'''
# TODO
# begin answer
relu = relu_feedforward(in_)
out_sensitivity = relu / in_ * in_sensitivity
# end answer
return out_sensitivity