def G_base(name, x, batch):
if name == 'G1':
first_out_channels = 128
else:
first_out_channels = 16
with tf.variable_scope(name):
conv1 = conv('conv1', x, 7 * 7, first_out_channels, 1, 3, False)
ins1 = ins_norm('ins1', conv1)
relu1 = relu('relu1', ins1)
conv2 = conv('conv2', relu1, 3 * 3, first_out_channels * 2, 2, 0, True)
ins2 = ins_norm('ins2', conv2)
relu2 = relu('relu2', ins2)
conv3 = conv('conv3', relu2, 3 * 3, first_out_channels * 4, 2, 0, True)
ins3 = ins_norm('ins3', conv3)
relu3 = relu('relu3', ins3)
conv4 = conv('conv4', relu3, 3 * 3, first_out_channels * 8, 2, 0, True)
ins4 = ins_norm('ins4', conv4)
relu4 = relu('relu4', ins4)
conv5 = conv('conv5', relu4, 3 * 3, first_out_channels * 16, 2, 0,
True)
ins5 = ins_norm('ins5', conv5)
relu5 = relu('relu5', ins5)
x_in = relu5
if name == 'G1':
for i in range(9):
name = 'res' + str(i + 1)
x_in = res_block(name, x_in)
up1 = conv_trans('up1', x_in, 3 * 3, first_out_channels * 8, 2, batch,
True)
ins_up1 = ins_norm('ins_up1', up1)
relu_up1 = relu('relu_up1', ins_up1)
up2 = conv_trans('up2', relu_up1, 3 * 3, first_out_channels * 4, 2,
batch, True)
ins_up2 = ins_norm('ins_up2', up2)
relu_up2 = relu('relu_up2', ins_up2)
up3 = conv_trans('up3', relu_up2, 3 * 3, first_out_channels * 2, 2,
batch, True)
ins_up3 = ins_norm('ins_up3', up3)
relu_up3 = relu('relu_up3', ins_up3)
up4 = conv_trans('up4', relu_up3, 3 * 3, first_out_channels, 2, batch,
True)
ins_up4 = ins_norm('ins_up4', up4)
relu_up4 = relu('relu_up4', ins_up4)
conv_end = conv('conv_end', relu_up4, 7 * 7, 3, 1, 3, False)
tanh_end = tanh('tanh_end', conv_end)
return tanh_end, relu_up4
def Convolutional_Forward(minibatch_X, parameters_conv, hparameters1,
hparameters2, hparameters3, hparameters4):
Z1, cache_conv1 = conv_forward(minibatch_X, parameters_conv["W1"],
parameters_conv["b1"], hparameters1)
A1, c1 = relu(Z1)
P1, cache_pool1 = pool_forward(A1, hparameters2)
Z2, cache_conv2 = conv_forward(P1, parameters_conv["W2"],
parameters_conv["b2"], hparameters3)
A2, c2 = relu(Z2)
P2, cache_pool2 = pool_forward(A2, hparameters4)
return P2, cache_pool2, cache_pool1, cache_conv1, cache_conv2, c1, c2
def res_block(
name,
x,
channels=2048): # conv('conv1', x, 3*3, 64, 2, ref_pad=1, pad=True)
with tf.variable_scope(name):
conv1 = conv('conv1', x, 3 * 3, channels, 1, 1, False)
conv1_ins = ins_norm('ins1', conv1)
conv1_relu = relu('relu1', conv1_ins)
conv2 = conv('conv2', conv1_relu, 3 * 3, channels, 1, 1, False)
conv2_ins = ins_norm('ins2', conv2)
conv2_relu = relu('relu2', conv2_ins)
return conv2_relu + x
def test_forward_propagation_manual(self):
# Tests forward propagation by comparing to manual calculations.
# Setup
data = [2, 2, 2]
inputs = [1, 2, 3]
outputs = [7]
connections = [
Connection(1, 5),
Connection(2, 5),
Connection(2, 4),
Connection(3, 4),
Connection(4, 5),
Connection(4, 6),
Connection(5, 6),
Connection(6, 7)
]
w0 = 1.5
w1 = 2
w2 = 2.5
w3 = -1.5
w4 = -2
w5 = 3
w6 = 3
w7 = 0.5
connections[0].weight = w0
connections[1].weight = w1
connections[2].weight = w2
connections[3].weight = w3
connections[4].weight = w4
connections[5].weight = w5
connections[6].weight = w6
connections[7].weight = w7
# Manual calculations
a4 = relu((w2 * data[1] + 1) + (w3 * data[2] + 1))
a5 = relu((w0 * data[0] + 1) + (w1 * data[1] + 1) + (w4 * a4 + 1))
a6 = relu((w6 * a5 + 1) + (w5 * a4 + 1))
a7 = sigmoid((w7 * a6 + 1))
# Run
graph = Graph()
a7_actual = graph.forward_propagate(data, inputs, outputs, connections)
# Assert
# TODO: Precision problem
self.assertTrue(abs(a7_actual[1] - a7) < 0.0005)
def _build_network(self):
"""Build a 2 hidden layer neural network with ReLU activations
and a softmax output layer"""
self.W1 = np.random.random((self.inp_shape, self.hidden_units))
self.b1 = np.zeros((self.hidden_units, ))
self.hid1 = lambda x: relu(np.dot(x, self.W1) + self.b1)
self.W2 = np.random.random((self.hidden_units, self.hidden_units))
self.b2 = np.zeros((self.hidden_units, ))
self.hid2 = lambda x: relu(np.dot(x, self.W2) + self.b2)
self.W3 = np.random.random((self.hidden_units, self.num_classes))
self.b3 = np.zeros((self.num_classes, ))
self.hid3 = lambda x: softmax(np.dot(x, self.W3) + self.b3)
def forward_pass(self, verbose=False, debug=False):
"""
Computes the values of all units (neurons) over ALL sample points (vectorized).
Args:
Returns:
"""
if (verbose): print("\n _______ Forward Pass _______")
# Zeroth step: Outputs of input units.
X = self.X_active
if (verbose): print("\t X.shape", X.shape)
# First step: Outputs (H) of hidden units.
if (verbose): print("\t Computing 1st layer . . . ")
S_h = util.withBias(X) @ self.V.T
H = activations.relu(S_h)
# Second step: Outputs (O) of output units.
if (verbose): print("\t Computing 2nd layer . . . ")
S_o = util.withBias(H) @ self.W.T
O = activations.softmax(S_o, verbose)
if debug: pdb.set_trace()
self.O = O
return X, S_h, H, S_o, O
def linear_activation_forward_with_dropout(A_prev,
W,
b,
activation,
keep_prob=0.5):
# Linear forward step
Z, linear_cache = linear_forward(A_prev, W, b)
# Activation forward step
if activation == 'relu':
A, activation_cache = relu(Z)
# Implementing dropout
D = np.random.rand(A.shape[0], A.shape[1])
D = (D < keep_prob).astype(
int
) # convert entries of D to 0 or 1 (using keep_prob as the threshold)
A = A * D # shut down some neurons of A
A = np.divide(
A, keep_prob
) # scale the value of neurons that haven't been shut down
cache = (linear_cache, activation_cache, D)
elif activation == 'sigmoid':
A, activation_cache = sigmoid(Z)
cache = (linear_cache, activation_cache, None)
return A, cache
def main():
src1, src2 = mpimg.imread('pic.jpg'), mpimg.imread('pic2.jpg')
src = np.array([src1, src2])
del src1, src2
print(src.shape)
# visualize(src, 3)
filter = np.array((
([-1, 0, 1], [-2, 0, 2], [-1, 0, 1]),
([-1, 0, 1], [-2, 0, 2], [-1, 0, 1]),
([-1, 0, 1], [-2, 0, 2], [-1, 0, 1]),
))
start = time.time()
result = convolve(src, filter)
end = time.time()
print('Time taken for vectorized: ', end - start)
result = relu(result)
plt.imshow(result[0, :, :], cmap='gray')
plt.show()
plt.imshow(result[1, :, :], cmap='gray')
plt.show()
print(result.shape)
start = time.time()
result = max_pool(result, pool_size=2)
end = time.time()
print('Time taken Vectorized Max Pool: ', end - start)
# plt.imshow(result[0], cmap = 'gray')
# plt.show()
# plt.imshow(result[1], cmap = 'gray')
# plt.show()
print(result.shape)
def forward(self, inputs):
assert inputs.shape == (inputs, 1)
s = np.matmul(np.transpose(weights), inputs)
if activationf == 'Relu':
self.activation = act.relu(s)
else:
self.activation = act.sigmoid(s)
def linear_activation_forward(A_prev, W, b, activation):
"""
Implement the forward propagation for the LINEAR->ACTIVATION layer
Arguments:
A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)
W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)
b -- bias vector, numpy array of shape (size of the current layer, 1)
activation -- the activation to be used in this layer, stored as a text string: "sigmoid" or "relu"
Returns:
A -- the output of the activation function, also called the post-activation value
cache -- a python dictionary containing "linear_cache" and "activation_cache";
stored for computing the backward pass efficiently
"""
if activation == "softmax":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = stable_softmax(Z)
elif activation == "relu":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = relu(Z)
assert (A.shape == (W.shape[0], A_prev.shape[1]))
cache = (linear_cache, activation_cache)
return A, cache
def linear_activation_forward(self, A_prev, W, b, activation):
if activation == "sigmoid":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache"
Z, linear_cache = self.linear_forward(A_prev, W, b)
A, activation_cache = activations.sigmoid(Z)
elif activation == "relu":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = self.linear_forward(A_prev, W, b)
A, activation_cache = activations.relu(Z)
elif activation == "softmax":
# Inputs: "A_prev, W, b". Outputs: "A, activation_cache".
Z, linear_cache = self.linear_forward(A_prev, W, b)
A, activation_cache = activations.softmax(Z)
elif activation == "euler":
Z, linear_cache = self.linear_forward(A_prev, W, b)
A, activation_cache = activations.euler(Z)
assert (A.shape == (W.shape[0], A_prev.shape[1]))
cache = (linear_cache, activation_cache)
return A, cache
def activation_forward(self, Z, activation="tanh"):
if activation is "tanh":
return [Z, tanh(Z)]
if activation is "relu":
return [Z, relu(Z)]
if activation is "sigmoid":
return [Z, sigmoid(Z)]
def linear_activation_forward(A_prev, W, b, activation):
'''
Implementa la propagación hacia adelante Linear->Activación de una capa
Arguments:
A_prev -- Activación de la capa anterior, de dimensiones (tamaño de la capa anterior, número de ejemplos)
W -- Pesos de la capa actual, de dimensiones (tamaño de la capa actual, tamaño de la capa anterior)
b -- sesgos de la capa actual, de dimensiones (tamaño de la capa actual, 1)
activation -- string con el nombre de la activación que será usada en esta capa: "Relu", "Sigmoid"
Returns:
A -- Activación de la capa actual, de dimensiones (tamaño de la capa actual, número de ejemplos)
cache -- python tupla que contiene el cache linear y de la activación
'''
if activation == "Sigmoid":
Z, linear_c = linear_forward(A_prev, W, b)
A, activacion_c = sigmoid(Z)
elif activation == "Relu":
Z, linear_c = linear_forward(A_prev, W, b)
A, activacion_c = relu(Z)
cache = (linear_c, activacion_c)
return A, cache
def forward_prop(X, params):
W1 = params['W1']
W2 = params['W2']
W3 = params['W3']
b1 = params['b1']
b2 = params['b2']
b3 = params['b3']
Z1 = np.dot(W1, X) + b1
A1 = relu(Z1)
Z2 = np.dot(W2, A1) + b2
A2 = relu(Z2)
Z3 = np.dot(W3, A2) + b3
A3 = sigmoid(Z3)
cache = [Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3]
return A3, cache
def build_G(self, x_bound, x_label, x_feat, x_k, x_b):
with tf.variable_scope('G'):
# 融合
x_feat_act = tf.add(tf.multiply(x_feat, x_k), x_b)
x_concat = tf.concat([x_bound, x_label, x_feat_act], 3)
#
input_downsampled = tf.nn.avg_pool(x_concat,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding="SAME")
# G1
_, G1_relu_up4 = G_base('G1', input_downsampled, self.batch)
# G2_1
G2_1_conv1 = conv('G2_1_conv1', x_concat, 7 * 7, 64, 1, None, True)
G2_1_ins1 = ins_norm('G2_1_ins1', G2_1_conv1)
G2_1_relu1 = relu('G2_1_relu1', G2_1_ins1)
G2_1_conv2 = conv('G2_1_conv2', G2_1_relu1, 3 * 3, 128, 2, 1,
False)
G2_1_ins2 = ins_norm('G2_1_ins2', G2_1_conv2)
G2_1_relu2 = relu('G2_1_relu2', G2_1_ins2)
# 融合G1的输出和G2_1的输出 128
G_add = tf.add(G1_relu_up4, G2_1_relu2, name='G_Add')
# G2_2
# res_block
for i in range(3):
name = 'G2_2_res' + str(i + 1)
G_add = res_block(name, G_add, channels=128)
#
G2_2_trans = conv_trans('G2_2_trans', G_add, 3 * 3, 64, 2,
self.batch, True)
G2_2_ins2 = ins_norm('G2_2_ins2', G2_2_trans)
G2_2_relu2 = relu('G2_2_relu2', G2_2_ins2)
# final convolution
G2_2_conv_end = conv('G2_2_conv_end', G2_2_relu2, 7 * 7, 3, 1,
None, True)
G2_2_tanh_end = tanh('G2_2_tanh_end', G2_2_conv_end)
return G2_2_tanh_end
def __one_layer_forward_prop(self, a_prev, layer_index, activation):
z = np.dot(self.w[layer_index], a_prev) + self.b[layer_index]
if activation == 'relu':
return relu(z), z
elif activation == 'sigmoid':
return sigmoid(z), z
elif activation == 'softmax':
return softmax(z), z
else:
raise "Invalid activation: {}".format(activation)
def linear_activation_forward(self, A, W, b, activation):
Z, linear_cache = self.linear_forward(A, W, b)
if activation == 'sigmoid':
A, activation_cache = sigmoid(Z), (linear_cache, Z)
elif activation == 'relu':
A, activation_cache = relu(Z), (linear_cache, Z)
else:
assert False
assert A.shape == (W.shape[0], Z.shape[1])
return A, activation_cache
def one_layer_forward_pass(input_activations, weights, bias, activation='R'):
output = np.dot(weights, input_activations) + bias
if activation is 'R':
activation_next = activations.relu(output)
elif activation is 'S':
activation_next = activations.sigmoid(output)
else:
raise Exception('Nahh!')
return activation_next, output
def forward(x, w1, b1, w2, b2):
z1 = x.dot(w1.T) + b1.T
a1 = relu(z1)
z2 = a1.dot(w2.T) + b2.T
a2 = softmax(z2)
return {
'a1': a1,
'z1': z1,
'a2': a2,
'z2': z2,
}
def linear_activation_forward(A_prev, W, b, activation):
# Linear forward step
Z, linear_cache = linear_forward(A_prev, W, b)
# Forward activation step
if activation == 'relu':
A, activation_cache = relu(Z)
elif activation == 'sigmoid':
A, activation_cache = sigmoid(Z)
cache = (linear_cache, activation_cache)
return A, cache
def linear_activation_forward(A_prev, W, b, activation):
Z, linear_cache = linear_forward(A_prev, W, b)
if activation == 'sigmoid':
A, activation_cache = sigmoid(Z)
elif activation == 'relu':
A, activation_cache = relu(Z)
else:
raise KeyError('No activation function found')
cache = (linear_cache, activation_cache)
return A, cache
def one_step_forward(A_prev, W, b, activation):
Z = np.dot(W, A_prev) + b
linear_cache = A_prev, W, b
# print(A_prev.shape,W.shape,b.shape)
if activation == 'relu':
A, activation_cache = relu(Z) # This relu function is for L-1 layers
if activation == 'softmax':
A, activation_cache = softmax(
Z) # The sigmoid function is for last Lth layer call
return A, (linear_cache, activation_cache)
def convolution_forward(X, filters, bias):
"""Funckja realizujaca warstwe konwolucyjna z liniowa funkcja aktywacji typu ReLU
Jest to ta sama f-cja co convolution w pliku splot.py z ta roznica, ze bierze pod uwage macierz progow
"""
feature_maps = []
for i in range(len(filters)):
feature_map = []
filter1 = filters[i]
depth = len(filter1)
for j in range(depth):
feature_map.append(signal.convolve2d(X[j], np.rot90(filter1[j],2) ,'valid'))
feature_map = sum(feature_map) + bias[i]*np.ones((feature_map[0].shape[0], feature_map[0].shape[1]))
feature_maps.append(act.relu(feature_map))
return np.asarray(feature_maps)
def forward_prop(self, X):
#print('X.shape',X.shape)
A_list = []
m, n = X.shape
A = X.T
A_list.append(A)
for i in range(len(self.w_list)):
Z = (np.dot(self.w_list[i], A)) + self.b_list[i]
if self.activ[i] == 'sigm':
A = activ.sigmoid(Z)
elif self.activ[i] == 'relu':
A = activ.relu(Z)
A_list.append(A)
return A_list
def linear_activation_forward(A_prev, W, b, activation):
Z, Z_cache = linear_forward(A_prev, W, b)
if activation == 'relu':
A, A_cache = activations.relu(Z)
elif activation == 'sigmoid':
A, A_cache = activations.sigmoid(Z)
elif activation == 'softmax':
A_cache = activations.softmax(Z)
cache = (Z_cache, A_cache)
return A, cache
def linear_act_forward(A, W, b, act):
"""
Implements the linear and activation functions of a single node
Also, returns the original values for storage in cache
"""
if act == "sigmoid":
Z, lin_cache = linear(A, W, b)
A, act_cache = sigmoid(Z)
elif act == "relu":
Z, lin_cache = linear(A, W, b)
A, act_cache = relu(Z)
elif act == "softmax":
Z, lin_cache = linear(A, W, b)
A, act_cache = softmax(Z)
cache = (lin_cache, act_cache)
return A, cache
def activation_forward(self,input,W,b,activation_type):
'''
:param input: the input of the current layer
:param W: the weights of the current layer
:param b: biases of the current layer
:param activation_type: Type of activation function used in the forward propagation
:return: - A --> the output of the activation function
- packet_of_packets --> Tuple of 2 elements which will be used in backward propagation :
1- linear packer : contains ( input , weights , biases ) of the current layer
2- activation packet : contains ( Z ) which is the input to the activation function
'''
if activation_type == "sigmoid":
Z, linear_packet = self.identity_forward(input, W, b) ## Z = input * w + b
temp=activations.Sigmoid()
A, activation_packet = temp.forward(Z) ## A = sig(z)
elif activation_type == "relu":
Z, linear_packet = self.identity_forward(input, W, b)
temp = activations.relu()
A, activation_packet = temp.forward(Z)
elif activation_type == "leaky_relu":
Z, linear_packet = self.identity_forward(input, W, b)
temp = activations.leaky_relu()
A, activation_packet = temp.forward(Z)
elif activation_type == "tanh":
Z, linear_packet = self.identity_forward(input, W, b)
temp = activations.tanh()
A, activation_packet = temp.forward(Z)
elif activation_type == "softmax":
Z, linear_packet = self.identity_forward(input, W, b)
#temp =
A, activation_packet = activations.Softmax().forward(Z)
elif activation_type == "linear":
Z, linear_packet = self.identity_forward(input, W, b)
# temp =
A, activation_packet = Z,Z
else:
raise ValueError("ERROR : Activation Function is Not Determined")
packet_of_packets = linear_packet, activation_packet
return A, packet_of_packets
def linear_activation_forward(A_prev, W, b, activation):
# function calculate A od units in single layer
if activation == "sigmoid":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = sigmoid(Z)
elif activation == "relu":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = relu(Z)
elif activation == "softmax":
Z, linear_cache = linear_forward(A_prev, W, b)
A, activation_cache = softmax(Z)
assert (A.shape == (W.shape[0], A_prev.shape[1]))
cache = (linear_cache, activation_cache)
return A, cache
def forward(self, inputData):
"""Funkcja realizacje propagacje danych w przod
Wejscie: obraz lub paczka obrazow ze zbioru danych
Wyjscie: produkt warstwy softmax: 10-dim wektor liczb
"""
weights = self.Weights
biases = self.Biases
poolParams = self.poolParams
cache = [
] #zmienna przechowujaca produkty warstw - pomocna do propagacji wstecznej
#warstwa wejsciowa
layer0 = np.asarray(inputData)
cache.append(layer0)
#pierwsza warstwa konwolucyjna
layer1 = convolution_forward(np.asarray([layer0]), weights[0],
biases[0])
cache.append(layer1)
#pierwsza warstwa max poolingu
layer2 = maxpool_forward(layer1, poolParams[0][0], poolParams[0][1])
cache.append(layer2)
#druga warstwa konwolucyjna
layer3 = convolution_forward(layer2, weights[1], biases[1])
cache.append(layer3)
#druga warstwa max poolingu
layer4 = maxpool_forward(layer3, poolParams[1][0], poolParams[1][1])
cache.append(layer4)
#pierwsza warstwa fully connected zrealizowana jako warstwa konwolucyjna
layer5 = convolution_forward(layer4, weights[2], biases[2])
cache.append(layer5)
#druga warstwa fully connected z funkcja aktywacji typu ReLU
layer6 = act.relu(
np.dot(weights[3], layer5[:, 0]).transpose() +
biases[3]).transpose()
cache.append(layer6)
#softmax
layer7 = np.dot(weights[4], layer6[:, 0]).transpose() + biases[4]
layer7 -= np.max(layer7)
layer7 = np.exp(layer7) / sum(np.exp(layer7))
return (layer7, cache)
def _step_forward(A_prev, W, b, activation="sigmoid", keep_prob=1):
# linear
# Z[l] = W[l] dot A[l-1] + b[l]
Z = np.dot(W, A_prev) + b
linear_cache = (A_prev, W, b)
# activation
# A[l] = g[l](Z[l])
if activation == "sigmoid":
A = sigmoid(Z)
elif activation == "relu":
A = relu(Z)
elif activation == "tanh":
A = np.tanh(Z)
D = np.random.rand(*A.shape) < keep_prob
A *= D
A /= keep_prob
activation_cache = (Z, activation)
return A, D, (linear_cache, activation_cache)
