def forward_propagation(self, nn, x, y):
"""
This function implements the forward propagation algorithm following
Deep Learning, pag. 205
Parameters
----------
nn: nn.NeuralNetwork:
a reference to the network
x: numpy.ndarray
a record, or batch, from the dataset
y: numpy.ndarray
the target array for the batch given in input
Returns
-------
The error between the predicted output and the target one.
"""
for i in range(nn.n_layers):
self.a[i] = nn.b[i] + (nn.W[i].dot(x.T if i == 0 else self.h[i -
1]))
self.h[i] = nn.activation[i](self.a[i])
if nn.task == 'classifier':
return lss.mean_squared_error(self.h[-1].T, y)
return lss.mean_squared_error(self.h[-1].T, y) # mee
def back_propagation(self, x, y):
"""
This function implements the back propagation algorithm following
Deep Learning, pag. 206
Parameters
----------
x: numpy.ndarray
a record, or batch, from the dataset
y: numpy.ndarray
the target array for the batch given in input
Returns
-------
"""
g = lss.mean_squared_error(self.h[-1], y.T, gradient=True)
for layer in reversed(range(self.n_layers)):
g = np.multiply(
g,
act.A_F[self.activation[layer]]['fdev'](self.a[layer]))
# update bias, sum over patterns
self.delta_b[layer] = g.sum(axis=1).reshape(-1, 1)
# the dot product is summing over patterns
self.delta_W[layer] = g.dot(self.h[layer - 1].T if layer != 0
else x)
# summing over previous layer units
g = self.W[layer].T.dot(g)
def forward_propagation(self, x, y):
"""
This function implements the forward propagation algorithm following
Deep Learning, pag. 205
Parameters
----------
x: numpy.ndarray
a record, or batch, from the dataset
y: numpy.ndarray
the target array for the batch given in input
Returns
-------
The error between the predicted output and the target one.
"""
for i in range(self.n_layers):
self.a[i] = self.b[i] + (self.W[i].dot(x.T if i == 0
else self.h[i - 1]))
self.h[i] = act.A_F[self.activation[i]]['f'](self.a[i])
return lss.mean_squared_error(self.h[-1].T, y)
def back_propagation(self, nn, x, y):
"""
This function implements the back propagation algorithm following
Deep Learning, pag. 206
Parameters
----------
nn: nn.NeuralNetwork:
a reference to the network
x: numpy.ndarray
a record, or batch, from the dataset
y: numpy.ndarray
the target array for the batch given in input
Returns
-------
"""
start_time = dt.datetime.now()
g = 0
if nn.task == 'classifier':
g = lss.mean_squared_error(self.h[-1], y.T, gradient=True)
else:
g = lss.mean_squared_error(self.h[-1], y.T, gradient=True) # mee
for layer in reversed(range(nn.n_layers)):
g = np.multiply(g, nn.activation[layer](self.a[layer], dev=True))
# update bias, sum over patterns
self.delta_b[layer] = g.sum(axis=1).reshape(-1, 1)
# the dot product is summing over patterns
self.delta_W[layer] = g.dot(self.h[layer -
1].T if layer != 0 else x)
# summing over previous layer units
g = nn.W[layer].T.dot(g)
self.statistics['time_bw'] += \
(dt.datetime.now() - start_time).total_seconds() * 1000
self.g = g
def train_online_iaf(model, optimizer, data_generator, iterations, batch_size, beta, p_bar=None,
clip_value=None, global_step=None, transform=None, beta_max=1.0,
beta_step=250, beta_increment=0.01):
"""
Utility function to perform the # number of training loops given by the itertations argument.
---------
Arguments:
model : tf.keras.Model -- the invertible chaoin with an optional summary net
both models are jointly trained
optimizer : tf.train.optimizers.Optimizer -- the optimizer used for backprop
data_generator : callable -- a function providing batches of X, theta (data, params)
iterations : int -- the number of training loops to perform
batch_size : int -- the batch_size used for training
p_bar : ProgressBar -- an instance for tracking the training progress
clip_value : float -- the value used for clipping the gradients
global_step : tf.EagerVariavle -- a scalar tensor tracking the number of
steps and used for learning rate decay
transform : callable ot None -- a function to transform X and theta, if given
n_smooth : int -- a value indicating how many values to use for computing the running ML loss
----------
Returns:
losses : dict -- a dictionary with the ml_loss and decay
"""
# Prepare a dictionary to track losses
losses = {
'kl_loss': [],
'rec_loss': [],
'decay': []
}
# Run training loop
for it in range(1, iterations+1):
with tf.GradientTape() as tape:
# Generate data and parameters
X_batch, theta_batch = data_generator(batch_size)
# Apply some transformation, if specified
if transform:
X_batch, theta_batch = transform(X_batch, theta_batch)
# Sanity check for non-empty tensors
if tf.equal(X_batch.shape[0], 0).numpy():
print('Iteration produced empty tensor, skipping...')
continue
# Forward pass
theta_hat, z, logqz_x = model(theta_batch, X_batch)
# Compute losses
kl = kullback_leibler_iaf(z, logqz_x, beta)
rec = mean_squared_error(theta_batch, theta_hat)
decay = tf.add_n(model.losses)
total_loss = kl + rec + decay
# Store losses
losses['kl_loss'].append(kl.numpy())
losses['rec_loss'].append(rec.numpy())
losses['decay'].append(decay.numpy())
# One step backprop
gradients = tape.gradient(total_loss, model.trainable_variables)
if clip_value is not None:
gradients, _ = tf.clip_by_global_norm(gradients, clip_value)
apply_gradients(optimizer, gradients, model.trainable_variables, global_step)
# Increase beta every beta_step iterations
if (global_step.numpy() + 1) % beta_step == 0:
tf.assign(beta, min(beta_max, beta.numpy() + beta_increment))
# Update progress bar
p_bar.set_postfix_str("Iteration: {0},Rec Loss: {1:.3f},KL Loss: {2:.3f},Regularization Loss: {3:.3f},Beta: {4:.2f}"
.format(it, rec.numpy(), kl.numpy(), decay.numpy(), beta.numpy()))
p_bar.update(1)
return losses
def forward(self, y, t):
self.y = y
self.t = t
return mean_squared_error(self.y, self.t)
# TODO: Not check.
if __name__ == "__main__":
img = Image.open(b"../../../LFW/match pairs/0001/Aaron_Peirsol_0001.jpg")
# print(img.size)
img = np.array(img, dtype=np.float32)
# print(img.shape)
z = np.random.rand(10, 51, 51, 3).astype(np.float32)
# print(z)
Conv1 = Convolution(z.shape, out_channels=4, kernel_size=2, stride=1, learning_rate=0.00001)
out1 = Conv1.forward(z)
pooling1 = Pooling(out1.shape, pool_size=2)
out2 = pooling1.forward(out1)
Conv2 = Convolution(out2.shape, out_channels=5 ,kernel_size=2, stride=1, learning_rate=0.00001)
y_pred = Conv2.forward(out2)
print(y_pred.shape)
y_true = np.ones((10, 24, 24, 5)).astype(np.float32)
for i in range(10):
error, dy = losses.mean_squared_error(y_pred, y_true)
print(error)
print("------")
eta3to2 = Conv2.gradient(dy)
Conv2.backward()
eta2to1 = pooling1.gradient(eta3to2)
_ = Conv1.gradient(eta2to1)
Conv1.backward()
out1 = Conv1.forward(z)
out2 = pooling1.forward(out1)
y_pred = Conv2.forward(out2)