def forward_propagation(self, nn, x, y): """ This function implements the forward propagation algorithm following Deep Learning, pag. 205 Parameters ---------- nn: nn.NeuralNetwork: a reference to the network x: numpy.ndarray a record, or batch, from the dataset y: numpy.ndarray the target array for the batch given in input Returns ------- The error between the predicted output and the target one. """ for i in range(nn.n_layers): self.a[i] = nn.b[i] + (nn.W[i].dot(x.T if i == 0 else self.h[i - 1])) self.h[i] = nn.activation[i](self.a[i]) if nn.task == 'classifier': return lss.mean_squared_error(self.h[-1].T, y) return lss.mean_squared_error(self.h[-1].T, y) # mee
def back_propagation(self, x, y): """ This function implements the back propagation algorithm following Deep Learning, pag. 206 Parameters ---------- x: numpy.ndarray a record, or batch, from the dataset y: numpy.ndarray the target array for the batch given in input Returns ------- """ g = lss.mean_squared_error(self.h[-1], y.T, gradient=True) for layer in reversed(range(self.n_layers)): g = np.multiply( g, act.A_F[self.activation[layer]]['fdev'](self.a[layer])) # update bias, sum over patterns self.delta_b[layer] = g.sum(axis=1).reshape(-1, 1) # the dot product is summing over patterns self.delta_W[layer] = g.dot(self.h[layer - 1].T if layer != 0 else x) # summing over previous layer units g = self.W[layer].T.dot(g)
def forward_propagation(self, x, y): """ This function implements the forward propagation algorithm following Deep Learning, pag. 205 Parameters ---------- x: numpy.ndarray a record, or batch, from the dataset y: numpy.ndarray the target array for the batch given in input Returns ------- The error between the predicted output and the target one. """ for i in range(self.n_layers): self.a[i] = self.b[i] + (self.W[i].dot(x.T if i == 0 else self.h[i - 1])) self.h[i] = act.A_F[self.activation[i]]['f'](self.a[i]) return lss.mean_squared_error(self.h[-1].T, y)
def back_propagation(self, nn, x, y): """ This function implements the back propagation algorithm following Deep Learning, pag. 206 Parameters ---------- nn: nn.NeuralNetwork: a reference to the network x: numpy.ndarray a record, or batch, from the dataset y: numpy.ndarray the target array for the batch given in input Returns ------- """ start_time = dt.datetime.now() g = 0 if nn.task == 'classifier': g = lss.mean_squared_error(self.h[-1], y.T, gradient=True) else: g = lss.mean_squared_error(self.h[-1], y.T, gradient=True) # mee for layer in reversed(range(nn.n_layers)): g = np.multiply(g, nn.activation[layer](self.a[layer], dev=True)) # update bias, sum over patterns self.delta_b[layer] = g.sum(axis=1).reshape(-1, 1) # the dot product is summing over patterns self.delta_W[layer] = g.dot(self.h[layer - 1].T if layer != 0 else x) # summing over previous layer units g = nn.W[layer].T.dot(g) self.statistics['time_bw'] += \ (dt.datetime.now() - start_time).total_seconds() * 1000 self.g = g
def train_online_iaf(model, optimizer, data_generator, iterations, batch_size, beta, p_bar=None, clip_value=None, global_step=None, transform=None, beta_max=1.0, beta_step=250, beta_increment=0.01): """ Utility function to perform the # number of training loops given by the itertations argument. --------- Arguments: model : tf.keras.Model -- the invertible chaoin with an optional summary net both models are jointly trained optimizer : tf.train.optimizers.Optimizer -- the optimizer used for backprop data_generator : callable -- a function providing batches of X, theta (data, params) iterations : int -- the number of training loops to perform batch_size : int -- the batch_size used for training p_bar : ProgressBar -- an instance for tracking the training progress clip_value : float -- the value used for clipping the gradients global_step : tf.EagerVariavle -- a scalar tensor tracking the number of steps and used for learning rate decay transform : callable ot None -- a function to transform X and theta, if given n_smooth : int -- a value indicating how many values to use for computing the running ML loss ---------- Returns: losses : dict -- a dictionary with the ml_loss and decay """ # Prepare a dictionary to track losses losses = { 'kl_loss': [], 'rec_loss': [], 'decay': [] } # Run training loop for it in range(1, iterations+1): with tf.GradientTape() as tape: # Generate data and parameters X_batch, theta_batch = data_generator(batch_size) # Apply some transformation, if specified if transform: X_batch, theta_batch = transform(X_batch, theta_batch) # Sanity check for non-empty tensors if tf.equal(X_batch.shape[0], 0).numpy(): print('Iteration produced empty tensor, skipping...') continue # Forward pass theta_hat, z, logqz_x = model(theta_batch, X_batch) # Compute losses kl = kullback_leibler_iaf(z, logqz_x, beta) rec = mean_squared_error(theta_batch, theta_hat) decay = tf.add_n(model.losses) total_loss = kl + rec + decay # Store losses losses['kl_loss'].append(kl.numpy()) losses['rec_loss'].append(rec.numpy()) losses['decay'].append(decay.numpy()) # One step backprop gradients = tape.gradient(total_loss, model.trainable_variables) if clip_value is not None: gradients, _ = tf.clip_by_global_norm(gradients, clip_value) apply_gradients(optimizer, gradients, model.trainable_variables, global_step) # Increase beta every beta_step iterations if (global_step.numpy() + 1) % beta_step == 0: tf.assign(beta, min(beta_max, beta.numpy() + beta_increment)) # Update progress bar p_bar.set_postfix_str("Iteration: {0},Rec Loss: {1:.3f},KL Loss: {2:.3f},Regularization Loss: {3:.3f},Beta: {4:.2f}" .format(it, rec.numpy(), kl.numpy(), decay.numpy(), beta.numpy())) p_bar.update(1) return losses
def forward(self, y, t): self.y = y self.t = t return mean_squared_error(self.y, self.t)
# TODO: Not check. if __name__ == "__main__": img = Image.open(b"../../../LFW/match pairs/0001/Aaron_Peirsol_0001.jpg") # print(img.size) img = np.array(img, dtype=np.float32) # print(img.shape) z = np.random.rand(10, 51, 51, 3).astype(np.float32) # print(z) Conv1 = Convolution(z.shape, out_channels=4, kernel_size=2, stride=1, learning_rate=0.00001) out1 = Conv1.forward(z) pooling1 = Pooling(out1.shape, pool_size=2) out2 = pooling1.forward(out1) Conv2 = Convolution(out2.shape, out_channels=5 ,kernel_size=2, stride=1, learning_rate=0.00001) y_pred = Conv2.forward(out2) print(y_pred.shape) y_true = np.ones((10, 24, 24, 5)).astype(np.float32) for i in range(10): error, dy = losses.mean_squared_error(y_pred, y_true) print(error) print("------") eta3to2 = Conv2.gradient(dy) Conv2.backward() eta2to1 = pooling1.gradient(eta3to2) _ = Conv1.gradient(eta2to1) Conv1.backward() out1 = Conv1.forward(z) out2 = pooling1.forward(out1) y_pred = Conv2.forward(out2)