def master_process_minibatch(self, A_indices, A_scaling_factors, segment): assert A_indices.shape == A_scaling_factors.shape, "Failed to assertion that %s == %s." % ( A_indices.shape, A_scaling_factors.shape) assert segment in ["train"] if self.model_config["normalize_data"]: X_minibatch = normalizeMatrix(self.data[segment][0][A_indices], self.mean, self.std) else: X_minibatch = self.data[segment][0][A_indices] Y_minibatch = self.data[segment][1][A_indices] assert np.all(np.isfinite(X_minibatch)) assert np.all(np.isfinite(Y_minibatch)) # These numpy arrays here have the same names as theano variables # elsewhere in this class. Don't get confused. (cost, accuracy, mean_gradient_square_norm, individual_gradient_square_norm) = self.func_master_process_minibatch( X_minibatch, Y_minibatch, A_scaling_factors) self.num_minibatches_processed_master += 1 # The mean_gradient_square_norm and mean_gradient_variance # are not written to the database, but they would be really nice to log to have a # good idea of what is happening on the master. # Returns nothing. The master should have used this call to # update its internal parameters. # return # For use in debugging, we return here the `individual_gradient_square_norm`. # This can change to suit our debugging needs. return individual_gradient_square_norm
def master_process_minibatch(self, A_indices, A_scaling_factors, segment): assert A_indices.shape == A_scaling_factors.shape, "Failed to assertion that %s == %s." % ( A_indices.shape, A_scaling_factors.shape, ) assert segment in ["train"] if self.model_config["normalize_data"]: X_minibatch = normalizeMatrix(self.data[segment][0][A_indices], self.mean, self.std) else: X_minibatch = self.data[segment][0][A_indices] Y_minibatch = self.data[segment][1][A_indices] assert np.all(np.isfinite(X_minibatch)) assert np.all(np.isfinite(Y_minibatch)) # These numpy arrays here have the same names as theano variables # elsewhere in this class. Don't get confused. ( cost, accuracy, mean_gradient_square_norm, individual_gradient_square_norm, ) = self.func_master_process_minibatch(X_minibatch, Y_minibatch, A_scaling_factors) self.num_minibatches_processed_master += 1 # The mean_gradient_square_norm and mean_gradient_variance # are not written to the database, but they would be really nice to log to have a # good idea of what is happening on the master. # Returns nothing. The master should have used this call to # update its internal parameters. # return # For use in debugging, we return here the `individual_gradient_square_norm`. # This can change to suit our debugging needs. return individual_gradient_square_norm
def worker_process_minibatch(self, A_indices, segment, L_measurements): assert segment in ["train", "valid", "test"] for key in L_measurements: assert key in [ "individual_importance_weight", "individual_gradient_square_norm", "individual_loss", "individual_accuracy", "minibatch_gradient_mean_square_norm", # old measurement names "importance_weight", "gradient_square_norm", "loss", "accuracy", ] if self.model_config["normalize_data"]: X_minibatch = normalizeMatrix(self.data[segment][0][A_indices], self.mean, self.std) else: X_minibatch = self.data[segment][0][A_indices] Y_minibatch = self.data[segment][1][A_indices] assert np.all(np.isfinite(X_minibatch)) assert np.all(np.isfinite(Y_minibatch)) # These numpy arrays here have the same names as theano variables # elsewhere in this class. Don't get confused. ( individual_cost, individual_accuracy, individual_gradient_square_norm, minibatch_gradient_mean_square_norm, ) = self.func_process_worker_minibatch(X_minibatch, Y_minibatch) individual_importance_weight = np.sqrt(individual_gradient_square_norm) # DEBUG : Set all those quantities to something random, and see what happens. # This is not the same thing as actually influencing the gradients. # We are just messing with the values in the database. # Tip : Use values that don't have expectation 0. The trouble with # things that have expectation 0 is that you get a mu2 term # that is very small and difficult to measure accurately. # With random values, and expectation 0, you are easily # misled to conclude that you cannot do anything useful # with the measurements available because the differences # to estimate Trace(Covariance) lead to negative values ! # print "Before override." # print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,) # print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,) # (N, d) = (individual_gradient_square_norm.shape[0], 10) # G = (np.random.randn(N, d) + np.tile(np.arange(d), (N,1))).astype(np.float32) # G = np.tile(np.arange(d), (N,1)).astype(np.float32) # individual_gradient_square_norm = (G**2).sum(axis=1) # minibatch_gradient_mean_square_norm = ((G.mean(axis=0))**2).sum() # print "After override." # print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,) # print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,) # The individual_gradient_square_norm.mean() and individual_gradient_variance.mean() # are not written to the database, but they would be really nice to log to have a # good idea of what is happening on the worker. self.num_minibatches_processed_worker += 1 # We can change the quantity that corresponds to 'importance_weight' # by changing the entry in the `mapping` dictionary below. mapping = { "individual_importance_weight": individual_importance_weight, "individual_cost": individual_cost, "individual_loss": individual_cost, "individual_accuracy": individual_accuracy.astype(dtype=np.float32), "individual_gradient_square_norm": individual_gradient_square_norm, "minibatch_gradient_mean_square_norm": np.array(minibatch_gradient_mean_square_norm), # old measurement names "importance_weight": individual_importance_weight, "cost": individual_cost, "loss": individual_cost, "accuracy": individual_accuracy.astype(dtype=np.float32), "gradient_square_norm": individual_gradient_square_norm, } # Returns a full array for every data point in the minibatch. res = dict((measurement, mapping[measurement]) for measurement in L_measurements) return res
def worker_process_minibatch(self, A_indices, segment, L_measurements): assert segment in ["train", "valid", "test"] for key in L_measurements: assert key in [ "individual_importance_weight", "individual_gradient_square_norm", "individual_loss", "individual_accuracy", "minibatch_gradient_mean_square_norm", # old measurement names "importance_weight", "gradient_square_norm", "loss", "accuracy" ] if self.model_config["normalize_data"]: X_minibatch = normalizeMatrix(self.data[segment][0][A_indices], self.mean, self.std) else: X_minibatch = self.data[segment][0][A_indices] Y_minibatch = self.data[segment][1][A_indices] assert np.all(np.isfinite(X_minibatch)) assert np.all(np.isfinite(Y_minibatch)) # These numpy arrays here have the same names as theano variables # elsewhere in this class. Don't get confused. (individual_cost, individual_accuracy, individual_gradient_square_norm, minibatch_gradient_mean_square_norm ) = self.func_process_worker_minibatch(X_minibatch, Y_minibatch) individual_importance_weight = np.sqrt(individual_gradient_square_norm) # DEBUG : Set all those quantities to something random, and see what happens. # This is not the same thing as actually influencing the gradients. # We are just messing with the values in the database. # Tip : Use values that don't have expectation 0. The trouble with # things that have expectation 0 is that you get a mu2 term # that is very small and difficult to measure accurately. # With random values, and expectation 0, you are easily # misled to conclude that you cannot do anything useful # with the measurements available because the differences # to estimate Trace(Covariance) lead to negative values ! #print "Before override." #print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,) #print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,) #(N, d) = (individual_gradient_square_norm.shape[0], 10) #G = (np.random.randn(N, d) + np.tile(np.arange(d), (N,1))).astype(np.float32) #G = np.tile(np.arange(d), (N,1)).astype(np.float32) #individual_gradient_square_norm = (G**2).sum(axis=1) #minibatch_gradient_mean_square_norm = ((G.mean(axis=0))**2).sum() #print "After override." #print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,) #print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,) # The individual_gradient_square_norm.mean() and individual_gradient_variance.mean() # are not written to the database, but they would be really nice to log to have a # good idea of what is happening on the worker. self.num_minibatches_processed_worker += 1 # We can change the quantity that corresponds to 'importance_weight' # by changing the entry in the `mapping` dictionary below. mapping = { 'individual_importance_weight': individual_importance_weight, 'individual_cost': individual_cost, 'individual_loss': individual_cost, 'individual_accuracy': individual_accuracy.astype(dtype=np.float32), 'individual_gradient_square_norm': individual_gradient_square_norm, 'minibatch_gradient_mean_square_norm': np.array(minibatch_gradient_mean_square_norm), # old measurement names 'importance_weight': individual_importance_weight, 'cost': individual_cost, 'loss': individual_cost, 'accuracy': individual_accuracy.astype(dtype=np.float32), 'gradient_square_norm': individual_gradient_square_norm } # Returns a full array for every data point in the minibatch. res = dict((measurement, mapping[measurement]) for measurement in L_measurements) return res