def master_process_minibatch(self, A_indices, A_scaling_factors, segment):
assert A_indices.shape == A_scaling_factors.shape, "Failed to assertion that %s == %s." % (
A_indices.shape, A_scaling_factors.shape)
assert segment in ["train"]
if self.model_config["normalize_data"]:
X_minibatch = normalizeMatrix(self.data[segment][0][A_indices],
self.mean, self.std)
else:
X_minibatch = self.data[segment][0][A_indices]
Y_minibatch = self.data[segment][1][A_indices]
assert np.all(np.isfinite(X_minibatch))
assert np.all(np.isfinite(Y_minibatch))
# These numpy arrays here have the same names as theano variables
# elsewhere in this class. Don't get confused.
(cost, accuracy, mean_gradient_square_norm,
individual_gradient_square_norm) = self.func_master_process_minibatch(
X_minibatch, Y_minibatch, A_scaling_factors)
self.num_minibatches_processed_master += 1
# The mean_gradient_square_norm and mean_gradient_variance
# are not written to the database, but they would be really nice to log to have a
# good idea of what is happening on the master.
# Returns nothing. The master should have used this call to
# update its internal parameters.
# return
# For use in debugging, we return here the `individual_gradient_square_norm`.
# This can change to suit our debugging needs.
return individual_gradient_square_norm
def master_process_minibatch(self, A_indices, A_scaling_factors, segment):
assert A_indices.shape == A_scaling_factors.shape, "Failed to assertion that %s == %s." % (
A_indices.shape,
A_scaling_factors.shape,
)
assert segment in ["train"]
if self.model_config["normalize_data"]:
X_minibatch = normalizeMatrix(self.data[segment][0][A_indices], self.mean, self.std)
else:
X_minibatch = self.data[segment][0][A_indices]
Y_minibatch = self.data[segment][1][A_indices]
assert np.all(np.isfinite(X_minibatch))
assert np.all(np.isfinite(Y_minibatch))
# These numpy arrays here have the same names as theano variables
# elsewhere in this class. Don't get confused.
(
cost,
accuracy,
mean_gradient_square_norm,
individual_gradient_square_norm,
) = self.func_master_process_minibatch(X_minibatch, Y_minibatch, A_scaling_factors)
self.num_minibatches_processed_master += 1
# The mean_gradient_square_norm and mean_gradient_variance
# are not written to the database, but they would be really nice to log to have a
# good idea of what is happening on the master.
# Returns nothing. The master should have used this call to
# update its internal parameters.
# return
# For use in debugging, we return here the `individual_gradient_square_norm`.
# This can change to suit our debugging needs.
return individual_gradient_square_norm
def worker_process_minibatch(self, A_indices, segment, L_measurements):
assert segment in ["train", "valid", "test"]
for key in L_measurements:
assert key in [
"individual_importance_weight",
"individual_gradient_square_norm",
"individual_loss",
"individual_accuracy",
"minibatch_gradient_mean_square_norm",
# old measurement names
"importance_weight",
"gradient_square_norm",
"loss",
"accuracy",
]
if self.model_config["normalize_data"]:
X_minibatch = normalizeMatrix(self.data[segment][0][A_indices], self.mean, self.std)
else:
X_minibatch = self.data[segment][0][A_indices]
Y_minibatch = self.data[segment][1][A_indices]
assert np.all(np.isfinite(X_minibatch))
assert np.all(np.isfinite(Y_minibatch))
# These numpy arrays here have the same names as theano variables
# elsewhere in this class. Don't get confused.
(
individual_cost,
individual_accuracy,
individual_gradient_square_norm,
minibatch_gradient_mean_square_norm,
) = self.func_process_worker_minibatch(X_minibatch, Y_minibatch)
individual_importance_weight = np.sqrt(individual_gradient_square_norm)
# DEBUG : Set all those quantities to something random, and see what happens.
# This is not the same thing as actually influencing the gradients.
# We are just messing with the values in the database.
# Tip : Use values that don't have expectation 0. The trouble with
# things that have expectation 0 is that you get a mu2 term
# that is very small and difficult to measure accurately.
# With random values, and expectation 0, you are easily
# misled to conclude that you cannot do anything useful
# with the measurements available because the differences
# to estimate Trace(Covariance) lead to negative values !
# print "Before override."
# print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,)
# print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,)
# (N, d) = (individual_gradient_square_norm.shape[0], 10)
# G = (np.random.randn(N, d) + np.tile(np.arange(d), (N,1))).astype(np.float32)
# G = np.tile(np.arange(d), (N,1)).astype(np.float32)
# individual_gradient_square_norm = (G**2).sum(axis=1)
# minibatch_gradient_mean_square_norm = ((G.mean(axis=0))**2).sum()
# print "After override."
# print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,)
# print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,)
# The individual_gradient_square_norm.mean() and individual_gradient_variance.mean()
# are not written to the database, but they would be really nice to log to have a
# good idea of what is happening on the worker.
self.num_minibatches_processed_worker += 1
# We can change the quantity that corresponds to 'importance_weight'
# by changing the entry in the `mapping` dictionary below.
mapping = {
"individual_importance_weight": individual_importance_weight,
"individual_cost": individual_cost,
"individual_loss": individual_cost,
"individual_accuracy": individual_accuracy.astype(dtype=np.float32),
"individual_gradient_square_norm": individual_gradient_square_norm,
"minibatch_gradient_mean_square_norm": np.array(minibatch_gradient_mean_square_norm),
# old measurement names
"importance_weight": individual_importance_weight,
"cost": individual_cost,
"loss": individual_cost,
"accuracy": individual_accuracy.astype(dtype=np.float32),
"gradient_square_norm": individual_gradient_square_norm,
}
# Returns a full array for every data point in the minibatch.
res = dict((measurement, mapping[measurement]) for measurement in L_measurements)
return res
def worker_process_minibatch(self, A_indices, segment, L_measurements):
assert segment in ["train", "valid", "test"]
for key in L_measurements:
assert key in [
"individual_importance_weight",
"individual_gradient_square_norm",
"individual_loss",
"individual_accuracy",
"minibatch_gradient_mean_square_norm",
# old measurement names
"importance_weight",
"gradient_square_norm",
"loss",
"accuracy"
]
if self.model_config["normalize_data"]:
X_minibatch = normalizeMatrix(self.data[segment][0][A_indices],
self.mean, self.std)
else:
X_minibatch = self.data[segment][0][A_indices]
Y_minibatch = self.data[segment][1][A_indices]
assert np.all(np.isfinite(X_minibatch))
assert np.all(np.isfinite(Y_minibatch))
# These numpy arrays here have the same names as theano variables
# elsewhere in this class. Don't get confused.
(individual_cost, individual_accuracy, individual_gradient_square_norm,
minibatch_gradient_mean_square_norm
) = self.func_process_worker_minibatch(X_minibatch, Y_minibatch)
individual_importance_weight = np.sqrt(individual_gradient_square_norm)
# DEBUG : Set all those quantities to something random, and see what happens.
# This is not the same thing as actually influencing the gradients.
# We are just messing with the values in the database.
# Tip : Use values that don't have expectation 0. The trouble with
# things that have expectation 0 is that you get a mu2 term
# that is very small and difficult to measure accurately.
# With random values, and expectation 0, you are easily
# misled to conclude that you cannot do anything useful
# with the measurements available because the differences
# to estimate Trace(Covariance) lead to negative values !
#print "Before override."
#print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,)
#print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,)
#(N, d) = (individual_gradient_square_norm.shape[0], 10)
#G = (np.random.randn(N, d) + np.tile(np.arange(d), (N,1))).astype(np.float32)
#G = np.tile(np.arange(d), (N,1)).astype(np.float32)
#individual_gradient_square_norm = (G**2).sum(axis=1)
#minibatch_gradient_mean_square_norm = ((G.mean(axis=0))**2).sum()
#print "After override."
#print "individual_gradient_square_norm.shape : %s" % str(individual_gradient_square_norm.shape,)
#print "minibatch_gradient_mean_square_norm.shape : %s" % str(minibatch_gradient_mean_square_norm.shape,)
# The individual_gradient_square_norm.mean() and individual_gradient_variance.mean()
# are not written to the database, but they would be really nice to log to have a
# good idea of what is happening on the worker.
self.num_minibatches_processed_worker += 1
# We can change the quantity that corresponds to 'importance_weight'
# by changing the entry in the `mapping` dictionary below.
mapping = {
'individual_importance_weight':
individual_importance_weight,
'individual_cost':
individual_cost,
'individual_loss':
individual_cost,
'individual_accuracy':
individual_accuracy.astype(dtype=np.float32),
'individual_gradient_square_norm':
individual_gradient_square_norm,
'minibatch_gradient_mean_square_norm':
np.array(minibatch_gradient_mean_square_norm),
# old measurement names
'importance_weight':
individual_importance_weight,
'cost':
individual_cost,
'loss':
individual_cost,
'accuracy':
individual_accuracy.astype(dtype=np.float32),
'gradient_square_norm':
individual_gradient_square_norm
}
# Returns a full array for every data point in the minibatch.
res = dict((measurement, mapping[measurement])
for measurement in L_measurements)
return res
