def compute_output(self,gpu_data):
"""
Computes p(y|x). Puts the result in self.gpu_p_y_given_x.
"""
cm.dot(self.W,gpu_data,self.gpu_act_from_x)
self.gpu_act_from_x.add_col_vec(self.c)
for c in range(self.n_classes):
cm.dot(self.U,self.gpu_target_vectors.slice(c,c+1),self.gpu_act_from_y)
# to avoid memory creation, using gpu_h
# and gpu_h_sample for these computations
self.gpu_act_from_x.add_col_vec(self.gpu_act_from_y,target=self.gpu_h)
cm.exp(self.gpu_h,self.gpu_h_sample)
self.gpu_h_sample.add(1.)
cm.log(self.gpu_h_sample,self.gpu_h)
self.gpu_h.sum(axis=0,target=self.gpu_negative_free_energy_for_y)
cm.dot(self.d.T,self.gpu_target_vectors.slice(c,c+1),target=self.gpu_bias_from_y)
self.gpu_negative_free_energy_for_y.add_col_vec(self.gpu_bias_from_y)
self.gpu_negative_free_energy_for_y.transpose(target=self.gpu_negative_free_energy.slice(c,c+1))
# Subtracting mean for more stable softmax computation
self.gpu_negative_free_energy.sum(axis=1,target=self.gpu_mean_negative_free_energy)
self.gpu_mean_negative_free_energy.divide(-self.n_classes)
self.gpu_negative_free_energy.add_col_vec(self.gpu_mean_negative_free_energy)
cm.exp(self.gpu_negative_free_energy,target=self.gpu_negative_free_energy)
self.gpu_negative_free_energy.sum(axis=1,target=self.gpu_p_y_given_x_norm)
for c in range(self.n_classes):
self.gpu_negative_free_energy.slice(c,c+1).divide(self.gpu_p_y_given_x_norm,
target=self.gpu_p_y_given_x.slice(c,c+1))
self.gpu_p_y_given_x.transpose(target=self.gpu_p_y_given_x_trans)
def _log_multivariate_normal_density_diag(X, means, covars, temp_gpu_mem):
'''Compute Gaussian log-density at X for a diagonal model'''
N, D = X.shape
K = means.shape[0]
lpr_NxK = temp_gpu_mem['posteriors_NxK']
inv_covars_KxD = temp_gpu_mem['temp_KxD_2']
temp_NxD = temp_gpu_mem['temp_NxD']
temp_KxD = temp_gpu_mem['temp_KxD']
temp_Kx1 = temp_gpu_mem['temp_Kx1']
# compute inverse variances
inv_covars_KxD.assign(1.0)
inv_covars_KxD.divide(covars)
# lpr = D * np.log(2*np.pi)
lpr_NxK.assign(D * np.log(2*np.pi))
# temp_Kx1 = np.sum(np.log(covars), 1)
cm.log(covars, target=temp_KxD)
temp_KxD.sum(axis=1, target=temp_Kx1)
# temp_Kx1 += np.sum((means**2)/covars, 1)
means.mult(means, target=temp_KxD)
temp_KxD.mult(inv_covars_KxD)
temp_Kx1.add_sums(temp_KxD, axis=1)
# lpr += temp_Kx1
temp_Kx1.reshape((1, K)) # transpose
lpr_NxK.add_row_vec(temp_Kx1)
temp_Kx1.reshape((K, 1)) # return to original shape
# lpr += -2*np.dot(X, (means / covars).T)
temp_KxD.assign(means)
temp_KxD.mult(inv_covars_KxD)
lpr_NxK.add_dot(X, temp_KxD.T, mult=-2.)
# lpr += np.dot(X**2, (1.0 / covars).T)
temp_NxD.assign(X)
temp_NxD.mult(temp_NxD)
lpr_NxK.add_dot(temp_NxD, inv_covars_KxD.T)
# lpr *= -0.5
lpr_NxK.mult(-0.5)
# lpr_NxK still in use
# lpr = -0.5 * (D * np.log(2 * np.pi) + np.sum(np.log(covars), 1)
# + np.sum((means ** 2) / covars, 1)
# - 2 * np.dot(X, (means / covars).T)
# + np.dot(X ** 2, (1.0 / covars).T))
return lpr_NxK
def logOnePlusExp(x, temp, targ = None):
"""
When this function is done, x should contain log(1+exp(x)). We
clobber the value of temp. We compute log(1+exp(x)) as x +
log(1+exp(-x)), which will hopefully be more finite-precision
friendly.
"""
assert(x.shape == temp.shape)
x.mult(-1, target = temp)
cm.exp(temp)
temp.add(1)
cm.log(temp)
x.add(temp, target = targ)
def logOnePlusExp(x, temp, targ=None):
"""
When this function is done, x should contain log(1+exp(x)). We
clobber the value of temp. We compute log(1+exp(x)) as x +
log(1+exp(-x)), which will hopefully be more finite-precision
friendly.
"""
assert (x.shape == temp.shape)
x.mult(-1, target=temp)
cm.exp(temp)
temp.add(1)
cm.log(temp)
x.add(temp, target=targ)
def negative_free_energy(self, gpu_data):
"""
Computes the negative free-energy.
Outputs a reference to a pre-allocated GPU variable
containing the result.
"""
cm.dot(self.W, gpu_data, self.gpu_h)
self.gpu_h.add_col_vec(self.c)
# to avoid memory creation, using gpu_h
# and gpu_h_sample for these computations
cm.exp(self.gpu_h, self.gpu_h_sample)
self.gpu_h_sample.add(1.)
cm.log(self.gpu_h_sample, self.gpu_h)
self.gpu_h.sum(axis=0, target=self.gpu_negative_free_energy)
self.gpu_negative_free_energy.add_dot(self.b.T, gpu_data)
return self.gpu_negative_free_energy
def negative_free_energy(self,gpu_data):
"""
Computes the negative free-energy.
Outputs a reference to a pre-allocated GPU variable
containing the result.
"""
cm.dot(self.W,gpu_data,self.gpu_h)
self.gpu_h.add_col_vec(self.c)
# to avoid memory creation, using gpu_h
# and gpu_h_sample for these computations
cm.exp(self.gpu_h,self.gpu_h_sample)
self.gpu_h_sample.add(1.)
cm.log(self.gpu_h_sample,self.gpu_h)
self.gpu_h.sum(axis=0,target=self.gpu_negative_free_energy)
self.gpu_negative_free_energy.add_dot(self.b.T,gpu_data)
return self.gpu_negative_free_energy
def test_log():
m = 256
n = 128
a = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float32, order='F')
b = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float32, order='F')
c = np.log(a)
m1 = cm.CUDAMatrix(a)
m2 = cm.CUDAMatrix(b)
cm.log(m1, target = m2)
cm.log(m1)
m1.copy_to_host()
m2.copy_to_host()
assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold"
assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold"
def compute_gamma_entropy(self, G):
if not self.gpu:
Prod = G * (np.log(G) - 1)
ent = np.nan_to_num(Prod).sum()
else:
Prod = cm.empty(G.shape)
Prod = G.mult(cm.log(G.copy()).subtract(1), target=Prod)
ent = np.nan_to_num(Prod.asarray()).sum()
return ent
def test_log():
m = 256
n = 128
a = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float32, order='F')
b = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float32, order='F')
c = np.log(a)
m1 = cm.CUDAMatrix(a)
m2 = cm.CUDAMatrix(b)
cm.log(m1, target = m2)
cm.log(m1)
m1.copy_to_host()
m2.copy_to_host()
assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold"
assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold"
def compute_energy_mcRBM_visual(self, data, normdata, energy, VF, FH,
bias_cov, bias_vis, w_mean, bias_mean, t1,
t2, t6, feat, featsq, feat_mean, length,
lengthsq, normcoeff, small, num_vis):
# normalize input data vectors
data.mult(data, target=t6) # DxP (nr input dims x nr samples)
t6.sum(axis=0, target=lengthsq) # 1xP
lengthsq.mult(0.5,
target=energy) # energy of quadratic regularization term
lengthsq.mult(1. /
num_vis) # normalize by number of components (like std)
lengthsq.add(small) # small prevents division by 0
cmt.sqrt(lengthsq, target=length)
length.reciprocal(target=normcoeff) # 1xP
data.mult_by_row(normcoeff, target=normdata) # normalized data
## potential
# covariance contribution
cmt.dot(VF.T, normdata, target=feat) # HxP (nr factors x nr samples)
feat.mult(feat, target=featsq) # HxP
cmt.dot(FH.T, featsq, target=t1) # OxP (nr cov hiddens x nr samples)
t1.mult(-0.5)
t1.add_col_vec(bias_cov) # OxP
cmt.exp(t1) # OxP
t1.add(1, target=t2) # OxP
cmt.log(t2)
t2.mult(-1)
energy.add_sums(t2, axis=0)
# mean contribution
cmt.dot(w_mean.T, data,
target=feat_mean) # HxP (nr mean hiddens x nr samples)
feat_mean.add_col_vec(bias_mean) # HxP
cmt.exp(feat_mean)
feat_mean.add(1)
cmt.log(feat_mean)
feat_mean.mult(-1)
energy.add_sums(feat_mean, axis=0)
# visible bias term
data.mult_by_col(bias_vis, target=t6)
t6.mult(-1) # DxP
energy.add_sums(t6, axis=0) # 1xP
# kinetic
data.mult(data, target=t6)
energy.add_sums(t6, axis=0, mult=.5)
def compute_output(self, gpu_data):
"""
Computes p(y|x). Puts the result in self.gpu_p_y_given_x.
"""
cm.dot(self.W, gpu_data, self.gpu_act_from_x)
self.gpu_act_from_x.add_col_vec(self.c)
for c in range(self.n_classes):
cm.dot(self.U, self.gpu_target_vectors.slice(c, c + 1),
self.gpu_act_from_y)
# to avoid memory creation, using gpu_h
# and gpu_h_sample for these computations
self.gpu_act_from_x.add_col_vec(self.gpu_act_from_y,
target=self.gpu_h)
cm.exp(self.gpu_h, self.gpu_h_sample)
self.gpu_h_sample.add(1.)
cm.log(self.gpu_h_sample, self.gpu_h)
self.gpu_h.sum(axis=0, target=self.gpu_negative_free_energy_for_y)
cm.dot(self.d.T,
self.gpu_target_vectors.slice(c, c + 1),
target=self.gpu_bias_from_y)
self.gpu_negative_free_energy_for_y.add_col_vec(
self.gpu_bias_from_y)
self.gpu_negative_free_energy_for_y.transpose(
target=self.gpu_negative_free_energy.slice(c, c + 1))
# Subtracting mean for more stable softmax computation
self.gpu_negative_free_energy.sum(
axis=1, target=self.gpu_mean_negative_free_energy)
self.gpu_mean_negative_free_energy.divide(-self.n_classes)
self.gpu_negative_free_energy.add_col_vec(
self.gpu_mean_negative_free_energy)
cm.exp(self.gpu_negative_free_energy,
target=self.gpu_negative_free_energy)
self.gpu_negative_free_energy.sum(axis=1,
target=self.gpu_p_y_given_x_norm)
for c in range(self.n_classes):
self.gpu_negative_free_energy.slice(c, c + 1).divide(
self.gpu_p_y_given_x_norm,
target=self.gpu_p_y_given_x.slice(c, c + 1))
self.gpu_p_y_given_x.transpose(target=self.gpu_p_y_given_x_trans)
def Sample(self):
logging.debug("Sample in %s", self.name)
state = self.state
sample = self.sample
if self.activation == deepnet_pb2.Hyperparams.LOGISTIC:
state.sample_binomial(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.TANH:
state.sample_binomial_tanh(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR:
state.sample_poisson(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH:
state.sample_poisson(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.LINEAR:
# sample.assign(state)
if self.learn_precision:
sample.fill_with_randn()
sample.div_by_col(self.params["precision"])
sample.add(state)
# state.sample_gaussian(target=sample, mult=0.01)
elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX:
sample.fill_with_rand()
cm.log(sample)
sample.mult(-1)
sample.reciprocal()
sample.mult(state)
sample.argmax(axis=0, target=self.temp)
self.expansion_matrix.select_columns(self.temp, target=sample)
elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX:
if self.proto.hyperparams.normalize:
sample.assign(0)
temp_sample = self.expanded_batch
numsamples = int(self.proto.hyperparams.normalize_to)
for i in range(numsamples):
temp_sample.fill_with_rand()
cm.log(temp_sample)
temp_sample.mult(-1)
state.divide(temp_sample, target=temp_sample)
temp_sample.max(axis=0, target=self.temp)
temp_sample.add_row_mult(self.temp, -1)
temp_sample.less_than(0)
sample.add(temp_sample)
sample.subtract(numsamples)
sample.mult(-1)
else: # This is an approximation.
sample.fill_with_rand()
cm.log(sample)
sample.mult(-1)
sample.reciprocal()
sample.mult(state)
sample.argmax(axis=0, target=self.temp)
self.expansion_matrix.select_columns(self.temp, target=sample)
sample.mult_by_row(self.NN)
else:
raise Exception("Unknown activation")
def normalizeM(self, norm):
if norm == "median":
self.M_GPU.divide(float(np.median(self.M_GPU.asarray())))
elif norm == "max":
self.M_GPU.divide(float(np.max(self.M_GPU.asarray())))
elif norm == "log":
self.M_GPU.add(1)
cudamat.log(self.M_GPU)
elif norm == "loglog":
self.M_GPU.add(1)
cudamat.log(self.M_GPU)
self.M_GPU.add(1)
cudamat.log(self.M_GPU)
def normalizeM(self, norm):
if norm == "median":
self.M_GPU.divide(float(np.median(self.M_GPU.asarray())))
elif norm == "max":
self.M_GPU.divide(float(np.max(self.M_GPU.asarray())))
elif norm == "log":
self.M_GPU.add(1)
cudamat.log(self.M_GPU)
elif norm == "loglog":
self.M_GPU.add(1)
cudamat.log(self.M_GPU)
self.M_GPU.add(1)
cudamat.log(self.M_GPU)
def Sample(self):
logging.debug('Sample in %s', self.name)
state = self.state
sample = self.sample
if self.activation == deepnet_pb2.Hyperparams.LOGISTIC:
state.sample_binomial(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.TANH:
state.sample_binomial_tanh(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR:
state.sample_poisson(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH:
state.sample_poisson(target=sample)
elif self.activation == deepnet_pb2.Hyperparams.LINEAR:
#sample.assign(state)
state.sample_gaussian(target=sample, mult=0.01)
elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX:
sample.fill_with_rand()
cm.log(sample)
sample.mult(-1)
sample.reciprocal()
sample.mult(state)
sample.argmax(axis=0, target=self.temp)
self.expansion_matrix.select_columns(self.temp, target=sample)
elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX:
if self.proto.hyperparams.normalize:
sample.assign(0)
temp_sample = self.expanded_batch
numsamples = int(self.proto.hyperparams.normalize_to)
for i in range(numsamples):
temp_sample.fill_with_rand()
cm.log(temp_sample)
temp_sample.mult(-1)
state.divide(temp_sample, target=temp_sample)
temp_sample.max(axis=0, target=self.temp)
temp_sample.add_row_mult(self.temp, -1)
temp_sample.less_than(0)
sample.add(temp_sample)
sample.subtract(numsamples)
sample.mult(-1)
else: # This is an approximation.
sample.fill_with_rand()
cm.log(sample)
sample.mult(-1)
sample.reciprocal()
sample.mult(state)
sample.argmax(axis=0, target=self.temp)
self.expansion_matrix.select_columns(self.temp, target=sample)
sample.mult_by_row(self.NN)
else:
raise Exception('Unknown activation')
def GetLoss(self, get_deriv=False):
"""Compute loss and also deriv w.r.t to it if asked for.
Compute the loss function. Targets should be in self.data, predictions
should be in self.state.
Args:
get_deriv: If True, compute the derivative w.r.t the loss function and put
it in self.deriv.
"""
perf = deepnet_pb2.Metrics()
perf.MergeFrom(self.proto.performance_stats)
perf.count = self.batchsize
tiny = self.tiny
if self.loss_function == deepnet_pb2.Layer.CROSS_ENTROPY:
if self.activation == deepnet_pb2.Hyperparams.LOGISTIC:
data = self.data
state = self.state
deriv = self.deriv
temp3 = self.temp3
unitcell = self.unitcell
cm.cross_entropy(data, state, target=deriv, tiny=self.tiny)
deriv.sum(axis=1, target=temp3)
temp3.sum(axis=0, target=unitcell)
cross_entropy = unitcell.euclid_norm()
cm.correct_preds(data, state, target=deriv, cutoff=0.5)
deriv.sum(axis=1, target=temp3)
temp3.sum(axis=0, target=unitcell)
correct_preds = unitcell.euclid_norm()
if get_deriv:
self.state.subtract(self.data, target=self.deriv)
perf.cross_entropy = cross_entropy
perf.correct_preds = correct_preds
elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX:
temp2 = self.temp2
temp = self.temp
batchsize = self.batchsize
dimensions = self.dimensions
numlabels = self.numlabels
state = self.state
data = self.data
unitcell = self.unitcell
indices = self.indices
# Optimized for space to handle large number of labels in a softmax.
data.reshape((1, batchsize * dimensions))
data.add(self.rowshift, target=indices)
state.reshape((numlabels, dimensions * batchsize))
state.max(axis=0, target=temp2)
state.reshape((1, batchsize * numlabels * dimensions))
state.select_columns(indices, temp)
temp2.subtract(temp)
temp2.sign(target=temp2)
temp2.sum(axis=1, target=unitcell)
correct_preds = batchsize - unitcell.euclid_norm()
if get_deriv:
temp.subtract(1, target=temp2)
state.set_selected_columns(indices, temp2)
state.reshape((numlabels * dimensions, batchsize))
self.deriv.assign(self.state)
state.reshape((numlabels * dimensions, batchsize))
temp.add(tiny)
cm.log(temp)
temp.sum(axis=1, target=unitcell)
cross_entropy = unitcell.euclid_norm()
perf.cross_entropy = cross_entropy
perf.correct_preds = correct_preds
elif self.loss_function == deepnet_pb2.Layer.SQUARED_LOSS:
if self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX and self.hyperparams.normalize:
self.data.sum(axis=0, target=self.temp)
self.temp.add(self.tiny)
self.data.div_by_row(self.temp, target=self.deriv)
self.deriv.mult(self.proto.hyperparams.normalize_to)
self.deriv.subtract(self.state)
elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX:
self.expansion_matrix.select_columns(
self.data, target=self.expanded_batch)
self.state.subtract(self.expanded_batch, target=self.deriv)
else:
self.state.subtract(self.data, target=self.deriv)
error = self.deriv.euclid_norm()**2
perf.error = error
if self.activation != deepnet_pb2.Hyperparams.SOFTMAX and \
self.activation != deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX:
self.ComputeDeriv()
else:
raise Exception('Unknown loss function.')
return perf
def GetLoss(self, get_deriv=False):
"""Compute loss and also deriv w.r.t to it if asked for.
Compute the loss function. Targets should be in self.data, predictions
should be in self.state.
Args:
get_deriv: If True, compute the derivative w.r.t the loss function and put
it in self.deriv.
"""
perf = deepnet_pb2.Metrics()
perf.MergeFrom(self.proto.performance_stats)
perf.count = self.batchsize
tiny = self.tiny
if self.loss_function == deepnet_pb2.Layer.CROSS_ENTROPY:
if self.activation == deepnet_pb2.Hyperparams.LOGISTIC:
data = self.data
state = self.state
deriv = self.deriv
temp3 = self.dimsize
unitcell = self.unitcell
cm.cross_entropy(data, state, target=deriv, tiny=self.tiny)
deriv.sum(axis=1, target=temp3)
temp3.sum(axis=0, target=unitcell)
cross_entropy = unitcell.euclid_norm()
cm.correct_preds(data, state, target=deriv, cutoff=0.5)
deriv.sum(axis=1, target=temp3)
temp3.sum(axis=0, target=unitcell)
correct_preds = unitcell.euclid_norm()
if get_deriv:
self.state.subtract(self.data, target=self.deriv)
perf.cross_entropy = cross_entropy
perf.correct_preds = correct_preds
elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX:
temp2 = self.temp2
temp = self.temp
batchsize = self.batchsize
dimensions = self.dimensions
numlabels = self.numlabels
state = self.state
data = self.data
unitcell = self.unitcell
indices = self.indices
# Optimized for space to handle large number of labels in a softmax.
data.reshape((1, batchsize * dimensions))
data.add(self.rowshift, target=indices)
state.reshape((numlabels, dimensions * batchsize))
state.max(axis=0, target=temp2)
state.reshape((1, batchsize * numlabels * dimensions))
state.select_columns(indices, temp)
temp2.subtract(temp)
temp2.sign(target=temp2)
temp2.sum(axis=1, target=unitcell)
correct_preds = batchsize - unitcell.euclid_norm()
if get_deriv:
temp.subtract(1, target=temp2)
state.set_selected_columns(indices, temp2)
state.reshape((numlabels * dimensions, batchsize))
self.deriv.assign(self.state)
state.reshape((numlabels * dimensions, batchsize))
temp.add(tiny)
cm.log(temp)
temp.sum(axis=1, target=unitcell)
cross_entropy = unitcell.euclid_norm()
perf.cross_entropy = cross_entropy
perf.correct_preds = correct_preds
elif self.loss_function == deepnet_pb2.Layer.SQUARED_LOSS:
if self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX and self.hyperparams.normalize:
self.data.sum(axis=0, target=self.temp)
self.temp.add(self.tiny)
self.data.div_by_row(self.temp, target=self.deriv)
self.deriv.mult(self.proto.hyperparams.normalize_to)
self.deriv.subtract(self.state)
elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX:
self.expansion_matrix.select_columns(self.data, target=self.expanded_batch)
self.state.subtract(self.expanded_batch, target=self.deriv)
else:
if "precision" in self.params:
self.data.mult_by_col(self.params["precision"], target=self.deriv)
self.deriv.subtract(self.state)
else:
self.state.subtract(self.data, target=self.deriv)
error = self.deriv.euclid_norm() ** 2
perf.error = error
if (
self.activation != deepnet_pb2.Hyperparams.SOFTMAX
and self.activation != deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX
):
self.ComputeDeriv()
else:
raise Exception("Unknown loss function.")
return perf
def ConvolveUp(self, inputs, edge, target):
w = edge.params["weight"]
conv = edge.conv_params
size = conv.size
stride = conv.stride
padding = conv.padding
num_filters = conv.num_filters
num_colors = conv.num_colors
f, numdims = w.shape
assert f == num_filters, "f is %d but num_filters is %d" % (f, num_filters)
if edge.conv:
assert numdims == size ** 2 * num_colors
input_t = edge.input_t
inputs.transpose(input_t)
# Convolve Up.
if conv.max_pool:
output_t = edge.unpooled_layer
elif conv.rnorm:
output_t = edge.unrnormalized_layer
else:
output_t = edge.output_t
numimages, numdims = input_t.shape
numimages2, numdims2 = output_t.shape
assert numimages == numimages2, "%d %d." % (numimages, numimages2)
assert numdims % num_colors == 0
x = int(np.sqrt(numdims / num_colors))
assert x ** 2 == numdims / num_colors
n_locs = (x + 2 * padding - size) / stride + 1
if edge.conv:
cc.convUp(input_t, w, output_t, n_locs, padding, stride, num_colors)
else:
cc.localUp(input_t, w, output_t, n_locs, padding, stride, num_colors)
# Do maxpooling
if conv.max_pool:
input_t = output_t
if conv.rnorm:
output_t = edge.unrnormalized_layer
else:
output_t = edge.output_t
n_locs = (n_locs - conv.pool_size) / conv.pool_stride + 1
if conv.prob:
# Get Gumbel variates.
rnd = edge.rnd
rnd.fill_with_rand()
cm.log(rnd)
rnd.mult(-1)
cm.log(rnd)
rnd.mult(-1)
cc.ProbMaxPool(input_t, rnd, output_t, num_filters, conv.pool_size, 0, conv.pool_stride, n_locs)
else:
cc.MaxPool(input_t, output_t, num_filters, conv.pool_size, 0, conv.pool_stride, n_locs)
if conv.rnorm:
input_t = output_t
output_t = edge.output_t
denoms = edge.denoms
sizeX = conv.norm_size
add_scale = conv.add_scale
pow_scale = conv.pow_scale
cc.ResponseNorm(input_t, denoms, output_t, num_filters, sizeX, add_scale, pow_scale)
output_t.transpose(target)
def ff(x0_cpu):
data_size = x0_cpu.shape[1]
x_l0 = cm.empty((num_input, data_size))
x_l0.assign(cm.CUDAMatrix(x0_cpu))
x_l1 = cm.empty((num_hid, data_size))
cm.dot(w1.T, x_l0, target = x_l1)
x_l1.add_col_vec(b1)
x_l1.apply_sigmoid()
x_l2 = cm.empty((num_hid, data_size))
del x_l0
cm.dot(w2.T, x_l1, target = x_l2)
x_l2.add_col_vec(b2)
x_l2.apply_sigmoid()
x_l3 = cm.empty((num_hid, data_size))
del x_l1
cm.dot(w3.T, x_l2, target = x_l3)
x_l3.add_col_vec(b3)
x_l3.apply_sigmoid()
x_l4 = cm.empty((num_hid, data_size))
del x_l2
cm.dot(w4.T, x_l3, target = x_l4)
x_l4.add_col_vec(b4)
x_l4.apply_sigmoid()
x_l5 = cm.empty((num_hid, data_size))
del x_l3
cm.dot(w5.T, x_l4, target = x_l5)
x_l5.add_col_vec(b5)
x_l5.apply_sigmoid()
x_output = cm.empty((num_output, data_size))
del x_l4
tmp_x_output = cm.empty((num_output, data_size))
tmp_x_output_sums = cm.empty((1, data_size))
cm.dot(wo.T, x_l5, target = tmp_x_output)
tmp_x_output.add_col_vec(bo)
cm.exp(tmp_x_output)
tmp_x_output.sum(axis=0, target = tmp_x_output_sums)
tmp_x_output_sums.reciprocal()
tmp_x_output.mult_by_row(tmp_x_output_sums)
x_output.assign(tmp_x_output)
x_output.mult_by_col(state_prior_gpu_rec)
cm.log(x_output)
x_output.mult(1./np.log(10))
xo = x_output.asarray()
return xo
def ff(x0_cpu):
data_size = x0_cpu.shape[1]
x_l0 = cm.empty((num_input, data_size))
x_l0.assign(cm.CUDAMatrix(x0_cpu))
x_l1 = cm.empty((num_hid, data_size))
cm.dot(w1.T, x_l0, target=x_l1)
x_l1.add_col_vec(b1)
x_l1.apply_sigmoid()
x_l2 = cm.empty((num_hid, data_size))
del x_l0
cm.dot(w2.T, x_l1, target=x_l2)
x_l2.add_col_vec(b2)
x_l2.apply_sigmoid()
x_l3 = cm.empty((num_hid, data_size))
del x_l1
cm.dot(w3.T, x_l2, target=x_l3)
x_l3.add_col_vec(b3)
x_l3.apply_sigmoid()
x_l4 = cm.empty((num_hid, data_size))
del x_l2
cm.dot(w4.T, x_l3, target=x_l4)
x_l4.add_col_vec(b4)
x_l4.apply_sigmoid()
x_l5 = cm.empty((num_hid, data_size))
del x_l3
cm.dot(w5.T, x_l4, target=x_l5)
x_l5.add_col_vec(b5)
x_l5.apply_sigmoid()
x_output = cm.empty((num_output, data_size))
del x_l4
tmp_x_output = cm.empty((num_output, data_size))
tmp_x_output_sums = cm.empty((1, data_size))
cm.dot(wo.T, x_l5, target=tmp_x_output)
tmp_x_output.add_col_vec(bo)
cm.exp(tmp_x_output)
tmp_x_output.sum(axis=0, target=tmp_x_output_sums)
tmp_x_output_sums.reciprocal()
tmp_x_output.mult_by_row(tmp_x_output_sums)
x_output.assign(tmp_x_output)
x_output.mult_by_col(state_prior_gpu_rec)
cm.log(x_output)
x_output.mult(1. / np.log(10))
xo = x_output.asarray()
return xo
def score_samples(self, X, temp_gpu_mem=None):
'''Return the per-sample likelihood of the data under the model.
Compute the log probability of X under the model and
return the posterior probability of each
mixture component for each element of X.
Parameters
----------
X: numpy.ndarray, shape (n_samples, n_dimensions)
Array of n_samples data points. Each row
corresponds to a single data point.
Returns
-------
logprob_Nx1 : array_like, shape (n_samples,)
Log probabilities of each data point in X.
posteriors : array_like, shape (n_samples, n_components)
Posterior probability of each mixture component for each
sample
'''
if None in (self.weights, self.means, self.covars):
raise ValueError('GMM parameters have not been initialized')
if X.shape[1] != self.n_dimensions:
raise ValueError(
'input data matrix X is of shape %s, should be %s'
% (X.shape, (X.shape[0], self.n_dimensions)))
N = X.shape[0]
if temp_gpu_mem is None:
temp_gpu_mem = TempGPUMem()
temp_gpu_mem.alloc(N, self.n_components, self.n_dimensions)
# lpr = log_multivariate_normal_density()
# + np.log(self.weights)[None, :]
# -----------------------------------------------------
posteriors_NxK = log_multivariate_normal_density(
X, self.means, self.covars,
self.covariance_type, temp_gpu_mem)
# lpr += np.log(self.weights)
temp_Kx1 = temp_gpu_mem['temp_Kx1']
cm.log(self.weights, target=temp_Kx1)
temp_Kx1.reshape((1, self.n_components)) # transpose
posteriors_NxK.add_row_vec(temp_Kx1)
temp_Kx1.reshape((self.n_components, 1)) # original shape
# in use: lpr -> 'NxK'
# logprob_Nx1 = np.log(np.sum(np.exp(lpr - vmax), axis=1))
# logprob_Nx1 += vmax
# ---------------------------------------------------------
vmax_Nx1 = temp_gpu_mem['vmax_Nx1']
logprob_Nx1 = temp_gpu_mem['logprob_Nx1']
# vmax_Nx1 = np.max(lpr, axis=1)
posteriors_NxK.max(axis=1, target=vmax_Nx1)
# lpr -= vmax_Nx1[:, None]
posteriors_NxK.add_col_mult(vmax_Nx1, -1.0)
# posteriors_NxK = np.exp(posteriors_NxK)
cm.exp(posteriors_NxK)
# logprob_Nx1 = np.sum(posteriors_NxK, axis=1)
posteriors_NxK.sum(axis=1, target=logprob_Nx1)
# posteriors_NxK /= logprob_Nx1[:, None]
posteriors_NxK.div_by_col(logprob_Nx1)
# logprob_Nx1 = np.log(logprob_Nx1)
cm.log(logprob_Nx1, target=logprob_Nx1)
# logprob_Nx1 += vmax_Nx1
logprob_Nx1.add(vmax_Nx1)
return logprob_Nx1, posteriors_NxK
