def compute_output(self,gpu_data): """ Computes p(y|x). Puts the result in self.gpu_p_y_given_x. """ cm.dot(self.W,gpu_data,self.gpu_act_from_x) self.gpu_act_from_x.add_col_vec(self.c) for c in range(self.n_classes): cm.dot(self.U,self.gpu_target_vectors.slice(c,c+1),self.gpu_act_from_y) # to avoid memory creation, using gpu_h # and gpu_h_sample for these computations self.gpu_act_from_x.add_col_vec(self.gpu_act_from_y,target=self.gpu_h) cm.exp(self.gpu_h,self.gpu_h_sample) self.gpu_h_sample.add(1.) cm.log(self.gpu_h_sample,self.gpu_h) self.gpu_h.sum(axis=0,target=self.gpu_negative_free_energy_for_y) cm.dot(self.d.T,self.gpu_target_vectors.slice(c,c+1),target=self.gpu_bias_from_y) self.gpu_negative_free_energy_for_y.add_col_vec(self.gpu_bias_from_y) self.gpu_negative_free_energy_for_y.transpose(target=self.gpu_negative_free_energy.slice(c,c+1)) # Subtracting mean for more stable softmax computation self.gpu_negative_free_energy.sum(axis=1,target=self.gpu_mean_negative_free_energy) self.gpu_mean_negative_free_energy.divide(-self.n_classes) self.gpu_negative_free_energy.add_col_vec(self.gpu_mean_negative_free_energy) cm.exp(self.gpu_negative_free_energy,target=self.gpu_negative_free_energy) self.gpu_negative_free_energy.sum(axis=1,target=self.gpu_p_y_given_x_norm) for c in range(self.n_classes): self.gpu_negative_free_energy.slice(c,c+1).divide(self.gpu_p_y_given_x_norm, target=self.gpu_p_y_given_x.slice(c,c+1)) self.gpu_p_y_given_x.transpose(target=self.gpu_p_y_given_x_trans)
def _log_multivariate_normal_density_diag(X, means, covars, temp_gpu_mem): '''Compute Gaussian log-density at X for a diagonal model''' N, D = X.shape K = means.shape[0] lpr_NxK = temp_gpu_mem['posteriors_NxK'] inv_covars_KxD = temp_gpu_mem['temp_KxD_2'] temp_NxD = temp_gpu_mem['temp_NxD'] temp_KxD = temp_gpu_mem['temp_KxD'] temp_Kx1 = temp_gpu_mem['temp_Kx1'] # compute inverse variances inv_covars_KxD.assign(1.0) inv_covars_KxD.divide(covars) # lpr = D * np.log(2*np.pi) lpr_NxK.assign(D * np.log(2*np.pi)) # temp_Kx1 = np.sum(np.log(covars), 1) cm.log(covars, target=temp_KxD) temp_KxD.sum(axis=1, target=temp_Kx1) # temp_Kx1 += np.sum((means**2)/covars, 1) means.mult(means, target=temp_KxD) temp_KxD.mult(inv_covars_KxD) temp_Kx1.add_sums(temp_KxD, axis=1) # lpr += temp_Kx1 temp_Kx1.reshape((1, K)) # transpose lpr_NxK.add_row_vec(temp_Kx1) temp_Kx1.reshape((K, 1)) # return to original shape # lpr += -2*np.dot(X, (means / covars).T) temp_KxD.assign(means) temp_KxD.mult(inv_covars_KxD) lpr_NxK.add_dot(X, temp_KxD.T, mult=-2.) # lpr += np.dot(X**2, (1.0 / covars).T) temp_NxD.assign(X) temp_NxD.mult(temp_NxD) lpr_NxK.add_dot(temp_NxD, inv_covars_KxD.T) # lpr *= -0.5 lpr_NxK.mult(-0.5) # lpr_NxK still in use # lpr = -0.5 * (D * np.log(2 * np.pi) + np.sum(np.log(covars), 1) # + np.sum((means ** 2) / covars, 1) # - 2 * np.dot(X, (means / covars).T) # + np.dot(X ** 2, (1.0 / covars).T)) return lpr_NxK
def logOnePlusExp(x, temp, targ = None): """ When this function is done, x should contain log(1+exp(x)). We clobber the value of temp. We compute log(1+exp(x)) as x + log(1+exp(-x)), which will hopefully be more finite-precision friendly. """ assert(x.shape == temp.shape) x.mult(-1, target = temp) cm.exp(temp) temp.add(1) cm.log(temp) x.add(temp, target = targ)
def logOnePlusExp(x, temp, targ=None): """ When this function is done, x should contain log(1+exp(x)). We clobber the value of temp. We compute log(1+exp(x)) as x + log(1+exp(-x)), which will hopefully be more finite-precision friendly. """ assert (x.shape == temp.shape) x.mult(-1, target=temp) cm.exp(temp) temp.add(1) cm.log(temp) x.add(temp, target=targ)
def negative_free_energy(self, gpu_data): """ Computes the negative free-energy. Outputs a reference to a pre-allocated GPU variable containing the result. """ cm.dot(self.W, gpu_data, self.gpu_h) self.gpu_h.add_col_vec(self.c) # to avoid memory creation, using gpu_h # and gpu_h_sample for these computations cm.exp(self.gpu_h, self.gpu_h_sample) self.gpu_h_sample.add(1.) cm.log(self.gpu_h_sample, self.gpu_h) self.gpu_h.sum(axis=0, target=self.gpu_negative_free_energy) self.gpu_negative_free_energy.add_dot(self.b.T, gpu_data) return self.gpu_negative_free_energy
def negative_free_energy(self,gpu_data): """ Computes the negative free-energy. Outputs a reference to a pre-allocated GPU variable containing the result. """ cm.dot(self.W,gpu_data,self.gpu_h) self.gpu_h.add_col_vec(self.c) # to avoid memory creation, using gpu_h # and gpu_h_sample for these computations cm.exp(self.gpu_h,self.gpu_h_sample) self.gpu_h_sample.add(1.) cm.log(self.gpu_h_sample,self.gpu_h) self.gpu_h.sum(axis=0,target=self.gpu_negative_free_energy) self.gpu_negative_free_energy.add_dot(self.b.T,gpu_data) return self.gpu_negative_free_energy
def test_log(): m = 256 n = 128 a = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float32, order='F') b = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float32, order='F') c = np.log(a) m1 = cm.CUDAMatrix(a) m2 = cm.CUDAMatrix(b) cm.log(m1, target = m2) cm.log(m1) m1.copy_to_host() m2.copy_to_host() assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold" assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold"
def compute_gamma_entropy(self, G): if not self.gpu: Prod = G * (np.log(G) - 1) ent = np.nan_to_num(Prod).sum() else: Prod = cm.empty(G.shape) Prod = G.mult(cm.log(G.copy()).subtract(1), target=Prod) ent = np.nan_to_num(Prod.asarray()).sum() return ent
def compute_energy_mcRBM_visual(self, data, normdata, energy, VF, FH, bias_cov, bias_vis, w_mean, bias_mean, t1, t2, t6, feat, featsq, feat_mean, length, lengthsq, normcoeff, small, num_vis): # normalize input data vectors data.mult(data, target=t6) # DxP (nr input dims x nr samples) t6.sum(axis=0, target=lengthsq) # 1xP lengthsq.mult(0.5, target=energy) # energy of quadratic regularization term lengthsq.mult(1. / num_vis) # normalize by number of components (like std) lengthsq.add(small) # small prevents division by 0 cmt.sqrt(lengthsq, target=length) length.reciprocal(target=normcoeff) # 1xP data.mult_by_row(normcoeff, target=normdata) # normalized data ## potential # covariance contribution cmt.dot(VF.T, normdata, target=feat) # HxP (nr factors x nr samples) feat.mult(feat, target=featsq) # HxP cmt.dot(FH.T, featsq, target=t1) # OxP (nr cov hiddens x nr samples) t1.mult(-0.5) t1.add_col_vec(bias_cov) # OxP cmt.exp(t1) # OxP t1.add(1, target=t2) # OxP cmt.log(t2) t2.mult(-1) energy.add_sums(t2, axis=0) # mean contribution cmt.dot(w_mean.T, data, target=feat_mean) # HxP (nr mean hiddens x nr samples) feat_mean.add_col_vec(bias_mean) # HxP cmt.exp(feat_mean) feat_mean.add(1) cmt.log(feat_mean) feat_mean.mult(-1) energy.add_sums(feat_mean, axis=0) # visible bias term data.mult_by_col(bias_vis, target=t6) t6.mult(-1) # DxP energy.add_sums(t6, axis=0) # 1xP # kinetic data.mult(data, target=t6) energy.add_sums(t6, axis=0, mult=.5)
def compute_output(self, gpu_data): """ Computes p(y|x). Puts the result in self.gpu_p_y_given_x. """ cm.dot(self.W, gpu_data, self.gpu_act_from_x) self.gpu_act_from_x.add_col_vec(self.c) for c in range(self.n_classes): cm.dot(self.U, self.gpu_target_vectors.slice(c, c + 1), self.gpu_act_from_y) # to avoid memory creation, using gpu_h # and gpu_h_sample for these computations self.gpu_act_from_x.add_col_vec(self.gpu_act_from_y, target=self.gpu_h) cm.exp(self.gpu_h, self.gpu_h_sample) self.gpu_h_sample.add(1.) cm.log(self.gpu_h_sample, self.gpu_h) self.gpu_h.sum(axis=0, target=self.gpu_negative_free_energy_for_y) cm.dot(self.d.T, self.gpu_target_vectors.slice(c, c + 1), target=self.gpu_bias_from_y) self.gpu_negative_free_energy_for_y.add_col_vec( self.gpu_bias_from_y) self.gpu_negative_free_energy_for_y.transpose( target=self.gpu_negative_free_energy.slice(c, c + 1)) # Subtracting mean for more stable softmax computation self.gpu_negative_free_energy.sum( axis=1, target=self.gpu_mean_negative_free_energy) self.gpu_mean_negative_free_energy.divide(-self.n_classes) self.gpu_negative_free_energy.add_col_vec( self.gpu_mean_negative_free_energy) cm.exp(self.gpu_negative_free_energy, target=self.gpu_negative_free_energy) self.gpu_negative_free_energy.sum(axis=1, target=self.gpu_p_y_given_x_norm) for c in range(self.n_classes): self.gpu_negative_free_energy.slice(c, c + 1).divide( self.gpu_p_y_given_x_norm, target=self.gpu_p_y_given_x.slice(c, c + 1)) self.gpu_p_y_given_x.transpose(target=self.gpu_p_y_given_x_trans)
def Sample(self): logging.debug("Sample in %s", self.name) state = self.state sample = self.sample if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: state.sample_binomial(target=sample) elif self.activation == deepnet_pb2.Hyperparams.TANH: state.sample_binomial_tanh(target=sample) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR: state.sample_poisson(target=sample) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH: state.sample_poisson(target=sample) elif self.activation == deepnet_pb2.Hyperparams.LINEAR: # sample.assign(state) if self.learn_precision: sample.fill_with_randn() sample.div_by_col(self.params["precision"]) sample.add(state) # state.sample_gaussian(target=sample, mult=0.01) elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: sample.fill_with_rand() cm.log(sample) sample.mult(-1) sample.reciprocal() sample.mult(state) sample.argmax(axis=0, target=self.temp) self.expansion_matrix.select_columns(self.temp, target=sample) elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX: if self.proto.hyperparams.normalize: sample.assign(0) temp_sample = self.expanded_batch numsamples = int(self.proto.hyperparams.normalize_to) for i in range(numsamples): temp_sample.fill_with_rand() cm.log(temp_sample) temp_sample.mult(-1) state.divide(temp_sample, target=temp_sample) temp_sample.max(axis=0, target=self.temp) temp_sample.add_row_mult(self.temp, -1) temp_sample.less_than(0) sample.add(temp_sample) sample.subtract(numsamples) sample.mult(-1) else: # This is an approximation. sample.fill_with_rand() cm.log(sample) sample.mult(-1) sample.reciprocal() sample.mult(state) sample.argmax(axis=0, target=self.temp) self.expansion_matrix.select_columns(self.temp, target=sample) sample.mult_by_row(self.NN) else: raise Exception("Unknown activation")
def normalizeM(self, norm): if norm == "median": self.M_GPU.divide(float(np.median(self.M_GPU.asarray()))) elif norm == "max": self.M_GPU.divide(float(np.max(self.M_GPU.asarray()))) elif norm == "log": self.M_GPU.add(1) cudamat.log(self.M_GPU) elif norm == "loglog": self.M_GPU.add(1) cudamat.log(self.M_GPU) self.M_GPU.add(1) cudamat.log(self.M_GPU)
def Sample(self): logging.debug('Sample in %s', self.name) state = self.state sample = self.sample if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: state.sample_binomial(target=sample) elif self.activation == deepnet_pb2.Hyperparams.TANH: state.sample_binomial_tanh(target=sample) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR: state.sample_poisson(target=sample) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH: state.sample_poisson(target=sample) elif self.activation == deepnet_pb2.Hyperparams.LINEAR: #sample.assign(state) state.sample_gaussian(target=sample, mult=0.01) elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: sample.fill_with_rand() cm.log(sample) sample.mult(-1) sample.reciprocal() sample.mult(state) sample.argmax(axis=0, target=self.temp) self.expansion_matrix.select_columns(self.temp, target=sample) elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX: if self.proto.hyperparams.normalize: sample.assign(0) temp_sample = self.expanded_batch numsamples = int(self.proto.hyperparams.normalize_to) for i in range(numsamples): temp_sample.fill_with_rand() cm.log(temp_sample) temp_sample.mult(-1) state.divide(temp_sample, target=temp_sample) temp_sample.max(axis=0, target=self.temp) temp_sample.add_row_mult(self.temp, -1) temp_sample.less_than(0) sample.add(temp_sample) sample.subtract(numsamples) sample.mult(-1) else: # This is an approximation. sample.fill_with_rand() cm.log(sample) sample.mult(-1) sample.reciprocal() sample.mult(state) sample.argmax(axis=0, target=self.temp) self.expansion_matrix.select_columns(self.temp, target=sample) sample.mult_by_row(self.NN) else: raise Exception('Unknown activation')
def GetLoss(self, get_deriv=False): """Compute loss and also deriv w.r.t to it if asked for. Compute the loss function. Targets should be in self.data, predictions should be in self.state. Args: get_deriv: If True, compute the derivative w.r.t the loss function and put it in self.deriv. """ perf = deepnet_pb2.Metrics() perf.MergeFrom(self.proto.performance_stats) perf.count = self.batchsize tiny = self.tiny if self.loss_function == deepnet_pb2.Layer.CROSS_ENTROPY: if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: data = self.data state = self.state deriv = self.deriv temp3 = self.temp3 unitcell = self.unitcell cm.cross_entropy(data, state, target=deriv, tiny=self.tiny) deriv.sum(axis=1, target=temp3) temp3.sum(axis=0, target=unitcell) cross_entropy = unitcell.euclid_norm() cm.correct_preds(data, state, target=deriv, cutoff=0.5) deriv.sum(axis=1, target=temp3) temp3.sum(axis=0, target=unitcell) correct_preds = unitcell.euclid_norm() if get_deriv: self.state.subtract(self.data, target=self.deriv) perf.cross_entropy = cross_entropy perf.correct_preds = correct_preds elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: temp2 = self.temp2 temp = self.temp batchsize = self.batchsize dimensions = self.dimensions numlabels = self.numlabels state = self.state data = self.data unitcell = self.unitcell indices = self.indices # Optimized for space to handle large number of labels in a softmax. data.reshape((1, batchsize * dimensions)) data.add(self.rowshift, target=indices) state.reshape((numlabels, dimensions * batchsize)) state.max(axis=0, target=temp2) state.reshape((1, batchsize * numlabels * dimensions)) state.select_columns(indices, temp) temp2.subtract(temp) temp2.sign(target=temp2) temp2.sum(axis=1, target=unitcell) correct_preds = batchsize - unitcell.euclid_norm() if get_deriv: temp.subtract(1, target=temp2) state.set_selected_columns(indices, temp2) state.reshape((numlabels * dimensions, batchsize)) self.deriv.assign(self.state) state.reshape((numlabels * dimensions, batchsize)) temp.add(tiny) cm.log(temp) temp.sum(axis=1, target=unitcell) cross_entropy = unitcell.euclid_norm() perf.cross_entropy = cross_entropy perf.correct_preds = correct_preds elif self.loss_function == deepnet_pb2.Layer.SQUARED_LOSS: if self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX and self.hyperparams.normalize: self.data.sum(axis=0, target=self.temp) self.temp.add(self.tiny) self.data.div_by_row(self.temp, target=self.deriv) self.deriv.mult(self.proto.hyperparams.normalize_to) self.deriv.subtract(self.state) elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: self.expansion_matrix.select_columns( self.data, target=self.expanded_batch) self.state.subtract(self.expanded_batch, target=self.deriv) else: self.state.subtract(self.data, target=self.deriv) error = self.deriv.euclid_norm()**2 perf.error = error if self.activation != deepnet_pb2.Hyperparams.SOFTMAX and \ self.activation != deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX: self.ComputeDeriv() else: raise Exception('Unknown loss function.') return perf
def GetLoss(self, get_deriv=False): """Compute loss and also deriv w.r.t to it if asked for. Compute the loss function. Targets should be in self.data, predictions should be in self.state. Args: get_deriv: If True, compute the derivative w.r.t the loss function and put it in self.deriv. """ perf = deepnet_pb2.Metrics() perf.MergeFrom(self.proto.performance_stats) perf.count = self.batchsize tiny = self.tiny if self.loss_function == deepnet_pb2.Layer.CROSS_ENTROPY: if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: data = self.data state = self.state deriv = self.deriv temp3 = self.dimsize unitcell = self.unitcell cm.cross_entropy(data, state, target=deriv, tiny=self.tiny) deriv.sum(axis=1, target=temp3) temp3.sum(axis=0, target=unitcell) cross_entropy = unitcell.euclid_norm() cm.correct_preds(data, state, target=deriv, cutoff=0.5) deriv.sum(axis=1, target=temp3) temp3.sum(axis=0, target=unitcell) correct_preds = unitcell.euclid_norm() if get_deriv: self.state.subtract(self.data, target=self.deriv) perf.cross_entropy = cross_entropy perf.correct_preds = correct_preds elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: temp2 = self.temp2 temp = self.temp batchsize = self.batchsize dimensions = self.dimensions numlabels = self.numlabels state = self.state data = self.data unitcell = self.unitcell indices = self.indices # Optimized for space to handle large number of labels in a softmax. data.reshape((1, batchsize * dimensions)) data.add(self.rowshift, target=indices) state.reshape((numlabels, dimensions * batchsize)) state.max(axis=0, target=temp2) state.reshape((1, batchsize * numlabels * dimensions)) state.select_columns(indices, temp) temp2.subtract(temp) temp2.sign(target=temp2) temp2.sum(axis=1, target=unitcell) correct_preds = batchsize - unitcell.euclid_norm() if get_deriv: temp.subtract(1, target=temp2) state.set_selected_columns(indices, temp2) state.reshape((numlabels * dimensions, batchsize)) self.deriv.assign(self.state) state.reshape((numlabels * dimensions, batchsize)) temp.add(tiny) cm.log(temp) temp.sum(axis=1, target=unitcell) cross_entropy = unitcell.euclid_norm() perf.cross_entropy = cross_entropy perf.correct_preds = correct_preds elif self.loss_function == deepnet_pb2.Layer.SQUARED_LOSS: if self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX and self.hyperparams.normalize: self.data.sum(axis=0, target=self.temp) self.temp.add(self.tiny) self.data.div_by_row(self.temp, target=self.deriv) self.deriv.mult(self.proto.hyperparams.normalize_to) self.deriv.subtract(self.state) elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: self.expansion_matrix.select_columns(self.data, target=self.expanded_batch) self.state.subtract(self.expanded_batch, target=self.deriv) else: if "precision" in self.params: self.data.mult_by_col(self.params["precision"], target=self.deriv) self.deriv.subtract(self.state) else: self.state.subtract(self.data, target=self.deriv) error = self.deriv.euclid_norm() ** 2 perf.error = error if ( self.activation != deepnet_pb2.Hyperparams.SOFTMAX and self.activation != deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX ): self.ComputeDeriv() else: raise Exception("Unknown loss function.") return perf
def ConvolveUp(self, inputs, edge, target): w = edge.params["weight"] conv = edge.conv_params size = conv.size stride = conv.stride padding = conv.padding num_filters = conv.num_filters num_colors = conv.num_colors f, numdims = w.shape assert f == num_filters, "f is %d but num_filters is %d" % (f, num_filters) if edge.conv: assert numdims == size ** 2 * num_colors input_t = edge.input_t inputs.transpose(input_t) # Convolve Up. if conv.max_pool: output_t = edge.unpooled_layer elif conv.rnorm: output_t = edge.unrnormalized_layer else: output_t = edge.output_t numimages, numdims = input_t.shape numimages2, numdims2 = output_t.shape assert numimages == numimages2, "%d %d." % (numimages, numimages2) assert numdims % num_colors == 0 x = int(np.sqrt(numdims / num_colors)) assert x ** 2 == numdims / num_colors n_locs = (x + 2 * padding - size) / stride + 1 if edge.conv: cc.convUp(input_t, w, output_t, n_locs, padding, stride, num_colors) else: cc.localUp(input_t, w, output_t, n_locs, padding, stride, num_colors) # Do maxpooling if conv.max_pool: input_t = output_t if conv.rnorm: output_t = edge.unrnormalized_layer else: output_t = edge.output_t n_locs = (n_locs - conv.pool_size) / conv.pool_stride + 1 if conv.prob: # Get Gumbel variates. rnd = edge.rnd rnd.fill_with_rand() cm.log(rnd) rnd.mult(-1) cm.log(rnd) rnd.mult(-1) cc.ProbMaxPool(input_t, rnd, output_t, num_filters, conv.pool_size, 0, conv.pool_stride, n_locs) else: cc.MaxPool(input_t, output_t, num_filters, conv.pool_size, 0, conv.pool_stride, n_locs) if conv.rnorm: input_t = output_t output_t = edge.output_t denoms = edge.denoms sizeX = conv.norm_size add_scale = conv.add_scale pow_scale = conv.pow_scale cc.ResponseNorm(input_t, denoms, output_t, num_filters, sizeX, add_scale, pow_scale) output_t.transpose(target)
def ff(x0_cpu): data_size = x0_cpu.shape[1] x_l0 = cm.empty((num_input, data_size)) x_l0.assign(cm.CUDAMatrix(x0_cpu)) x_l1 = cm.empty((num_hid, data_size)) cm.dot(w1.T, x_l0, target = x_l1) x_l1.add_col_vec(b1) x_l1.apply_sigmoid() x_l2 = cm.empty((num_hid, data_size)) del x_l0 cm.dot(w2.T, x_l1, target = x_l2) x_l2.add_col_vec(b2) x_l2.apply_sigmoid() x_l3 = cm.empty((num_hid, data_size)) del x_l1 cm.dot(w3.T, x_l2, target = x_l3) x_l3.add_col_vec(b3) x_l3.apply_sigmoid() x_l4 = cm.empty((num_hid, data_size)) del x_l2 cm.dot(w4.T, x_l3, target = x_l4) x_l4.add_col_vec(b4) x_l4.apply_sigmoid() x_l5 = cm.empty((num_hid, data_size)) del x_l3 cm.dot(w5.T, x_l4, target = x_l5) x_l5.add_col_vec(b5) x_l5.apply_sigmoid() x_output = cm.empty((num_output, data_size)) del x_l4 tmp_x_output = cm.empty((num_output, data_size)) tmp_x_output_sums = cm.empty((1, data_size)) cm.dot(wo.T, x_l5, target = tmp_x_output) tmp_x_output.add_col_vec(bo) cm.exp(tmp_x_output) tmp_x_output.sum(axis=0, target = tmp_x_output_sums) tmp_x_output_sums.reciprocal() tmp_x_output.mult_by_row(tmp_x_output_sums) x_output.assign(tmp_x_output) x_output.mult_by_col(state_prior_gpu_rec) cm.log(x_output) x_output.mult(1./np.log(10)) xo = x_output.asarray() return xo
def ff(x0_cpu): data_size = x0_cpu.shape[1] x_l0 = cm.empty((num_input, data_size)) x_l0.assign(cm.CUDAMatrix(x0_cpu)) x_l1 = cm.empty((num_hid, data_size)) cm.dot(w1.T, x_l0, target=x_l1) x_l1.add_col_vec(b1) x_l1.apply_sigmoid() x_l2 = cm.empty((num_hid, data_size)) del x_l0 cm.dot(w2.T, x_l1, target=x_l2) x_l2.add_col_vec(b2) x_l2.apply_sigmoid() x_l3 = cm.empty((num_hid, data_size)) del x_l1 cm.dot(w3.T, x_l2, target=x_l3) x_l3.add_col_vec(b3) x_l3.apply_sigmoid() x_l4 = cm.empty((num_hid, data_size)) del x_l2 cm.dot(w4.T, x_l3, target=x_l4) x_l4.add_col_vec(b4) x_l4.apply_sigmoid() x_l5 = cm.empty((num_hid, data_size)) del x_l3 cm.dot(w5.T, x_l4, target=x_l5) x_l5.add_col_vec(b5) x_l5.apply_sigmoid() x_output = cm.empty((num_output, data_size)) del x_l4 tmp_x_output = cm.empty((num_output, data_size)) tmp_x_output_sums = cm.empty((1, data_size)) cm.dot(wo.T, x_l5, target=tmp_x_output) tmp_x_output.add_col_vec(bo) cm.exp(tmp_x_output) tmp_x_output.sum(axis=0, target=tmp_x_output_sums) tmp_x_output_sums.reciprocal() tmp_x_output.mult_by_row(tmp_x_output_sums) x_output.assign(tmp_x_output) x_output.mult_by_col(state_prior_gpu_rec) cm.log(x_output) x_output.mult(1. / np.log(10)) xo = x_output.asarray() return xo
def score_samples(self, X, temp_gpu_mem=None): '''Return the per-sample likelihood of the data under the model. Compute the log probability of X under the model and return the posterior probability of each mixture component for each element of X. Parameters ---------- X: numpy.ndarray, shape (n_samples, n_dimensions) Array of n_samples data points. Each row corresponds to a single data point. Returns ------- logprob_Nx1 : array_like, shape (n_samples,) Log probabilities of each data point in X. posteriors : array_like, shape (n_samples, n_components) Posterior probability of each mixture component for each sample ''' if None in (self.weights, self.means, self.covars): raise ValueError('GMM parameters have not been initialized') if X.shape[1] != self.n_dimensions: raise ValueError( 'input data matrix X is of shape %s, should be %s' % (X.shape, (X.shape[0], self.n_dimensions))) N = X.shape[0] if temp_gpu_mem is None: temp_gpu_mem = TempGPUMem() temp_gpu_mem.alloc(N, self.n_components, self.n_dimensions) # lpr = log_multivariate_normal_density() # + np.log(self.weights)[None, :] # ----------------------------------------------------- posteriors_NxK = log_multivariate_normal_density( X, self.means, self.covars, self.covariance_type, temp_gpu_mem) # lpr += np.log(self.weights) temp_Kx1 = temp_gpu_mem['temp_Kx1'] cm.log(self.weights, target=temp_Kx1) temp_Kx1.reshape((1, self.n_components)) # transpose posteriors_NxK.add_row_vec(temp_Kx1) temp_Kx1.reshape((self.n_components, 1)) # original shape # in use: lpr -> 'NxK' # logprob_Nx1 = np.log(np.sum(np.exp(lpr - vmax), axis=1)) # logprob_Nx1 += vmax # --------------------------------------------------------- vmax_Nx1 = temp_gpu_mem['vmax_Nx1'] logprob_Nx1 = temp_gpu_mem['logprob_Nx1'] # vmax_Nx1 = np.max(lpr, axis=1) posteriors_NxK.max(axis=1, target=vmax_Nx1) # lpr -= vmax_Nx1[:, None] posteriors_NxK.add_col_mult(vmax_Nx1, -1.0) # posteriors_NxK = np.exp(posteriors_NxK) cm.exp(posteriors_NxK) # logprob_Nx1 = np.sum(posteriors_NxK, axis=1) posteriors_NxK.sum(axis=1, target=logprob_Nx1) # posteriors_NxK /= logprob_Nx1[:, None] posteriors_NxK.div_by_col(logprob_Nx1) # logprob_Nx1 = np.log(logprob_Nx1) cm.log(logprob_Nx1, target=logprob_Nx1) # logprob_Nx1 += vmax_Nx1 logprob_Nx1.add(vmax_Nx1) return logprob_Nx1, posteriors_NxK