def updateLayer(self, layernum, sample = True): L = self.layers[layernum] if layernum == 0: self.downPass(layernum+1, sample = sample) if layernum == len(self.layers)-1: self.upPass(layernum-1, sample) if layernum<len(self.layers)-1 and layernum>0: hi = self.layers[layernum+1] lo = self.layers[layernum-1] wlo = self.weights[layernum-1] whi = self.weights[layernum] cp.prod(L.act, whi.mat, hi.act, 'n', 'n') cp.matrix_plus_col(L.act, whi.bias_lo) tmp = L.act.copy() cp.prod(L.act, wlo.mat, lo.act, 't', 'n') cp.matrix_plus_col(L.act, wlo.bias_hi) # add parts from above/below cp.apply_binary_functor(L.act, tmp, cp.binary_functor.AXPBY, 0.5, 0.5) tmp.dealloc() L.nonlinearity() if sample: L.sample()
def updateLayer(self, layernum, sample=True): L = self.layers[layernum] if layernum == 0: self.downPass(layernum + 1, sample=sample) if layernum == len(self.layers) - 1: self.upPass(layernum - 1, sample) if layernum < len(self.layers) - 1 and layernum > 0: hi = self.layers[layernum + 1] lo = self.layers[layernum - 1] wlo = self.weights[layernum - 1] whi = self.weights[layernum] cp.prod(L.act, whi.mat, hi.act, 'n', 'n') cp.matrix_plus_col(L.act, whi.bias_lo) tmp = L.act.copy() cp.prod(L.act, wlo.mat, lo.act, 't', 'n') cp.matrix_plus_col(L.act, wlo.bias_hi) # add parts from above/below cp.apply_binary_functor(L.act, tmp, cp.binary_functor.AXPBY, 0.5, 0.5) tmp.dealloc() L.nonlinearity() if sample: L.sample()
def forward(self, input, weight, bias, linear=False): result = cp.dev_tensor_float_cm([weight.shape[1], input.shape[1]]) cp.fill(result, 0) cp.prod(result, weight, input, "t", "n") cp.matrix_plus_col(result, bias) if not linear: cp.apply_scalar_functor(result, cp.scalar_functor.SIGM) return result
def forward(self, input, weight, bias,linear=False): result = cp.dev_tensor_float_cm([weight.shape[1], input.shape[1]]) cp.fill(result,0) cp.prod(result, weight, input, "t", "n") cp.matrix_plus_col(result, bias) if not linear: cp.apply_scalar_functor(result, cp.scalar_functor.SIGM) return result
def get_distance_matrix(self, test): t = cp.dev_tensor_float_cm(test) assert t.shape[1] == self.data.shape[1] tsq = cp.dev_tensor_float(t.shape[0]) cp.reduce_to_col(tsq,t,cp.reduce_functor.ADD_SQUARED) p = cp.dev_tensor_float_cm([self.data.shape[0], t.shape[0]]) cp.prod(p, self.data, t, 'n','t',-2, 0) cp.matrix_plus_col(p,self.dsq) cp.matrix_plus_row(p,tsq) return p
def get_distance_matrix(self, test): t = cp.dev_tensor_float_cm(test) assert t.shape[1] == self.data.shape[1] tsq = cp.dev_tensor_float(t.shape[0]) cp.reduce_to_col(tsq, t, cp.reduce_functor.ADD_SQUARED) p = cp.dev_tensor_float_cm([self.data.shape[0], t.shape[0]]) cp.prod(p, self.data, t, "n", "t", -2, 0) cp.matrix_plus_col(p, self.dsq) cp.matrix_plus_row(p, tsq) return p
class weight_layer: """ weight layer of the MLP represented by a matrix.""" def __init__(self, source_layer, target_layer): """Constructor @param source_layer pointer to the previous neuron layer. @param target_layer pointer to the next neuron layer. """ self.source=source_layer self.target=target_layer dim1 = self.target.activations.h dim2 = self.source.activations.h self.weight = cp.get_filled_matrix(dim1, dim2, 0.0) cp.fill_rnd_uniform(self.weight) cp.apply_scalar_functor(self.weight, cp.scalar_functor.SUBTRACT, 0.5) cp.apply_scalar_functor(self.weight, cp.scalar_functor.DIV, 10) self.bias = cp.get_filled_matrix(dim1, 1, 0) def forward(self): """Forward pass, calculates the activations of next neuron layer.""" cp.prod(self.target.activations, self.weight, self.source.activations) cp.matrix_plus_col(self.target.activations, self.bias.vec) self.target.nonlinearity(self.target.activations)
def p_k(self,beta,tmp,tmp2,collect): cp.prod(tmp,self.v,self.baserate_bias,'t','n') cp.apply_scalar_functor(tmp,cp.scalar_functor.MULT,(1-beta)) collect(tmp) cp.prod(tmp2,self.w,self.v,'t','n') cp.matrix_plus_col(tmp2,self.bias_hi) cp.apply_scalar_functor(tmp2,cp.scalar_functor.MULT,beta) # RECT computes log(1+exp(x)) cp.apply_scalar_functor(tmp2,cp.scalar_functor.RECT,1) cp.reduce_to_row(tmp.T,tmp2,cp.reduce_functor.ADD) # tmp.T is an evil hack. it makes tmp into row major, which doesn't change anything since it's a vector any way. But vectors are always assumed to be row major. collect(tmp) cp.prod(tmp,self.v,self.bias_lo.T,'t','n') cp.apply_scalar_functor(tmp,cp.scalar_functor.MULT,beta) collect(tmp)
def partialsumV(self, actv, acth, row): """ sums out hidden variables for given v exp( log(exp(bh + actv*W)+1).sum(axis=0) + (v*bv).sum(axis=0) ) """ # acth = bv + actv*W cp.prod(acth, self.weight, actv, 't', 'n') cp.matrix_plus_col(acth, self.bh) # acth = log(exp(acth)+1) cp.apply_scalar_functor(acth, cp.scalar_functor.RECT, 1.0) # row = actv.sum(axis=0) cp.reduce_to_row(row, acth, cp.reduce_functor.ADD) # row += h*bh cp.matrix_times_col(actv, self.bv) cp.reduce_to_row(row, actv, cp.reduce_functor.ADD, 1.0, 1.0) # exp(row) m = row.np.astype("float64") return math.fsum(m.flatten()) / actv.shape[1]
def partialsumV(self, actv, acth, row): """ sums out hidden variables for given v exp( log(exp(bh + actv*W)+1).sum(axis=0) + (v*bv).sum(axis=0) ) """ # acth = bv + actv*W cp.prod(acth, self.weight, actv, "t", "n") cp.matrix_plus_col(acth, self.bh) # acth = log(exp(acth)+1) cp.apply_scalar_functor(acth, cp.scalar_functor.RECT, 1.0) # row = actv.sum(axis=0) cp.reduce_to_row(row, acth, cp.reduce_functor.ADD) # row += h*bh cp.matrix_times_col(actv, self.bv) cp.reduce_to_row(row, actv, cp.reduce_functor.ADD, 1.0, 1.0) # exp(row) m = row.np.astype("float64") return math.fsum(m.flatten()) / actv.shape[1]
def partialsum(self, acth, actv, row): """ sums out visible variables for given hidden variables exp( log(exp(bv + acth*W)+1).sum(axis=0) + (h*bh).sum(axis=0) ) """ # actv = bv + acth*W cp.prod(actv, self.weight, acth, "n", "n") cp.matrix_plus_col(actv, self.bv) # actv = log(exp(actv)+1) cp.apply_scalar_functor(actv, cp.scalar_functor.RECT, 1.0) # row = actv.sum(axis=0) cp.reduce_to_row(row, actv, cp.reduce_functor.ADD) # row += h*bh cp.matrix_times_col(acth, self.bh) cp.reduce_to_row(row, acth, cp.reduce_functor.ADD, 1.0, 1.0) # cp.prod(row,self.bv,actv,'t','n',1.0,1.0) # exp(row) m = row.np.astype("float64") return math.fsum(np.exp(m).flatten())
def partialsum(self, acth, actv, row): """ sums out visible variables for given hidden variables exp( log(exp(bv + acth*W)+1).sum(axis=0) + (h*bh).sum(axis=0) ) """ # actv = bv + acth*W cp.prod(actv, self.weight, acth, 'n', 'n') cp.matrix_plus_col(actv, self.bv) # actv = log(exp(actv)+1) cp.apply_scalar_functor(actv, cp.scalar_functor.RECT, 1.0) # row = actv.sum(axis=0) cp.reduce_to_row(row, actv, cp.reduce_functor.ADD) # row += h*bh cp.matrix_times_col(acth, self.bh) cp.reduce_to_row(row, acth, cp.reduce_functor.ADD, 1.0, 1.0) #cp.prod(row,self.bv,actv,'t','n',1.0,1.0) # exp(row) m = row.np.astype("float64") return math.fsum(np.exp(m).flatten())
def sample_markov_chains(self,beta,step): cp.prod(self.h,self.w,self.v,'t','n') cp.matrix_plus_col(self.h,self.bias_hi) cp.apply_scalar_functor(self.h,cp.scalar_functor.MULT,beta) cp.apply_scalar_functor(self.h,cp.scalar_functor.SIGM) cp.rnd_binarize(self.h) cp.prod(self.v,self.w,self.h,'n','n') cp.matrix_plus_col(self.v,self.bias_lo) cp.apply_scalar_functor(self.v,cp.scalar_functor.MULT,beta) cp.apply_scalar_functor(self.baserate_bias,cp.scalar_functor.MULT,1-beta) cp.matrix_plus_col(self.v,self.baserate_bias) cp.apply_scalar_functor(self.baserate_bias,cp.scalar_functor.MULT,1.0/(1-beta)) cp.apply_scalar_functor(self.v,cp.scalar_functor.SIGM) #if step % 100 == 0: #plt.figure(1) #self.v_=self.v.np #showthis = self.v_.copy() #plt.matshow(showthis[:,0].reshape((28,28))) #plt.draw() #if not os.path.exists("/tmp/%s"%os.getlogin()): #os.mkdir("/tmp/%s"%os.getlogin()) #plt.savefig("/tmp/%s/chain_%05d.png"%(os.getlogin(),step)) cp.rnd_binarize(self.v)
def normalize_zmuv(self, batch): """ Zero Mean, Unit Variance based on recorded statistics """ cp.matrix_plus_col(batch, self.negative_mean) cp.matrix_divide_col(batch, self.std)
def postUpdateFromBelow(self, sample, bias): cp.matrix_plus_col(self.act, bias) self.nonlinearity() if sample: self.sample()
def forward(self): """Forward pass, calculates the activations of next neuron layer.""" cp.prod(self.target.activations, self.weight, self.source.activations) cp.matrix_plus_col(self.target.activations, self.bias) self.target.nonlinearity(self.target.activations)
def normalize_minmax(self, batch): """ normalize by subtracting min and dividing by range""" cp.matrix_plus_col(batch, self.negative_min) cp.matrix_divide_col(batch, self.range)
def normalize_zmuv(self,batch): """ Zero Mean, Unit Variance based on recorded statistics """ cp.matrix_plus_col(batch,self.negative_mean) cp.matrix_divide_col(batch,self.std)
def normalize_minmax(self,batch): """ normalize by subtracting min and dividing by range""" cp.matrix_plus_col(batch,self.negative_min) cp.matrix_divide_col(batch,self.range)