def ApplyActivation(self): state = self.state if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: cm.sigmoid(state) elif self.activation == deepnet_pb2.Hyperparams.TANH: cm.tanh(state) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR: state.greater_than(0, target=self.temp) state.mult(self.temp) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH: cm.log_1_plus_exp(state) elif self.activation == deepnet_pb2.Hyperparams.LINEAR: pass elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.temp.reciprocal() state.mult_by_row(self.temp) elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.NN.divide(self.temp, target=self.temp) state.mult_by_row(self.temp) else: raise Exception('Unknown activation')
def tests(): a = np.random.rand(300,500) b = np.random.rand(500,300) start = timer() c = np.dot(a,b) nptime = timer()-start print('nptime',nptime) x = np.array(np.random.rand(600,1500),dtype='float32',order='F') y = np.array(np.random.rand(1500,300),dtype='float32',order='F') z = np.zeros((1000,1000),order='F',dtype='float32') stream = cuda.stream() dx = cuda.to_device(x) dy = cuda.to_device(y) dz = cuda.to_device(z) start = timer() blas.gemm('N','N',1000,1500,1000,1.0,dx,dy,0.0,dz) cutime = timer()-start print('cutime',cutime) #dz.copy_to_host(z) print(dz[0]) c = np.ones((1000,1000),order='F',dtype='float32') print(c.shape) dc = cuda.to_device(c) # blockDim = (256,256) #gridDim = (((1000 + blockDim[0]-1)/blockDim[0]),((1000 + blockDim[1]-1)/blockDim[1])) blockDim = (30,30) gridDim = ((((c.shape[0] + blockDim[0]) - 1) / blockDim[0]), (((c.shape[1] + blockDim[1]) - 1) / blockDim[1])) start = timer() mtanh[gridDim,blockDim,stream](dc) tantime = timer() - start print('tantime',tantime) dc.copy_to_host(c,stream=stream) stream.synchronize() print(c) y = cm.CUDAMatrix(np.ones((1000,1000))) start = timer() cm.tanh(y) cmtan = timer()-start print('cmtan',cmtan) x = cm.CUDAMatrix(np.random.rand(1000,1500)) y = cm.CUDAMatrix(np.random.rand(1500,1000)) start = timer() cm.dot(x,y) cmtime = timer()-start print('cmtime',cmtime)
def ApplyActivation(self): state = self.state if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: cm.sigmoid(state) elif self.activation == deepnet_pb2.Hyperparams.TANH: cm.tanh(state) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR: state.greater_than(0, target=self.temp) state.mult(self.temp) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH: cm.log_1_plus_exp(state) elif self.activation == deepnet_pb2.Hyperparams.LINEAR: pass elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.temp.reciprocal() state.mult_by_row(self.temp) elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.NN.divide(self.temp, target=self.temp) state.mult_by_row(self.temp) else: raise Exception("Unknown activation")
def Update(self,input): # input has to have same size # as n_in. Returns the output as shape (2,1), so if yo # u want to plot the data, a buffer is mandatory. input = np.atleast_2d(input) Input = cm.CUDAMatrix(input) Input = Input.reshape([self.n_in,1]) self.a = self.a.mult(1-self.leakrate).add(cm.tanh(cm.dot(self.Wrr,self.a).add(cm.dot(self.Wir,Input).add(self.Wbr))).mult(self.leakrate)) y = cm.dot(self.Wro,self.a) return y.asarray()
def step(self, dt=None, ctl=False): if dt is None: dt = self.DT self.xdt = self.x.mult(1.0 - dt) self.rdt = self.r.mult(dt) self.zdt = self.z.mult(dt) self.x = self.xdt.add(cm.dot(self.M, self.rdt)).add(cm.dot(self.wf, self.zdt)) # if ctl: # self.x = self.x + cm.dot(self.wfc,(self.z_ctl*dt)) self.r = cm.tanh(self.x) self.z = cm.dot(self.wo.T, self.r)
def __init__(self, N=1000, pz=1, pg=0.1, g=1.5, alpha=1, dt=0.1, num_fits=1, num_inputs=0, state=None): cm.cublas_init() if state is not None: self.from_dict(state) else: self.N = N self.pg = pg self.pz = pz self.g = g self.alpha = alpha self.DT = dt self.num_fits = num_fits scale = 1.0 / np.sqrt(self.pg * self.N) M_rvs = stats.norm(loc=0, scale=scale).rvs self.M = sp.sparse.random(N, N, pg, data_rvs=M_rvs) * g self.M = cm.CUDAMatrix(self.M.toarray()) self.P = (1.0 / self.alpha) * np.identity(N) self.wf = cm.CUDAMatrix(np.random.uniform(-1, 1, (N, num_fits))) #self.wo = np.expand_dims(stats.norm(loc=0,scale=(1.0/np.sqrt(N))).rvs(N),num_fits) self.wo = cm.CUDAMatrix(np.zeros((N, num_fits))) self.dw = np.zeros((N, num_fits)) self.woc = np.zeros((N, 1)) self.wfc = np.random.uniform(-1, 1, (N, 1)) self.x = cm.CUDAMatrix(np.expand_dims(0.5 * np.random.randn(N), 1)) self.xdt = cm.empty(self.x.shape).assign(0) self.r = cm.tanh(self.x) self.rdt = cm.empty(self.r.shape).assign(0) self.z = cm.CUDAMatrix( np.expand_dims(0.5 * np.random.randn(num_fits), 1)) self.zdt = cm.empty(self.z.shape).assign(0) self.z_ctl = np.expand_dims(0.5 * np.random.randn(1), 1)
def d_tanh(self, x): return 1 - cm.tanh(x)**2
def tanh(self, x): return cm.tanh(x)