def ApplyActivation(self): state = self.state if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: cm.sigmoid(state) elif self.activation == deepnet_pb2.Hyperparams.TANH: cm.tanh(state) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR: state.greater_than(0, target=self.temp) state.mult(self.temp) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH: cm.log_1_plus_exp(state) elif self.activation == deepnet_pb2.Hyperparams.LINEAR: pass elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.temp.reciprocal() state.mult_by_row(self.temp) elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.NN.divide(self.temp, target=self.temp) state.mult_by_row(self.temp) else: raise Exception("Unknown activation")
def ExactZ_binary_binary(model): assert len(model.layer) == 2, 'Only implemented for RBMs.' steps = len(schedule) input_layer = model.layer[0] hidden_layer = model.layer[1] edge = model.edge[0] w = edge.params['weight'] a = hidden_layer.params['bias'] b = input_layer.params['bias'] numvis, numhid = w.shape batchsize = 2**numvis input_layer.AllocateBatchsizeDependentMemory(batchsize) hidden_layer.AllocateBatchsizeDependentMemory(batchsize) all_inputs = GetAll(numvis) w_ais = cm.CUDAMatrix(np.zeros((1, batchsize))) input_layer.sample.overwrite(all_inputs) cm.dot(w.T, input_layer.sample, target=hidden_layer.state) hidden_layer.state.add_col_vec(a) cm.log_1_plus_exp(hidden_layer.state) w_ais.add_sums(hidden_layer.state, axis=0) w_ais.add_dot(b.T, input_layer.state) offset = float(w_ais.asarray().max()) w_ais.subtract(offset) cm.exp(w_ais) z = offset + np.log(w_ais.asarray().sum()) return z
def ApplyActivation(self): state = self.state if self.activation == deepnet_pb2.Hyperparams.LOGISTIC: cm.sigmoid(state) elif self.activation == deepnet_pb2.Hyperparams.TANH: cm.tanh(state) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR: state.greater_than(0, target=self.temp) state.mult(self.temp) elif self.activation == deepnet_pb2.Hyperparams.RECTIFIED_LINEAR_SMOOTH: cm.log_1_plus_exp(state) elif self.activation == deepnet_pb2.Hyperparams.LINEAR: pass elif self.activation == deepnet_pb2.Hyperparams.SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.temp.reciprocal() state.mult_by_row(self.temp) elif self.activation == deepnet_pb2.Hyperparams.REPLICATED_SOFTMAX: state.max(axis=0, target=self.temp) state.add_row_mult(self.temp, -1) cm.exp(state) state.sum(axis=0, target=self.temp) self.NN.divide(self.temp, target=self.temp) state.mult_by_row(self.temp) else: raise Exception('Unknown activation')
def AISBinaryRbm(model, schedule): cm.CUDAMatrix.init_random(seed=int(time.time())) assert len(model.layer) == 2, 'Only implemented for RBMs.' steps = len(schedule) input_layer = model.layer[0] hidden_layer = model.layer[1] edge = model.edge[0] batchsize = model.t_op.batchsize w = edge.params['weight'] a = hidden_layer.params['bias'] b = input_layer.params['bias'] numvis, numhid = w.shape # INITIALIZE TO UNIFORM RANDOM. input_layer.state.assign(0) input_layer.ApplyActivation() input_layer.Sample() w_ais = cm.CUDAMatrix(np.zeros((1, batchsize))) unitcell = cm.empty((1, 1)) # RUN AIS. for i in range(1, steps): cm.dot(w.T, input_layer.sample, target=hidden_layer.state) hidden_layer.state.add_col_vec(a) hidden_layer.state.mult(schedule[i - 1], target=hidden_layer.temp) hidden_layer.state.mult(schedule[i]) cm.log_1_plus_exp(hidden_layer.state, target=hidden_layer.deriv) cm.log_1_plus_exp(hidden_layer.temp) hidden_layer.deriv.subtract(hidden_layer.temp) w_ais.add_sums(hidden_layer.deriv, axis=0) w_ais.add_dot(b.T, input_layer.state, mult=schedule[i] - schedule[i - 1]) hidden_layer.ApplyActivation() hidden_layer.Sample() cm.dot(w, hidden_layer.sample, target=input_layer.state) input_layer.state.add_col_vec(b) input_layer.state.mult(schedule[i]) input_layer.ApplyActivation() input_layer.Sample() z = LogMeanExp(w_ais.asarray()) + numvis * np.log(2) + numhid * np.log(2) return z
def AISBinaryRbm(model, schedule): cm.CUDAMatrix.init_random(seed=int(time.time())) assert len(model.layer) == 2, 'Only implemented for RBMs.' steps = len(schedule) input_layer = model.layer[0] hidden_layer = model.layer[1] edge = model.edge[0] batchsize = model.t_op.batchsize w = edge.params['weight'] a = hidden_layer.params['bias'] b = input_layer.params['bias'] numvis, numhid = w.shape # INITIALIZE TO UNIFORM RANDOM. input_layer.state.assign(0) input_layer.ApplyActivation() input_layer.Sample() w_ais = cm.CUDAMatrix(np.zeros((1, batchsize))) unitcell = cm.empty((1, 1)) # RUN AIS. for i in range(1, steps): cm.dot(w.T, input_layer.sample, target=hidden_layer.state) hidden_layer.state.add_col_vec(a) hidden_layer.state.mult(schedule[i-1], target=hidden_layer.temp) hidden_layer.state.mult(schedule[i]) cm.log_1_plus_exp(hidden_layer.state, target=hidden_layer.deriv) cm.log_1_plus_exp(hidden_layer.temp) hidden_layer.deriv.subtract(hidden_layer.temp) w_ais.add_sums(hidden_layer.deriv, axis=0) w_ais.add_dot(b.T, input_layer.state, mult=schedule[i]-schedule[i-1]) hidden_layer.ApplyActivation() hidden_layer.Sample() cm.dot(w, hidden_layer.sample, target=input_layer.state) input_layer.state.add_col_vec(b) input_layer.state.mult(schedule[i]) input_layer.ApplyActivation() input_layer.Sample() z = LogMeanExp(w_ais.asarray()) + numvis * np.log(2) + numhid * np.log(2) return z
def AISReplicatedSoftmax(model, D, num_chains, display=False): schedule = np.concatenate(( #np.arange(0.0, 1.0, 0.01), #np.arange(0.0, 1.0, 0.001), np.arange(0.0, 0.7, 0.001), # 700 np.arange(0.7, 0.9, 0.0001), # 2000 np.arange(0.9, 1.0, 0.00002) # 5000 )) #schedule = np.array([0.]) cm.CUDAMatrix.init_random(seed=0) assert len(model.layer) == 2, 'Only implemented for RBMs.' steps = len(schedule) input_layer = model.layer[0] hidden_layer = model.layer[1] edge = model.edge[0] batchsize = num_chains w = edge.params['weight'] a = hidden_layer.params['bias'] b = input_layer.params['bias'] numvis, numhid = w.shape f = 0.1 input_layer.AllocateBatchsizeDependentMemory(num_chains) hidden_layer.AllocateBatchsizeDependentMemory(num_chains) # INITIALIZE TO SAMPLES FROM BASE MODEL. input_layer.state.assign(0) input_layer.NN.assign(D) input_layer.state.add_col_mult(b, f) SampleEnergySoftmax(input_layer, D) w_ais = cm.CUDAMatrix(np.zeros((1, batchsize))) #pdb.set_trace() w_variance = [] x_axis = [] if display: Display(w_ais, hidden_layer.state, input_layer.state, w_variance, x_axis) #raw_input('Press Enter.') #pdb.set_trace() # RUN AIS. for i in range(steps-1): sys.stdout.write('\r%d' % (i+1)) sys.stdout.flush() cm.dot(w.T, input_layer.sample, target=hidden_layer.state) hidden_layer.state.add_col_mult(a, D) hidden_layer.state.mult(schedule[i], target=hidden_layer.temp) hidden_layer.state.mult(schedule[i+1]) cm.log_1_plus_exp(hidden_layer.state, target=hidden_layer.deriv) cm.log_1_plus_exp(hidden_layer.temp) hidden_layer.deriv.subtract(hidden_layer.temp) w_ais.add_sums(hidden_layer.deriv, axis=0) w_ais.add_dot(b.T, input_layer.sample, mult=(1-f)*(schedule[i+1]-schedule[i])) hidden_layer.ApplyActivation() hidden_layer.Sample() cm.dot(w, hidden_layer.sample, target=input_layer.state) input_layer.state.add_col_vec(b) input_layer.state.mult(schedule[i+1]) input_layer.state.add_col_mult(b, f*(1-schedule[i+1])) SampleEnergySoftmax(input_layer, D) if display and (i % 100 == 0 or i == steps - 2): w_variance.append(w_ais.asarray().var()) x_axis.append(i) Display(w_ais, hidden_layer.state, input_layer.sample, w_variance, x_axis) sys.stdout.write('\n') z = LogMeanExp(w_ais.asarray()) + D * LogSumExp(f * b.asarray()) + numhid * np.log(2) return z
def AISReplicatedSoftmax(model, D, num_chains, display=False): schedule = np.concatenate(( #np.arange(0.0, 1.0, 0.01), #np.arange(0.0, 1.0, 0.001), np.arange(0.0, 0.7, 0.001), # 700 np.arange(0.7, 0.9, 0.0001), # 2000 np.arange(0.9, 1.0, 0.00002) # 5000 )) #schedule = np.array([0.]) cm.CUDAMatrix.init_random(seed=0) assert len(model.layer) == 2, 'Only implemented for RBMs.' steps = len(schedule) input_layer = model.layer[0] hidden_layer = model.layer[1] edge = model.edge[0] batchsize = num_chains w = edge.params['weight'] a = hidden_layer.params['bias'] b = input_layer.params['bias'] numvis, numhid = w.shape f = 0.1 input_layer.AllocateBatchsizeDependentMemory(num_chains) hidden_layer.AllocateBatchsizeDependentMemory(num_chains) # INITIALIZE TO SAMPLES FROM BASE MODEL. input_layer.state.assign(0) input_layer.NN.assign(D) input_layer.state.add_col_mult(b, f) SampleEnergySoftmax(input_layer, D) w_ais = cm.CUDAMatrix(np.zeros((1, batchsize))) #pdb.set_trace() w_variance = [] x_axis = [] if display: Display(w_ais, hidden_layer.state, input_layer.state, w_variance, x_axis) #raw_input('Press Enter.') #pdb.set_trace() # RUN AIS. for i in range(steps - 1): sys.stdout.write('\r%d' % (i + 1)) sys.stdout.flush() cm.dot(w.T, input_layer.sample, target=hidden_layer.state) hidden_layer.state.add_col_mult(a, D) hidden_layer.state.mult(schedule[i], target=hidden_layer.temp) hidden_layer.state.mult(schedule[i + 1]) cm.log_1_plus_exp(hidden_layer.state, target=hidden_layer.deriv) cm.log_1_plus_exp(hidden_layer.temp) hidden_layer.deriv.subtract(hidden_layer.temp) w_ais.add_sums(hidden_layer.deriv, axis=0) w_ais.add_dot(b.T, input_layer.sample, mult=(1 - f) * (schedule[i + 1] - schedule[i])) hidden_layer.ApplyActivation() hidden_layer.Sample() cm.dot(w, hidden_layer.sample, target=input_layer.state) input_layer.state.add_col_vec(b) input_layer.state.mult(schedule[i + 1]) input_layer.state.add_col_mult(b, f * (1 - schedule[i + 1])) SampleEnergySoftmax(input_layer, D) if display and (i % 100 == 0 or i == steps - 2): w_variance.append(w_ais.asarray().var()) x_axis.append(i) Display(w_ais, hidden_layer.state, input_layer.sample, w_variance, x_axis) sys.stdout.write('\n') z = LogMeanExp( w_ais.asarray()) + D * LogSumExp(f * b.asarray()) + numhid * np.log(2) return z
def rbmPredict(m, X): """using trained rbm model to do prediction""" nClass = m.labels.size numCase = X.shape[0] # This part is executed on CPU # define the free energy # FF = np.zeros((numCase, nClass)) # FFcol = np.zeros((numCase, 1)) # for index in range(nClass) : # temp = np.zeros((numCase, nClass)) # temp[:, index] = 1 # # tt = np.emath.log(np.exp(np.dot(X, m.weight)+ np.dot(temp, m.weightLabel) + m.biasH)+1) # # FFcol = temp[:,index] * m.biasLabel[0,index] + np.sum(tt,axis = 1) # # FF[:, index] = FFcol # # [x, y] = np.where(np.abs(FF - np.max(FF, axis=1, keepdims=True)) < 1e-5) # result = np.zeros(y.shape) # for index in range(y.size) : # result[index] = m.labels[y[index]] # The following part runs on GPU cm.cublas_init() # copy data to GPU data = cm.CUDAMatrix(cm.reformat(X)) weight = cm.CUDAMatrix(cm.reformat(m.weight)) biasH = cm.CUDAMatrix(cm.reformat(m.biasH)) weightLabel = cm.CUDAMatrix(cm.reformat(m.weightLabel)) biasLabel = cm.CUDAMatrix(cm.reformat(m.biasLabel)) F = cm.CUDAMatrix(np.zeros((numCase, nClass))) Fcol = cm.CUDAMatrix(np.zeros((numCase, 1))) temp = cm.CUDAMatrix(np.zeros((numCase, nClass))) tt = cm.CUDAMatrix(np.zeros((numCase, biasH.asarray().size))) for index in range(nClass): temp.assign(0) temp.set_col_slice(index, index + 1, 1) tt = cm.dot(data, weight) tt.add_dot(temp, weightLabel) tt.add_row_vec(biasH) cm.log_1_plus_exp(tt, target=tt, exact=True) Fcol = cm.sum(tt, axis=1) Fcol.add_mult(temp.get_col_slice(index, index + 1), biasLabel.numpy_array[0, index]) F.set_col_slice(index, index + 1, Fcol) tt.free_device_memory() F.copy_to_host() [x, y] = np.where(np.abs(F.numpy_array - np.max(F.numpy_array, axis=1, keepdims=True)) < 1e-5) # free device memory data.free_device_memory() weight.free_device_memory() biasH.free_device_memory() biasLabel.free_device_memory() weightLabel.free_device_memory() F.free_device_memory() Fcol.free_device_memory() temp.free_device_memory() cm.shutdown() result = np.zeros(y.shape) for index in range(y.size): result[index] = m.labels[y[index]] return [result, F.numpy_array]
def rbmPredict(m, X): """using trained rbm model to do prediction""" nClass = m.labels.size numCase = X.shape[0] # This part is executed on CPU # define the free energy # FF = np.zeros((numCase, nClass)) # FFcol = np.zeros((numCase, 1)) # for index in range(nClass) : # temp = np.zeros((numCase, nClass)) # temp[:, index] = 1 # # tt = np.emath.log(np.exp(np.dot(X, m.weight)+ np.dot(temp, m.weightLabel) + m.biasH)+1) # # FFcol = temp[:,index] * m.biasLabel[0,index] + np.sum(tt,axis = 1) # # FF[:, index] = FFcol # # [x, y] = np.where(np.abs(FF - np.max(FF, axis=1, keepdims=True)) < 1e-5) # result = np.zeros(y.shape) # for index in range(y.size) : # result[index] = m.labels[y[index]] # The following part runs on GPU cm.cublas_init() # copy data to GPU data = cm.CUDAMatrix(cm.reformat(X)) weight = cm.CUDAMatrix(cm.reformat(m.weight)) biasH = cm.CUDAMatrix(cm.reformat(m.biasH)) weightLabel = cm.CUDAMatrix(cm.reformat(m.weightLabel)) biasLabel = cm.CUDAMatrix(cm.reformat(m.biasLabel)) F = cm.CUDAMatrix(np.zeros((numCase, nClass))) Fcol = cm.CUDAMatrix(np.zeros((numCase, 1))) temp = cm.CUDAMatrix(np.zeros((numCase, nClass))) tt = cm.CUDAMatrix(np.zeros((numCase, biasH.asarray().size))) for index in range(nClass): temp.assign(0) temp.set_col_slice(index, index + 1, 1) tt = cm.dot(data, weight) tt.add_dot(temp, weightLabel) tt.add_row_vec(biasH) cm.log_1_plus_exp(tt, target=tt, exact=True) Fcol = cm.sum(tt, axis=1) Fcol.add_mult(temp.get_col_slice(index, index + 1), biasLabel.numpy_array[0, index]) F.set_col_slice(index, index + 1, Fcol) tt.free_device_memory() F.copy_to_host() [x, y] = np.where( np.abs(F.numpy_array - np.max(F.numpy_array, axis=1, keepdims=True)) < 1e-5) # free device memory data.free_device_memory() weight.free_device_memory() biasH.free_device_memory() biasLabel.free_device_memory() weightLabel.free_device_memory() F.free_device_memory() Fcol.free_device_memory() temp.free_device_memory() cm.shutdown() result = np.zeros(y.shape) for index in range(y.size): result[index] = m.labels[y[index]] return [result, F.numpy_array]