def use(self,dataset): outputs = [] for example in dataset: input,dummy = example if len(input.shape) == 1: input = input.reshape((1,-1)) data_vis_bias = np.dot(input,self.V)+self.b if self.n_inference_iterations > 0: if self.approximate_inference == 'mean_field': target_singles = 1./(1+np.exp(-data_vis_bias)) for k in range(self.n_inference_iterations): if self.damping_factor > 0: target_singles *= self.damping_factor target_singles += (1.-self.damping_factor)/(1+np.exp(-data_vis_bias-np.dot(target_singles,self.U))) else: target_singles = 1./(1+np.exp(-data_vis_bias-np.dot(target_singles,self.U))) elif self.approximate_inference == 'loopy_belief_propagation': target_singles = np.zeros((input.shape[0],self.target_size)) log_messages = np.zeros((self.target_size,self.target_size)) softplus_out1 = np.zeros((self.target_size,self.target_size)) softplus_out2 = np.zeros((self.target_size,self.target_size)) for i in range(input.shape[0]): # Loopy BP data_bias = data_vis_bias[i] log_messages[:] = 0 for k in range(self.n_inference_iterations): # Update messages acc_log_messages = data_bias + log_messages.sum(axis=1) - log_messages if self.damping_factor > 0: log_messages *= self.damping_factor mlnonlin.softplus(acc_log_messages,softplus_out2) acc_log_messages += self.U mlnonlin.softplus(acc_log_messages,softplus_out1) log_messages += (1-self.damping_factor) * (softplus_out1-softplus_out2) else: mlnonlin.softplus(acc_log_messages,softplus_out2) acc_log_messages += self.U mlnonlin.softplus(acc_log_messages,softplus_out1) log_messages = softplus_out1-softplus_out2 # Single marginals target_singles[i] = 1./(1+np.exp(-data_bias-log_messages.sum(axis=1))) else: raise ValueError('approximate_inference \'%\' unknown' % self.approximate_inference) else: target_singles = 1./(1+np.exp(-data_vis_bias)) if self.binary_outputs: outputs += [ target_singles > 0.5 ] else: outputs += [ target_singles ] return outputs
def test_softplus(): """ Testing nonlinear softplus. """ input = np.random.randn(20) output = np.zeros((20)) nonlinear.softplus(input,output) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(output-np.log(1+np.exp(input)))) print 'Testing nonlinear reclin' input = np.random.randn(30,20) output = np.zeros((30,20)) nonlinear.reclin(input,output) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(output-(input>0)*input)) print 'Testing nonlinear reclin deriv.' dinput = np.zeros((30,20)) doutput = np.random.randn(30,20) nonlinear.dreclin(output,doutput,dinput) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(dinput-(input>0)*doutput))
def test_softplus(): """ Testing nonlinear softplus. """ input = np.random.randn(20) output = np.zeros((20)) nonlinear.softplus(input, output) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum( np.abs(output - np.log(1 + np.exp(input)))) print 'Testing nonlinear reclin' input = np.random.randn(30, 20) output = np.zeros((30, 20)) nonlinear.reclin(input, output) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum( np.abs(output - (input > 0) * input)) print 'Testing nonlinear reclin deriv.' dinput = np.zeros((30, 20)) doutput = np.random.randn(30, 20) nonlinear.dreclin(output, doutput, dinput) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum( np.abs(dinput - (input > 0) * doutput))
def train(self,trainset): if self.stage == 0: self.initialize(trainset) for it in range(self.stage,self.n_stages): for example in trainset: input,target = example if len(input.shape) == 1: input = input.reshape((1,-1)) target = target.reshape((1,-1)) data_vis_bias = np.dot(input,self.V)+self.b if self.n_inference_iterations > 0: if self.approximate_inference == 'mean_field': target_singles = 1./(1+np.exp(-data_vis_bias)) for k in range(self.n_inference_iterations): if self.damping_factor > 0: target_singles *= self.damping_factor target_singles += (1.-self.damping_factor)/(1+np.exp(-data_vis_bias-np.dot(target_singles,self.U))) else: target_singles = 1./(1+np.exp(-data_vis_bias-np.dot(target_singles,self.U))) sum_target_pairs = np.dot(target_singles.T,target_singles) elif self.approximate_inference == 'loopy_belief_propagation': target_singles = np.zeros((input.shape[0],self.target_size)) sum_target_pairs = np.zeros((self.target_size,self.target_size)) log_messages = np.zeros((self.target_size,self.target_size)) softplus_out1 = np.zeros((self.target_size,self.target_size)) softplus_out2 = np.zeros((self.target_size,self.target_size)) for i in range(input.shape[0]): # Loopy BP data_bias = data_vis_bias[i] log_messages[:] = 0 for k in range(self.n_inference_iterations): # Update messages acc_log_messages = data_bias + log_messages.sum(axis=1) - log_messages if self.damping_factor > 0: log_messages *= self.damping_factor mlnonlin.softplus(acc_log_messages,softplus_out2) acc_log_messages += self.U mlnonlin.softplus(acc_log_messages,softplus_out1) log_messages += (1-self.damping_factor) * (softplus_out1-softplus_out2) else: mlnonlin.softplus(acc_log_messages,softplus_out2) acc_log_messages += self.U mlnonlin.softplus(acc_log_messages,softplus_out1) log_messages = softplus_out1-softplus_out2 # Single marginals target_singles[i] = 1./(1+np.exp(-data_bias-log_messages.sum(axis=1))) # Pair-wise marginals acc_log_messages = data_bias + log_messages.sum(axis=1) - log_messages p11 = np.exp(self.U+acc_log_messages.T+acc_log_messages) p10 = np.exp(acc_log_messages) p01 = np.exp(acc_log_messages.T) sum_p = 1 + p11 + p10 + p01 sum_target_pairs += p11 / sum_p else: raise ValueError('approximate_inference \'%\' unknown' % self.approximate_inference) else: target_singles = 1./(1+np.exp(-data_vis_bias)) # apply CRF gradient update db = np.sum(target_singles,axis=0) - np.sum(target,axis=0) dV = np.dot(input.T,target_singles-target) self.b -= self.lr/input.shape[0] * db self.V -= self.lr/input.shape[0] * dV if self.n_inference_iterations > 0: dU = sum_target_pairs - np.dot(target.T,target) self.U -= self.lr/input.shape[0] * dU # Ensuring symmetry and 0 diagonal self.U = 0.5*(self.U+self.U.T) self.U -= np.diag(np.diag(self.U)) self.stage = self.n_stages
doutput = np.random.randn(30, 20) nonlinear.dsigmoid(output, doutput, dinput) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum( np.abs(dinput - doutput * output * (1 - output))) print 'Testing nonlinear softmax' input = np.random.randn(20) output = np.zeros((20)) nonlinear.softmax(input, output) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum( np.abs(output - np.exp(input) / np.sum(np.exp(input)))) print 'Testing nonlinear softplus' input = np.random.randn(20) output = np.zeros((20)) nonlinear.softplus(input, output) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum( np.abs(output - np.log(1 + np.exp(input)))) print 'Testing nonlinear reclin' input = np.random.randn(30, 20) output = np.zeros((30, 20)) nonlinear.reclin(input, output) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum( np.abs(output - (input > 0) * input)) print 'Testing nonlinear reclin deriv.' dinput = np.zeros((30, 20)) doutput = np.random.randn(30, 20) nonlinear.dreclin(output, doutput, dinput) print 'NumPy vs mathutils.nonlinear diff. output:', np.sum(
print 'Testing nonlinear sigmoid deriv.' dinput = np.zeros((30,20)) doutput = np.random.randn(30,20) nonlinear.dsigmoid(output,doutput,dinput) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(dinput-doutput*output*(1-output))) print 'Testing nonlinear softmax' input = np.random.randn(20) output = np.zeros((20)) nonlinear.softmax(input,output) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(output-np.exp(input)/np.sum(np.exp(input)))) print 'Testing nonlinear softplus' input = np.random.randn(20) output = np.zeros((20)) nonlinear.softplus(input,output) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(output-np.log(1+np.exp(input)))) print 'Testing nonlinear reclin' input = np.random.randn(30,20) output = np.zeros((30,20)) nonlinear.reclin(input,output) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(output-(input>0)*input)) print 'Testing nonlinear reclin deriv.' dinput = np.zeros((30,20)) doutput = np.random.randn(30,20) nonlinear.dreclin(output,doutput,dinput) print 'NumPy vs mathutils.nonlinear diff. output:',np.sum(np.abs(dinput-(input>0)*doutput))