def verify_sums_to_one(self): print 'WARNING: calling verify_sum_to_one reinitializes the learner' self.hidden_size = 6 input_size = 10 self.learning_rate = 1 epsilon=1e-6 word_ids = np.array([0,3,7,8,9]) word_counts = np.array([3,2,1,7,20]) trainset = mlpb.MLProblem([(word_counts,word_ids)], {'input_size':input_size}) self.initialize(trainset) log_sum_p = -np.inf sum_p = 0 for i in range(input_size): for j in range(input_size): for k in range(input_size): self.words = np.array([i,j,k],dtype="int32") self.act = np.zeros((len(self.words),self.hidden_size)) np.add.accumulate(self.W[self.words[:-1],:],axis=0,out=self.act[1:,:]) if self.hidden_bias_scaled_by_document_size: self.act += self.c*len(self.words) else: self.act += self.c self.h = np.zeros((len(self.words),self.hidden_size)) self.apply_activation(self.act,self.h) self.fprop_word_probs() log_p_i = np.log(self.probs).sum() m = max(log_p_i,log_sum_p) log_sum_p = m + np.log(np.exp(log_p_i-m)+np.exp(log_sum_p-m)) sum_p += np.exp(log_p_i) print "Sums to",np.exp(log_sum_p)#,sum_p
def verify_learning(self): print 'WARNING: calling verify_sum_to_one reinitializes the learner' self.hidden_size = 6 input_size = 10 self.learning_rate = 0.01 words = np.zeros((10,)) words[3] = 1 words[7] = 1 trainset = mlpb.MLProblem([words], {'input_size':input_size}) self.initialize(trainset) self.b[:] = 0 words_neg = np.zeros((10,)) for t in range(1,10001): p = np.exp(np.dot(words,self.b)+np.sum(np.log(1+np.exp(np.dot(self.W,words)+words.sum()*self.c)))) s = 0 words_neg[:] = 0 for i in range(input_size): for j in range(input_size): words_neg[:] = 0 words_neg[i] += 1 words_neg[j] += 1 s += np.exp(np.dot(words_neg,self.b)+np.sum(np.log(1+np.exp(np.dot(self.W,words_neg)+words_neg.sum()*self.c)))) print p/s self.n_stages = t self.train(trainset)
def get(dataset_name): # List of datasets that works with the current model ? datasets = ['adult', 'binarized_mnist', 'connect4', 'dna', 'mushrooms', 'nips', 'ocr_letters', 'rcv1', 'rcv2_russ', 'web'] # Setup dataset env if dataset_name not in datasets: raise ValueError('Dataset unknown: ' + dataset_name) # mldataset = __import__('mlpython.datasets.' + dataset_name, globals(), locals(), [dataset_name], -1) datadir = os.path.join(os.getenv("MLPYTHON_DATASET_REPO"), dataset_name) # Verify if dataset exist and if not, download it if(not os.path.exists(datadir)): dataset_store.download(dataset_name) print('### Loading dataset [{0}] ...'.format(dataset_name)) start_time = t.time() all_data = mldataset.load(datadir, load_to_memory=True) train_data, train_metadata = all_data['train'] if dataset_name == 'binarized_mnist' or dataset_name == 'nips': trainset = mlpb.MLProblem(train_data, train_metadata) else: trainset = mlpb.SubsetFieldsProblem(train_data, train_metadata) trainset.setup() valid_data, valid_metadata = all_data['valid'] validset = trainset.apply_on(valid_data, valid_metadata) test_data, test_metadata = all_data['test'] testset = trainset.apply_on(test_data, test_metadata) # Cleaning up, packaging and theanized full_dataset = {'input_size': trainset.metadata['input_size']} trainset_theano = theano.shared(value=Dataset._clean(trainset), borrow=True) validset_theano = theano.shared(value=Dataset._clean(validset), borrow=True) testset_theano = theano.shared(value=Dataset._clean(testset), borrow=True) full_dataset['train'] = {'data': trainset_theano, 'length': all_data['train'][1]['length']} full_dataset['valid'] = {'data': validset_theano, 'length': all_data['valid'][1]['length']} full_dataset['test'] = {'data': testset_theano, 'length': all_data['test'][1]['length']} print("(Dim:{0} Train:{1} Valid:{2} Test:{3})".format(trainset.metadata['input_size'], full_dataset['train']['length'], full_dataset['valid']['length'], full_dataset['test']['length'])) print(get_done_text(start_time), "###") return full_dataset
def load_data(dataset_name): datadir = root + '/data/' exec 'import mlpython.datasets.' + dataset_name + ' as mldataset' exec 'datadir = datadir + \'' + dataset_name + '/\'' all_data = mldataset.load(datadir, load_to_memory=True) train_data, train_metadata = all_data['train'] if dataset_name == 'binarized_mnist' or dataset_name == 'nips': trainset = mlpb.MLProblem(train_data, train_metadata) else: trainset = mlpb.SubsetFieldsProblem(train_data, train_metadata) trainset.setup() valid_data, valid_metadata = all_data['valid'] validset = trainset.apply_on(valid_data, valid_metadata) test_data, test_metadata = all_data['test'] testset = trainset.apply_on(test_data, test_metadata) train_X = trainset.data.mem_data[0] valid_X = validset.data.mem_data[0] test_X = testset.data.mem_data[0] return train_X, valid_X, test_X
name = 'ocr_letters_sequential' import os repo = os.environ.get('MLPYTHON_DATASET_REPO') if repo is None: raise ValueError( 'environment variable MLPYTHON_DATASET_REPO is not defined') dataset_dir = os.environ.get('MLPYTHON_DATASET_REPO') + '/' + name all_data = mldataset.load(dataset_dir, load_to_memory=load_to_memory) train_data, train_metadata = all_data['train'] valid_data, valid_metadata = all_data['valid'] test_data, test_metadata = all_data['test'] import mlpython.mlproblems.generic as mlpb trainset = mlpb.MLProblem(train_data, train_metadata) validset = trainset.apply_on(valid_data, valid_metadata) testset = trainset.apply_on(test_data, test_metadata) def compute_error_mean_and_sterror(costs): classif_errors = np.hstack([c[0] for c in costs]) classif_mean = classif_errors.mean() classif_sterror = classif_errors.std(ddof=1) / np.sqrt( classif_errors.shape[0]) nll_errors = [c[1] for c in costs] nll_mean = np.mean(nll_errors) nll_sterror = np.std(nll_errors, ddof=1) / np.sqrt(len(nll_errors)) return classif_mean, nll_mean, classif_sterror, nll_sterror
testSetPath = sys.argv[6] inputSize = int(sys.argv[7]) # Create DocNADE learner object docNadeObject = DocNADE(n_stages=1, hidden_size=hidden_size, learning_rate=learning_rate, activation_function=activation_function) # Load the data train_data, train_metadata = load(trainSetPath, inputSize) valid_data, valid_metadata = load(validSetPath, inputSize) test_data, test_metadata = load(testSetPath, inputSize) # Create MLProblems (data structure used for data sets in MLPython) trainset = mlpb.MLProblem(train_data, train_metadata) validset = mlpb.MLProblem(valid_data, valid_metadata) testset = mlpb.MLProblem(test_data, test_metadata) # Training wiht early-stopping best_val_error = np.inf best_it = 0 look_ahead = 10 n_incr_error = 0 # Nb. of consecutive increase in error for stage in range(1, max_iter + 1, 1): if not n_incr_error < look_ahead: break docNadeObject.n_stages = stage # Ask for one more training iteration docNadeObject.train(trainset) # Train some more n_incr_error += 1
def verify_gradients(self): print 'WARNING: calling verify_gradients reinitializes the learner' self.activation_function = "softmax" self.hidden_size = 6 input_size = 10 self.learning_rate = 0.01 epsilon=1e-6 word_ids = np.array([0,3,7,8,9]) word_counts = np.array([3,2,1,7,20]) trainset = mlpb.MLProblem([(word_counts,word_ids)], {'input_size':input_size}) self.initialize(trainset) self.learning_rate = 1 self.c = self.rng.rand(self.hidden_size) # Compute all derivatives rng_test_time = np.random.mtrand.RandomState(1234) tmp_rng = self.rng self.rng = rng_test_time self.fprop(word_ids,word_counts) self.bprop() self.rng = tmp_rng # Estimate derivatives by finite differences W_copy = np.array(self.W) lim_dW = np.zeros(self.W.shape) for i in range(self.W.shape[0]): for j in range(self.W.shape[1]): self.W[i,j] += epsilon outputs,costs = self.test(trainset) a = costs[0][0] self.W[i,j] -= 2.*epsilon outputs,costs = self.test(trainset) b = costs[0][0] self.W[i,j] += epsilon lim_dW[i,j] = (a-b)/(2.*epsilon) print 'dW diff.:',np.sum(np.abs(self.dW.ravel()-lim_dW.ravel()))/self.W.ravel().shape[0] b_copy = np.array(self.b) lim_db = np.zeros(self.b.shape) for i in range(self.b.shape[0]): self.b[i] += epsilon outputs,costs = self.test(trainset) a = costs[0][0] self.b[i] -= 2.*epsilon outputs,costs = self.test(trainset) b = costs[0][0] self.b[i] += epsilon lim_db[i] = (a-b)/(2.*epsilon) print 'db diff.:',np.sum(np.abs(self.db.ravel()-lim_db.ravel()))/self.b.ravel().shape[0] V_copy = np.array(self.V) lim_dV = np.zeros(self.V.shape) for i in range(self.V.shape[0]): for j in range(self.V.shape[1]): self.V[i,j] += epsilon outputs,costs = self.test(trainset) a = costs[0][0] self.V[i,j] -= 2.*epsilon outputs,costs = self.test(trainset) b = costs[0][0] self.V[i,j] += epsilon lim_dV[i,j] = (a-b)/(2.*epsilon) print 'dV diff.:',np.sum(np.abs(self.dV.ravel()-lim_dV.ravel()))/self.V.ravel().shape[0] c_copy = np.array(self.c) lim_dh = np.zeros(self.c.shape) for i in range(self.c.shape[0]): self.c[i] += epsilon outputs,costs = self.test(trainset) a = costs[0][0] self.c[i] -= 2.*epsilon outputs,costs = self.test(trainset) b = costs[0][0] self.c[i] += epsilon lim_dh[i] = (a-b)/(2.*epsilon) print 'dc diff.:',np.sum(np.abs(self.dc.ravel()-lim_dh.ravel()))/self.c.ravel().shape[0]
target = 2 * example[1] - 1 # Targets are 0/1 output = np.dot(self.w, input) + self.b if np.sign(output) != target: self.w += self.lr * target * input self.b += self.lr * target def use_learner(self, example): return np.sign(np.dot(self.w, example[0]) + self.b) def cost(self, output, example): return int(output != 2 * example[1] - 1) inputs = np.array([[0, 0, 1, 1], [1, 1, 0, 0]]) targets = np.array([0, 1]).T metadata = {'input_size': 4, 'targets': set([0, 1])} trainset = mlpb.MLProblem(zip(inputs, targets), metadata) perceptron = Perceptron(2, lr=0.01) perceptron.train(trainset) print perceptron.use(trainset) print perceptron.test(trainset) inputs = np.array([[0, 0, 0, 1], [1, 0, 0, 0]]) targets = np.array([0, 1]).T metadata = {'input_size': 4, 'targets': set([0, 1])} testset = mlpb.MLProblem(zip(inputs, targets), metadata) print perceptron.use(testset) print perceptron.test(testset)
# authors and should not be interpreted as representing official policies, either expressed # or implied, of Hugo Larochelle. import mlpython.mlproblems.generic as mlpbgen import mlpython.mlproblems.classification as mlpbclass raw_data = zip(range(6), ['A', 'A', 'B', 'C', 'A', 'B']) metadata = {'length': 6, 'targets': ['A', 'B', 'C'], 'input_size': 1} def features(example, metadata): metadata['input_size'] = 2 return ((example[0], example[0]), example[1]) pb1 = mlpbgen.MLProblem(raw_data, metadata) print 'pb1:' for example in pb1: print example print 'metadata:', pb1.metadata print '' pb2 = mlpbgen.SubsetProblem(pb1, subset=set([1, 3, 5])) print 'pb2:' for example in pb2: print example print 'metadata:', pb2.metadata print '' pb3 = mlpbgen.MergedProblem([pb2, pb1]) print 'pb3:'
def test_mlproblem_combinations(): """ Test a combination of many different MLProblems. """ raw_data = zip(range(6), ['A', 'A', 'B', 'C', 'A', 'B']) metadata = {'length': 6, 'targets': ['A', 'B', 'C'], 'input_size': 1} def features(example, metadata): metadata['input_size'] = 2 return ((example[0], example[0]), example[1]) pb1 = mlpbgen.MLProblem(raw_data, metadata) print 'pb1', pb1.metadata pb2 = mlpbgen.SubsetProblem(pb1, subset=set([1, 3, 5])) print 'pb2', pb2.metadata pb3 = mlpbgen.MergedProblem([pb2, pb1]) print 'pb3', pb3.metadata pb4 = mlpbgen.PreprocessedProblem(pb3, preprocess=features) print 'pb4', pb4.metadata pb5 = mlpbclass.ClassificationProblem(pb4) print 'pb5', pb5.metadata pb6 = mlpbclass.ClassSubsetProblem(pb5, subset=set(['A', 'C'])) print 'pb6', pb6.metadata pb7 = mlpbgen.SubsetFieldsProblem(pb6, fields=[0, 0, 1]) print 'pb7', pb7.metadata final_data = [[(1, 1), (1, 1), 0], [(3, 3), (3, 3), 1], [(0, 0), (0, 0), 0], [(1, 1), (1, 1), 0], [(3, 3), (3, 3), 1], [(4, 4), (4, 4), 0]] final_metadata = { 'input_size': 2, 'targets': set(['A', 'C']), 'class_to_id': { 'A': 0, 'C': 1 } } for ex1, ex2 in zip(pb7, final_data): assert cmp(ex1, ex2) == 0 print pb7.metadata, final_metadata assert cmp(pb7.metadata, final_metadata) == 0 raw_data2 = zip(range(6, 10), ['C', 'B', 'A', 'C']) metadata2 = {'length': 4, 'targets': ['A', 'B', 'C'], 'input_size': 1} pbtest = pb7.apply_on(raw_data2, metadata2) final_test_data = [[(6, 6), (6, 6), 1], [(8, 8), (8, 8), 0], [(9, 9), (9, 9), 1]] final_test_metadata = { 'input_size': 2, 'targets': set(['A', 'C']), 'class_to_id': { 'A': 0, 'C': 1 } } for ex1, ex2 in zip(pbtest, final_test_data): assert cmp(ex1, ex2) == 0 assert cmp(pbtest.metadata, final_test_metadata) == 0