def regularization_test(): with open("log_regular.txt", "w+") as f: f.write("Regularization | lambda | Area | R2\n") f.write("-" * 50 + "\n") for reg in ['l2', 'l1']: for lamb in [0.0001, 0.001, 0.01, 0.1, 1.0]: logreg = LogReg(X, Y) logreg.optimize(m=100, epochs=5000, eta=0.01, regularization=reg, lamb=lamb) ypred_train = logreg.p_train ypred_test = logreg.p_test area = gain_chart(logreg.Y_test, ypred_test, plot=False) R2 = prob_acc(logreg.Y_test, ypred_test, plot=False) f.write(" %s | %g | %.4f | %.4f \n" % (reg, lamb, area, R2)) f.write("-" * 50 + "\n")
def setUp(self): self.mockV = Mock() self.mockM = Mock() self.test_obj = LogReg([('summer', { 'text': "some_text!" }), ('winter', { 'text': 'some_text2!' })], vectorizer=self.mockV, model=self.mockM)
class TestKnn(unittest.TestCase): def setUp(self): self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0) self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0) def test_reg(self): print(self.logreg_reg.beta) print(kPOS.x) beta = self.logreg_reg.sg_update(kPOS, 1) self.assertAlmostEqual(beta[0], .5) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 1.5) self.assertAlmostEqual(beta[3], 0.5) self.assertAlmostEqual(beta[4], 0.0) print(self.logreg_reg.beta) print(kNEG.x) beta = self.logreg_reg.sg_update(kNEG, 2) self.assertAlmostEqual(beta[0], -0.72068776924864364) self.assertAlmostEqual(beta[1], 1.0) self.assertAlmostEqual(beta[2], -0.22068776924864364) self.assertAlmostEqual(beta[3], -2.6620633077459308) self.assertAlmostEqual(beta[4], -3.8827510769945746) print("Finalize") print(self.logreg_reg.last_update) beta = self.logreg_reg.finalize_lazy(2) self.assertAlmostEqual(beta[0], -0.72068776924864364) self.assertAlmostEqual(beta[1], 1.0) self.assertAlmostEqual(beta[2], -0.22068776924864364) self.assertAlmostEqual(beta[3], -2.6620633077459308) self.assertAlmostEqual(beta[4], -3.8827510769945746) def test_unreg(self): print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kPOS, 1) self.assertAlmostEqual(beta[0], .5) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 1.5) self.assertAlmostEqual(beta[3], 0.5) self.assertAlmostEqual(beta[4], 0.0) print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kNEG, 2) self.assertAlmostEqual(beta[0], -0.47068776924864364) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 0.5293122307513564) self.assertAlmostEqual(beta[3], -2.4120633077459308) self.assertAlmostEqual(beta[4], -3.8827510769945746)
class TestKnn(unittest.TestCase): def setUp(self): self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0) self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0) def test_reg(self): #print("\nREGULARIZED:\n") print(self.logreg_reg.beta) print(kPOS.x) beta = self.logreg_reg.sg_update(kPOS, 0) self.assertAlmostEqual(beta[0], .25) self.assertAlmostEqual(beta[1], 1.0) self.assertAlmostEqual(beta[2], 0.75) self.assertAlmostEqual(beta[3], 0.25) self.assertAlmostEqual(beta[4], 0.0) #print(self.logreg_reg.beta) #print(kPOS.x) beta = self.logreg_reg.sg_update(kNEG, 1) self.assertAlmostEqual(beta[0], -0.30097640098415529) self.assertAlmostEqual(beta[1], 1.0) self.assertAlmostEqual(beta[2], -0.05097640098415529) self.assertAlmostEqual(beta[3], -1.1529292029524658) self.assertAlmostEqual(beta[4], -0.85195280196831058) def test_unreg(self): #print("\nUNREGULARIZED:\n\n") print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kPOS, 0) self.assertAlmostEqual(beta[0], .5) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 1.5) self.assertAlmostEqual(beta[3], 0.5) self.assertAlmostEqual(beta[4], 0.0) print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kNEG, 1) self.assertAlmostEqual(beta[0], -0.47068776924864364) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 0.5293122307513564) self.assertAlmostEqual(beta[3], -2.4120633077459308) self.assertAlmostEqual(beta[4], -3.8827510769945746)
class TestLogReg(unittest.TestCase): def setUp(self): self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0) self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0) def test_unreg(self): print(self.logreg_unreg.w) print(kPOS.x) w = self.logreg_unreg.sg_update(kPOS, 0) self.assertAlmostEqual(w[0], .5) self.assertAlmostEqual(w[1], 2.0) self.assertAlmostEqual(w[2], 1.5) self.assertAlmostEqual(w[3], 0.5) self.assertAlmostEqual(w[4], 0.0) print(self.logreg_unreg.w) print(kNEG.x) w = self.logreg_unreg.sg_update(kNEG, 1) self.assertAlmostEqual(w[0], -0.49330714907571527) self.assertAlmostEqual(w[1], 2.0) self.assertAlmostEqual(w[2], -0.48661429815143054) self.assertAlmostEqual(w[3], -2.479921447227146) self.assertAlmostEqual(w[4], -0.99330714907571527) def test_reg(self): print(self.logreg_reg.w) print(kPOS.x) w = self.logreg_reg.sg_update(kPOS, 0) self.assertAlmostEqual(w[0], .5) self.assertAlmostEqual(w[1], 1.0) self.assertAlmostEqual(w[2], 0.75) self.assertAlmostEqual(w[3], 0.25) self.assertAlmostEqual(w[4], 0.0) print(self.logreg_reg.w) print(kNEG.x) w = self.logreg_reg.sg_update(kNEG, 1) print w self.assertAlmostEqual(w[0], -0.43991334982599239) self.assertAlmostEqual(w[1], 1.0) self.assertAlmostEqual(w[2], -0.56491334982599239) self.assertAlmostEqual(w[3], -1.2848700247389886) self.assertAlmostEqual(w[4], -0.2349783374564981)
class TestKnn(unittest.TestCase): def setUp(self): self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0) self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0) def test_reg(self): print(self.logreg_reg.beta) print(kPOS.x) beta = self.logreg_reg.sg_update(kPOS, 0) self.assertAlmostEqual(beta[0], .25) self.assertAlmostEqual(beta[1], 1.0) self.assertAlmostEqual(beta[2], 0.75) self.assertAlmostEqual(beta[3], 0.25) self.assertAlmostEqual(beta[4], 0.0) print(self.logreg_reg.beta) print(kPOS.x) beta = self.logreg_reg.sg_update(kNEG, 1) self.assertAlmostEqual(beta[0], -0.30097640098415529) self.assertAlmostEqual(beta[1], 1.0) self.assertAlmostEqual(beta[2], -0.05097640098415529) self.assertAlmostEqual(beta[3], -1.1529292029524658) self.assertAlmostEqual(beta[4], -0.85195280196831058) def test_unreg(self): print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kPOS, 0) self.assertAlmostEqual(beta[0], .5) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 1.5) self.assertAlmostEqual(beta[3], 0.5) self.assertAlmostEqual(beta[4], 0.0) print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kNEG, 1) self.assertAlmostEqual(beta[0], -0.47068776924864364) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 0.5293122307513564) self.assertAlmostEqual(beta[3], -2.4120633077459308) self.assertAlmostEqual(beta[4], -3.8827510769945746)
def setUp(self): self.mockV = Mock() self.mockM = Mock() self.test_obj = LogReg([ ('summer', {'text': "some_text!"}), ('winter', {'text': 'some_text2!'}) ], vectorizer = self.mockV, model = self.mockM )
def find_epoch(dims, nb_classes, train_embs, train_labels, test_embs, test_labels, cuda): log = LogReg(dims, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) xent = nn.BCEWithLogitsLoss() sigmoid = nn.Sigmoid() if cuda: log.cuda() epoch_flag = 0 epoch_win = 0 best_f1 = torch.zeros(1) tmp_th = 0.5 if cuda: best_f1 = best_f1.cuda() for e in range(2000): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_labels) loss.backward() opt.step() if (e + 1) % 100 == 0: log.eval() logits = sigmoid(log(test_embs)) zero = torch.zeros_like(logits) one = torch.ones_like(logits) preds = torch.where(logits >= tmp_th, one, zero) if cuda: test_labels = test_labels.cpu() preds = preds.cpu() f1 = f1_score(test_labels.numpy(), preds.numpy(), average='micro') if f1 >= best_f1: epoch_flag = e + 1 best_f1 = f1 epoch_win = 0 else: epoch_win += 1 if epoch_win == 10: break return epoch_flag
class TestKnn(unittest.TestCase): def setUp(self): self.logreg_unreg = LogReg(5, 1.0) def test_unreg(self): print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kPOS) self.assertAlmostEqual(beta[0], .5) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 1.5) self.assertAlmostEqual(beta[3], 0.5) self.assertAlmostEqual(beta[4], 0.0) print(self.logreg_unreg.beta) print(kPOS.x) beta = self.logreg_unreg.sg_update(kNEG) self.assertAlmostEqual(beta[0], -0.47068776924864364) self.assertAlmostEqual(beta[1], 2.0) self.assertAlmostEqual(beta[2], 0.5293122307513564) self.assertAlmostEqual(beta[3], -2.4120633077459308) self.assertAlmostEqual(beta[4], -3.8827510769945746)
def __init__(self, inp, layer_sizes): self.x = inp self.layer_sizes = layer_sizes self.layers = [] prev = self.x for i in range(1, len(self.layer_sizes)): new_shape = (self.layer_sizes[i], self.layer_sizes[i - 1]) self.layers.append(LogReg(prev, new_shape)) prev = self.layers[-1].a self.a = self.layers[-1].a self.params = sum((l.params for l in self.layers), [])
class TestLogReg(unittest.TestCase): def setUp(self): self.logreg_learnrate = LogReg(784, 0.5) self.logreg_nolearnrate = LogReg(784, 1.0) def test_learnrate(self): print("\nTesting: Learning Rate Update") w = self.logreg_learnrate.sgd_update(pos_example_x, pos_example_y) self.assertAlmostEqual(w[208], 0.05371094) self.assertAlmostEqual(w[209], 0.14453125) self.assertAlmostEqual(w[210], 0.20507812) self.assertAlmostEqual(w[211], 0.24707031) self.assertAlmostEqual(w[212], 0.24707031) def test_nolearnrate(self): print("\nTesting: No Learning Rate Update") w = self.logreg_nolearnrate.sgd_update(neg_example_x, neg_example_y) self.assertAlmostEqual(w[208], 0.10742188) self.assertAlmostEqual(w[209], 0.2890625) self.assertAlmostEqual(w[210], 0.41015625) self.assertAlmostEqual(w[211], 0.49414062) self.assertAlmostEqual(w[212], 0.49414062)
def evaluate_lenet5(self, learning_rate=0.1, n_epochs=1, dataset='mnist.pkl.gz', nkerns=[20, 50], batch_size=500, testing=0): rng = numpy.random.RandomState(32324) datasets = self.data train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] n_train_batches = train_set_x.get_value( borrow=True).shape[0] // batch_size n_valid_batches = valid_set_x.get_value( borrow=True).shape[0] // batch_size n_test_batches = test_set_x.get_value( borrow=True).shape[0] // batch_size index = T.lscalar() # index for each mini batch x = T.matrix('x') y = T.ivector('y') # ------------------------------- Building Model ---------------------------------- if testing == 0: print "...Building the model" # output image size = (28-5+1)/2 = 12 layer_0_input = x.reshape((batch_size, 1, 28, 28)) layer_0 = LeNetConvPoolLayer(rng, input=layer_0_input, image_shape=(batch_size, 1, 28, 28), filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2, 2)) #output image size = (12-5+1)/2 = 4 layer_1 = LeNetConvPoolLayer(rng, input=layer_0.output, image_shape=(batch_size, nkerns[0], 12, 12), filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2, 2)) # make the input to hidden layer 2 dimensional layer_2_input = layer_1.output.flatten(2) layer_2 = HiddenLayer(rng, input=layer_2_input, n_in=nkerns[1] * 4 * 4, n_out=500, activation=T.tanh) layer_3 = LogReg(input=layer_2.output, n_in=500, n_out=10) self.cost = layer_3.neg_log_likelihood(y) self.s = layer_3.s self.test_model = theano.function( [index], layer_3.errors(y), givens={ x: test_set_x[index * batch_size:(index + 1) * batch_size], y: test_set_y[index * batch_size:(index + 1) * batch_size] }) self.validate_model = theano.function( [index], layer_3.errors(y), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size] }) self.train_predic = theano.function( [index], layer_3.prob_y_given_x(), givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size] }) # list of parameters self.params = layer_3.params + layer_2.params + layer_1.params + layer_0.params grads = T.grad(self.cost, self.params) self.coefficient = 1 self.shapes = [i.get_value().shape for i in self.params] symbolic_types = T.scalar, T.vector, T.matrix, T.tensor3, T.tensor4 v = [symbolic_types[len(i)]() for i in self.shapes] #import pdb #pdb.set_trace() gauss_vector = Gv(self.cost, self.s, self.params, v, self.coefficient) self.get_cost = theano.function( [ index, ], self.cost, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size] }, on_unused_input='ignore') self.get_grad = theano.function( [ index, ], grads, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size] }, on_unused_input='ignore') self.get_s = theano.function( [ index, ], self.s, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], }, on_unused_input='ignore') self.function_Gv = theano.function( [index], gauss_vector, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size] }, on_unused_input='ignore') # # Using stochastic gradient updates # updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ] # train_model = theano.function([index],cost, updates=updates, # givens={ # x: train_set_x[index*batch_size:(index+1)*batch_size], # y: train_set_y[index*batch_size:(index+1)*batch_size] # }) # Using conjugate gradient updates # 'cg_ = cg(cost,output,params,coefficient,v) # updated_params = [(param_i, param_j) for param_i,param_j in zip(params,cg_)]' #self.update_parameters= theano.function([updated_params],updates=[params,updated_params]) # -----------------------------------------Starting Training ------------------------------ if testing == 0: print('..... Training ') # for early stopping patience = 10000 patience_increase = 2 improvement_threshold = 0.95 validation_frequency = min(n_train_batches, patience // 2) self.best_validation_loss = numpy.inf # initialising loss to be inifinite best_itr = 0 test_score = 0 start_time = timeit.default_timer() epoch = 0 done_looping = False while (epoch < n_epochs): epoch = epoch + 1 for minibatch_index in range(n_train_batches): iter = (epoch - 1) * n_train_batches + minibatch_index if iter % 1 == 0: print('training @ iter = ', iter) self.cg(minibatch_index) if testing == 0: print('Optimization complete') print( 'Best validation score of %f %% obtained at iteration %i,' 'with test performance %f %%' % (best_validation_loss * 100., best_itr, test_score * 100)) print('The code ran for %.2fm' % ((end_time - start_time) / 60.))
def setUp(self): self.logreg_unreg = LogReg(5, 1.0)
def setUp(self): self.logreg_learnrate = LogReg(5, 0.0, lambda x: 0.5) self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0) self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def setUp(self): self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0) self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
fn = 'Skin_NonSkin.txt' data = np.loadtxt(fn) data[:,-1] -= 1 np.random.shuffle(data) data = np.c_[np.ones(data.shape[0]), data] # adding offset term #%% Data X = data[:ndat, :-1] y = data[:ndat, -1] #%% Prior mean = np.zeros(X.shape[-1]) Sigma = np.eye(X.shape[-1]) * 100. prior = LogReg(mean=mean, Sigma=Sigma) #%% Estimation for xt, yt in zip(X, y): prior.update(yt, xt) prior.log() #%% Confusion matrix CM = ConfusionMatrix(prior.true_vals, prior.binary_preds) CM.print_stats() #%% Plots beta_log = np.array(prior.mean_log) plt.figure(figsize=(8, 4)) plt.plot(prior.brier_score_log)
def evaluate_lenet5(learning_rate=0.1, n_epochs=200, dataset='mnist.pkl.gz', nkerns=[16, 16, 16], batch_size=500): rng = numpy.random.RandomState(32324) datasets = load_data(dataset) train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size index = T.lscalar() # index for each mini batch train_epoch = T.lscalar() x = T.matrix('x') y = T.ivector('y') # ------------------------------- Building Model ---------------------------------- print "...Building the model" # output image size = (28-5+1+4)/2 = 14 layer_0_input = x.reshape((batch_size, 1, 28, 28)) layer_0 = LeNetConvPoolLayer(rng, input=layer_0_input, image_shape=(batch_size, 1, 28, 28), filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2, 2), border_mode=2) #output image size = (14-3+1)/2 = 6 layer_1 = LeNetConvPoolLayer(rng, input=layer_0.output, image_shape=(batch_size, nkerns[0], 14, 14), filter_shape=(nkerns[1], nkerns[0], 3, 3), poolsize=(2, 2)) #output image size = (6-3+1)/2 = 2 layer_2 = LeNetConvPoolLayer(rng, input=layer_1.output, image_shape=(batch_size, nkerns[1], 6, 6), filter_shape=(nkerns[2], nkerns[1], 3, 3), poolsize=(2, 2)) # make the input to hidden layer 2 dimensional layer_3_input = layer_2.output.flatten(2) layer_3 = HiddenLayer(rng, input=layer_3_input, n_in=nkerns[2] * 2 * 2, n_out=200, activation=T.tanh) layer_4 = LogReg(input=layer_3.output, n_in=200, n_out=10) teacher_p_y_given_x = theano.shared(numpy.asarray( pickle.load(open('prob_best_model.pkl', 'rb')), dtype=theano.config.floatX), borrow=True) #cost = layer_4.neg_log_likelihood(y) + T.mean((teacher_W - layer_4.W)**2)/(2.*(1+epoch*2)) + T.mean((teacher_b-layer_4.b)**2)/(2.*(1+epoch*2)) # import pdb # pdb.set_trace() p_y_given_x = T.matrix('p_y_given_x') e = theano.shared(value=0, name='e', borrow=True) #cost = layer_4.neg_log_likelihood(y) + 1.0/(e)*T.mean((layer_4.p_y_given_x - p_y_given_x)**2) cost = layer_4.neg_log_likelihood( y) + 2.0 / (e) * T.mean(-T.log(layer_4.p_y_given_x) * p_y_given_x - layer_4.p_y_given_x * T.log(p_y_given_x)) test_model = theano.function( [index], layer_4.errors(y), givens={ x: test_set_x[index * batch_size:(index + 1) * batch_size], y: test_set_y[index * batch_size:(index + 1) * batch_size] }) validate_model = theano.function( [index], layer_4.errors(y), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size] }) # list of parameters params = layer_4.params + layer_3.params + layer_2.params + layer_1.params + layer_0.params grads = T.grad(cost, params) updates = [(param_i, param_i - learning_rate * grad_i) for param_i, grad_i in zip(params, grads)] train_model = theano.function( [index, train_epoch], cost, updates=updates, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size], p_y_given_x: teacher_p_y_given_x[index], e: train_epoch }) # -----------------------------------------Starting Training ------------------------------ print('..... Training ') # for early stopping patience = 10000 patience_increase = 2 improvement_threshold = 0.95 validation_frequency = min(n_train_batches, patience // 2) best_validation_loss = numpy.inf # initialising loss to be inifinite best_itr = 0 test_score = 0 start_time = timeit.default_timer() #epo = theano.shared('epo') epoch = 0 done_looping = False while (epoch < n_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in range(n_train_batches): iter = (epoch - 1) * n_train_batches + minibatch_index if iter % 100 == 0: print('training @ iter = ', iter) cost_ij = train_model(minibatch_index, epoch) if (iter + 1) % validation_frequency == 0: # compute loss on validation set validation_losses = [ validate_model(i) for i in range(n_valid_batches) ] this_validation_loss = numpy.mean(validation_losses) # import pdb # pdb.set_trace() print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) # check with best validation score till now if this_validation_loss < best_validation_loss: # improve if this_validation_loss < best_validation_loss * improvement_threshold: patience = max(patience, iter * patience_increase) best_validation_loss = this_validation_loss best_itr = iter test_losses = [ test_model(i) for i in range(n_test_batches) ] test_score = numpy.mean(test_losses) print('epoch %i, minibatch %i/%i, testing error %f %%' % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.)) with open('best_model_3layer.pkl', 'wb') as f: pickle.dump(params, f) with open('Results_student_3.txt', 'wb') as f2: f2.write(str(test_score * 100) + '\n') #if patience <= iter: # done_looping = True # break end_time = timeit.default_timer() print('Optimization complete') print( 'Best validation score of %f %% obtained at iteration %i,' 'with test performance %f %%' % (best_validation_loss * 100., best_itr, test_score * 100)) print('The code ran for %.2fm' % ((end_time - start_time) / 60.))
from logreg import LogReg from initdata import InitData from DNN import NeuralNetwork """ In this part of the project, we assess the predictive ability of a feed-forward Neural network on determining default based on credit card data. To explore hyperparameter-space, chose explore = True To run self-written NN network, chose sklearn = False Chose metric for saving best score with metric (accuracy/rocauc) For credit card data, cost=cross_entropy is chosen """ ## Get data from InitData Class --------------------------------------- logreg = LogReg() # init Logreg class data = InitData() ## Params ------------------------------------------------------------- explore = False sklearn = False metric = "accuracy" #"rocauc" cost = "binary_cross_entropy" data_size = "full" if data_size == "full": XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot = data.credit_data(trainingShare=0.5) else: XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot = data.credit_data(trainingShare=0.5, return_cols=False, drop_zero=True,drop_neg2=True) if explore==True: # Explore parameter space for credit card data
import matplotlib.pyplot as plt from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, classification_report #plt.style.use(u"~/.matplotlib/stylelib/trygveplot_astro.mplstyle") plt.style.use(u"../trygveplot_astro.mplstyle") from logreg import LogReg from initdata import InitData """ In this part of the project, we assess the predictive ability of logistic regression on determining default based on credit card data. The weights are trained using a gradient solver and compared with Scikit-Learns Logistic regression method. """ # unit test (exits afterwards) if (False): logreg = LogReg(cost='cross_entropy') # init Logreg class logreg.unit_test() # see logreg for expected output ## Get data from InitData Class data = InitData() ##initialize all data, splitting gender, education and marital status #XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot = data.credit_data(trainingShare=0.5,per_col=True,drop_zero=True,drop_neg2=True) ##initialize all data (with some bill_amt and pay_amt), split education and marital status XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot, data_cols = data.credit_data( trainingShare=0.5, drop_zero=False, drop_neg2=False, per_col=True, return_cols=True,
import numpy as np from read_data import get_data import sys sys.path.append("network/") from logreg import LogReg from metrics import gain_chart, prob_acc #X, Y = get_data(normalized = False, standardized = False) X, Y = get_data() logreg = LogReg(X, Y) logreg.optimize(m=100, epochs=5000, eta=0.01) #, regularization='l2', lamb=0.0001) ypred_train = logreg.p_train ypred_test = logreg.p_test gain_chart(logreg.Y_test, ypred_test) prob_acc(logreg.Y_test, ypred_test) def regularization_test(): with open("log_regular.txt", "w+") as f: f.write("Regularization | lambda | Area | R2\n") f.write("-" * 50 + "\n") for reg in ['l2', 'l1']: for lamb in [0.0001, 0.001, 0.01, 0.1, 1.0]: logreg = LogReg(X, Y) logreg.optimize(m=100,
def evaluate_lenet5(learning_rate = 0.1, n_epochs=200, dataset='mnist.pkl.gz',nkerns = [20,50], batch_size = 500 , testing =0): rng = numpy.random.RandomState(32324) datasets = load_data(dataset) train_set_x,train_set_y = datasets[0] valid_set_x,valid_set_y = datasets[1] test_set_x,test_set_y = datasets[2] n_train_batches = train_set_x.get_value(borrow=True).shape[0]//batch_size n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]//batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0]//batch_size index = T.lscalar() # index for each mini batch x = T.matrix('x') y = T.ivector('y') # ------------------------------- Building Model ---------------------------------- if testing ==0: print "...Building the model" # output image size = (28-5+1)/2 = 12 layer_0_input = x.reshape((batch_size,1,28,28)) layer_0 = LeNetConvPoolLayer(rng,input = layer_0_input, image_shape=(batch_size,1,28,28),filter_shape=(nkerns[0],1,5,5),poolsize=(2,2)) #output image size = (12-5+1)/2 = 4 layer_1 = LeNetConvPoolLayer(rng, input = layer_0.output, image_shape = (batch_size, nkerns[0],12,12), filter_shape = (nkerns[1],nkerns[0],5,5), poolsize=(2,2) ) # make the input to hidden layer 2 dimensional layer_2_input = layer_1.output.flatten(2) layer_2 = HiddenLayer(rng,input = layer_2_input, n_in = nkerns[1]*4*4, n_out = 500, activation = T.tanh) layer_3 = LogReg(input = layer_2.output, n_in=500, n_out = 10) cost = layer_3.neg_log_likelihood(y) test_model = theano.function([index],layer_3.errors(y), givens={ x: test_set_x[index*batch_size:(index+1)*batch_size], y: test_set_y[index*batch_size:(index+1)*batch_size] }) validate_model = theano.function([index],layer_3.errors(y), givens={ x: valid_set_x[index*batch_size:(index+1)*batch_size], y: valid_set_y[index*batch_size:(index+1)*batch_size] }) train_predic = theano.function([index], layer_3.prob_y_given_x(), givens={ x: train_set_x[index*batch_size:(index+1)*batch_size] }) # list of parameters layer_guided = theano.function([index], layer_1.output, givens={ x: train_set_x[index*batch_size:(index+1)*batch_size] }) params = layer_3.params + layer_2.params + layer_1.params + layer_0.params grads = T.grad(cost,params) updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ] train_model = theano.function([index],cost, updates=updates, givens={ x: train_set_x[index*batch_size:(index+1)*batch_size], y: train_set_y[index*batch_size:(index+1)*batch_size] }) # -----------------------------------------Starting Training ------------------------------ if testing ==0: print ('..... Training ' ) # for early stopping patience = 10000 patience_increase = 2 improvement_threshold = 0.95 validation_frequency = min(n_train_batches, patience//2) best_validation_loss = numpy.inf # initialising loss to be inifinite best_itr = 0 test_score = 0 start_time = timeit.default_timer() epoch = 0 done_looping = False while (epoch < n_epochs) and (not done_looping) and testing ==0: epoch = epoch+1 for minibatch_index in range(n_train_batches): iter = (epoch - 1)*n_train_batches + minibatch_index if iter%100 ==0: print ('training @ iter = ', iter) cost_ij = train_model(minibatch_index) if(iter +1)%validation_frequency ==0: # compute loss on validation set validation_losses = [validate_model(i) for i in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) # import pdb # pdb.set_trace() print ('epoch %i, minibatch %i/%i, validation error %f %%' %(epoch,minibatch_index+1,n_train_batches,this_validation_loss*100. )) # check with best validation score till now if this_validation_loss<best_validation_loss: # improve # if this_validation_loss < best_validation_loss * improvement_threshold: # patience = max(patience, iter*patience_increase) best_validation_loss = this_validation_loss best_itr = iter test_losses = [test_model(i) for i in range(n_test_batches)] test_score = numpy.mean(test_losses) print ('epoch %i, minibatch %i/%i, testing error %f %%' %(epoch, minibatch_index+1,n_train_batches,test_score*100.)) with open('best_model.pkl', 'wb') as f: pickle.dump(params, f) with open('Results_teacher.txt','wb') as f2: f2.write(str(test_score*100) + '\n') p_y_given_x = [train_predic(i) for i in range(n_train_batches)] with open ('prob_best_model.pkl','wb') as f1: pickle.dump(p_y_given_x,f1) # if patience <= iter: # done_looping = True # break layer_2_op_dump = [layer_guided(i) for i in range(n_train_batches)] with open ('layer_guided.pkl','wb') as lg: pickle.dump(layer_2_op_dump,lg) end_time = timeit.default_timer() # p_y_given_x = [train_model(i) for i in range(n_train_batches)] # with open ('prob_best_model.pkl') as f: # pickle.dump(p_y_given_x) if testing ==0 : print ('Optimization complete') print ('Best validation score of %f %% obtained at iteration %i,' 'with test performance %f %%' % (best_validation_loss*100., best_itr, test_score*100 )) print('The code ran for %.2fm' %((end_time - start_time)/60.))
class TestLogReg(unittest.TestCase): def setUp(self): self.logreg_learnrate = LogReg(5, 0.0, lambda x: 0.5) self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0) self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0) def test_learnrate(self): print("\nTesting: Learning Rate Usage") print(self.logreg_learnrate.w) print(kPOS.x) w = self.logreg_learnrate.sg_update(kPOS, 0) self.assertAlmostEqual(w[0], 0.25) self.assertAlmostEqual(w[1], 1.00) self.assertAlmostEqual(w[2], 0.75) self.assertAlmostEqual(w[3], 0.25) self.assertAlmostEqual(w[4], 0.0) print(self.logreg_learnrate.w) print(kNEG.x) w = self.logreg_learnrate.sg_update(kNEG, 1) self.assertAlmostEqual(w[0], -0.21207090998937828) self.assertAlmostEqual(w[1], 1.0) self.assertAlmostEqual(w[2], -0.17414181997875655) self.assertAlmostEqual(w[3], -1.1362127299681348) self.assertAlmostEqual(w[4], -0.46207090998937828) def test_unreg(self): print("\nTesting: Unregularized Update") print(self.logreg_unreg.w) print(kPOS.x) w = self.logreg_unreg.sg_update(kPOS, 0) self.assertAlmostEqual(w[0], .5) self.assertAlmostEqual(w[1], 2.0) self.assertAlmostEqual(w[2], 1.5) self.assertAlmostEqual(w[3], 0.5) self.assertAlmostEqual(w[4], 0.0) print(self.logreg_unreg.w) print(kNEG.x) w = self.logreg_unreg.sg_update(kNEG, 1) self.assertAlmostEqual(w[0], -0.49330714907571527) self.assertAlmostEqual(w[1], 2.0) self.assertAlmostEqual(w[2], -0.48661429815143054) self.assertAlmostEqual(w[3], -2.479921447227146) self.assertAlmostEqual(w[4], -0.99330714907571527) def test_reg(self): print("\nTesting: Regularized Update") print(self.logreg_reg.w) print(kPOS.x) w = self.logreg_reg.sg_update(kPOS, 0) self.assertAlmostEqual(w[0], .5) self.assertAlmostEqual(w[1], 1.0) self.assertAlmostEqual(w[2], 0.75) self.assertAlmostEqual(w[3], 0.25) self.assertAlmostEqual(w[4], 0.0) print(self.logreg_reg.w) print(kNEG.x) w = self.logreg_reg.sg_update(kNEG, 1) self.assertAlmostEqual(w[0], -0.43991334982599239) self.assertAlmostEqual(w[1], 1.0) self.assertAlmostEqual(w[2], -0.56491334982599239) self.assertAlmostEqual(w[3], -1.2848700247389886) self.assertAlmostEqual(w[4], -0.2349783374564981)
""" In this part of the project, we assess the predictive ability of a feed-forward Neural network on fitting a polynomial to the franke function. To explore hyperparameter-space, chose explore = True To run self-written NN network, chose sklearn = False Chose metric for saving best score (R2/mse) For credit card data, cost=mse is chosen """ ## Get data from InitData Class --------------------------------------- data = InitData() noise = 0.05 XTrain, yTrain, XTest, yTest, X, y, b, x_, y_ = data.franke_data(noise=noise, N=20) logreg = LogReg() # init Logreg class cf = CostFunctions() # ----- PARAMETERS ------ sklearn = False NN = True explore = False metric = "R2" # "mse" if explore == True: # Explore parameter space for franke function eta_vals = np.logspace(-3, -1, 3) lmbd_vals = np.logspace(-3, -1, 3) #acts_hidden = ["sigmoid", "tanh", "relu", "elu"] acts_hidden = ["logistic", "tanh", "relu"] # Supported by sklearn act_o = "identity" hidden_neurons = [4, 8, 12, 16, 50, 100]
for line in json_file: json_obj = json.loads(line) reviews += [(classifier, json_obj)] with open("winter-" + classifier + ".json") as json_file: for line in json_file: json_obj = json.loads(line) reviews += [(classifier, json_obj)] # Creating model objects model = args.model if (model == "baseline"): model_obj = BaseLine(reviews, categories) elif (model == "logreg"): model_obj = LogReg(reviews) elif (model == "multinomialNB"): model_obj = NaiveBayes(reviews, "multinomial") elif (model == "lda"): model_obj = TopicModel(reviews) elif (model == "kNearestNeighbors"): model_obj = knn(reviews, target) else: # put additional models here. print("Argument Error: invalid model specified") sys.exit() model_classified = [] # classifications stored here
##### START PARAMETERS ##### ## Comment out the preferred goal # goal = 'ICU admission' goal = 'Mortality' # goal = 'Duration of stay at ICU' ## Select which variables to include here variables_to_include = { 'Form Collection Name': [], # groups 'Form Name': [], # variable subgroups 'Field Variable Name': ['days_since_onset', 'age_yrs'] + ['ethnic_cat_' + str(i) for i in range(1, 11)] + get_feature_set() # single variables } model = LogReg() # Initialize one of the model in .\Classifiers ##### END PARAMETERS ##### config = configparser.ConfigParser() config.read('user_settings.ini') path_creds = config['CastorCredentials']['local_private_path'] model.goal = goal data, data_struct = load_data(path_creds) data, data_struct = preprocess(data, data_struct) x, y, data = prepare_for_learning(data, data_struct, variables_to_include, goal) # Y = [event, days_to_event]. Select first column for logreg (or similar) # y = y.iloc[:, 0] # FIXME: handle different inputs per model
class testLogReg(unittest.TestCase): """ Every method needs a LogReg object with the vectorizer and model mocked out, as those objects would perform potentially slow methods otherwise. """ def setUp(self): self.mockV = Mock() self.mockM = Mock() self.test_obj = LogReg([('summer', { 'text': "some_text!" }), ('winter', { 'text': 'some_text2!' })], vectorizer=self.mockV, model=self.mockM) """ After object creation, a LogReg object should have: a TfidVectorizer named vectorizer a list of labels named labels a LogisticRegression model named model. model should have called the "fit" method to ensure that the initial model has been trained. vectorizer should have called the fit_transform method to create a vector for the model.fit method. This test uses mocks to test that the initial training methods are being called without actually performing the associated operations. """ def test_init_trained(self): assert self.mockV.fit_transform.called #vectorizer "trained" assert self.mockM.fit.called #model trained """ After object creation, the labels variable should contain a list of labels given in the reviews tuple list. This test mocks the vectorizer and model and asserts that the expected labels are present after initialization. """ def test_init_labels(self): assert self.test_obj.labels == ['summer', 'winter'] """ After object creation, the corpus variable should contain a list of training data given in the reviews tuple list. This test mocks the vectorizer and model and asserts that the expected review text is present after initialization. """ def test_init_corpus(self): assert self.test_obj.corpus == ['some_text!', 'some_text2!'] """ After object creation, the review variable should contain the reviews given upon initialization. This test mocks the vectorizer and model and asserts that the expected reviews are present after initialization. """ def test_init_reviews(self): assert self.test_obj.reviews == [('summer', { 'text': "some_text!" }), ('winter', { 'text': 'some_text2!' })] """ After calling the train method the corpus should be expanded to include the text for those reviews that are passed into the method. Test mocks the vectorizer and model created in the initialization of the object and checks to see that the new reviews are parsed and saved correctly in the corpus variable. """ def test_train_corpus(self): #setting initial corpus so we don't rely on initialization setting it #correctly self.test_obj.corpus = ["Hello world!"] self.test_obj.train([('summer', {'text': 'some_text!'})]) assert self.test_obj.corpus == ["Hello world!", "some_text!"] """ After calling the train method the labels should be expanded to include the text for those reviews that are passed into the method. Test mocks the vectorizer and model created in the initialization of the object and checks to see that the new reviews are parsed and saved correctly in the labels variable. """ def test_train_labels(self): #setting initial labels so we don't rely on initialization setting it #correctly self.test_obj.labels = ["winter"] self.test_obj.train([('fall', {'text': 'some_text!'})]) assert self.test_obj.labels == ["winter", "fall"] """ The train method should call the model's fit method after parsing out the appropriate data from the given input. This test asserts that the model calls fit with the expect arguments when the train method is executed. """ def test_train_trained(self): #Mocking the return value for fit_transform, since that result is passed to #the models' fit method. self.mockV.fit_transform.return_value = ["THIS IS A VECTOR!"] #Setting initial labels and corpus self.test_obj.labels = ["winter"] self.test_obj.corpus = ["Hello world!"] self.test_obj.train([('fall', {'text': 'some_text!'})]) self.mockV.fit_transform.assert_called_with( ["Hello world!", "some_text!"]) self.mockM.fit.assert_called_with(["THIS IS A VECTOR!"], ["winter", "fall"]) """ classify_all should complete with a call to the object's model's predict method, which does the work of predicting the season for a given test corpus. This test sets the return value for model.predict and asserts that the returned value of classify_all matches that. """ def test_classify_all(self): self.mockM.predict.return_value = [ "summer", "fall", "winter", "spring" ] assert self.test_obj.classify_all([("some", { "text": "data" })]) == ["summer", "fall", "winter", "spring"] """ The vocabulary method is a WIP method that should return a list of strings, with each string representing a word that can be used to predict a particular season. This test checks to make sure that at the very least the vocabulary method is returning a list of strings. """ def test_vocabulary(self): #Mocking the selector used in the method to avoid a lot of computation mockS = Mock() mockS.transform.return_value = ["fireplace", "warmth"] result = self.test_obj.vocabulary([("summer", { "text": "data" })], selector=mockS) for i in result: assert isinstance(i, str)
def evaluate_lenet5(learning_rate = 0.10, n_epochs=200, dataset='mnist.pkl.gz',nkerns = [16,16,16,12,12,12], batch_size = 500): rng = numpy.random.RandomState(32324) datasets = load_data(dataset) train_set_x,train_set_y = datasets[0] valid_set_x,valid_set_y = datasets[1] test_set_x,test_set_y = datasets[2] n_train_batches = train_set_x.get_value(borrow=True).shape[0]//batch_size n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]//batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0]//batch_size index = T.lscalar() # index for each mini batch train_epoch = T.lscalar('train_epoch') x = T.matrix('x') y = T.ivector('y') # ------------------------------- Building Model ---------------------------------- print "...Building the model" layer_0_input = x.reshape((batch_size,1,28,28)) # output image size = (28-5+1+)/1 = 24 layer_0 = LeNetConvPoolLayer(rng,input = layer_0_input, image_shape=(batch_size,1,28,28), filter_shape=(nkerns[0],1,5,5),poolsize=(1,1)) #output image size = (24-3+1) = 22 layer_1 = LeNetConvPoolLayer(rng, input = layer_0.output, image_shape = (batch_size, nkerns[0],24,24), filter_shape = (nkerns[1],nkerns[0],3,3), poolsize=(1,1) ) #output image size = (22-3+1)/2 = 10 layer_2 = LeNetConvPoolLayer(rng, input = layer_1.output, image_shape = (batch_size, nkerns[1],22,22), filter_shape = (nkerns[2],nkerns[1],3,3), poolsize=(2,2) ) #output image size = (10-3+1)/2 = 4 layer_3 = LeNetConvPoolLayer(rng, input = layer_2.output, image_shape = (batch_size, nkerns[2],10,10), filter_shape = (nkerns[3], nkerns[2],3,3), poolsize=(2,2) ) #output image size = (4-3+2+1) = 4 layer_4 = LeNetConvPoolLayer(rng, input = layer_3.output, image_shape = (batch_size, nkerns[3],4,4), filter_shape = (nkerns[4], nkerns[3],3,3), poolsize=(1,1), border_mode = 1 ) #output image size = (4-3+1)/2 = 2 layer_5 = LeNetConvPoolLayer(rng, input = layer_4.output, image_shape = (batch_size, nkerns[4],4,4), filter_shape = (nkerns[5], nkerns[4],3,3), poolsize=(2,2), border_mode = 1 ) # make the input to hidden layer 2 dimensional layer_6_input = layer_5.output.flatten(2) layer_6 = HiddenLayer(rng,input = layer_6_input, n_in = nkerns[5]*2*2, n_out = 200, activation = T.tanh) layer_7 = LogReg(input = layer_6.output, n_in=200, n_out = 10) teacher_p_y_given_x = theano.shared(numpy.asarray(pickle.load(open('prob_best_model.pkl','rb')),dtype =theano.config.floatX), borrow=True) p_y_given_x = T.matrix('p_y_given_x') e = theano.shared(value = 0, name = 'e', borrow = True) cost = layer_7.neg_log_likelihood(y) + 2.0/(e)*T.mean(-T.log(layer_7.p_y_given_x)*p_y_given_x - layer_7.p_y_given_x*T.log(p_y_given_x)) tg = theano.shared(numpy.asarray(pickle.load(open('modified_guided_data.pkl','rb')),dtype =theano.config.floatX), borrow=True) guiding_weights = T.tensor4('guiding_weights') #guide_cost = T.mean(-T.log(layer_3.output)*guiding_weights - layer_3.output*T.log(guiding_weights)) guide_cost = T.mean((layer_3.output-guiding_weights)**2) test_model = theano.function([index],layer_7.errors(y), givens={ x: test_set_x[index*batch_size:(index+1)*batch_size], y: test_set_y[index*batch_size:(index+1)*batch_size] }) validate_model = theano.function([index],layer_7.errors(y), givens={ x: valid_set_x[index*batch_size:(index+1)*batch_size], y: valid_set_y[index*batch_size:(index+1)*batch_size] }) # list of parameters params = layer_7.params + layer_6.params + layer_5.params + layer_4.params + layer_3.params + layer_2.params + layer_1.params + layer_0.params params_gl = layer_3.params + layer_2.params + layer_1.params + layer_0.params # import pdb # pdb.set_trace() grads_gl = T.grad(guide_cost,params_gl) updates_gl = [ (param_i,param_i-learning_rate/10*grad_i) for param_i,grad_i in zip(params_gl,grads_gl) ] grads = T.grad(cost,params) updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ] train_model = theano.function([index,train_epoch],cost, updates=updates, givens={ x: train_set_x[index*batch_size:(index+1)*batch_size], y: train_set_y[index*batch_size:(index+1)*batch_size], p_y_given_x: teacher_p_y_given_x[index], e: train_epoch }) train_till_guided_layer = theano.function([index],guide_cost,updates = updates_gl, givens={ x: train_set_x[index*batch_size:(index+1)*batch_size], y: train_set_y[index*batch_size:(index+1)*batch_size], guiding_weights : tg[index] },on_unused_input='ignore') # -----------------------------------------Starting Training ------------------------------ print ('..... Training ' ) # for early stopping patience = 10000 patience_increase = 2 improvement_threshold = 0.95 validation_frequency = min(n_train_batches, patience//2) best_validation_loss = numpy.inf # initialising loss to be inifinite best_itr = 0 test_score = 0 start_time = timeit.default_timer() epoch = 0 done_looping = False while (epoch < n_epochs) and (not done_looping) : epoch = epoch+1 for minibatch_index in range(n_train_batches): iter = (epoch - 1)*n_train_batches + minibatch_index if iter%100==0: print ('training @ iter = ', iter) if epoch < n_epochs/5: cost_ij_guided = train_till_guided_layer(minibatch_index) cost_ij = train_model(minibatch_index,epoch) if(iter +1)%validation_frequency ==0: # compute loss on validation set validation_losses = [validate_model(i) for i in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) # import pdb # pdb.set_trace() with open('Student_6_terminal_out','a+') as f_: f_.write('epoch %i, minibatch %i/%i, validation error %f %% \n' %(epoch,minibatch_index+1,n_train_batches,this_validation_loss*100. )) # check with best validation score till now if this_validation_loss<best_validation_loss: # improve if this_validation_loss < best_validation_loss * improvement_threshold: patience = max(patience, iter*patience_increase) best_validation_loss = this_validation_loss best_itr = iter test_losses = [test_model(i) for i in range(n_test_batches)] test_score = numpy.mean(test_losses) with open('Student_6_terminal_out','a+') as f_: f_.write('epoch %i, minibatch %i/%i, testing error %f %%\n' %(epoch, minibatch_index+1,n_train_batches,test_score*100.)) with open('best_model_7layer.pkl', 'wb') as f: pickle.dump(params, f) with open('Results_student_6.txt', 'wb') as f1: f1.write(str(test_score*100)+'\n') #if patience <= iter: # done_looping = True # break end_time = timeit.default_timer() with open('Student_6_terminal_out','a+') as f_: f_.write('Optimization complete\n') f_.write('Best validation score of %f %% obtained at iteration %i, with test performance %f %% \n' % (best_validation_loss*100., best_itr, test_score*100 )) f_.write('The code ran for %.2fm \n' %((end_time - start_time)/60.))
def setUp(self): self.logreg_learnrate = LogReg(784, 0.5) self.logreg_nolearnrate = LogReg(784, 1.0)
class testLogReg(unittest.TestCase): """ Every method needs a LogReg object with the vectorizer and model mocked out, as those objects would perform potentially slow methods otherwise. """ def setUp(self): self.mockV = Mock() self.mockM = Mock() self.test_obj = LogReg([ ('summer', {'text': "some_text!"}), ('winter', {'text': 'some_text2!'}) ], vectorizer = self.mockV, model = self.mockM ) """ After object creation, a LogReg object should have: a TfidVectorizer named vectorizer a list of labels named labels a LogisticRegression model named model. model should have called the "fit" method to ensure that the initial model has been trained. vectorizer should have called the fit_transform method to create a vector for the model.fit method. This test uses mocks to test that the initial training methods are being called without actually performing the associated operations. """ def test_init_trained(self): assert self.mockV.fit_transform.called #vectorizer "trained" assert self.mockM.fit.called #model trained """ After object creation, the labels variable should contain a list of labels given in the reviews tuple list. This test mocks the vectorizer and model and asserts that the expected labels are present after initialization. """ def test_init_labels(self): assert self.test_obj.labels == ['summer', 'winter'] """ After object creation, the corpus variable should contain a list of training data given in the reviews tuple list. This test mocks the vectorizer and model and asserts that the expected review text is present after initialization. """ def test_init_corpus(self): assert self.test_obj.corpus == ['some_text!', 'some_text2!'] """ After object creation, the review variable should contain the reviews given upon initialization. This test mocks the vectorizer and model and asserts that the expected reviews are present after initialization. """ def test_init_reviews(self): assert self.test_obj.reviews == [ ('summer', {'text': "some_text!"}), ('winter', {'text': 'some_text2!'}) ] """ After calling the train method the corpus should be expanded to include the text for those reviews that are passed into the method. Test mocks the vectorizer and model created in the initialization of the object and checks to see that the new reviews are parsed and saved correctly in the corpus variable. """ def test_train_corpus(self): #setting initial corpus so we don't rely on initialization setting it #correctly self.test_obj.corpus = ["Hello world!"] self.test_obj.train([('summer', {'text': 'some_text!'})]) assert self.test_obj.corpus == ["Hello world!", "some_text!"] """ After calling the train method the labels should be expanded to include the text for those reviews that are passed into the method. Test mocks the vectorizer and model created in the initialization of the object and checks to see that the new reviews are parsed and saved correctly in the labels variable. """ def test_train_labels(self): #setting initial labels so we don't rely on initialization setting it #correctly self.test_obj.labels = ["winter"] self.test_obj.train([('fall', {'text': 'some_text!'})]) assert self.test_obj.labels == ["winter", "fall"] """ The train method should call the model's fit method after parsing out the appropriate data from the given input. This test asserts that the model calls fit with the expect arguments when the train method is executed. """ def test_train_trained(self): #Mocking the return value for fit_transform, since that result is passed to #the models' fit method. self.mockV.fit_transform.return_value = ["THIS IS A VECTOR!"] #Setting initial labels and corpus self.test_obj.labels = ["winter"] self.test_obj.corpus = ["Hello world!"] self.test_obj.train([('fall', {'text': 'some_text!'})]) self.mockV.fit_transform.assert_called_with(["Hello world!", "some_text!"]) self.mockM.fit.assert_called_with( ["THIS IS A VECTOR!"], ["winter", "fall"] ) """ classify_all should complete with a call to the object's model's predict method, which does the work of predicting the season for a given test corpus. This test sets the return value for model.predict and asserts that the returned value of classify_all matches that. """ def test_classify_all(self): self.mockM.predict.return_value = ["summer", "fall", "winter", "spring"] assert self.test_obj.classify_all([("some", {"text" : "data"})]) == [ "summer", "fall", "winter", "spring" ] """ The vocabulary method is a WIP method that should return a list of strings, with each string representing a word that can be used to predict a particular season. This test checks to make sure that at the very least the vocabulary method is returning a list of strings. """ def test_vocabulary(self): #Mocking the selector used in the method to avoid a lot of computation mockS = Mock() mockS.transform.return_value = ["fireplace", "warmth"] result = self.test_obj.vocabulary([("summer", {"text": "data"})], selector = mockS) for i in result: assert isinstance(i, str)
def run_logreg(train_embs, train_labels, test_embs, test_labels, cuda=False): train_embs = torch.Tensor(train_embs) train_labels = torch.Tensor(train_labels) test_embs = torch.Tensor(test_embs) test_labels = torch.Tensor(test_labels) if cuda: train_embs = train_embs.cuda() test_embs = test_embs.cuda() train_labels = train_labels.cuda() test_labels = test_labels.cuda() tot = torch.zeros(1) if cuda: tot = tot.cuda() res = [] xent = nn.BCEWithLogitsLoss() sigmoid = nn.Sigmoid() dims = train_embs.shape[1] nb_classes = train_labels.shape[1] best_epoch = find_epoch(dims, nb_classes, train_embs, train_labels, test_embs, test_labels, cuda) print('best epoch', best_epoch) best_th = 0.0 repeats = 50 for i in range(repeats): log = LogReg(dims, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) if cuda: log.cuda() for _ in range(best_epoch): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_labels) loss.backward() opt.step() if i == 0: train_logits = sigmoid(log(train_embs)) if cuda: best_th = find_best_th(train_logits.cpu(), train_labels.cpu()) else: best_th = find_best_th(train_logits, train_labels) print('best threshold:', best_th) logits = sigmoid(log(test_embs)) zero = torch.zeros_like(logits) one = torch.ones_like(logits) preds = torch.where( logits >= best_th, one, zero) # ppi is a multi-label classification problem if cuda: test_labels = test_labels.cpu() preds = preds.cpu() f1 = f1_score(test_labels.numpy(), preds.numpy(), average='micro') res.append(f1) print(f1) tot += f1 print('Average f1:', tot / repeats) res = np.stack(res) print(np.mean(res)) print(np.std(res))
'Diastolic blood pressure (mm Hg)', 'Triceps skin fold thickness (mm)', '2-Hour serum insulin (mu U/ml)', 'Body mass index (weight in kg/(height in m)^2)', 'Diabetes pedigree function', 'Age (years)', 'Class variable (0: no diabetes, 1: diabetes)'] # Load data and perform train test split. print('[INFO] Loading data.') data = load_data(file_name, columns) X_train, X_test, y_train, y_test = train_test_split(data, frac=0.8, seed=5) # Logistic Model print('\n[INFO] Training logistic regression model.') logreg = LogReg(args.lr_logreg, args.epochs, len(X_train[0])) bias_logreg, weights_logreg = logreg.train(X_train, y_train) y_logistic = [round(logreg.predict(example)) for example in X_test] # Perceptron Model print('[INFO] Training Perceptron model.') perceptron = Perceptron(args.lr_perceptron, args.epochs, len(X_train[0])) bias_perceptron, weights_perceptron = perceptron.train(X_train, y_train) y_perceptron = [round(perceptron.predict(example)) for example in X_test] # Compare accuracies accuracy_logistic = get_accuracy(y_logistic, y_test) accuracy_perceptron = get_accuracy(y_perceptron, y_test) print('\n[INFO] Logistic Regression Accuracy: {:0.3f}'.format( accuracy_logistic)) print('[INFO] Perceptron Accuracy: {:0.3f}'.format(accuracy_perceptron))
#processor data = proc.load_csv(args.CSV_FILE) data = proc.normalize_col(data, 'Amount') data = data.drop(['Time'], axis=1) print data[args.YCOL].value_counts() X = proc.get_xvals(data, args.YCOL) y = proc.get_yvals(data, args.YCOL) #processor xfolds Xu, yu = proc.under_sample(data, args.YCOL) Xu_train, Xu_test, yu_train, yu_test = proc.cross_validation_sets( Xu, yu, .3, 0) X_train, X_test, y_train, y_test = proc.cross_validation_sets(X, y, .3, 0) if args.LR_DRIVE: lin = LogReg() #under sampled data c = lin.printing_Kfold_scores(Xu_train, yu_train) lin.logistic_regression(Xu_train, Xu_test, yu_train, yu_test, c) lin.logistic_regression(Xu_train, X_test, yu_train, y_test, c) lin.get_roc_curve(Xu_train, Xu_test, yu_train, yu_test, c) #regular data c = lin.printing_Kfold_scores(X_train, y_train) lin.logistic_regression(X_train, X_test, y_train, y_test, c) lin.get_roc_curve(X_train, X_test, y_train, y_test, c) if args.SVM_DRIVE: sv = SVM() sv.svm_run(Xu_train, Xu_test, yu_train, yu_test) if args.SVML_DRIVE: sv = SVM()