def regularization_test():
with open("log_regular.txt", "w+") as f:
f.write("Regularization | lambda | Area | R2\n")
f.write("-" * 50 + "\n")
for reg in ['l2', 'l1']:
for lamb in [0.0001, 0.001, 0.01, 0.1, 1.0]:
logreg = LogReg(X, Y)
logreg.optimize(m=100,
epochs=5000,
eta=0.01,
regularization=reg,
lamb=lamb)
ypred_train = logreg.p_train
ypred_test = logreg.p_test
area = gain_chart(logreg.Y_test, ypred_test, plot=False)
R2 = prob_acc(logreg.Y_test, ypred_test, plot=False)
f.write(" %s | %g | %.4f | %.4f \n" %
(reg, lamb, area, R2))
f.write("-" * 50 + "\n")
def setUp(self):
self.mockV = Mock()
self.mockM = Mock()
self.test_obj = LogReg([('summer', {
'text': "some_text!"
}), ('winter', {
'text': 'some_text2!'
})],
vectorizer=self.mockV,
model=self.mockM)
def __init__(self, inp, layer_sizes):
self.x = inp
self.layer_sizes = layer_sizes
self.layers = []
prev = self.x
for i in range(1, len(self.layer_sizes)):
new_shape = (self.layer_sizes[i], self.layer_sizes[i - 1])
self.layers.append(LogReg(prev, new_shape))
prev = self.layers[-1].a
self.a = self.layers[-1].a
self.params = sum((l.params for l in self.layers), [])
def find_epoch(dims, nb_classes, train_embs, train_labels, test_embs,
test_labels, cuda):
log = LogReg(dims, nb_classes)
opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0)
xent = nn.BCEWithLogitsLoss()
sigmoid = nn.Sigmoid()
if cuda:
log.cuda()
epoch_flag = 0
epoch_win = 0
best_f1 = torch.zeros(1)
tmp_th = 0.5
if cuda:
best_f1 = best_f1.cuda()
for e in range(2000):
log.train()
opt.zero_grad()
logits = log(train_embs)
loss = xent(logits, train_labels)
loss.backward()
opt.step()
if (e + 1) % 100 == 0:
log.eval()
logits = sigmoid(log(test_embs))
zero = torch.zeros_like(logits)
one = torch.ones_like(logits)
preds = torch.where(logits >= tmp_th, one, zero)
if cuda:
test_labels = test_labels.cpu()
preds = preds.cpu()
f1 = f1_score(test_labels.numpy(), preds.numpy(), average='micro')
if f1 >= best_f1:
epoch_flag = e + 1
best_f1 = f1
epoch_win = 0
else:
epoch_win += 1
if epoch_win == 10:
break
return epoch_flag
def evaluate_lenet5(learning_rate=0.1,
n_epochs=200,
dataset='mnist.pkl.gz',
nkerns=[16, 16, 16],
batch_size=500):
rng = numpy.random.RandomState(32324)
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size
index = T.lscalar() # index for each mini batch
train_epoch = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
print "...Building the model"
# output image size = (28-5+1+4)/2 = 14
layer_0_input = x.reshape((batch_size, 1, 28, 28))
layer_0 = LeNetConvPoolLayer(rng,
input=layer_0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2),
border_mode=2)
#output image size = (14-3+1)/2 = 6
layer_1 = LeNetConvPoolLayer(rng,
input=layer_0.output,
image_shape=(batch_size, nkerns[0], 14, 14),
filter_shape=(nkerns[1], nkerns[0], 3, 3),
poolsize=(2, 2))
#output image size = (6-3+1)/2 = 2
layer_2 = LeNetConvPoolLayer(rng,
input=layer_1.output,
image_shape=(batch_size, nkerns[1], 6, 6),
filter_shape=(nkerns[2], nkerns[1], 3, 3),
poolsize=(2, 2))
# make the input to hidden layer 2 dimensional
layer_3_input = layer_2.output.flatten(2)
layer_3 = HiddenLayer(rng,
input=layer_3_input,
n_in=nkerns[2] * 2 * 2,
n_out=200,
activation=T.tanh)
layer_4 = LogReg(input=layer_3.output, n_in=200, n_out=10)
teacher_p_y_given_x = theano.shared(numpy.asarray(
pickle.load(open('prob_best_model.pkl', 'rb')),
dtype=theano.config.floatX),
borrow=True)
#cost = layer_4.neg_log_likelihood(y) + T.mean((teacher_W - layer_4.W)**2)/(2.*(1+epoch*2)) + T.mean((teacher_b-layer_4.b)**2)/(2.*(1+epoch*2))
# import pdb
# pdb.set_trace()
p_y_given_x = T.matrix('p_y_given_x')
e = theano.shared(value=0, name='e', borrow=True)
#cost = layer_4.neg_log_likelihood(y) + 1.0/(e)*T.mean((layer_4.p_y_given_x - p_y_given_x)**2)
cost = layer_4.neg_log_likelihood(
y) + 2.0 / (e) * T.mean(-T.log(layer_4.p_y_given_x) * p_y_given_x -
layer_4.p_y_given_x * T.log(p_y_given_x))
test_model = theano.function(
[index],
layer_4.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
layer_4.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
# list of parameters
params = layer_4.params + layer_3.params + layer_2.params + layer_1.params + layer_0.params
grads = T.grad(cost, params)
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index, train_epoch],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size],
p_y_given_x: teacher_p_y_given_x[index],
e: train_epoch
})
# -----------------------------------------Starting Training ------------------------------
print('..... Training ')
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience // 2)
best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
#epo = theano.shared('epo')
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print('training @ iter = ', iter)
cost_ij = train_model(minibatch_index, epoch)
if (iter + 1) % validation_frequency == 0:
# compute loss on validation set
validation_losses = [
validate_model(i) for i in range(n_valid_batches)
]
this_validation_loss = numpy.mean(validation_losses)
# import pdb
# pdb.set_trace()
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# check with best validation score till now
if this_validation_loss < best_validation_loss:
# improve
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_itr = iter
test_losses = [
test_model(i) for i in range(n_test_batches)
]
test_score = numpy.mean(test_losses)
print('epoch %i, minibatch %i/%i, testing error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
with open('best_model_3layer.pkl', 'wb') as f:
pickle.dump(params, f)
with open('Results_student_3.txt', 'wb') as f2:
f2.write(str(test_score * 100) + '\n')
#if patience <= iter:
# done_looping = True
# break
end_time = timeit.default_timer()
print('Optimization complete')
print(
'Best validation score of %f %% obtained at iteration %i,'
'with test performance %f %%' %
(best_validation_loss * 100., best_itr, test_score * 100))
print('The code ran for %.2fm' % ((end_time - start_time) / 60.))
"""
In this part of the project, we assess the predictive ability of a feed-forward Neural
network on fitting a polynomial to the franke function.
To explore hyperparameter-space, chose explore = True
To run self-written NN network, chose sklearn = False
Chose metric for saving best score (R2/mse)
For credit card data, cost=mse is chosen
"""
## Get data from InitData Class ---------------------------------------
data = InitData()
noise = 0.05
XTrain, yTrain, XTest, yTest, X, y, b, x_, y_ = data.franke_data(noise=noise,
N=20)
logreg = LogReg() # init Logreg class
cf = CostFunctions()
# ----- PARAMETERS ------
sklearn = False
NN = True
explore = False
metric = "R2" # "mse"
if explore == True: # Explore parameter space for franke function
eta_vals = np.logspace(-3, -1, 3)
lmbd_vals = np.logspace(-3, -1, 3)
#acts_hidden = ["sigmoid", "tanh", "relu", "elu"]
acts_hidden = ["logistic", "tanh", "relu"] # Supported by sklearn
act_o = "identity"
hidden_neurons = [4, 8, 12, 16, 50, 100]
def setUp(self):
self.logreg_unreg = LogReg(5, 1.0)
def evaluate_lenet5(learning_rate = 0.10, n_epochs=200, dataset='mnist.pkl.gz',nkerns = [16,16,16,12,12,12], batch_size = 500):
rng = numpy.random.RandomState(32324)
datasets = load_data(dataset)
train_set_x,train_set_y = datasets[0]
valid_set_x,valid_set_y = datasets[1]
test_set_x,test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0]//batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]//batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]//batch_size
index = T.lscalar() # index for each mini batch
train_epoch = T.lscalar('train_epoch')
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
print "...Building the model"
layer_0_input = x.reshape((batch_size,1,28,28))
# output image size = (28-5+1+)/1 = 24
layer_0 = LeNetConvPoolLayer(rng,input = layer_0_input, image_shape=(batch_size,1,28,28),
filter_shape=(nkerns[0],1,5,5),poolsize=(1,1))
#output image size = (24-3+1) = 22
layer_1 = LeNetConvPoolLayer(rng, input = layer_0.output, image_shape = (batch_size, nkerns[0],24,24),
filter_shape = (nkerns[1],nkerns[0],3,3), poolsize=(1,1) )
#output image size = (22-3+1)/2 = 10
layer_2 = LeNetConvPoolLayer(rng, input = layer_1.output, image_shape = (batch_size, nkerns[1],22,22),
filter_shape = (nkerns[2],nkerns[1],3,3), poolsize=(2,2) )
#output image size = (10-3+1)/2 = 4
layer_3 = LeNetConvPoolLayer(rng, input = layer_2.output, image_shape = (batch_size, nkerns[2],10,10),
filter_shape = (nkerns[3], nkerns[2],3,3), poolsize=(2,2) )
#output image size = (4-3+2+1) = 4
layer_4 = LeNetConvPoolLayer(rng, input = layer_3.output, image_shape = (batch_size, nkerns[3],4,4),
filter_shape = (nkerns[4], nkerns[3],3,3), poolsize=(1,1), border_mode = 1 )
#output image size = (4-3+1)/2 = 2
layer_5 = LeNetConvPoolLayer(rng, input = layer_4.output, image_shape = (batch_size, nkerns[4],4,4),
filter_shape = (nkerns[5], nkerns[4],3,3), poolsize=(2,2), border_mode = 1 )
# make the input to hidden layer 2 dimensional
layer_6_input = layer_5.output.flatten(2)
layer_6 = HiddenLayer(rng,input = layer_6_input, n_in = nkerns[5]*2*2, n_out = 200, activation = T.tanh)
layer_7 = LogReg(input = layer_6.output, n_in=200, n_out = 10)
teacher_p_y_given_x = theano.shared(numpy.asarray(pickle.load(open('prob_best_model.pkl','rb')),dtype =theano.config.floatX), borrow=True)
p_y_given_x = T.matrix('p_y_given_x')
e = theano.shared(value = 0, name = 'e', borrow = True)
cost = layer_7.neg_log_likelihood(y) + 2.0/(e)*T.mean(-T.log(layer_7.p_y_given_x)*p_y_given_x - layer_7.p_y_given_x*T.log(p_y_given_x))
tg = theano.shared(numpy.asarray(pickle.load(open('modified_guided_data.pkl','rb')),dtype =theano.config.floatX), borrow=True)
guiding_weights = T.tensor4('guiding_weights')
#guide_cost = T.mean(-T.log(layer_3.output)*guiding_weights - layer_3.output*T.log(guiding_weights))
guide_cost = T.mean((layer_3.output-guiding_weights)**2)
test_model = theano.function([index],layer_7.errors(y),
givens={
x: test_set_x[index*batch_size:(index+1)*batch_size],
y: test_set_y[index*batch_size:(index+1)*batch_size]
})
validate_model = theano.function([index],layer_7.errors(y),
givens={
x: valid_set_x[index*batch_size:(index+1)*batch_size],
y: valid_set_y[index*batch_size:(index+1)*batch_size]
})
# list of parameters
params = layer_7.params + layer_6.params + layer_5.params + layer_4.params + layer_3.params + layer_2.params + layer_1.params + layer_0.params
params_gl = layer_3.params + layer_2.params + layer_1.params + layer_0.params
# import pdb
# pdb.set_trace()
grads_gl = T.grad(guide_cost,params_gl)
updates_gl = [ (param_i,param_i-learning_rate/10*grad_i) for param_i,grad_i in zip(params_gl,grads_gl) ]
grads = T.grad(cost,params)
updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ]
train_model = theano.function([index,train_epoch],cost, updates=updates,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size],
y: train_set_y[index*batch_size:(index+1)*batch_size],
p_y_given_x: teacher_p_y_given_x[index],
e: train_epoch
})
train_till_guided_layer = theano.function([index],guide_cost,updates = updates_gl,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size],
y: train_set_y[index*batch_size:(index+1)*batch_size],
guiding_weights : tg[index]
},on_unused_input='ignore')
# -----------------------------------------Starting Training ------------------------------
print ('..... Training ' )
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience//2)
best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping) :
epoch = epoch+1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1)*n_train_batches + minibatch_index
if iter%100==0:
print ('training @ iter = ', iter)
if epoch < n_epochs/5:
cost_ij_guided = train_till_guided_layer(minibatch_index)
cost_ij = train_model(minibatch_index,epoch)
if(iter +1)%validation_frequency ==0:
# compute loss on validation set
validation_losses = [validate_model(i) for i in range(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
# import pdb
# pdb.set_trace()
with open('Student_6_terminal_out','a+') as f_:
f_.write('epoch %i, minibatch %i/%i, validation error %f %% \n' %(epoch,minibatch_index+1,n_train_batches,this_validation_loss*100. ))
# check with best validation score till now
if this_validation_loss<best_validation_loss:
# improve
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter*patience_increase)
best_validation_loss = this_validation_loss
best_itr = iter
test_losses = [test_model(i) for i in range(n_test_batches)]
test_score = numpy.mean(test_losses)
with open('Student_6_terminal_out','a+') as f_:
f_.write('epoch %i, minibatch %i/%i, testing error %f %%\n' %(epoch, minibatch_index+1,n_train_batches,test_score*100.))
with open('best_model_7layer.pkl', 'wb') as f:
pickle.dump(params, f)
with open('Results_student_6.txt', 'wb') as f1:
f1.write(str(test_score*100)+'\n')
#if patience <= iter:
# done_looping = True
# break
end_time = timeit.default_timer()
with open('Student_6_terminal_out','a+') as f_:
f_.write('Optimization complete\n')
f_.write('Best validation score of %f %% obtained at iteration %i, with test performance %f %% \n' % (best_validation_loss*100., best_itr, test_score*100 ))
f_.write('The code ran for %.2fm \n' %((end_time - start_time)/60.))
for line in json_file:
json_obj = json.loads(line)
reviews += [(classifier, json_obj)]
with open("winter-" + classifier + ".json") as json_file:
for line in json_file:
json_obj = json.loads(line)
reviews += [(classifier, json_obj)]
# Creating model objects
model = args.model
if (model == "baseline"):
model_obj = BaseLine(reviews, categories)
elif (model == "logreg"):
model_obj = LogReg(reviews)
elif (model == "multinomialNB"):
model_obj = NaiveBayes(reviews, "multinomial")
elif (model == "lda"):
model_obj = TopicModel(reviews)
elif (model == "kNearestNeighbors"):
model_obj = knn(reviews, target)
else: # put additional models here.
print("Argument Error: invalid model specified")
sys.exit()
model_classified = [] # classifications stored here
def setUp(self):
self.logreg_learnrate = LogReg(784, 0.5)
self.logreg_nolearnrate = LogReg(784, 1.0)
import numpy as np
from read_data import get_data
import sys
sys.path.append("network/")
from logreg import LogReg
from metrics import gain_chart, prob_acc
#X, Y = get_data(normalized = False, standardized = False)
X, Y = get_data()
logreg = LogReg(X, Y)
logreg.optimize(m=100, epochs=5000,
eta=0.01) #, regularization='l2', lamb=0.0001)
ypred_train = logreg.p_train
ypred_test = logreg.p_test
gain_chart(logreg.Y_test, ypred_test)
prob_acc(logreg.Y_test, ypred_test)
def regularization_test():
with open("log_regular.txt", "w+") as f:
f.write("Regularization | lambda | Area | R2\n")
f.write("-" * 50 + "\n")
for reg in ['l2', 'l1']:
for lamb in [0.0001, 0.001, 0.01, 0.1, 1.0]:
logreg = LogReg(X, Y)
logreg.optimize(m=100,
def evaluate_lenet5(learning_rate = 0.1, n_epochs=200, dataset='mnist.pkl.gz',nkerns = [20,50], batch_size = 500 , testing =0):
rng = numpy.random.RandomState(32324)
datasets = load_data(dataset)
train_set_x,train_set_y = datasets[0]
valid_set_x,valid_set_y = datasets[1]
test_set_x,test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0]//batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]//batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]//batch_size
index = T.lscalar() # index for each mini batch
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
if testing ==0:
print "...Building the model"
# output image size = (28-5+1)/2 = 12
layer_0_input = x.reshape((batch_size,1,28,28))
layer_0 = LeNetConvPoolLayer(rng,input = layer_0_input, image_shape=(batch_size,1,28,28),filter_shape=(nkerns[0],1,5,5),poolsize=(2,2))
#output image size = (12-5+1)/2 = 4
layer_1 = LeNetConvPoolLayer(rng, input = layer_0.output, image_shape = (batch_size, nkerns[0],12,12),
filter_shape = (nkerns[1],nkerns[0],5,5), poolsize=(2,2) )
# make the input to hidden layer 2 dimensional
layer_2_input = layer_1.output.flatten(2)
layer_2 = HiddenLayer(rng,input = layer_2_input, n_in = nkerns[1]*4*4, n_out = 500, activation = T.tanh)
layer_3 = LogReg(input = layer_2.output, n_in=500, n_out = 10)
cost = layer_3.neg_log_likelihood(y)
test_model = theano.function([index],layer_3.errors(y),
givens={
x: test_set_x[index*batch_size:(index+1)*batch_size],
y: test_set_y[index*batch_size:(index+1)*batch_size]
})
validate_model = theano.function([index],layer_3.errors(y),
givens={
x: valid_set_x[index*batch_size:(index+1)*batch_size],
y: valid_set_y[index*batch_size:(index+1)*batch_size]
})
train_predic = theano.function([index], layer_3.prob_y_given_x(),
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size]
})
# list of parameters
layer_guided = theano.function([index], layer_1.output,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size]
})
params = layer_3.params + layer_2.params + layer_1.params + layer_0.params
grads = T.grad(cost,params)
updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ]
train_model = theano.function([index],cost, updates=updates,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size],
y: train_set_y[index*batch_size:(index+1)*batch_size]
})
# -----------------------------------------Starting Training ------------------------------
if testing ==0:
print ('..... Training ' )
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience//2)
best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping) and testing ==0:
epoch = epoch+1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1)*n_train_batches + minibatch_index
if iter%100 ==0:
print ('training @ iter = ', iter)
cost_ij = train_model(minibatch_index)
if(iter +1)%validation_frequency ==0:
# compute loss on validation set
validation_losses = [validate_model(i) for i in range(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
# import pdb
# pdb.set_trace()
print ('epoch %i, minibatch %i/%i, validation error %f %%' %(epoch,minibatch_index+1,n_train_batches,this_validation_loss*100. ))
# check with best validation score till now
if this_validation_loss<best_validation_loss:
# improve
# if this_validation_loss < best_validation_loss * improvement_threshold:
# patience = max(patience, iter*patience_increase)
best_validation_loss = this_validation_loss
best_itr = iter
test_losses = [test_model(i) for i in range(n_test_batches)]
test_score = numpy.mean(test_losses)
print ('epoch %i, minibatch %i/%i, testing error %f %%' %(epoch, minibatch_index+1,n_train_batches,test_score*100.))
with open('best_model.pkl', 'wb') as f:
pickle.dump(params, f)
with open('Results_teacher.txt','wb') as f2:
f2.write(str(test_score*100) + '\n')
p_y_given_x = [train_predic(i) for i in range(n_train_batches)]
with open ('prob_best_model.pkl','wb') as f1:
pickle.dump(p_y_given_x,f1)
# if patience <= iter:
# done_looping = True
# break
layer_2_op_dump = [layer_guided(i) for i in range(n_train_batches)]
with open ('layer_guided.pkl','wb') as lg:
pickle.dump(layer_2_op_dump,lg)
end_time = timeit.default_timer()
# p_y_given_x = [train_model(i) for i in range(n_train_batches)]
# with open ('prob_best_model.pkl') as f:
# pickle.dump(p_y_given_x)
if testing ==0 :
print ('Optimization complete')
print ('Best validation score of %f %% obtained at iteration %i,'
'with test performance %f %%' % (best_validation_loss*100., best_itr, test_score*100 ))
print('The code ran for %.2fm' %((end_time - start_time)/60.))
def setUp(self):
self.logreg_learnrate = LogReg(5, 0.0, lambda x: 0.5)
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def run_logreg(train_embs, train_labels, test_embs, test_labels, cuda=False):
train_embs = torch.Tensor(train_embs)
train_labels = torch.Tensor(train_labels)
test_embs = torch.Tensor(test_embs)
test_labels = torch.Tensor(test_labels)
if cuda:
train_embs = train_embs.cuda()
test_embs = test_embs.cuda()
train_labels = train_labels.cuda()
test_labels = test_labels.cuda()
tot = torch.zeros(1)
if cuda:
tot = tot.cuda()
res = []
xent = nn.BCEWithLogitsLoss()
sigmoid = nn.Sigmoid()
dims = train_embs.shape[1]
nb_classes = train_labels.shape[1]
best_epoch = find_epoch(dims, nb_classes, train_embs, train_labels,
test_embs, test_labels, cuda)
print('best epoch', best_epoch)
best_th = 0.0
repeats = 50
for i in range(repeats):
log = LogReg(dims, nb_classes)
opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0)
if cuda:
log.cuda()
for _ in range(best_epoch):
log.train()
opt.zero_grad()
logits = log(train_embs)
loss = xent(logits, train_labels)
loss.backward()
opt.step()
if i == 0:
train_logits = sigmoid(log(train_embs))
if cuda:
best_th = find_best_th(train_logits.cpu(), train_labels.cpu())
else:
best_th = find_best_th(train_logits, train_labels)
print('best threshold:', best_th)
logits = sigmoid(log(test_embs))
zero = torch.zeros_like(logits)
one = torch.ones_like(logits)
preds = torch.where(
logits >= best_th, one,
zero) # ppi is a multi-label classification problem
if cuda:
test_labels = test_labels.cpu()
preds = preds.cpu()
f1 = f1_score(test_labels.numpy(), preds.numpy(), average='micro')
res.append(f1)
print(f1)
tot += f1
print('Average f1:', tot / repeats)
res = np.stack(res)
print(np.mean(res))
print(np.std(res))
##### START PARAMETERS #####
## Comment out the preferred goal
# goal = 'ICU admission'
goal = 'Mortality'
# goal = 'Duration of stay at ICU'
## Select which variables to include here
variables_to_include = {
'Form Collection Name': [], # groups
'Form Name': [], # variable subgroups
'Field Variable Name': ['days_since_onset', 'age_yrs'] +
['ethnic_cat_' + str(i)
for i in range(1, 11)] + get_feature_set() # single variables
}
model = LogReg() # Initialize one of the model in .\Classifiers
##### END PARAMETERS #####
config = configparser.ConfigParser()
config.read('user_settings.ini')
path_creds = config['CastorCredentials']['local_private_path']
model.goal = goal
data, data_struct = load_data(path_creds)
data, data_struct = preprocess(data, data_struct)
x, y, data = prepare_for_learning(data, data_struct, variables_to_include,
goal)
# Y = [event, days_to_event]. Select first column for logreg (or similar)
# y = y.iloc[:, 0] # FIXME: handle different inputs per model
fn = 'Skin_NonSkin.txt'
data = np.loadtxt(fn)
data[:,-1] -= 1
np.random.shuffle(data)
data = np.c_[np.ones(data.shape[0]), data] # adding offset term
#%% Data
X = data[:ndat, :-1]
y = data[:ndat, -1]
#%% Prior
mean = np.zeros(X.shape[-1])
Sigma = np.eye(X.shape[-1]) * 100.
prior = LogReg(mean=mean, Sigma=Sigma)
#%% Estimation
for xt, yt in zip(X, y):
prior.update(yt, xt)
prior.log()
#%% Confusion matrix
CM = ConfusionMatrix(prior.true_vals, prior.binary_preds)
CM.print_stats()
#%% Plots
beta_log = np.array(prior.mean_log)
plt.figure(figsize=(8, 4))
plt.plot(prior.brier_score_log)
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, classification_report
#plt.style.use(u"~/.matplotlib/stylelib/trygveplot_astro.mplstyle")
plt.style.use(u"../trygveplot_astro.mplstyle")
from logreg import LogReg
from initdata import InitData
"""
In this part of the project, we assess the predictive ability of logistic regression on
determining default based on credit card data. The weights are trained using a gradient
solver and compared with Scikit-Learns Logistic regression method.
"""
# unit test (exits afterwards)
if (False):
logreg = LogReg(cost='cross_entropy') # init Logreg class
logreg.unit_test() # see logreg for expected output
## Get data from InitData Class
data = InitData()
##initialize all data, splitting gender, education and marital status
#XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot = data.credit_data(trainingShare=0.5,per_col=True,drop_zero=True,drop_neg2=True)
##initialize all data (with some bill_amt and pay_amt), split education and marital status
XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot, data_cols = data.credit_data(
trainingShare=0.5,
drop_zero=False,
drop_neg2=False,
per_col=True,
return_cols=True,
def setUp(self):
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def evaluate_lenet5(self,
learning_rate=0.1,
n_epochs=1,
dataset='mnist.pkl.gz',
nkerns=[20, 50],
batch_size=500,
testing=0):
rng = numpy.random.RandomState(32324)
datasets = self.data
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(
borrow=True).shape[0] // batch_size
n_valid_batches = valid_set_x.get_value(
borrow=True).shape[0] // batch_size
n_test_batches = test_set_x.get_value(
borrow=True).shape[0] // batch_size
index = T.lscalar() # index for each mini batch
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
if testing == 0:
print "...Building the model"
# output image size = (28-5+1)/2 = 12
layer_0_input = x.reshape((batch_size, 1, 28, 28))
layer_0 = LeNetConvPoolLayer(rng,
input=layer_0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2))
#output image size = (12-5+1)/2 = 4
layer_1 = LeNetConvPoolLayer(rng,
input=layer_0.output,
image_shape=(batch_size, nkerns[0], 12,
12),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2))
# make the input to hidden layer 2 dimensional
layer_2_input = layer_1.output.flatten(2)
layer_2 = HiddenLayer(rng,
input=layer_2_input,
n_in=nkerns[1] * 4 * 4,
n_out=500,
activation=T.tanh)
layer_3 = LogReg(input=layer_2.output, n_in=500, n_out=10)
self.cost = layer_3.neg_log_likelihood(y)
self.s = layer_3.s
self.test_model = theano.function(
[index],
layer_3.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
self.validate_model = theano.function(
[index],
layer_3.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
self.train_predic = theano.function(
[index],
layer_3.prob_y_given_x(),
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size]
})
# list of parameters
self.params = layer_3.params + layer_2.params + layer_1.params + layer_0.params
grads = T.grad(self.cost, self.params)
self.coefficient = 1
self.shapes = [i.get_value().shape for i in self.params]
symbolic_types = T.scalar, T.vector, T.matrix, T.tensor3, T.tensor4
v = [symbolic_types[len(i)]() for i in self.shapes]
#import pdb
#pdb.set_trace()
gauss_vector = Gv(self.cost, self.s, self.params, v, self.coefficient)
self.get_cost = theano.function(
[
index,
],
self.cost,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
},
on_unused_input='ignore')
self.get_grad = theano.function(
[
index,
],
grads,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
},
on_unused_input='ignore')
self.get_s = theano.function(
[
index,
],
self.s,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
},
on_unused_input='ignore')
self.function_Gv = theano.function(
[index],
gauss_vector,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
},
on_unused_input='ignore')
# # Using stochastic gradient updates
# updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ]
# train_model = theano.function([index],cost, updates=updates,
# givens={
# x: train_set_x[index*batch_size:(index+1)*batch_size],
# y: train_set_y[index*batch_size:(index+1)*batch_size]
# })
# Using conjugate gradient updates
# 'cg_ = cg(cost,output,params,coefficient,v)
# updated_params = [(param_i, param_j) for param_i,param_j in zip(params,cg_)]'
#self.update_parameters= theano.function([updated_params],updates=[params,updated_params])
# -----------------------------------------Starting Training ------------------------------
if testing == 0:
print('..... Training ')
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience // 2)
self.best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 1 == 0:
print('training @ iter = ', iter)
self.cg(minibatch_index)
if testing == 0:
print('Optimization complete')
print(
'Best validation score of %f %% obtained at iteration %i,'
'with test performance %f %%' %
(best_validation_loss * 100., best_itr, test_score * 100))
print('The code ran for %.2fm' % ((end_time - start_time) / 60.))
'Diastolic blood pressure (mm Hg)',
'Triceps skin fold thickness (mm)',
'2-Hour serum insulin (mu U/ml)',
'Body mass index (weight in kg/(height in m)^2)',
'Diabetes pedigree function',
'Age (years)',
'Class variable (0: no diabetes, 1: diabetes)']
# Load data and perform train test split.
print('[INFO] Loading data.')
data = load_data(file_name, columns)
X_train, X_test, y_train, y_test = train_test_split(data, frac=0.8, seed=5)
# Logistic Model
print('\n[INFO] Training logistic regression model.')
logreg = LogReg(args.lr_logreg, args.epochs, len(X_train[0]))
bias_logreg, weights_logreg = logreg.train(X_train, y_train)
y_logistic = [round(logreg.predict(example)) for example in X_test]
# Perceptron Model
print('[INFO] Training Perceptron model.')
perceptron = Perceptron(args.lr_perceptron, args.epochs, len(X_train[0]))
bias_perceptron, weights_perceptron = perceptron.train(X_train, y_train)
y_perceptron = [round(perceptron.predict(example)) for example in X_test]
# Compare accuracies
accuracy_logistic = get_accuracy(y_logistic, y_test)
accuracy_perceptron = get_accuracy(y_perceptron, y_test)
print('\n[INFO] Logistic Regression Accuracy: {:0.3f}'.format(
accuracy_logistic))
print('[INFO] Perceptron Accuracy: {:0.3f}'.format(accuracy_perceptron))
#processor
data = proc.load_csv(args.CSV_FILE)
data = proc.normalize_col(data, 'Amount')
data = data.drop(['Time'], axis=1)
print data[args.YCOL].value_counts()
X = proc.get_xvals(data, args.YCOL)
y = proc.get_yvals(data, args.YCOL)
#processor xfolds
Xu, yu = proc.under_sample(data, args.YCOL)
Xu_train, Xu_test, yu_train, yu_test = proc.cross_validation_sets(
Xu, yu, .3, 0)
X_train, X_test, y_train, y_test = proc.cross_validation_sets(X, y, .3, 0)
if args.LR_DRIVE:
lin = LogReg()
#under sampled data
c = lin.printing_Kfold_scores(Xu_train, yu_train)
lin.logistic_regression(Xu_train, Xu_test, yu_train, yu_test, c)
lin.logistic_regression(Xu_train, X_test, yu_train, y_test, c)
lin.get_roc_curve(Xu_train, Xu_test, yu_train, yu_test, c)
#regular data
c = lin.printing_Kfold_scores(X_train, y_train)
lin.logistic_regression(X_train, X_test, y_train, y_test, c)
lin.get_roc_curve(X_train, X_test, y_train, y_test, c)
if args.SVM_DRIVE:
sv = SVM()
sv.svm_run(Xu_train, Xu_test, yu_train, yu_test)
if args.SVML_DRIVE:
sv = SVM()
