def regularization_test():
with open("log_regular.txt", "w+") as f:
f.write("Regularization | lambda | Area | R2\n")
f.write("-" * 50 + "\n")
for reg in ['l2', 'l1']:
for lamb in [0.0001, 0.001, 0.01, 0.1, 1.0]:
logreg = LogReg(X, Y)
logreg.optimize(m=100,
epochs=5000,
eta=0.01,
regularization=reg,
lamb=lamb)
ypred_train = logreg.p_train
ypred_test = logreg.p_test
area = gain_chart(logreg.Y_test, ypred_test, plot=False)
R2 = prob_acc(logreg.Y_test, ypred_test, plot=False)
f.write(" %s | %g | %.4f | %.4f \n" %
(reg, lamb, area, R2))
f.write("-" * 50 + "\n")
def setUp(self):
self.mockV = Mock()
self.mockM = Mock()
self.test_obj = LogReg([('summer', {
'text': "some_text!"
}), ('winter', {
'text': 'some_text2!'
})],
vectorizer=self.mockV,
model=self.mockM)
class TestKnn(unittest.TestCase):
def setUp(self):
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def test_reg(self):
print(self.logreg_reg.beta)
print(kPOS.x)
beta = self.logreg_reg.sg_update(kPOS, 1)
self.assertAlmostEqual(beta[0], .5)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 1.5)
self.assertAlmostEqual(beta[3], 0.5)
self.assertAlmostEqual(beta[4], 0.0)
print(self.logreg_reg.beta)
print(kNEG.x)
beta = self.logreg_reg.sg_update(kNEG, 2)
self.assertAlmostEqual(beta[0], -0.72068776924864364)
self.assertAlmostEqual(beta[1], 1.0)
self.assertAlmostEqual(beta[2], -0.22068776924864364)
self.assertAlmostEqual(beta[3], -2.6620633077459308)
self.assertAlmostEqual(beta[4], -3.8827510769945746)
print("Finalize")
print(self.logreg_reg.last_update)
beta = self.logreg_reg.finalize_lazy(2)
self.assertAlmostEqual(beta[0], -0.72068776924864364)
self.assertAlmostEqual(beta[1], 1.0)
self.assertAlmostEqual(beta[2], -0.22068776924864364)
self.assertAlmostEqual(beta[3], -2.6620633077459308)
self.assertAlmostEqual(beta[4], -3.8827510769945746)
def test_unreg(self):
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kPOS, 1)
self.assertAlmostEqual(beta[0], .5)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 1.5)
self.assertAlmostEqual(beta[3], 0.5)
self.assertAlmostEqual(beta[4], 0.0)
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kNEG, 2)
self.assertAlmostEqual(beta[0], -0.47068776924864364)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 0.5293122307513564)
self.assertAlmostEqual(beta[3], -2.4120633077459308)
self.assertAlmostEqual(beta[4], -3.8827510769945746)
class TestKnn(unittest.TestCase):
def setUp(self):
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def test_reg(self):
#print("\nREGULARIZED:\n")
print(self.logreg_reg.beta)
print(kPOS.x)
beta = self.logreg_reg.sg_update(kPOS, 0)
self.assertAlmostEqual(beta[0], .25)
self.assertAlmostEqual(beta[1], 1.0)
self.assertAlmostEqual(beta[2], 0.75)
self.assertAlmostEqual(beta[3], 0.25)
self.assertAlmostEqual(beta[4], 0.0)
#print(self.logreg_reg.beta)
#print(kPOS.x)
beta = self.logreg_reg.sg_update(kNEG, 1)
self.assertAlmostEqual(beta[0], -0.30097640098415529)
self.assertAlmostEqual(beta[1], 1.0)
self.assertAlmostEqual(beta[2], -0.05097640098415529)
self.assertAlmostEqual(beta[3], -1.1529292029524658)
self.assertAlmostEqual(beta[4], -0.85195280196831058)
def test_unreg(self):
#print("\nUNREGULARIZED:\n\n")
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kPOS, 0)
self.assertAlmostEqual(beta[0], .5)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 1.5)
self.assertAlmostEqual(beta[3], 0.5)
self.assertAlmostEqual(beta[4], 0.0)
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kNEG, 1)
self.assertAlmostEqual(beta[0], -0.47068776924864364)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 0.5293122307513564)
self.assertAlmostEqual(beta[3], -2.4120633077459308)
self.assertAlmostEqual(beta[4], -3.8827510769945746)
class TestLogReg(unittest.TestCase):
def setUp(self):
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def test_unreg(self):
print(self.logreg_unreg.w)
print(kPOS.x)
w = self.logreg_unreg.sg_update(kPOS, 0)
self.assertAlmostEqual(w[0], .5)
self.assertAlmostEqual(w[1], 2.0)
self.assertAlmostEqual(w[2], 1.5)
self.assertAlmostEqual(w[3], 0.5)
self.assertAlmostEqual(w[4], 0.0)
print(self.logreg_unreg.w)
print(kNEG.x)
w = self.logreg_unreg.sg_update(kNEG, 1)
self.assertAlmostEqual(w[0], -0.49330714907571527)
self.assertAlmostEqual(w[1], 2.0)
self.assertAlmostEqual(w[2], -0.48661429815143054)
self.assertAlmostEqual(w[3], -2.479921447227146)
self.assertAlmostEqual(w[4], -0.99330714907571527)
def test_reg(self):
print(self.logreg_reg.w)
print(kPOS.x)
w = self.logreg_reg.sg_update(kPOS, 0)
self.assertAlmostEqual(w[0], .5)
self.assertAlmostEqual(w[1], 1.0)
self.assertAlmostEqual(w[2], 0.75)
self.assertAlmostEqual(w[3], 0.25)
self.assertAlmostEqual(w[4], 0.0)
print(self.logreg_reg.w)
print(kNEG.x)
w = self.logreg_reg.sg_update(kNEG, 1)
print w
self.assertAlmostEqual(w[0], -0.43991334982599239)
self.assertAlmostEqual(w[1], 1.0)
self.assertAlmostEqual(w[2], -0.56491334982599239)
self.assertAlmostEqual(w[3], -1.2848700247389886)
self.assertAlmostEqual(w[4], -0.2349783374564981)
class TestKnn(unittest.TestCase):
def setUp(self):
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def test_reg(self):
print(self.logreg_reg.beta)
print(kPOS.x)
beta = self.logreg_reg.sg_update(kPOS, 0)
self.assertAlmostEqual(beta[0], .25)
self.assertAlmostEqual(beta[1], 1.0)
self.assertAlmostEqual(beta[2], 0.75)
self.assertAlmostEqual(beta[3], 0.25)
self.assertAlmostEqual(beta[4], 0.0)
print(self.logreg_reg.beta)
print(kPOS.x)
beta = self.logreg_reg.sg_update(kNEG, 1)
self.assertAlmostEqual(beta[0], -0.30097640098415529)
self.assertAlmostEqual(beta[1], 1.0)
self.assertAlmostEqual(beta[2], -0.05097640098415529)
self.assertAlmostEqual(beta[3], -1.1529292029524658)
self.assertAlmostEqual(beta[4], -0.85195280196831058)
def test_unreg(self):
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kPOS, 0)
self.assertAlmostEqual(beta[0], .5)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 1.5)
self.assertAlmostEqual(beta[3], 0.5)
self.assertAlmostEqual(beta[4], 0.0)
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kNEG, 1)
self.assertAlmostEqual(beta[0], -0.47068776924864364)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 0.5293122307513564)
self.assertAlmostEqual(beta[3], -2.4120633077459308)
self.assertAlmostEqual(beta[4], -3.8827510769945746)
def setUp(self):
self.mockV = Mock()
self.mockM = Mock()
self.test_obj = LogReg([
('summer', {'text': "some_text!"}),
('winter', {'text': 'some_text2!'})
],
vectorizer = self.mockV,
model = self.mockM
)
def find_epoch(dims, nb_classes, train_embs, train_labels, test_embs,
test_labels, cuda):
log = LogReg(dims, nb_classes)
opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0)
xent = nn.BCEWithLogitsLoss()
sigmoid = nn.Sigmoid()
if cuda:
log.cuda()
epoch_flag = 0
epoch_win = 0
best_f1 = torch.zeros(1)
tmp_th = 0.5
if cuda:
best_f1 = best_f1.cuda()
for e in range(2000):
log.train()
opt.zero_grad()
logits = log(train_embs)
loss = xent(logits, train_labels)
loss.backward()
opt.step()
if (e + 1) % 100 == 0:
log.eval()
logits = sigmoid(log(test_embs))
zero = torch.zeros_like(logits)
one = torch.ones_like(logits)
preds = torch.where(logits >= tmp_th, one, zero)
if cuda:
test_labels = test_labels.cpu()
preds = preds.cpu()
f1 = f1_score(test_labels.numpy(), preds.numpy(), average='micro')
if f1 >= best_f1:
epoch_flag = e + 1
best_f1 = f1
epoch_win = 0
else:
epoch_win += 1
if epoch_win == 10:
break
return epoch_flag
class TestKnn(unittest.TestCase):
def setUp(self):
self.logreg_unreg = LogReg(5, 1.0)
def test_unreg(self):
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kPOS)
self.assertAlmostEqual(beta[0], .5)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 1.5)
self.assertAlmostEqual(beta[3], 0.5)
self.assertAlmostEqual(beta[4], 0.0)
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kNEG)
self.assertAlmostEqual(beta[0], -0.47068776924864364)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 0.5293122307513564)
self.assertAlmostEqual(beta[3], -2.4120633077459308)
self.assertAlmostEqual(beta[4], -3.8827510769945746)
class TestKnn(unittest.TestCase):
def setUp(self):
self.logreg_unreg = LogReg(5, 1.0)
def test_unreg(self):
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kPOS)
self.assertAlmostEqual(beta[0], .5)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 1.5)
self.assertAlmostEqual(beta[3], 0.5)
self.assertAlmostEqual(beta[4], 0.0)
print(self.logreg_unreg.beta)
print(kPOS.x)
beta = self.logreg_unreg.sg_update(kNEG)
self.assertAlmostEqual(beta[0], -0.47068776924864364)
self.assertAlmostEqual(beta[1], 2.0)
self.assertAlmostEqual(beta[2], 0.5293122307513564)
self.assertAlmostEqual(beta[3], -2.4120633077459308)
self.assertAlmostEqual(beta[4], -3.8827510769945746)
def __init__(self, inp, layer_sizes):
self.x = inp
self.layer_sizes = layer_sizes
self.layers = []
prev = self.x
for i in range(1, len(self.layer_sizes)):
new_shape = (self.layer_sizes[i], self.layer_sizes[i - 1])
self.layers.append(LogReg(prev, new_shape))
prev = self.layers[-1].a
self.a = self.layers[-1].a
self.params = sum((l.params for l in self.layers), [])
class TestLogReg(unittest.TestCase):
def setUp(self):
self.logreg_learnrate = LogReg(784, 0.5)
self.logreg_nolearnrate = LogReg(784, 1.0)
def test_learnrate(self):
print("\nTesting: Learning Rate Update")
w = self.logreg_learnrate.sgd_update(pos_example_x, pos_example_y)
self.assertAlmostEqual(w[208], 0.05371094)
self.assertAlmostEqual(w[209], 0.14453125)
self.assertAlmostEqual(w[210], 0.20507812)
self.assertAlmostEqual(w[211], 0.24707031)
self.assertAlmostEqual(w[212], 0.24707031)
def test_nolearnrate(self):
print("\nTesting: No Learning Rate Update")
w = self.logreg_nolearnrate.sgd_update(neg_example_x, neg_example_y)
self.assertAlmostEqual(w[208], 0.10742188)
self.assertAlmostEqual(w[209], 0.2890625)
self.assertAlmostEqual(w[210], 0.41015625)
self.assertAlmostEqual(w[211], 0.49414062)
self.assertAlmostEqual(w[212], 0.49414062)
def evaluate_lenet5(self,
learning_rate=0.1,
n_epochs=1,
dataset='mnist.pkl.gz',
nkerns=[20, 50],
batch_size=500,
testing=0):
rng = numpy.random.RandomState(32324)
datasets = self.data
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(
borrow=True).shape[0] // batch_size
n_valid_batches = valid_set_x.get_value(
borrow=True).shape[0] // batch_size
n_test_batches = test_set_x.get_value(
borrow=True).shape[0] // batch_size
index = T.lscalar() # index for each mini batch
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
if testing == 0:
print "...Building the model"
# output image size = (28-5+1)/2 = 12
layer_0_input = x.reshape((batch_size, 1, 28, 28))
layer_0 = LeNetConvPoolLayer(rng,
input=layer_0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2))
#output image size = (12-5+1)/2 = 4
layer_1 = LeNetConvPoolLayer(rng,
input=layer_0.output,
image_shape=(batch_size, nkerns[0], 12,
12),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2))
# make the input to hidden layer 2 dimensional
layer_2_input = layer_1.output.flatten(2)
layer_2 = HiddenLayer(rng,
input=layer_2_input,
n_in=nkerns[1] * 4 * 4,
n_out=500,
activation=T.tanh)
layer_3 = LogReg(input=layer_2.output, n_in=500, n_out=10)
self.cost = layer_3.neg_log_likelihood(y)
self.s = layer_3.s
self.test_model = theano.function(
[index],
layer_3.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
self.validate_model = theano.function(
[index],
layer_3.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
self.train_predic = theano.function(
[index],
layer_3.prob_y_given_x(),
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size]
})
# list of parameters
self.params = layer_3.params + layer_2.params + layer_1.params + layer_0.params
grads = T.grad(self.cost, self.params)
self.coefficient = 1
self.shapes = [i.get_value().shape for i in self.params]
symbolic_types = T.scalar, T.vector, T.matrix, T.tensor3, T.tensor4
v = [symbolic_types[len(i)]() for i in self.shapes]
#import pdb
#pdb.set_trace()
gauss_vector = Gv(self.cost, self.s, self.params, v, self.coefficient)
self.get_cost = theano.function(
[
index,
],
self.cost,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
},
on_unused_input='ignore')
self.get_grad = theano.function(
[
index,
],
grads,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
},
on_unused_input='ignore')
self.get_s = theano.function(
[
index,
],
self.s,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
},
on_unused_input='ignore')
self.function_Gv = theano.function(
[index],
gauss_vector,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
},
on_unused_input='ignore')
# # Using stochastic gradient updates
# updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ]
# train_model = theano.function([index],cost, updates=updates,
# givens={
# x: train_set_x[index*batch_size:(index+1)*batch_size],
# y: train_set_y[index*batch_size:(index+1)*batch_size]
# })
# Using conjugate gradient updates
# 'cg_ = cg(cost,output,params,coefficient,v)
# updated_params = [(param_i, param_j) for param_i,param_j in zip(params,cg_)]'
#self.update_parameters= theano.function([updated_params],updates=[params,updated_params])
# -----------------------------------------Starting Training ------------------------------
if testing == 0:
print('..... Training ')
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience // 2)
self.best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 1 == 0:
print('training @ iter = ', iter)
self.cg(minibatch_index)
if testing == 0:
print('Optimization complete')
print(
'Best validation score of %f %% obtained at iteration %i,'
'with test performance %f %%' %
(best_validation_loss * 100., best_itr, test_score * 100))
print('The code ran for %.2fm' % ((end_time - start_time) / 60.))
def setUp(self):
self.logreg_unreg = LogReg(5, 1.0)
def setUp(self):
self.logreg_learnrate = LogReg(5, 0.0, lambda x: 0.5)
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def setUp(self):
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
fn = 'Skin_NonSkin.txt'
data = np.loadtxt(fn)
data[:,-1] -= 1
np.random.shuffle(data)
data = np.c_[np.ones(data.shape[0]), data] # adding offset term
#%% Data
X = data[:ndat, :-1]
y = data[:ndat, -1]
#%% Prior
mean = np.zeros(X.shape[-1])
Sigma = np.eye(X.shape[-1]) * 100.
prior = LogReg(mean=mean, Sigma=Sigma)
#%% Estimation
for xt, yt in zip(X, y):
prior.update(yt, xt)
prior.log()
#%% Confusion matrix
CM = ConfusionMatrix(prior.true_vals, prior.binary_preds)
CM.print_stats()
#%% Plots
beta_log = np.array(prior.mean_log)
plt.figure(figsize=(8, 4))
plt.plot(prior.brier_score_log)
def evaluate_lenet5(learning_rate=0.1,
n_epochs=200,
dataset='mnist.pkl.gz',
nkerns=[16, 16, 16],
batch_size=500):
rng = numpy.random.RandomState(32324)
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size
index = T.lscalar() # index for each mini batch
train_epoch = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
print "...Building the model"
# output image size = (28-5+1+4)/2 = 14
layer_0_input = x.reshape((batch_size, 1, 28, 28))
layer_0 = LeNetConvPoolLayer(rng,
input=layer_0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2),
border_mode=2)
#output image size = (14-3+1)/2 = 6
layer_1 = LeNetConvPoolLayer(rng,
input=layer_0.output,
image_shape=(batch_size, nkerns[0], 14, 14),
filter_shape=(nkerns[1], nkerns[0], 3, 3),
poolsize=(2, 2))
#output image size = (6-3+1)/2 = 2
layer_2 = LeNetConvPoolLayer(rng,
input=layer_1.output,
image_shape=(batch_size, nkerns[1], 6, 6),
filter_shape=(nkerns[2], nkerns[1], 3, 3),
poolsize=(2, 2))
# make the input to hidden layer 2 dimensional
layer_3_input = layer_2.output.flatten(2)
layer_3 = HiddenLayer(rng,
input=layer_3_input,
n_in=nkerns[2] * 2 * 2,
n_out=200,
activation=T.tanh)
layer_4 = LogReg(input=layer_3.output, n_in=200, n_out=10)
teacher_p_y_given_x = theano.shared(numpy.asarray(
pickle.load(open('prob_best_model.pkl', 'rb')),
dtype=theano.config.floatX),
borrow=True)
#cost = layer_4.neg_log_likelihood(y) + T.mean((teacher_W - layer_4.W)**2)/(2.*(1+epoch*2)) + T.mean((teacher_b-layer_4.b)**2)/(2.*(1+epoch*2))
# import pdb
# pdb.set_trace()
p_y_given_x = T.matrix('p_y_given_x')
e = theano.shared(value=0, name='e', borrow=True)
#cost = layer_4.neg_log_likelihood(y) + 1.0/(e)*T.mean((layer_4.p_y_given_x - p_y_given_x)**2)
cost = layer_4.neg_log_likelihood(
y) + 2.0 / (e) * T.mean(-T.log(layer_4.p_y_given_x) * p_y_given_x -
layer_4.p_y_given_x * T.log(p_y_given_x))
test_model = theano.function(
[index],
layer_4.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
layer_4.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
# list of parameters
params = layer_4.params + layer_3.params + layer_2.params + layer_1.params + layer_0.params
grads = T.grad(cost, params)
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index, train_epoch],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size],
p_y_given_x: teacher_p_y_given_x[index],
e: train_epoch
})
# -----------------------------------------Starting Training ------------------------------
print('..... Training ')
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience // 2)
best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
#epo = theano.shared('epo')
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print('training @ iter = ', iter)
cost_ij = train_model(minibatch_index, epoch)
if (iter + 1) % validation_frequency == 0:
# compute loss on validation set
validation_losses = [
validate_model(i) for i in range(n_valid_batches)
]
this_validation_loss = numpy.mean(validation_losses)
# import pdb
# pdb.set_trace()
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# check with best validation score till now
if this_validation_loss < best_validation_loss:
# improve
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_itr = iter
test_losses = [
test_model(i) for i in range(n_test_batches)
]
test_score = numpy.mean(test_losses)
print('epoch %i, minibatch %i/%i, testing error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
with open('best_model_3layer.pkl', 'wb') as f:
pickle.dump(params, f)
with open('Results_student_3.txt', 'wb') as f2:
f2.write(str(test_score * 100) + '\n')
#if patience <= iter:
# done_looping = True
# break
end_time = timeit.default_timer()
print('Optimization complete')
print(
'Best validation score of %f %% obtained at iteration %i,'
'with test performance %f %%' %
(best_validation_loss * 100., best_itr, test_score * 100))
print('The code ran for %.2fm' % ((end_time - start_time) / 60.))
from logreg import LogReg
from initdata import InitData
from DNN import NeuralNetwork
"""
In this part of the project, we assess the predictive ability of a feed-forward Neural
network on determining default based on credit card data.
To explore hyperparameter-space, chose explore = True
To run self-written NN network, chose sklearn = False
Chose metric for saving best score with metric (accuracy/rocauc)
For credit card data, cost=cross_entropy is chosen
"""
## Get data from InitData Class ---------------------------------------
logreg = LogReg() # init Logreg class
data = InitData()
## Params -------------------------------------------------------------
explore = False
sklearn = False
metric = "accuracy" #"rocauc"
cost = "binary_cross_entropy"
data_size = "full"
if data_size == "full":
XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot = data.credit_data(trainingShare=0.5)
else:
XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot = data.credit_data(trainingShare=0.5, return_cols=False, drop_zero=True,drop_neg2=True)
if explore==True: # Explore parameter space for credit card data
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, classification_report
#plt.style.use(u"~/.matplotlib/stylelib/trygveplot_astro.mplstyle")
plt.style.use(u"../trygveplot_astro.mplstyle")
from logreg import LogReg
from initdata import InitData
"""
In this part of the project, we assess the predictive ability of logistic regression on
determining default based on credit card data. The weights are trained using a gradient
solver and compared with Scikit-Learns Logistic regression method.
"""
# unit test (exits afterwards)
if (False):
logreg = LogReg(cost='cross_entropy') # init Logreg class
logreg.unit_test() # see logreg for expected output
## Get data from InitData Class
data = InitData()
##initialize all data, splitting gender, education and marital status
#XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot = data.credit_data(trainingShare=0.5,per_col=True,drop_zero=True,drop_neg2=True)
##initialize all data (with some bill_amt and pay_amt), split education and marital status
XTrain, yTrain, XTest, yTest, Y_train_onehot, Y_test_onehot, data_cols = data.credit_data(
trainingShare=0.5,
drop_zero=False,
drop_neg2=False,
per_col=True,
return_cols=True,
import numpy as np
from read_data import get_data
import sys
sys.path.append("network/")
from logreg import LogReg
from metrics import gain_chart, prob_acc
#X, Y = get_data(normalized = False, standardized = False)
X, Y = get_data()
logreg = LogReg(X, Y)
logreg.optimize(m=100, epochs=5000,
eta=0.01) #, regularization='l2', lamb=0.0001)
ypred_train = logreg.p_train
ypred_test = logreg.p_test
gain_chart(logreg.Y_test, ypred_test)
prob_acc(logreg.Y_test, ypred_test)
def regularization_test():
with open("log_regular.txt", "w+") as f:
f.write("Regularization | lambda | Area | R2\n")
f.write("-" * 50 + "\n")
for reg in ['l2', 'l1']:
for lamb in [0.0001, 0.001, 0.01, 0.1, 1.0]:
logreg = LogReg(X, Y)
logreg.optimize(m=100,
def evaluate_lenet5(learning_rate = 0.1, n_epochs=200, dataset='mnist.pkl.gz',nkerns = [20,50], batch_size = 500 , testing =0):
rng = numpy.random.RandomState(32324)
datasets = load_data(dataset)
train_set_x,train_set_y = datasets[0]
valid_set_x,valid_set_y = datasets[1]
test_set_x,test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0]//batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]//batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]//batch_size
index = T.lscalar() # index for each mini batch
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
if testing ==0:
print "...Building the model"
# output image size = (28-5+1)/2 = 12
layer_0_input = x.reshape((batch_size,1,28,28))
layer_0 = LeNetConvPoolLayer(rng,input = layer_0_input, image_shape=(batch_size,1,28,28),filter_shape=(nkerns[0],1,5,5),poolsize=(2,2))
#output image size = (12-5+1)/2 = 4
layer_1 = LeNetConvPoolLayer(rng, input = layer_0.output, image_shape = (batch_size, nkerns[0],12,12),
filter_shape = (nkerns[1],nkerns[0],5,5), poolsize=(2,2) )
# make the input to hidden layer 2 dimensional
layer_2_input = layer_1.output.flatten(2)
layer_2 = HiddenLayer(rng,input = layer_2_input, n_in = nkerns[1]*4*4, n_out = 500, activation = T.tanh)
layer_3 = LogReg(input = layer_2.output, n_in=500, n_out = 10)
cost = layer_3.neg_log_likelihood(y)
test_model = theano.function([index],layer_3.errors(y),
givens={
x: test_set_x[index*batch_size:(index+1)*batch_size],
y: test_set_y[index*batch_size:(index+1)*batch_size]
})
validate_model = theano.function([index],layer_3.errors(y),
givens={
x: valid_set_x[index*batch_size:(index+1)*batch_size],
y: valid_set_y[index*batch_size:(index+1)*batch_size]
})
train_predic = theano.function([index], layer_3.prob_y_given_x(),
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size]
})
# list of parameters
layer_guided = theano.function([index], layer_1.output,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size]
})
params = layer_3.params + layer_2.params + layer_1.params + layer_0.params
grads = T.grad(cost,params)
updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ]
train_model = theano.function([index],cost, updates=updates,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size],
y: train_set_y[index*batch_size:(index+1)*batch_size]
})
# -----------------------------------------Starting Training ------------------------------
if testing ==0:
print ('..... Training ' )
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience//2)
best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping) and testing ==0:
epoch = epoch+1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1)*n_train_batches + minibatch_index
if iter%100 ==0:
print ('training @ iter = ', iter)
cost_ij = train_model(minibatch_index)
if(iter +1)%validation_frequency ==0:
# compute loss on validation set
validation_losses = [validate_model(i) for i in range(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
# import pdb
# pdb.set_trace()
print ('epoch %i, minibatch %i/%i, validation error %f %%' %(epoch,minibatch_index+1,n_train_batches,this_validation_loss*100. ))
# check with best validation score till now
if this_validation_loss<best_validation_loss:
# improve
# if this_validation_loss < best_validation_loss * improvement_threshold:
# patience = max(patience, iter*patience_increase)
best_validation_loss = this_validation_loss
best_itr = iter
test_losses = [test_model(i) for i in range(n_test_batches)]
test_score = numpy.mean(test_losses)
print ('epoch %i, minibatch %i/%i, testing error %f %%' %(epoch, minibatch_index+1,n_train_batches,test_score*100.))
with open('best_model.pkl', 'wb') as f:
pickle.dump(params, f)
with open('Results_teacher.txt','wb') as f2:
f2.write(str(test_score*100) + '\n')
p_y_given_x = [train_predic(i) for i in range(n_train_batches)]
with open ('prob_best_model.pkl','wb') as f1:
pickle.dump(p_y_given_x,f1)
# if patience <= iter:
# done_looping = True
# break
layer_2_op_dump = [layer_guided(i) for i in range(n_train_batches)]
with open ('layer_guided.pkl','wb') as lg:
pickle.dump(layer_2_op_dump,lg)
end_time = timeit.default_timer()
# p_y_given_x = [train_model(i) for i in range(n_train_batches)]
# with open ('prob_best_model.pkl') as f:
# pickle.dump(p_y_given_x)
if testing ==0 :
print ('Optimization complete')
print ('Best validation score of %f %% obtained at iteration %i,'
'with test performance %f %%' % (best_validation_loss*100., best_itr, test_score*100 ))
print('The code ran for %.2fm' %((end_time - start_time)/60.))
class TestLogReg(unittest.TestCase):
def setUp(self):
self.logreg_learnrate = LogReg(5, 0.0, lambda x: 0.5)
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def test_learnrate(self):
print("\nTesting: Learning Rate Usage")
print(self.logreg_learnrate.w)
print(kPOS.x)
w = self.logreg_learnrate.sg_update(kPOS, 0)
self.assertAlmostEqual(w[0], 0.25)
self.assertAlmostEqual(w[1], 1.00)
self.assertAlmostEqual(w[2], 0.75)
self.assertAlmostEqual(w[3], 0.25)
self.assertAlmostEqual(w[4], 0.0)
print(self.logreg_learnrate.w)
print(kNEG.x)
w = self.logreg_learnrate.sg_update(kNEG, 1)
self.assertAlmostEqual(w[0], -0.21207090998937828)
self.assertAlmostEqual(w[1], 1.0)
self.assertAlmostEqual(w[2], -0.17414181997875655)
self.assertAlmostEqual(w[3], -1.1362127299681348)
self.assertAlmostEqual(w[4], -0.46207090998937828)
def test_unreg(self):
print("\nTesting: Unregularized Update")
print(self.logreg_unreg.w)
print(kPOS.x)
w = self.logreg_unreg.sg_update(kPOS, 0)
self.assertAlmostEqual(w[0], .5)
self.assertAlmostEqual(w[1], 2.0)
self.assertAlmostEqual(w[2], 1.5)
self.assertAlmostEqual(w[3], 0.5)
self.assertAlmostEqual(w[4], 0.0)
print(self.logreg_unreg.w)
print(kNEG.x)
w = self.logreg_unreg.sg_update(kNEG, 1)
self.assertAlmostEqual(w[0], -0.49330714907571527)
self.assertAlmostEqual(w[1], 2.0)
self.assertAlmostEqual(w[2], -0.48661429815143054)
self.assertAlmostEqual(w[3], -2.479921447227146)
self.assertAlmostEqual(w[4], -0.99330714907571527)
def test_reg(self):
print("\nTesting: Regularized Update")
print(self.logreg_reg.w)
print(kPOS.x)
w = self.logreg_reg.sg_update(kPOS, 0)
self.assertAlmostEqual(w[0], .5)
self.assertAlmostEqual(w[1], 1.0)
self.assertAlmostEqual(w[2], 0.75)
self.assertAlmostEqual(w[3], 0.25)
self.assertAlmostEqual(w[4], 0.0)
print(self.logreg_reg.w)
print(kNEG.x)
w = self.logreg_reg.sg_update(kNEG, 1)
self.assertAlmostEqual(w[0], -0.43991334982599239)
self.assertAlmostEqual(w[1], 1.0)
self.assertAlmostEqual(w[2], -0.56491334982599239)
self.assertAlmostEqual(w[3], -1.2848700247389886)
self.assertAlmostEqual(w[4], -0.2349783374564981)
"""
In this part of the project, we assess the predictive ability of a feed-forward Neural
network on fitting a polynomial to the franke function.
To explore hyperparameter-space, chose explore = True
To run self-written NN network, chose sklearn = False
Chose metric for saving best score (R2/mse)
For credit card data, cost=mse is chosen
"""
## Get data from InitData Class ---------------------------------------
data = InitData()
noise = 0.05
XTrain, yTrain, XTest, yTest, X, y, b, x_, y_ = data.franke_data(noise=noise,
N=20)
logreg = LogReg() # init Logreg class
cf = CostFunctions()
# ----- PARAMETERS ------
sklearn = False
NN = True
explore = False
metric = "R2" # "mse"
if explore == True: # Explore parameter space for franke function
eta_vals = np.logspace(-3, -1, 3)
lmbd_vals = np.logspace(-3, -1, 3)
#acts_hidden = ["sigmoid", "tanh", "relu", "elu"]
acts_hidden = ["logistic", "tanh", "relu"] # Supported by sklearn
act_o = "identity"
hidden_neurons = [4, 8, 12, 16, 50, 100]
for line in json_file:
json_obj = json.loads(line)
reviews += [(classifier, json_obj)]
with open("winter-" + classifier + ".json") as json_file:
for line in json_file:
json_obj = json.loads(line)
reviews += [(classifier, json_obj)]
# Creating model objects
model = args.model
if (model == "baseline"):
model_obj = BaseLine(reviews, categories)
elif (model == "logreg"):
model_obj = LogReg(reviews)
elif (model == "multinomialNB"):
model_obj = NaiveBayes(reviews, "multinomial")
elif (model == "lda"):
model_obj = TopicModel(reviews)
elif (model == "kNearestNeighbors"):
model_obj = knn(reviews, target)
else: # put additional models here.
print("Argument Error: invalid model specified")
sys.exit()
model_classified = [] # classifications stored here
##### START PARAMETERS #####
## Comment out the preferred goal
# goal = 'ICU admission'
goal = 'Mortality'
# goal = 'Duration of stay at ICU'
## Select which variables to include here
variables_to_include = {
'Form Collection Name': [], # groups
'Form Name': [], # variable subgroups
'Field Variable Name': ['days_since_onset', 'age_yrs'] +
['ethnic_cat_' + str(i)
for i in range(1, 11)] + get_feature_set() # single variables
}
model = LogReg() # Initialize one of the model in .\Classifiers
##### END PARAMETERS #####
config = configparser.ConfigParser()
config.read('user_settings.ini')
path_creds = config['CastorCredentials']['local_private_path']
model.goal = goal
data, data_struct = load_data(path_creds)
data, data_struct = preprocess(data, data_struct)
x, y, data = prepare_for_learning(data, data_struct, variables_to_include,
goal)
# Y = [event, days_to_event]. Select first column for logreg (or similar)
# y = y.iloc[:, 0] # FIXME: handle different inputs per model
class testLogReg(unittest.TestCase):
"""
Every method needs a LogReg object with the vectorizer and model mocked out,
as those objects would perform potentially slow methods otherwise.
"""
def setUp(self):
self.mockV = Mock()
self.mockM = Mock()
self.test_obj = LogReg([('summer', {
'text': "some_text!"
}), ('winter', {
'text': 'some_text2!'
})],
vectorizer=self.mockV,
model=self.mockM)
"""
After object creation, a LogReg object should have:
a TfidVectorizer named vectorizer
a list of labels named labels
a LogisticRegression model named model.
model should have called the "fit" method to ensure that the initial model
has been trained.
vectorizer should have called the fit_transform method to create a vector for
the model.fit method.
This test uses mocks to test that the initial training methods are being
called without actually performing the associated operations.
"""
def test_init_trained(self):
assert self.mockV.fit_transform.called #vectorizer "trained"
assert self.mockM.fit.called #model trained
"""
After object creation, the labels variable should contain a list of labels
given in the reviews tuple list. This test mocks the vectorizer and model
and asserts that the expected labels are present after initialization.
"""
def test_init_labels(self):
assert self.test_obj.labels == ['summer', 'winter']
"""
After object creation, the corpus variable should contain a list of training
data given in the reviews tuple list. This test mocks the vectorizer and model
and asserts that the expected review text is present after initialization.
"""
def test_init_corpus(self):
assert self.test_obj.corpus == ['some_text!', 'some_text2!']
"""
After object creation, the review variable should contain the reviews given
upon initialization. This test mocks the vectorizer and model
and asserts that the expected reviews are present after initialization.
"""
def test_init_reviews(self):
assert self.test_obj.reviews == [('summer', {
'text': "some_text!"
}), ('winter', {
'text': 'some_text2!'
})]
"""
After calling the train method the corpus should be expanded to include the
text for those reviews that are passed into the method. Test mocks the
vectorizer and model created in the initialization of the object and checks
to see that the new reviews are parsed and saved correctly in the corpus
variable.
"""
def test_train_corpus(self):
#setting initial corpus so we don't rely on initialization setting it
#correctly
self.test_obj.corpus = ["Hello world!"]
self.test_obj.train([('summer', {'text': 'some_text!'})])
assert self.test_obj.corpus == ["Hello world!", "some_text!"]
"""
After calling the train method the labels should be expanded to include the
text for those reviews that are passed into the method. Test mocks the
vectorizer and model created in the initialization of the object and checks
to see that the new reviews are parsed and saved correctly in the labels
variable.
"""
def test_train_labels(self):
#setting initial labels so we don't rely on initialization setting it
#correctly
self.test_obj.labels = ["winter"]
self.test_obj.train([('fall', {'text': 'some_text!'})])
assert self.test_obj.labels == ["winter", "fall"]
"""
The train method should call the model's fit method after parsing out the
appropriate data from the given input. This test asserts that the model calls
fit with the expect arguments when the train method is executed.
"""
def test_train_trained(self):
#Mocking the return value for fit_transform, since that result is passed to
#the models' fit method.
self.mockV.fit_transform.return_value = ["THIS IS A VECTOR!"]
#Setting initial labels and corpus
self.test_obj.labels = ["winter"]
self.test_obj.corpus = ["Hello world!"]
self.test_obj.train([('fall', {'text': 'some_text!'})])
self.mockV.fit_transform.assert_called_with(
["Hello world!", "some_text!"])
self.mockM.fit.assert_called_with(["THIS IS A VECTOR!"],
["winter", "fall"])
"""
classify_all should complete with a call to the
object's model's predict method, which does the work of predicting the season
for a given test corpus. This test sets the return value for model.predict
and asserts that the returned value of classify_all matches that.
"""
def test_classify_all(self):
self.mockM.predict.return_value = [
"summer", "fall", "winter", "spring"
]
assert self.test_obj.classify_all([("some", {
"text": "data"
})]) == ["summer", "fall", "winter", "spring"]
"""
The vocabulary method is a WIP method that should
return a list of strings, with each string representing
a word that can be used to predict a particular season.
This test checks to make sure that at the very least the
vocabulary method is returning a list of strings.
"""
def test_vocabulary(self):
#Mocking the selector used in the method to avoid a lot of computation
mockS = Mock()
mockS.transform.return_value = ["fireplace", "warmth"]
result = self.test_obj.vocabulary([("summer", {
"text": "data"
})],
selector=mockS)
for i in result:
assert isinstance(i, str)
def setUp(self):
self.logreg_unreg = LogReg(5, 0.0, lambda x: 1.0)
self.logreg_reg = LogReg(5, 0.25, lambda x: 1.0)
def evaluate_lenet5(learning_rate = 0.10, n_epochs=200, dataset='mnist.pkl.gz',nkerns = [16,16,16,12,12,12], batch_size = 500):
rng = numpy.random.RandomState(32324)
datasets = load_data(dataset)
train_set_x,train_set_y = datasets[0]
valid_set_x,valid_set_y = datasets[1]
test_set_x,test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0]//batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]//batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]//batch_size
index = T.lscalar() # index for each mini batch
train_epoch = T.lscalar('train_epoch')
x = T.matrix('x')
y = T.ivector('y')
# ------------------------------- Building Model ----------------------------------
print "...Building the model"
layer_0_input = x.reshape((batch_size,1,28,28))
# output image size = (28-5+1+)/1 = 24
layer_0 = LeNetConvPoolLayer(rng,input = layer_0_input, image_shape=(batch_size,1,28,28),
filter_shape=(nkerns[0],1,5,5),poolsize=(1,1))
#output image size = (24-3+1) = 22
layer_1 = LeNetConvPoolLayer(rng, input = layer_0.output, image_shape = (batch_size, nkerns[0],24,24),
filter_shape = (nkerns[1],nkerns[0],3,3), poolsize=(1,1) )
#output image size = (22-3+1)/2 = 10
layer_2 = LeNetConvPoolLayer(rng, input = layer_1.output, image_shape = (batch_size, nkerns[1],22,22),
filter_shape = (nkerns[2],nkerns[1],3,3), poolsize=(2,2) )
#output image size = (10-3+1)/2 = 4
layer_3 = LeNetConvPoolLayer(rng, input = layer_2.output, image_shape = (batch_size, nkerns[2],10,10),
filter_shape = (nkerns[3], nkerns[2],3,3), poolsize=(2,2) )
#output image size = (4-3+2+1) = 4
layer_4 = LeNetConvPoolLayer(rng, input = layer_3.output, image_shape = (batch_size, nkerns[3],4,4),
filter_shape = (nkerns[4], nkerns[3],3,3), poolsize=(1,1), border_mode = 1 )
#output image size = (4-3+1)/2 = 2
layer_5 = LeNetConvPoolLayer(rng, input = layer_4.output, image_shape = (batch_size, nkerns[4],4,4),
filter_shape = (nkerns[5], nkerns[4],3,3), poolsize=(2,2), border_mode = 1 )
# make the input to hidden layer 2 dimensional
layer_6_input = layer_5.output.flatten(2)
layer_6 = HiddenLayer(rng,input = layer_6_input, n_in = nkerns[5]*2*2, n_out = 200, activation = T.tanh)
layer_7 = LogReg(input = layer_6.output, n_in=200, n_out = 10)
teacher_p_y_given_x = theano.shared(numpy.asarray(pickle.load(open('prob_best_model.pkl','rb')),dtype =theano.config.floatX), borrow=True)
p_y_given_x = T.matrix('p_y_given_x')
e = theano.shared(value = 0, name = 'e', borrow = True)
cost = layer_7.neg_log_likelihood(y) + 2.0/(e)*T.mean(-T.log(layer_7.p_y_given_x)*p_y_given_x - layer_7.p_y_given_x*T.log(p_y_given_x))
tg = theano.shared(numpy.asarray(pickle.load(open('modified_guided_data.pkl','rb')),dtype =theano.config.floatX), borrow=True)
guiding_weights = T.tensor4('guiding_weights')
#guide_cost = T.mean(-T.log(layer_3.output)*guiding_weights - layer_3.output*T.log(guiding_weights))
guide_cost = T.mean((layer_3.output-guiding_weights)**2)
test_model = theano.function([index],layer_7.errors(y),
givens={
x: test_set_x[index*batch_size:(index+1)*batch_size],
y: test_set_y[index*batch_size:(index+1)*batch_size]
})
validate_model = theano.function([index],layer_7.errors(y),
givens={
x: valid_set_x[index*batch_size:(index+1)*batch_size],
y: valid_set_y[index*batch_size:(index+1)*batch_size]
})
# list of parameters
params = layer_7.params + layer_6.params + layer_5.params + layer_4.params + layer_3.params + layer_2.params + layer_1.params + layer_0.params
params_gl = layer_3.params + layer_2.params + layer_1.params + layer_0.params
# import pdb
# pdb.set_trace()
grads_gl = T.grad(guide_cost,params_gl)
updates_gl = [ (param_i,param_i-learning_rate/10*grad_i) for param_i,grad_i in zip(params_gl,grads_gl) ]
grads = T.grad(cost,params)
updates = [ (param_i, param_i-learning_rate*grad_i) for param_i, grad_i in zip(params,grads) ]
train_model = theano.function([index,train_epoch],cost, updates=updates,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size],
y: train_set_y[index*batch_size:(index+1)*batch_size],
p_y_given_x: teacher_p_y_given_x[index],
e: train_epoch
})
train_till_guided_layer = theano.function([index],guide_cost,updates = updates_gl,
givens={
x: train_set_x[index*batch_size:(index+1)*batch_size],
y: train_set_y[index*batch_size:(index+1)*batch_size],
guiding_weights : tg[index]
},on_unused_input='ignore')
# -----------------------------------------Starting Training ------------------------------
print ('..... Training ' )
# for early stopping
patience = 10000
patience_increase = 2
improvement_threshold = 0.95
validation_frequency = min(n_train_batches, patience//2)
best_validation_loss = numpy.inf # initialising loss to be inifinite
best_itr = 0
test_score = 0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping) :
epoch = epoch+1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1)*n_train_batches + minibatch_index
if iter%100==0:
print ('training @ iter = ', iter)
if epoch < n_epochs/5:
cost_ij_guided = train_till_guided_layer(minibatch_index)
cost_ij = train_model(minibatch_index,epoch)
if(iter +1)%validation_frequency ==0:
# compute loss on validation set
validation_losses = [validate_model(i) for i in range(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
# import pdb
# pdb.set_trace()
with open('Student_6_terminal_out','a+') as f_:
f_.write('epoch %i, minibatch %i/%i, validation error %f %% \n' %(epoch,minibatch_index+1,n_train_batches,this_validation_loss*100. ))
# check with best validation score till now
if this_validation_loss<best_validation_loss:
# improve
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter*patience_increase)
best_validation_loss = this_validation_loss
best_itr = iter
test_losses = [test_model(i) for i in range(n_test_batches)]
test_score = numpy.mean(test_losses)
with open('Student_6_terminal_out','a+') as f_:
f_.write('epoch %i, minibatch %i/%i, testing error %f %%\n' %(epoch, minibatch_index+1,n_train_batches,test_score*100.))
with open('best_model_7layer.pkl', 'wb') as f:
pickle.dump(params, f)
with open('Results_student_6.txt', 'wb') as f1:
f1.write(str(test_score*100)+'\n')
#if patience <= iter:
# done_looping = True
# break
end_time = timeit.default_timer()
with open('Student_6_terminal_out','a+') as f_:
f_.write('Optimization complete\n')
f_.write('Best validation score of %f %% obtained at iteration %i, with test performance %f %% \n' % (best_validation_loss*100., best_itr, test_score*100 ))
f_.write('The code ran for %.2fm \n' %((end_time - start_time)/60.))
def setUp(self):
self.logreg_learnrate = LogReg(784, 0.5)
self.logreg_nolearnrate = LogReg(784, 1.0)
def setUp(self):
self.logreg_unreg = LogReg(5, 1.0)
class testLogReg(unittest.TestCase):
"""
Every method needs a LogReg object with the vectorizer and model mocked out,
as those objects would perform potentially slow methods otherwise.
"""
def setUp(self):
self.mockV = Mock()
self.mockM = Mock()
self.test_obj = LogReg([
('summer', {'text': "some_text!"}),
('winter', {'text': 'some_text2!'})
],
vectorizer = self.mockV,
model = self.mockM
)
"""
After object creation, a LogReg object should have:
a TfidVectorizer named vectorizer
a list of labels named labels
a LogisticRegression model named model.
model should have called the "fit" method to ensure that the initial model
has been trained.
vectorizer should have called the fit_transform method to create a vector for
the model.fit method.
This test uses mocks to test that the initial training methods are being
called without actually performing the associated operations.
"""
def test_init_trained(self):
assert self.mockV.fit_transform.called #vectorizer "trained"
assert self.mockM.fit.called #model trained
"""
After object creation, the labels variable should contain a list of labels
given in the reviews tuple list. This test mocks the vectorizer and model
and asserts that the expected labels are present after initialization.
"""
def test_init_labels(self):
assert self.test_obj.labels == ['summer', 'winter']
"""
After object creation, the corpus variable should contain a list of training
data given in the reviews tuple list. This test mocks the vectorizer and model
and asserts that the expected review text is present after initialization.
"""
def test_init_corpus(self):
assert self.test_obj.corpus == ['some_text!', 'some_text2!']
"""
After object creation, the review variable should contain the reviews given
upon initialization. This test mocks the vectorizer and model
and asserts that the expected reviews are present after initialization.
"""
def test_init_reviews(self):
assert self.test_obj.reviews == [
('summer', {'text': "some_text!"}),
('winter', {'text': 'some_text2!'})
]
"""
After calling the train method the corpus should be expanded to include the
text for those reviews that are passed into the method. Test mocks the
vectorizer and model created in the initialization of the object and checks
to see that the new reviews are parsed and saved correctly in the corpus
variable.
"""
def test_train_corpus(self):
#setting initial corpus so we don't rely on initialization setting it
#correctly
self.test_obj.corpus = ["Hello world!"]
self.test_obj.train([('summer', {'text': 'some_text!'})])
assert self.test_obj.corpus == ["Hello world!", "some_text!"]
"""
After calling the train method the labels should be expanded to include the
text for those reviews that are passed into the method. Test mocks the
vectorizer and model created in the initialization of the object and checks
to see that the new reviews are parsed and saved correctly in the labels
variable.
"""
def test_train_labels(self):
#setting initial labels so we don't rely on initialization setting it
#correctly
self.test_obj.labels = ["winter"]
self.test_obj.train([('fall', {'text': 'some_text!'})])
assert self.test_obj.labels == ["winter", "fall"]
"""
The train method should call the model's fit method after parsing out the
appropriate data from the given input. This test asserts that the model calls
fit with the expect arguments when the train method is executed.
"""
def test_train_trained(self):
#Mocking the return value for fit_transform, since that result is passed to
#the models' fit method.
self.mockV.fit_transform.return_value = ["THIS IS A VECTOR!"]
#Setting initial labels and corpus
self.test_obj.labels = ["winter"]
self.test_obj.corpus = ["Hello world!"]
self.test_obj.train([('fall', {'text': 'some_text!'})])
self.mockV.fit_transform.assert_called_with(["Hello world!", "some_text!"])
self.mockM.fit.assert_called_with(
["THIS IS A VECTOR!"],
["winter", "fall"]
)
"""
classify_all should complete with a call to the
object's model's predict method, which does the work of predicting the season
for a given test corpus. This test sets the return value for model.predict
and asserts that the returned value of classify_all matches that.
"""
def test_classify_all(self):
self.mockM.predict.return_value = ["summer", "fall", "winter", "spring"]
assert self.test_obj.classify_all([("some", {"text" : "data"})]) == [
"summer",
"fall",
"winter",
"spring"
]
"""
The vocabulary method is a WIP method that should
return a list of strings, with each string representing
a word that can be used to predict a particular season.
This test checks to make sure that at the very least the
vocabulary method is returning a list of strings.
"""
def test_vocabulary(self):
#Mocking the selector used in the method to avoid a lot of computation
mockS = Mock()
mockS.transform.return_value = ["fireplace", "warmth"]
result = self.test_obj.vocabulary([("summer", {"text": "data"})], selector
= mockS)
for i in result:
assert isinstance(i, str)
def run_logreg(train_embs, train_labels, test_embs, test_labels, cuda=False):
train_embs = torch.Tensor(train_embs)
train_labels = torch.Tensor(train_labels)
test_embs = torch.Tensor(test_embs)
test_labels = torch.Tensor(test_labels)
if cuda:
train_embs = train_embs.cuda()
test_embs = test_embs.cuda()
train_labels = train_labels.cuda()
test_labels = test_labels.cuda()
tot = torch.zeros(1)
if cuda:
tot = tot.cuda()
res = []
xent = nn.BCEWithLogitsLoss()
sigmoid = nn.Sigmoid()
dims = train_embs.shape[1]
nb_classes = train_labels.shape[1]
best_epoch = find_epoch(dims, nb_classes, train_embs, train_labels,
test_embs, test_labels, cuda)
print('best epoch', best_epoch)
best_th = 0.0
repeats = 50
for i in range(repeats):
log = LogReg(dims, nb_classes)
opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0)
if cuda:
log.cuda()
for _ in range(best_epoch):
log.train()
opt.zero_grad()
logits = log(train_embs)
loss = xent(logits, train_labels)
loss.backward()
opt.step()
if i == 0:
train_logits = sigmoid(log(train_embs))
if cuda:
best_th = find_best_th(train_logits.cpu(), train_labels.cpu())
else:
best_th = find_best_th(train_logits, train_labels)
print('best threshold:', best_th)
logits = sigmoid(log(test_embs))
zero = torch.zeros_like(logits)
one = torch.ones_like(logits)
preds = torch.where(
logits >= best_th, one,
zero) # ppi is a multi-label classification problem
if cuda:
test_labels = test_labels.cpu()
preds = preds.cpu()
f1 = f1_score(test_labels.numpy(), preds.numpy(), average='micro')
res.append(f1)
print(f1)
tot += f1
print('Average f1:', tot / repeats)
res = np.stack(res)
print(np.mean(res))
print(np.std(res))
'Diastolic blood pressure (mm Hg)',
'Triceps skin fold thickness (mm)',
'2-Hour serum insulin (mu U/ml)',
'Body mass index (weight in kg/(height in m)^2)',
'Diabetes pedigree function',
'Age (years)',
'Class variable (0: no diabetes, 1: diabetes)']
# Load data and perform train test split.
print('[INFO] Loading data.')
data = load_data(file_name, columns)
X_train, X_test, y_train, y_test = train_test_split(data, frac=0.8, seed=5)
# Logistic Model
print('\n[INFO] Training logistic regression model.')
logreg = LogReg(args.lr_logreg, args.epochs, len(X_train[0]))
bias_logreg, weights_logreg = logreg.train(X_train, y_train)
y_logistic = [round(logreg.predict(example)) for example in X_test]
# Perceptron Model
print('[INFO] Training Perceptron model.')
perceptron = Perceptron(args.lr_perceptron, args.epochs, len(X_train[0]))
bias_perceptron, weights_perceptron = perceptron.train(X_train, y_train)
y_perceptron = [round(perceptron.predict(example)) for example in X_test]
# Compare accuracies
accuracy_logistic = get_accuracy(y_logistic, y_test)
accuracy_perceptron = get_accuracy(y_perceptron, y_test)
print('\n[INFO] Logistic Regression Accuracy: {:0.3f}'.format(
accuracy_logistic))
print('[INFO] Perceptron Accuracy: {:0.3f}'.format(accuracy_perceptron))
#processor
data = proc.load_csv(args.CSV_FILE)
data = proc.normalize_col(data, 'Amount')
data = data.drop(['Time'], axis=1)
print data[args.YCOL].value_counts()
X = proc.get_xvals(data, args.YCOL)
y = proc.get_yvals(data, args.YCOL)
#processor xfolds
Xu, yu = proc.under_sample(data, args.YCOL)
Xu_train, Xu_test, yu_train, yu_test = proc.cross_validation_sets(
Xu, yu, .3, 0)
X_train, X_test, y_train, y_test = proc.cross_validation_sets(X, y, .3, 0)
if args.LR_DRIVE:
lin = LogReg()
#under sampled data
c = lin.printing_Kfold_scores(Xu_train, yu_train)
lin.logistic_regression(Xu_train, Xu_test, yu_train, yu_test, c)
lin.logistic_regression(Xu_train, X_test, yu_train, y_test, c)
lin.get_roc_curve(Xu_train, Xu_test, yu_train, yu_test, c)
#regular data
c = lin.printing_Kfold_scores(X_train, y_train)
lin.logistic_regression(X_train, X_test, y_train, y_test, c)
lin.get_roc_curve(X_train, X_test, y_train, y_test, c)
if args.SVM_DRIVE:
sv = SVM()
sv.svm_run(Xu_train, Xu_test, yu_train, yu_test)
if args.SVML_DRIVE:
sv = SVM()
