def demo():
X,y = utils.load_mnist()
y = utils.makeMultiClass(y)
# building the SDA
sDA = StackedDA([100])
# pre-trainning the SDA
sDA.pre_train(X[:100], noise_rate=0.3, epochs=1)
# saving a PNG representation of the first layer
W = sDA.Layers[0].W.T[:, 1:]
utils.saveTiles(W, img_shape= (28,28), tile_shape=(10,10), filename="results/res_dA.png")
# adding the final layer
sDA.finalLayer(X[:37500], y[:37500], epochs=2)
# trainning the whole network
sDA.fine_tune(X[:37500], y[:37500], epochs=2)
# predicting using the SDA
pred = sDA.predict(X[37500:]).argmax(1)
# let's see how the network did
y = y[37500:].argmax(1)
e = 0.0
for i in range(len(y)):
e += y[i]==pred[i]
# printing the result, this structure should result in 80% accuracy
print "accuracy: %2.2f%%"%(100*e/len(y))
return sDA
def demo(structure = [25**2, 23**2, 21**2,19**2,16**2, 15**2]):
# Getting the data
X,y = utils.load_mnist()
autoencoder = StackedDA([100], alpha=0.01)
autoencoder.pre_train(X[:1000], 10)
y = utils.makeMultiClass(y)
autoencoder.fine_tune(X[:1000], y[:1000], learning_layer=200, n_iters=20, alpha=0.01)
W = autoencoder.W[0].T[:, 1:]
W = utils.saveTiles(W, img_shape= (28,28), tile_shape=(10,10), filename="Results/res_dA.png")
def useLayers():
X,y = utils.load_mnist()
y = utils.makeMultiClass(y)
# Layers
sDA = StackedDA([100])
sDA.pre_train(X[:1000], rate=0.5, n_iters=500)
sDA.finalLayer(y[:1000], learner_size=200, n_iters=1)
sDA.fine_tune(X[:1000], y[:1000], n_iters=1)
pred = sDA.predict(X)
W = sDA.Layers[0].W.T[:, 1:]
W = utils.saveTiles(W, img_shape= (28,28), tile_shape=(10,10), filename="Results/res_dA.png")
return pred, y
def useLayers():
X, y = utils.load_mnist()
y = utils.makeMultiClass(y)
# Layers
sDA = StackedDA([100])
sDA.pre_train(X[:1000], rate=0.5, n_iters=500)
sDA.finalLayer(y[:1000], learner_size=200, n_iters=1)
sDA.fine_tune(X[:1000], y[:1000], n_iters=1)
pred = sDA.predict(X)
W = sDA.Layers[0].W.T[:, 1:]
W = utils.saveTiles(W,
img_shape=(28, 28),
tile_shape=(10, 10),
filename="Results/res_dA.png")
return pred, y
def demo(structure=[25**2, 23**2, 21**2, 19**2, 16**2, 15**2]):
# Getting the data
X, y = utils.load_mnist()
autoencoder = StackedDA([100], alpha=0.01)
autoencoder.pre_train(X[:1000], 10)
y = utils.makeMultiClass(y)
autoencoder.fine_tune(X[:1000],
y[:1000],
learning_layer=200,
n_iters=20,
alpha=0.01)
W = autoencoder.W[0].T[:, 1:]
W = utils.saveTiles(W,
img_shape=(28, 28),
tile_shape=(10, 10),
filename="Results/res_dA.png")
def test_DBN(self, finetune_lr=0.1, pretraining_epochs=10,
pretrain_lr=0.01, k=1, training_epochs=10, batch_size=10,
hidden_layers_sizes=[400,400], sidelen=20,
output_folder='DBN_plots', item='test'):
"""
Demonstrates how to train and test a Deep Belief Network.
This is demonstrated on MNIST.
:type finetune_lr: float
:param finetune_lr: learning rate used in the finetune stage
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type k: int
:param k: number of Gibbs steps in CD/PCD
:type training_epochs: int
:param training_epochs: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
:type batch_size: int
:param batch_size: the size of a minibatch
from DLRBM_2D_temporal import *
funcs = DLRBM_2D_temporal()
funcs.test_DBN()
DBN.test_DBN()
"""
traindata_path= 'allLpatches_10x10x5.pklz' #'allLpatches_subs_smaller.pklz' #'allLpatches.pklz'
trainUdata_path= 'allUpatches_10x10x5.pklz'#'allUpatches_subs_smaller.pklz' #'allUpatches.pklz'
labeldata_path= 'allLabels_10x10x5.pklz' #'allLabels_subs_smaller.pklz' #'allLabels.pklz'
#############
## LOAD datasets
#############
datasets = self.load_wUdata(traindata_path, labeldata_path, trainUdata_path)
train_set_x, train_set_y = datasets[0]
np_train_x, np_train_y = datasets[3]
valid_set_x, valid_set_y = datasets[1]
np_valid_x, np_valid_y = datasets[4]
test_set_x, test_set_y = datasets[2]
np_test_x, np_test_y = datasets[5]
#########################
# FORMAT THE DATA
#########################
## transform to pixel intensities between 0 and 1
tnp_train_x= list( utils.scale_to_unit_interval( np.asarray(np_train_x) ) )
tnp_valid_x = list( utils.scale_to_unit_interval( np.asarray(np_valid_x) ) )
tnp_test_x = list( utils.scale_to_unit_interval( np.asarray(np_test_x) ) )
#################
# Substract pathces from pre-contrast
#################
# subsnp_train_x = []
# subsnp_valid_x = []
# subsnp_test_x = []
#
# for img in np_train_x:
# Vol = img.reshape(5,10,10)
# imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
# subslicestime = []
# for k in range(1,5):
# # substract volume
# subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
# subslicestime.append(subVol)
# #append
# subsnp_train_x.append( np.asarray(subslicestime).reshape(4*10*10) )
# for img in np_valid_x:
# Vol = img.reshape(5,10,10)
# imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
# subslicestime = []
# for k in range(1,5):
# # substract volume
# subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
# subslicestime.append(subVol)
# #append
# subsnp_valid_x.append( np.asarray(subslicestime).reshape(4*10*10) )
# for img in np_test_x:
# Vol = img.reshape(5,10,10)
# imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
# subslicestime = []
# for k in range(1,5):
# # substract volume
# subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
# subslicestime.append(subVol)
# #append
# subsnp_test_x.append( np.asarray(subslicestime).reshape(4*10*10) )
#
#
#
# #########################
# # FORMAT THE DATA
# #########################
# ## transform to pixel intensities between 0 and 1
# tsubsnp_train_x = list( utils.scale_to_unit_interval( np.asarray(subsnp_train_x) ) )
# tsubsnp_valid_x = list( utils.scale_to_unit_interval( np.asarray(subsnp_valid_x) ) )
# tsubsnp_test_x = list( utils.scale_to_unit_interval( np.asarray(subsnp_test_x) ) )
#
# # inpect one image class 1/0
# tVol = tsubsnp_train_x[29594].reshape(4,10,10)
# timgslicestime = [tVol[0,:,:], tVol[1,:,:], tVol[2,:,:], tVol[3,:,:]]
#
# # inpect one image class 1/0
# Vol = np_train_x[29594].reshape(5,10,10)
# imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
#
# # Display image
# fig, ax = plt.subplots(nrows=4, ncols=6, figsize=(16, 8))
# for k in range(1,5):
# ax[k-1,0].imshow(imgslicestime[k], cmap=plt.cm.gray)
# ax[k-1,0].set_axis_off()
# ax[k-1,0].set_adjustable('box-forced')
#
# # Display Original histogram
# ax[k-1,1].hist(imgslicestime[k].ravel(), bins=50, color='black')
# ax[k-1,1].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0))
# ax[k-1,1].set_xlabel('original')
#
# # substract volume
# subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
# subslicestime.append(subVol)
#
# # Display subtracted histogram
# ax[k-1,2].hist(subVol.ravel(), bins=50, color='black')
# ax[k-1,2].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0))
# ax[k-1,2].set_xlabel('substracted histogram')
#
# # display subtracted
# ax[k-1,3].imshow(subVol, cmap=plt.cm.gray)
# ax[k-1,3].set_axis_off()
# ax[k-1,3].set_adjustable('box-forced')
#
# # display pixels 0-1 subtracted
# ax[k-1,4].imshow(timgslicestime[k-1], cmap=plt.cm.gray)
# ax[k-1,4].set_axis_off()
# ax[k-1,4].set_adjustable('box-forced')
#
# # display pixels 0-1 subtracted histogram
# ax[k-1,5].hist(timgslicestime[k-1].ravel(), bins=50, color='black')
# ax[k-1,5].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0))
# ax[k-1,5].set_xlabel(' pixels 0-1 subtracted histogram')
#
# plt.show(block=False)
train_set_x, train_set_y = self.shared_dataset( tnp_train_x, np_train_y )
valid_set_x, valid_set_y = self.shared_dataset( tnp_valid_x, np_valid_y )
test_set_x, test_set_y = self.shared_dataset( tnp_test_x, np_test_y )
datasets = [ (train_set_x, train_set_y),
(valid_set_x, valid_set_y),
(test_set_x, test_set_y) ]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
# construct the Deep Belief Network
dbn = DBN(numpy_rng=numpy_rng, n_ins=5*10*10,
hidden_layers_sizes=hidden_layers_sizes,
n_outs=2)
#########################
# PRETRAINING THE MODEL #
#########################
DBN_avg_costs = []
DBN_iter = []
layer_i = []
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = timeit.default_timer()
## Pre-train layer-wise
for i in range(dbn.n_layers):
# go through pretraining epochs
for epoch in range(pretraining_epochs):
# go through the training set
c = []
if( epoch % 50 == 0):
pretrain_lr = pretrain_lr/10
for batch_index in range(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
lr=pretrain_lr))
# append
DBN_avg_costs.append( np.mean(c) )
DBN_iter.append(epoch)
layer_i.append(i)
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = timeit.default_timer()
print 'The pretraining code ran for %.2fm' % ((end_time - start_time) / 60.)
#####################################
# Plot images in 2D
#####################################
Wrbm = dbn.rbm_layers[0].W.get_value(borrow=True).T
fig, axes = plt.subplots(ncols=2, nrows=2, figsize=(16, 6))
axes = axes.flatten()
for k in range(1,5):
image = Image.fromarray(
utils.tile_raster_images(
X=Wrbm.reshape( Wrbm.shape[0], 5, 10, 10)[:,k,:,:],
img_shape=(10,10),
tile_shape=(sidelen,sidelen),
tile_spacing=(1, 1) ))
im = axes[k-1].imshow(image, cmap="Greys_r")
axes[k-1].get_xaxis().set_visible(False)
axes[k-1].get_yaxis().set_visible(False)
cax,kw = mpl.colorbar.make_axes([ax for ax in axes.flat])
plt.colorbar(im, cax=cax, **kw)
fig.savefig(output_folder+'/filters_dbn'+str(item)+'.pdf')
##############
# Format
#################
LLdata = [float(L) for L in DBN_avg_costs]
LLiter = [float(it) for it in DBN_iter]
LLilayer = [ilayer for ilayer in layer_i]
dfpredata = pd.DataFrame( LLdata )
dfpredata.columns = ['LL_iter']
dfpredata['iter'] = LLiter
dfpredata['layer'] = LLilayer
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = dbn.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr)
############
### for plotting likelihood or cost, accumulate returns of train_model
############
minibatch_avg_costs = []
minibatch_iter = []
minibatch_loss = []
print '... finetuning the model'
# early-stopping parameters
patience = 1000 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatches before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = timeit.default_timer()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1,
n_train_batches,
this_validation_loss * 100.))
##############
# append
#################
minibatch_avg_costs.append(minibatch_avg_cost)
minibatch_iter.append(iter)
minibatch_loss.append(this_validation_loss*100)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print('Optimization complete with best validation score of %f %%, '
'obtained at iteration %i, '
'with test performance %f %%') % (best_validation_loss * 100., best_iter + 1, test_score * 100.)
print 'The fine tuning code ran for %.2fm' % ((end_time - start_time)/ 60.)
##############
# Format
#################
LLdata = [float(L) for L in minibatch_avg_costs]
LLiter = [float(it) for it in minibatch_iter]
LLoss = [float(l) for l in minibatch_loss]
dfinedata = pd.DataFrame( LLdata )
dfinedata.columns = ['LL_iter']
dfinedata['iter'] = LLiter
dfinedata['loss'] = LLoss
###############
## Predictions
###############
# get training data in numpy format
X,y = tnp_train_x, np_train_y # datasets[3]
Xtrain = np.asarray(X)
ytrain = utils.makeMultiClass(y)
# get valid data in numpy format
X,y = tnp_valid_x, np_valid_y # datasets[4]
Xvalid = np.asarray(X)
yvalid = utils.makeMultiClass(y)
# get test data in numpy format
X,y = tnp_test_x, np_test_y # datasets[5]
Xtest = np.asarray(X)
ytest = utils.makeMultiClass(y)
###############
# predicting using the SDA
###############
# in train
predtrain = dbn.predict_functions(Xtrain).argmax(1)
# let's see how the network did
y = ytrain.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predtrain[i]
if(y[i] == 0):
e0 += y[i]==predtrain[i]
# printing the result, this structure should result in 80% accuracy
Acutrain0=100*e0/y0
Acutrain1=100*e1/y1
print "Train Accuracy for class 0: %2.2f%%"%(Acutrain0)
print "Train Accuracy for class 1: %2.2f%%"%(Acutrain1)
# in Valid
predvalid = dbn.predict_functions(Xvalid).argmax(1)
# let's see how the network did
y = yvalid.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predvalid[i]
if(y[i] == 0):
e0 += y[i]==predvalid[i]
# printing the result, this structure should result in 80% accuracy
Acuvalid0=100*e0/y0
Acuvalid1=100*e1/y1
print "Valid Accuracy for class 0: %2.2f%%"%(Acuvalid0)
print "Valid Accuracy for class 1: %2.2f%%"%(Acuvalid1)
# in Xtest
predtest = dbn.predict_functions(Xtest).argmax(1)
# let's see how the network did
y = ytest.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predtest[i]
if(y[i] == 0):
e0 += y[i]==predtest[i]
# printing the result, this structure should result in 80% accuracy
Acutest0=100*e0/y0
Acutest1=100*e1/y1
print "Test Accuracy for class 0: %2.2f%%"%(Acutest0)
print "Test Accuracy for class 1: %2.2f%%"%(Acutest1)
return [dfpredata, dfinedata, dbn,
Acutrain0, Acutrain1,
Acuvalid0, Acuvalid1,
Acutest0, Acutest1]
def test_SdA_timep(self, pretraining_epochs, pretrain_lr, batch_size,
training_epochs, finetune_lr,
corruption_levels,
hidden_layers_sizes, hidden_layers_sidelen, output_folder):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
:type learning_rate: float
:param learning_rate: learning rate used in the finetune stage
(factor for the stochastic gradient)
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
"""
traindata_path='Z://Cristina//Section3//SdA_experiments//allLpatches.pklz'
trainUdata_path='Z://Cristina//Section3//SdA_experiments//allUpatches.pklz'
labeldata_path='Z://Cristina//Section3//SdA_experiments//allLabels.pklz'
#############
## LOAD datasets
#############
datasets = self.load_wUdata(traindata_path, labeldata_path, trainUdata_path)
train_set_x, train_set_y = datasets[0]
np_train_x, np_train_y = datasets[3]
valid_set_x, valid_set_y = datasets[1]
np_valid_x, np_valid_y = datasets[4]
test_set_x, test_set_y = datasets[2]
np_test_x, np_test_y = datasets[5]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
# numpy random generator
numpy_rng = np.random.RandomState(89677)
print('... building the Stacked Autoenconder model')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins = 30*30,
hidden_layers_sizes=hidden_layers_sizes,
corruption_levels=corruption_levels,
n_outs=2
)
#########################
# PRETRAINING THE MODEL #
#########################
dA_avg_costs = []
dA_iter = []
layer_i = []
print('... getting the pretraining functions')
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x,
np_train_y=np_train_y,
batch_size=batch_size)
print('... pre-training the model')
start_time = timeit.default_timer()
## Pre-train layer-wise
for i in range(sda.n_layers):
# go through pretraining epochs
meanc = []
for epoch in range(pretraining_epochs):
# go through the training set
c = [];
for batch_index in range(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
# append
dA_avg_costs.append( np.mean(c) )
dA_iter.append(epoch)
layer_i.append(i)
print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, np.mean(c)))
# check that Cost does not improve more than 1% on the training set after an epoch
meanc.append( np.mean(c) )
if (epoch > 100):
decrCost = abs(meanc[epoch-1] - meanc[epoch])/meanc[epoch]*100
if(decrCost <= 0.01):
break
#####################################
# Plot images in 2D
#####################################
Xtmp = sda.dA_layers[0].W.get_value(borrow=True).T
#imgX = Xtmp.reshape( Xtmp.shape[0], 30, 30)
image = Image.fromarray(
tile_raster_images(X=Xtmp , img_shape=(30, 30),
tile_shape=(10, 10),
tile_spacing=(1, 1)))
end_time = timeit.default_timer()
print(('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.)))
##############
# Format
#################
LLdata = [float(L) for L in dA_avg_costs]
LLiter = [float(it) for it in dA_iter]
LLilayer = [ilayer for ilayer in layer_i]
dfpredata = pd.DataFrame( LLdata )
dfpredata.columns = ['LL_iter']
dfpredata['iter'] = LLiter
dfpredata['layer'] = LLilayer
########################
# FINETUNING THE MODEL #
########################
# for fine tunning make minibatch size = 1
# compute number of minibatches for training, validation and testing
#batch_size = 4
#n_train_batches = train_set_x.get_value(borrow=True).shape[0]//4
# get the training, validation and testing function for the model
print('... getting the finetuning functions')
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
print('... finetunning the Stacked model')
############
### for plotting likelihood or cost, accumulate returns of train_model
############
minibatch_avg_costs = []
minibatch_iter = []
minibatch_loss = []
# early-stopping parameters
patience = 1000 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience // 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = np.inf
test_score = 0.
start_time = timeit.default_timer()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = np.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1,
n_train_batches,
this_validation_loss * 100.))
##############
# append
#################
minibatch_avg_costs.append(minibatch_avg_cost)
minibatch_iter.append(iter)
minibatch_loss.append(this_validation_loss*100)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = np.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print 'Optimization complete with best validation score of %f, on iteration %i, with test performance %f' % ((best_validation_loss * 100.), (best_iter + 1), (test_score * 100.))
print(('The training code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.)))
##############
# Format
#################
LLdata = [float(L) for L in minibatch_avg_costs]
LLiter = [float(it) for it in minibatch_iter]
LLoss = [float(l) for l in minibatch_loss]
dfinedata = pd.DataFrame( LLdata )
dfinedata.columns = ['LL_iter']
dfinedata['iter'] = LLiter
dfinedata['loss'] = LLoss
###############
## Predictions
###############
# get training data in numpy format
X,y = datasets[3]
Xtrain = np.asarray(X)
# extract one img
Xtrain = Xtrain.reshape(Xtrain.shape[0], 4, 900)
Xtrain = Xtrain[:,0,:]
ytrain = utils.makeMultiClass(y)
# get valid data in numpy format
X,y = datasets[4]
Xvalid = np.asarray(X)
# extract one img
Xvalid = Xvalid.reshape(Xvalid.shape[0], 4, 900)
Xvalid = Xvalid[:,0,:]
yvalid = utils.makeMultiClass(y)
X,y = datasets[5]
Xtest = np.asarray(X)
# extract one img
Xtest = Xtest.reshape(Xtest.shape[0], 4, 900)
Xtest = Xtest[:,0,:]
ytest = utils.makeMultiClass(y)
###############
# predicting using the SDA
###############
# in train
predtrain = sda.predict_functions(Xtrain).argmax(1)
# let's see how the network did
y = ytrain.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predtrain[i]
if(y[i] == 0):
e0 += y[i]==predtrain[i]
# printing the result, this structure should result in 80% accuracy
Acutrain0=100*e0/y0
Acutrain1=100*e1/y1
print "Train Accuracy for class 0: %2.2f%%"%(Acutrain0)
print "Train Accuracy for class 1: %2.2f%%"%(Acutrain1)
# in Valid
predvalid = sda.predict_functions(Xvalid).argmax(1)
# let's see how the network did
y = yvalid.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predvalid[i]
if(y[i] == 0):
e0 += y[i]==predvalid[i]
# printing the result, this structure should result in 80% accuracy
Acuvalid0=100*e0/y0
Acuvalid1=100*e1/y1
print "Valid Accuracy for class 0: %2.2f%%"%(Acuvalid0)
print "Valid Accuracy for class 1: %2.2f%%"%(Acuvalid1)
# in Xtest
predtest = sda.predict_functions(Xtest).argmax(1)
# let's see how the network did
y = ytest.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predtest[i]
if(y[i] == 0):
e0 += y[i]==predtest[i]
# printing the result, this structure should result in 80% accuracy
Acutest0=100*e0/y0
Acutest1=100*e1/y1
print "Test Accuracy for class 0: %2.2f%%"%(Acutest0)
print "Test Accuracy for class 1: %2.2f%%"%(Acutest1)
return [dfpredata, dfinedata, image, sda,
Acutrain0, Acutrain1,
Acuvalid0, Acuvalid1,
Acutest0, Acutest1]
def test_SdA_timep(self, pretraining_epochs, pretrain_lr, batch_size,
training_epochs, finetune_lr,
corruption_levels,
hidden_layers_sizes, hidden_layers_sidelen, output_folder):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
:type learning_rate: float
:param learning_rate: learning rate used in the finetune stage
(factor for the stochastic gradient)
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
"""
traindata_path= 'allLpatches_10x10x5.pklz' #'allLpatches_subs_smaller.pklz' #'allLpatches.pklz'
trainUdata_path= 'allUpatches_10x10x5.pklz'#'allUpatches_subs_smaller.pklz' #'allUpatches.pklz'
labeldata_path= 'allLabels_10x10x5.pklz' #'allLabels_subs_smaller.pklz' #'allLabels.pklz'
#############
## LOAD datasets
#############
datasets = self.load_wUdata(traindata_path, labeldata_path, trainUdata_path)
train_set_x, train_set_y = datasets[0]
np_train_x, np_train_y = datasets[3]
valid_set_x, valid_set_y = datasets[1]
np_valid_x, np_valid_y = datasets[4]
test_set_x, test_set_y = datasets[2]
np_test_x, np_test_y = datasets[5]
# inpect one image class 1
Vol = np_train_x[0].reshape(5,10,10)
imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
subslicestime = []
# Display image
fig, ax = plt.subplots(nrows=4, ncols=4, figsize=(12, 4))
for k in range(1,5):
ax[k-1,0].imshow(imgslicestime[k], cmap=plt.cm.gray)
ax[k-1,0].set_axis_off()
ax[k-1,0].set_adjustable('box-forced')
# Display Original histogram
ax[k-1,1].hist(imgslicestime[k].ravel(), bins=50, color='black')
ax[k-1,1].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0))
ax[k-1,1].set_xlabel('original')
# substract volume
subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
subslicestime.append(subVol)
# Display normalized histogram
ax[k-1,2].hist(subVol.ravel(), bins=50, color='black')
ax[k-1,2].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0))
ax[k-1,2].set_xlabel('substracted histogram')
# display normalized 0-1
ax[k-1,3].imshow(subVol, cmap=plt.cm.gray)
ax[k-1,3].set_axis_off()
ax[k-1,3].set_adjustable('box-forced')
plt.close()
#################
# Substract pathces from pre-contrast
#################
subsnp_train_x = []
subsnp_valid_x = []
subsnp_test_x = []
for img in np_train_x:
Vol = img.reshape(5,10,10)
imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
subslicestime = []
for k in range(1,5):
# substract volume
subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
subslicestime.append(subVol)
#append
subsnp_train_x.append( np.asarray(subslicestime).reshape(4*10*10) )
for img in np_valid_x:
Vol = img.reshape(5,10,10)
imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
subslicestime = []
for k in range(1,5):
# substract volume
subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
subslicestime.append(subVol)
#append
subsnp_valid_x.append( np.asarray(subslicestime).reshape(4*10*10) )
for img in np_test_x:
Vol = img.reshape(5,10,10)
imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]]
subslicestime = []
for k in range(1,5):
# substract volume
subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0])
subslicestime.append(subVol)
#append
subsnp_test_x.append( np.asarray(subslicestime).reshape(4*10*10) )
train_set_x, train_set_y = self.shared_dataset(subsnp_train_x, np_train_y)
valid_set_x, valid_set_y = self.shared_dataset(subsnp_valid_x, np_valid_y)
test_set_x, test_set_y = self.shared_dataset(subsnp_test_x, np_test_y)
subsdatasets = [(train_set_x, train_set_y),
(valid_set_x, valid_set_y),
(test_set_x, test_set_y) ]
#################
# To normalize find mean and std, then convert to float in [0-1] range
# ref on mininatch normalization: chrome-extension://oemmndcbldboiebfnladdacbdfmadadm/https://arxiv.org/pdf/1502.03167v3.pdf
# stackedOver: http://stats.stackexchange.com/questions/211436/why-do-we-normalize-images-by-subtracting-the-datasets-image-mean-and-not-the-c?rq=1
#################
# Xtrain = np.asarray(np_train_x)
# muXtrain = np.mean(Xtrain)
# varXtrain = np.var(Xtrain)
# print(muXtrain,varXtrain)
#
# Xvalid = np.asarray(np_valid_x)
# muXvalid = np.mean(Xvalid)
# varXvalid = np.var(Xvalid)
# print(muXvalid,varXvalid)
#
# Xtest = np.asarray(np_test_x)
# muXtest = np.mean(Xtest)
# varXtest = np.var(Xtest)
# print(muXtest,varXtest)
#
# fnp_train_x = []
# fnp_valid_x = []
# fnp_test_x = []
# for img in np_train_x:
# #img=np_train_x[1]
# fimg = (img - muXtrain)/np.sqrt(varXtrain+0.00001)
# rVol = exposure.rescale_intensity(fimg.reshape(4,10,10), out_range=(0, 1))
# img = sitk.GetImageFromArray(rVol)
# aimg = sitk.Cast(img,sitk.sitkFloat32)
# sitkVol = sitk.GetArrayFromImage(aimg)
# img = sitkVol.reshape(400)
# fnp_train_x.append(img)
# for img in np_valid_x:
# fimg = (img - muXvalid)/np.sqrt(varXvalid+0.00001)
# rVol = exposure.rescale_intensity(fimg.reshape(4,10,10), out_range=(0, 1))
# img = sitk.GetImageFromArray(rVol)
# aimg = sitk.Cast(img,sitk.sitkFloat32)
# sitkVol = sitk.GetArrayFromImage(aimg)
# img = sitkVol.reshape(400)
# fnp_valid_x.append(img)
# for img in np_test_x:
# fimg = (img - muXtest)/np.sqrt(varXtest+0.00001)
# rVol = exposure.rescale_intensity(fimg.reshape(4,10,10), out_range=(0, 1))
# img = sitk.GetImageFromArray(rVol)
# aimg = sitk.Cast(img,sitk.sitkFloat32)
# sitkVol = sitk.GetArrayFromImage(aimg)
# img = sitkVol.reshape(400)
# fnp_test_x.append(img)
#
#################
## plot ooptional
#################
# fig, axes = plt.subplots(ncols=4, nrows=2)
# axes[0,0].imshow(np_train_x[1].reshape(5,10,10)[0,:,:], cmap="Greys_r")
# axes[0,1].imshow(np_train_x[1].reshape(5,10,10)[1,:,:], cmap="Greys_r")
# axes[0,2].imshow(np_train_x[1].reshape(5,10,10)[2,:,:], cmap="Greys_r")
# axes[0,3].imshow(np_train_x[1].reshape(5,10,10)[3,:,:], cmap="Greys_r")
# axes[1,0].imshow( fnp_train_x[1].reshape(5,10,10)[0,:,:], cmap="Greys_r")
# axes[1,1].imshow( fnp_train_x[1].reshape(5,10,10)[1,:,:], cmap="Greys_r")
# axes[1,2].imshow( fnp_train_x[1].reshape(5,10,10)[2,:,:], cmap="Greys_r")
# axes[1,3].imshow( fnp_train_x[1].reshape(5,10,10)[3,:,:], cmap="Greys_r")
# train_set_x, train_set_y = self.shared_dataset(fnp_train_x, np_train_y)
# valid_set_x, valid_set_y = self.shared_dataset(fnp_valid_x, np_valid_y)
# test_set_x, test_set_y = self.shared_dataset(fnp_test_x, np_test_y)
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
# numpy random generator
numpy_rng = np.random.RandomState(89677)
print('... building the Stacked Autoenconder model')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins = 4*10*10, # before 30*30*4
hidden_layers_sizes=hidden_layers_sizes,
corruption_levels=corruption_levels,
n_outs=2
)
#########################
# PRETRAINING THE MODEL #
#########################
dA_avg_costs = []
dA_iter = []
layer_i = []
print('... getting the pretraining functions')
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x,
np_train_y=np_train_y,
batch_size=batch_size)
print('... pre-training the model')
start_time = timeit.default_timer()
## Pre-train layer-wise
for i in range(sda.n_layers):
# go through pretraining epochs
meanc = []
for epoch in range(pretraining_epochs):
# go through the training set
c = [];
if( epoch % 100 == 0):
pretrain_lr = pretrain_lr/10
for batch_index in range(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
# append
dA_avg_costs.append( np.mean(c) )
dA_iter.append(epoch)
layer_i.append(i)
print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, np.mean(c)))
# check that Cost does not improve more than 1% on the training set after an epoch
meanc.append( np.mean(c) )
if (epoch > 195):
decrCost = abs(meanc[epoch-1] - meanc[epoch])/meanc[epoch]*100
if(decrCost <= 0.01):
break
end_time = timeit.default_timer()
print(('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.)))
#####################################
# Plot images in 2D
#####################################
Xtmp = sda.dA_layers[0].W.get_value(borrow=True).T
imagefilters = []
for k in range(4):
imgX = Xtmp.reshape( Xtmp.shape[0], 4, 10, 10)[:,k,:,:]
image = tile_images(X=imgX , img_shape=(10, 10),
tile_shape=(10, 10),
tile_spacing=(1, 1))
# append to all
imagefilters.append( image )
##############
# Format
#################
LLdata = [float(L) for L in dA_avg_costs]
LLiter = [float(it) for it in dA_iter]
LLilayer = [ilayer for ilayer in layer_i]
dfpredata = pd.DataFrame( LLdata )
dfpredata.columns = ['LL_iter']
dfpredata['iter'] = LLiter
dfpredata['layer'] = LLilayer
########################
# FINETUNING THE MODEL #
########################
# for fine tunning make minibatch size = 1
# compute number of minibatches for training, validation and testing
#batch_size = 4
#n_train_batches = train_set_x.get_value(borrow=True).shape[0]//4
# get the training, validation and testing function for the model
print('... getting the finetuning functions')
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=subsdatasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
print('... finetunning the Stacked model')
############
### for plotting likelihood or cost, accumulate returns of train_model
############
minibatch_avg_costs = []
minibatch_iter = []
minibatch_loss = []
# early-stopping parameters
patience = 1000 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience // 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = np.inf
test_score = 0.
start_time = timeit.default_timer()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = np.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1,
n_train_batches,
this_validation_loss * 100.))
##############
# append
#################
minibatch_avg_costs.append(minibatch_avg_cost)
minibatch_iter.append(iter)
minibatch_loss.append(this_validation_loss*100)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = np.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print 'Optimization complete with best validation score of %f, on iteration %i, with test performance %f' % ((best_validation_loss * 100.), (best_iter + 1), (test_score * 100.))
print(('The training code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.)))
##############
# Format
#################
LLdata = [float(L) for L in minibatch_avg_costs]
LLiter = [float(it) for it in minibatch_iter]
LLoss = [float(l) for l in minibatch_loss]
dfinedata = pd.DataFrame( LLdata )
dfinedata.columns = ['LL_iter']
dfinedata['iter'] = LLiter
dfinedata['loss'] = LLoss
###############
## Predictions
###############
# get training data in numpy format
X,y = subsnp_train_x, np_train_y #datasets[3]
Xtrain = np.asarray(X)
ytrain = utils.makeMultiClass(y)
# get valid data in numpy format
X,y = subsnp_valid_x, np_valid_y # #datasets[4]
Xvalid = np.asarray(X)
yvalid = utils.makeMultiClass(y)
# get test data in numpy format
X,y = subsnp_test_x, np_test_y #datasets[5]
Xtest = np.asarray(X)
ytest = utils.makeMultiClass(y)
###############
# predicting using the SDA
###############
# in train
predtrain = sda.predict_functions(Xtrain).argmax(1)
# let's see how the network did
y = ytrain.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predtrain[i]
if(y[i] == 0):
e0 += y[i]==predtrain[i]
# printing the result, this structure should result in 80% accuracy
Acutrain0=100*e0/y0
Acutrain1=100*e1/y1
print "Train Accuracy for class 0: %2.2f%%"%(Acutrain0)
print "Train Accuracy for class 1: %2.2f%%"%(Acutrain1)
# in Valid
predvalid = sda.predict_functions(Xvalid).argmax(1)
# let's see how the network did
y = yvalid.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predvalid[i]
if(y[i] == 0):
e0 += y[i]==predvalid[i]
# printing the result, this structure should result in 80% accuracy
Acuvalid0=100*e0/y0
Acuvalid1=100*e1/y1
print "Valid Accuracy for class 0: %2.2f%%"%(Acuvalid0)
print "Valid Accuracy for class 1: %2.2f%%"%(Acuvalid1)
# in Xtest
predtest = sda.predict_functions(Xtest).argmax(1)
# let's see how the network did
y = ytest.argmax(1)
e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0])
e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1])
for i in range(len(y)):
if(y[i] == 1):
e1 += y[i]==predtest[i]
if(y[i] == 0):
e0 += y[i]==predtest[i]
# printing the result, this structure should result in 80% accuracy
Acutest0=100*e0/y0
Acutest1=100*e1/y1
print "Test Accuracy for class 0: %2.2f%%"%(Acutest0)
print "Test Accuracy for class 1: %2.2f%%"%(Acutest1)
return [dfpredata, dfinedata, imagefilters, sda,
Acutrain0, Acutrain1,
Acuvalid0, Acuvalid1,
Acutest0, Acutest1]
Train[x][len(Train[x])-1] = int(Train[x][len(Train[x])-1])
ClassTrain += [Train[x][len(Train[x])-1]]
del(Train[x][len(Train[x])-1])
emg = nk.emg_process(Train[x])
DataTrain += [emg["df"]["EMG_Envelope"]]
with open("../../Apps/test.csv","r") as f:
reader = csv.reader(f)
Test = list(reader)
for x in (range(0,len(Test))):
Test[x] =[float(n.strip('"')) for n in Test[x]]
Test[x][len(Test[x])-1] = int(Test[x][len(Test[x])-1])
ClassTest += [Test[x][len(Test[x])-1]]
del(Test[x][len(Test[x])-1])
emg = nk.emg_process(Test[x])
DataTest += [emg["df"]["EMG_Envelope"]]
ClassTest = makeMultiClass(ClassTest)
ClassTrain = makeMultiClass(ClassTrain)
print(ClassTrain[0])
my_df = pd.DataFrame(DataTrain)
my_df.to_csv('../data/trainX.csv', index=False, header=False)
my_df = pd.DataFrame(ClassTrain)
my_df.to_csv('../data/trainY.csv', index=False, header=False)
my_df = pd.DataFrame(DataTest)
my_df.to_csv('../data/testX.csv', index=False, header=False)
my_df = pd.DataFrame(ClassTest)
my_df.to_csv('../data/testY.csv', index=False, header=False)