def demo(): X,y = utils.load_mnist() y = utils.makeMultiClass(y) # building the SDA sDA = StackedDA([100]) # pre-trainning the SDA sDA.pre_train(X[:100], noise_rate=0.3, epochs=1) # saving a PNG representation of the first layer W = sDA.Layers[0].W.T[:, 1:] utils.saveTiles(W, img_shape= (28,28), tile_shape=(10,10), filename="results/res_dA.png") # adding the final layer sDA.finalLayer(X[:37500], y[:37500], epochs=2) # trainning the whole network sDA.fine_tune(X[:37500], y[:37500], epochs=2) # predicting using the SDA pred = sDA.predict(X[37500:]).argmax(1) # let's see how the network did y = y[37500:].argmax(1) e = 0.0 for i in range(len(y)): e += y[i]==pred[i] # printing the result, this structure should result in 80% accuracy print "accuracy: %2.2f%%"%(100*e/len(y)) return sDA
def demo(structure = [25**2, 23**2, 21**2,19**2,16**2, 15**2]): # Getting the data X,y = utils.load_mnist() autoencoder = StackedDA([100], alpha=0.01) autoencoder.pre_train(X[:1000], 10) y = utils.makeMultiClass(y) autoencoder.fine_tune(X[:1000], y[:1000], learning_layer=200, n_iters=20, alpha=0.01) W = autoencoder.W[0].T[:, 1:] W = utils.saveTiles(W, img_shape= (28,28), tile_shape=(10,10), filename="Results/res_dA.png")
def useLayers(): X,y = utils.load_mnist() y = utils.makeMultiClass(y) # Layers sDA = StackedDA([100]) sDA.pre_train(X[:1000], rate=0.5, n_iters=500) sDA.finalLayer(y[:1000], learner_size=200, n_iters=1) sDA.fine_tune(X[:1000], y[:1000], n_iters=1) pred = sDA.predict(X) W = sDA.Layers[0].W.T[:, 1:] W = utils.saveTiles(W, img_shape= (28,28), tile_shape=(10,10), filename="Results/res_dA.png") return pred, y
def useLayers(): X, y = utils.load_mnist() y = utils.makeMultiClass(y) # Layers sDA = StackedDA([100]) sDA.pre_train(X[:1000], rate=0.5, n_iters=500) sDA.finalLayer(y[:1000], learner_size=200, n_iters=1) sDA.fine_tune(X[:1000], y[:1000], n_iters=1) pred = sDA.predict(X) W = sDA.Layers[0].W.T[:, 1:] W = utils.saveTiles(W, img_shape=(28, 28), tile_shape=(10, 10), filename="Results/res_dA.png") return pred, y
def demo(structure=[25**2, 23**2, 21**2, 19**2, 16**2, 15**2]): # Getting the data X, y = utils.load_mnist() autoencoder = StackedDA([100], alpha=0.01) autoencoder.pre_train(X[:1000], 10) y = utils.makeMultiClass(y) autoencoder.fine_tune(X[:1000], y[:1000], learning_layer=200, n_iters=20, alpha=0.01) W = autoencoder.W[0].T[:, 1:] W = utils.saveTiles(W, img_shape=(28, 28), tile_shape=(10, 10), filename="Results/res_dA.png")
def test_DBN(self, finetune_lr=0.1, pretraining_epochs=10, pretrain_lr=0.01, k=1, training_epochs=10, batch_size=10, hidden_layers_sizes=[400,400], sidelen=20, output_folder='DBN_plots', item='test'): """ Demonstrates how to train and test a Deep Belief Network. This is demonstrated on MNIST. :type finetune_lr: float :param finetune_lr: learning rate used in the finetune stage :type pretraining_epochs: int :param pretraining_epochs: number of epoch to do pretraining :type pretrain_lr: float :param pretrain_lr: learning rate to be used during pre-training :type k: int :param k: number of Gibbs steps in CD/PCD :type training_epochs: int :param training_epochs: maximal number of iterations ot run the optimizer :type dataset: string :param dataset: path the the pickled dataset :type batch_size: int :param batch_size: the size of a minibatch from DLRBM_2D_temporal import * funcs = DLRBM_2D_temporal() funcs.test_DBN() DBN.test_DBN() """ traindata_path= 'allLpatches_10x10x5.pklz' #'allLpatches_subs_smaller.pklz' #'allLpatches.pklz' trainUdata_path= 'allUpatches_10x10x5.pklz'#'allUpatches_subs_smaller.pklz' #'allUpatches.pklz' labeldata_path= 'allLabels_10x10x5.pklz' #'allLabels_subs_smaller.pklz' #'allLabels.pklz' ############# ## LOAD datasets ############# datasets = self.load_wUdata(traindata_path, labeldata_path, trainUdata_path) train_set_x, train_set_y = datasets[0] np_train_x, np_train_y = datasets[3] valid_set_x, valid_set_y = datasets[1] np_valid_x, np_valid_y = datasets[4] test_set_x, test_set_y = datasets[2] np_test_x, np_test_y = datasets[5] ######################### # FORMAT THE DATA ######################### ## transform to pixel intensities between 0 and 1 tnp_train_x= list( utils.scale_to_unit_interval( np.asarray(np_train_x) ) ) tnp_valid_x = list( utils.scale_to_unit_interval( np.asarray(np_valid_x) ) ) tnp_test_x = list( utils.scale_to_unit_interval( np.asarray(np_test_x) ) ) ################# # Substract pathces from pre-contrast ################# # subsnp_train_x = [] # subsnp_valid_x = [] # subsnp_test_x = [] # # for img in np_train_x: # Vol = img.reshape(5,10,10) # imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] # subslicestime = [] # for k in range(1,5): # # substract volume # subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) # subslicestime.append(subVol) # #append # subsnp_train_x.append( np.asarray(subslicestime).reshape(4*10*10) ) # for img in np_valid_x: # Vol = img.reshape(5,10,10) # imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] # subslicestime = [] # for k in range(1,5): # # substract volume # subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) # subslicestime.append(subVol) # #append # subsnp_valid_x.append( np.asarray(subslicestime).reshape(4*10*10) ) # for img in np_test_x: # Vol = img.reshape(5,10,10) # imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] # subslicestime = [] # for k in range(1,5): # # substract volume # subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) # subslicestime.append(subVol) # #append # subsnp_test_x.append( np.asarray(subslicestime).reshape(4*10*10) ) # # # # ######################### # # FORMAT THE DATA # ######################### # ## transform to pixel intensities between 0 and 1 # tsubsnp_train_x = list( utils.scale_to_unit_interval( np.asarray(subsnp_train_x) ) ) # tsubsnp_valid_x = list( utils.scale_to_unit_interval( np.asarray(subsnp_valid_x) ) ) # tsubsnp_test_x = list( utils.scale_to_unit_interval( np.asarray(subsnp_test_x) ) ) # # # inpect one image class 1/0 # tVol = tsubsnp_train_x[29594].reshape(4,10,10) # timgslicestime = [tVol[0,:,:], tVol[1,:,:], tVol[2,:,:], tVol[3,:,:]] # # # inpect one image class 1/0 # Vol = np_train_x[29594].reshape(5,10,10) # imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] # # # Display image # fig, ax = plt.subplots(nrows=4, ncols=6, figsize=(16, 8)) # for k in range(1,5): # ax[k-1,0].imshow(imgslicestime[k], cmap=plt.cm.gray) # ax[k-1,0].set_axis_off() # ax[k-1,0].set_adjustable('box-forced') # # # Display Original histogram # ax[k-1,1].hist(imgslicestime[k].ravel(), bins=50, color='black') # ax[k-1,1].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0)) # ax[k-1,1].set_xlabel('original') # # # substract volume # subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) # subslicestime.append(subVol) # # # Display subtracted histogram # ax[k-1,2].hist(subVol.ravel(), bins=50, color='black') # ax[k-1,2].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0)) # ax[k-1,2].set_xlabel('substracted histogram') # # # display subtracted # ax[k-1,3].imshow(subVol, cmap=plt.cm.gray) # ax[k-1,3].set_axis_off() # ax[k-1,3].set_adjustable('box-forced') # # # display pixels 0-1 subtracted # ax[k-1,4].imshow(timgslicestime[k-1], cmap=plt.cm.gray) # ax[k-1,4].set_axis_off() # ax[k-1,4].set_adjustable('box-forced') # # # display pixels 0-1 subtracted histogram # ax[k-1,5].hist(timgslicestime[k-1].ravel(), bins=50, color='black') # ax[k-1,5].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0)) # ax[k-1,5].set_xlabel(' pixels 0-1 subtracted histogram') # # plt.show(block=False) train_set_x, train_set_y = self.shared_dataset( tnp_train_x, np_train_y ) valid_set_x, valid_set_y = self.shared_dataset( tnp_valid_x, np_valid_y ) test_set_x, test_set_y = self.shared_dataset( tnp_test_x, np_test_y ) datasets = [ (train_set_x, train_set_y), (valid_set_x, valid_set_y), (test_set_x, test_set_y) ] # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size # numpy random generator numpy_rng = numpy.random.RandomState(123) print '... building the model' # construct the Deep Belief Network dbn = DBN(numpy_rng=numpy_rng, n_ins=5*10*10, hidden_layers_sizes=hidden_layers_sizes, n_outs=2) ######################### # PRETRAINING THE MODEL # ######################### DBN_avg_costs = [] DBN_iter = [] layer_i = [] print '... getting the pretraining functions' pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x, batch_size=batch_size, k=k) print '... pre-training the model' start_time = timeit.default_timer() ## Pre-train layer-wise for i in range(dbn.n_layers): # go through pretraining epochs for epoch in range(pretraining_epochs): # go through the training set c = [] if( epoch % 50 == 0): pretrain_lr = pretrain_lr/10 for batch_index in range(n_train_batches): c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr)) # append DBN_avg_costs.append( np.mean(c) ) DBN_iter.append(epoch) layer_i.append(i) print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), print numpy.mean(c) end_time = timeit.default_timer() print 'The pretraining code ran for %.2fm' % ((end_time - start_time) / 60.) ##################################### # Plot images in 2D ##################################### Wrbm = dbn.rbm_layers[0].W.get_value(borrow=True).T fig, axes = plt.subplots(ncols=2, nrows=2, figsize=(16, 6)) axes = axes.flatten() for k in range(1,5): image = Image.fromarray( utils.tile_raster_images( X=Wrbm.reshape( Wrbm.shape[0], 5, 10, 10)[:,k,:,:], img_shape=(10,10), tile_shape=(sidelen,sidelen), tile_spacing=(1, 1) )) im = axes[k-1].imshow(image, cmap="Greys_r") axes[k-1].get_xaxis().set_visible(False) axes[k-1].get_yaxis().set_visible(False) cax,kw = mpl.colorbar.make_axes([ax for ax in axes.flat]) plt.colorbar(im, cax=cax, **kw) fig.savefig(output_folder+'/filters_dbn'+str(item)+'.pdf') ############## # Format ################# LLdata = [float(L) for L in DBN_avg_costs] LLiter = [float(it) for it in DBN_iter] LLilayer = [ilayer for ilayer in layer_i] dfpredata = pd.DataFrame( LLdata ) dfpredata.columns = ['LL_iter'] dfpredata['iter'] = LLiter dfpredata['layer'] = LLilayer ######################## # FINETUNING THE MODEL # ######################## # get the training, validation and testing function for the model print '... getting the finetuning functions' train_fn, validate_model, test_model = dbn.build_finetune_functions( datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr) ############ ### for plotting likelihood or cost, accumulate returns of train_model ############ minibatch_avg_costs = [] minibatch_iter = [] minibatch_loss = [] print '... finetuning the model' # early-stopping parameters patience = 1000 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience / 2) # go through this many # minibatches before checking the network # on the validation set; in this case we # check every epoch best_validation_loss = numpy.inf test_score = 0. start_time = timeit.default_timer() done_looping = False epoch = 0 while (epoch < training_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_fn(minibatch_index) iter = (epoch - 1) * n_train_batches + minibatch_index if (iter + 1) % validation_frequency == 0: validation_losses = validate_model() this_validation_loss = numpy.mean(validation_losses) print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) ############## # append ################# minibatch_avg_costs.append(minibatch_avg_cost) minibatch_iter.append(iter) minibatch_loss.append(this_validation_loss*100) # if we got the best validation score until now if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough if ( this_validation_loss < best_validation_loss * improvement_threshold ): patience = max(patience, iter * patience_increase) # save best validation score and iteration number best_validation_loss = this_validation_loss best_iter = iter # test it on the test set test_losses = test_model() test_score = numpy.mean(test_losses) print((' epoch %i, minibatch %i/%i, test error of ' 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.)) if patience <= iter: done_looping = True break end_time = timeit.default_timer() print('Optimization complete with best validation score of %f %%, ' 'obtained at iteration %i, ' 'with test performance %f %%') % (best_validation_loss * 100., best_iter + 1, test_score * 100.) print 'The fine tuning code ran for %.2fm' % ((end_time - start_time)/ 60.) ############## # Format ################# LLdata = [float(L) for L in minibatch_avg_costs] LLiter = [float(it) for it in minibatch_iter] LLoss = [float(l) for l in minibatch_loss] dfinedata = pd.DataFrame( LLdata ) dfinedata.columns = ['LL_iter'] dfinedata['iter'] = LLiter dfinedata['loss'] = LLoss ############### ## Predictions ############### # get training data in numpy format X,y = tnp_train_x, np_train_y # datasets[3] Xtrain = np.asarray(X) ytrain = utils.makeMultiClass(y) # get valid data in numpy format X,y = tnp_valid_x, np_valid_y # datasets[4] Xvalid = np.asarray(X) yvalid = utils.makeMultiClass(y) # get test data in numpy format X,y = tnp_test_x, np_test_y # datasets[5] Xtest = np.asarray(X) ytest = utils.makeMultiClass(y) ############### # predicting using the SDA ############### # in train predtrain = dbn.predict_functions(Xtrain).argmax(1) # let's see how the network did y = ytrain.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predtrain[i] if(y[i] == 0): e0 += y[i]==predtrain[i] # printing the result, this structure should result in 80% accuracy Acutrain0=100*e0/y0 Acutrain1=100*e1/y1 print "Train Accuracy for class 0: %2.2f%%"%(Acutrain0) print "Train Accuracy for class 1: %2.2f%%"%(Acutrain1) # in Valid predvalid = dbn.predict_functions(Xvalid).argmax(1) # let's see how the network did y = yvalid.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predvalid[i] if(y[i] == 0): e0 += y[i]==predvalid[i] # printing the result, this structure should result in 80% accuracy Acuvalid0=100*e0/y0 Acuvalid1=100*e1/y1 print "Valid Accuracy for class 0: %2.2f%%"%(Acuvalid0) print "Valid Accuracy for class 1: %2.2f%%"%(Acuvalid1) # in Xtest predtest = dbn.predict_functions(Xtest).argmax(1) # let's see how the network did y = ytest.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predtest[i] if(y[i] == 0): e0 += y[i]==predtest[i] # printing the result, this structure should result in 80% accuracy Acutest0=100*e0/y0 Acutest1=100*e1/y1 print "Test Accuracy for class 0: %2.2f%%"%(Acutest0) print "Test Accuracy for class 1: %2.2f%%"%(Acutest1) return [dfpredata, dfinedata, dbn, Acutrain0, Acutrain1, Acuvalid0, Acuvalid1, Acutest0, Acutest1]
def test_SdA_timep(self, pretraining_epochs, pretrain_lr, batch_size, training_epochs, finetune_lr, corruption_levels, hidden_layers_sizes, hidden_layers_sidelen, output_folder): """ Demonstrates how to train and test a stochastic denoising autoencoder. :type learning_rate: float :param learning_rate: learning rate used in the finetune stage (factor for the stochastic gradient) :type pretraining_epochs: int :param pretraining_epochs: number of epoch to do pretraining :type pretrain_lr: float :param pretrain_lr: learning rate to be used during pre-training :type n_iter: int :param n_iter: maximal number of iterations ot run the optimizer :type dataset: string :param dataset: path the the pickled dataset """ traindata_path='Z://Cristina//Section3//SdA_experiments//allLpatches.pklz' trainUdata_path='Z://Cristina//Section3//SdA_experiments//allUpatches.pklz' labeldata_path='Z://Cristina//Section3//SdA_experiments//allLabels.pklz' ############# ## LOAD datasets ############# datasets = self.load_wUdata(traindata_path, labeldata_path, trainUdata_path) train_set_x, train_set_y = datasets[0] np_train_x, np_train_y = datasets[3] valid_set_x, valid_set_y = datasets[1] np_valid_x, np_valid_y = datasets[4] test_set_x, test_set_y = datasets[2] np_test_x, np_test_y = datasets[5] # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] n_train_batches //= batch_size # numpy random generator numpy_rng = np.random.RandomState(89677) print('... building the Stacked Autoenconder model') if not os.path.isdir(output_folder): os.makedirs(output_folder) # construct the stacked denoising autoencoder class sda = SdA( numpy_rng=numpy_rng, n_ins = 30*30, hidden_layers_sizes=hidden_layers_sizes, corruption_levels=corruption_levels, n_outs=2 ) ######################### # PRETRAINING THE MODEL # ######################### dA_avg_costs = [] dA_iter = [] layer_i = [] print('... getting the pretraining functions') pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x, np_train_y=np_train_y, batch_size=batch_size) print('... pre-training the model') start_time = timeit.default_timer() ## Pre-train layer-wise for i in range(sda.n_layers): # go through pretraining epochs meanc = [] for epoch in range(pretraining_epochs): # go through the training set c = []; for batch_index in range(n_train_batches): c.append(pretraining_fns[i](index=batch_index, corruption=corruption_levels[i], lr=pretrain_lr)) # append dA_avg_costs.append( np.mean(c) ) dA_iter.append(epoch) layer_i.append(i) print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, np.mean(c))) # check that Cost does not improve more than 1% on the training set after an epoch meanc.append( np.mean(c) ) if (epoch > 100): decrCost = abs(meanc[epoch-1] - meanc[epoch])/meanc[epoch]*100 if(decrCost <= 0.01): break ##################################### # Plot images in 2D ##################################### Xtmp = sda.dA_layers[0].W.get_value(borrow=True).T #imgX = Xtmp.reshape( Xtmp.shape[0], 30, 30) image = Image.fromarray( tile_raster_images(X=Xtmp , img_shape=(30, 30), tile_shape=(10, 10), tile_spacing=(1, 1))) end_time = timeit.default_timer() print(('The pretraining code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.))) ############## # Format ################# LLdata = [float(L) for L in dA_avg_costs] LLiter = [float(it) for it in dA_iter] LLilayer = [ilayer for ilayer in layer_i] dfpredata = pd.DataFrame( LLdata ) dfpredata.columns = ['LL_iter'] dfpredata['iter'] = LLiter dfpredata['layer'] = LLilayer ######################## # FINETUNING THE MODEL # ######################## # for fine tunning make minibatch size = 1 # compute number of minibatches for training, validation and testing #batch_size = 4 #n_train_batches = train_set_x.get_value(borrow=True).shape[0]//4 # get the training, validation and testing function for the model print('... getting the finetuning functions') train_fn, validate_model, test_model = sda.build_finetune_functions( datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr ) print('... finetunning the Stacked model') ############ ### for plotting likelihood or cost, accumulate returns of train_model ############ minibatch_avg_costs = [] minibatch_iter = [] minibatch_loss = [] # early-stopping parameters patience = 1000 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience // 2) # go through this many # minibatche before checking the network # on the validation set; in this case we # check every epoch best_validation_loss = np.inf test_score = 0. start_time = timeit.default_timer() done_looping = False epoch = 0 while (epoch < training_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_fn(minibatch_index) iter = (epoch - 1) * n_train_batches + minibatch_index if (iter + 1) % validation_frequency == 0: validation_losses = validate_model() this_validation_loss = np.mean(validation_losses) print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) ############## # append ################# minibatch_avg_costs.append(minibatch_avg_cost) minibatch_iter.append(iter) minibatch_loss.append(this_validation_loss*100) # if we got the best validation score until now if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough if ( this_validation_loss < best_validation_loss * improvement_threshold ): patience = max(patience, iter * patience_increase) # save best validation score and iteration number best_validation_loss = this_validation_loss best_iter = iter # test it on the test set test_losses = test_model() test_score = np.mean(test_losses) print((' epoch %i, minibatch %i/%i, test error of ' 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.)) if patience <= iter: done_looping = True break end_time = timeit.default_timer() print 'Optimization complete with best validation score of %f, on iteration %i, with test performance %f' % ((best_validation_loss * 100.), (best_iter + 1), (test_score * 100.)) print(('The training code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.))) ############## # Format ################# LLdata = [float(L) for L in minibatch_avg_costs] LLiter = [float(it) for it in minibatch_iter] LLoss = [float(l) for l in minibatch_loss] dfinedata = pd.DataFrame( LLdata ) dfinedata.columns = ['LL_iter'] dfinedata['iter'] = LLiter dfinedata['loss'] = LLoss ############### ## Predictions ############### # get training data in numpy format X,y = datasets[3] Xtrain = np.asarray(X) # extract one img Xtrain = Xtrain.reshape(Xtrain.shape[0], 4, 900) Xtrain = Xtrain[:,0,:] ytrain = utils.makeMultiClass(y) # get valid data in numpy format X,y = datasets[4] Xvalid = np.asarray(X) # extract one img Xvalid = Xvalid.reshape(Xvalid.shape[0], 4, 900) Xvalid = Xvalid[:,0,:] yvalid = utils.makeMultiClass(y) X,y = datasets[5] Xtest = np.asarray(X) # extract one img Xtest = Xtest.reshape(Xtest.shape[0], 4, 900) Xtest = Xtest[:,0,:] ytest = utils.makeMultiClass(y) ############### # predicting using the SDA ############### # in train predtrain = sda.predict_functions(Xtrain).argmax(1) # let's see how the network did y = ytrain.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predtrain[i] if(y[i] == 0): e0 += y[i]==predtrain[i] # printing the result, this structure should result in 80% accuracy Acutrain0=100*e0/y0 Acutrain1=100*e1/y1 print "Train Accuracy for class 0: %2.2f%%"%(Acutrain0) print "Train Accuracy for class 1: %2.2f%%"%(Acutrain1) # in Valid predvalid = sda.predict_functions(Xvalid).argmax(1) # let's see how the network did y = yvalid.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predvalid[i] if(y[i] == 0): e0 += y[i]==predvalid[i] # printing the result, this structure should result in 80% accuracy Acuvalid0=100*e0/y0 Acuvalid1=100*e1/y1 print "Valid Accuracy for class 0: %2.2f%%"%(Acuvalid0) print "Valid Accuracy for class 1: %2.2f%%"%(Acuvalid1) # in Xtest predtest = sda.predict_functions(Xtest).argmax(1) # let's see how the network did y = ytest.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predtest[i] if(y[i] == 0): e0 += y[i]==predtest[i] # printing the result, this structure should result in 80% accuracy Acutest0=100*e0/y0 Acutest1=100*e1/y1 print "Test Accuracy for class 0: %2.2f%%"%(Acutest0) print "Test Accuracy for class 1: %2.2f%%"%(Acutest1) return [dfpredata, dfinedata, image, sda, Acutrain0, Acutrain1, Acuvalid0, Acuvalid1, Acutest0, Acutest1]
def test_SdA_timep(self, pretraining_epochs, pretrain_lr, batch_size, training_epochs, finetune_lr, corruption_levels, hidden_layers_sizes, hidden_layers_sidelen, output_folder): """ Demonstrates how to train and test a stochastic denoising autoencoder. :type learning_rate: float :param learning_rate: learning rate used in the finetune stage (factor for the stochastic gradient) :type pretraining_epochs: int :param pretraining_epochs: number of epoch to do pretraining :type pretrain_lr: float :param pretrain_lr: learning rate to be used during pre-training :type n_iter: int :param n_iter: maximal number of iterations ot run the optimizer :type dataset: string :param dataset: path the the pickled dataset """ traindata_path= 'allLpatches_10x10x5.pklz' #'allLpatches_subs_smaller.pklz' #'allLpatches.pklz' trainUdata_path= 'allUpatches_10x10x5.pklz'#'allUpatches_subs_smaller.pklz' #'allUpatches.pklz' labeldata_path= 'allLabels_10x10x5.pklz' #'allLabels_subs_smaller.pklz' #'allLabels.pklz' ############# ## LOAD datasets ############# datasets = self.load_wUdata(traindata_path, labeldata_path, trainUdata_path) train_set_x, train_set_y = datasets[0] np_train_x, np_train_y = datasets[3] valid_set_x, valid_set_y = datasets[1] np_valid_x, np_valid_y = datasets[4] test_set_x, test_set_y = datasets[2] np_test_x, np_test_y = datasets[5] # inpect one image class 1 Vol = np_train_x[0].reshape(5,10,10) imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] subslicestime = [] # Display image fig, ax = plt.subplots(nrows=4, ncols=4, figsize=(12, 4)) for k in range(1,5): ax[k-1,0].imshow(imgslicestime[k], cmap=plt.cm.gray) ax[k-1,0].set_axis_off() ax[k-1,0].set_adjustable('box-forced') # Display Original histogram ax[k-1,1].hist(imgslicestime[k].ravel(), bins=50, color='black') ax[k-1,1].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0)) ax[k-1,1].set_xlabel('original') # substract volume subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) subslicestime.append(subVol) # Display normalized histogram ax[k-1,2].hist(subVol.ravel(), bins=50, color='black') ax[k-1,2].ticklabel_format(axis='y', style='scientific', scilimits=(0, 0)) ax[k-1,2].set_xlabel('substracted histogram') # display normalized 0-1 ax[k-1,3].imshow(subVol, cmap=plt.cm.gray) ax[k-1,3].set_axis_off() ax[k-1,3].set_adjustable('box-forced') plt.close() ################# # Substract pathces from pre-contrast ################# subsnp_train_x = [] subsnp_valid_x = [] subsnp_test_x = [] for img in np_train_x: Vol = img.reshape(5,10,10) imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] subslicestime = [] for k in range(1,5): # substract volume subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) subslicestime.append(subVol) #append subsnp_train_x.append( np.asarray(subslicestime).reshape(4*10*10) ) for img in np_valid_x: Vol = img.reshape(5,10,10) imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] subslicestime = [] for k in range(1,5): # substract volume subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) subslicestime.append(subVol) #append subsnp_valid_x.append( np.asarray(subslicestime).reshape(4*10*10) ) for img in np_test_x: Vol = img.reshape(5,10,10) imgslicestime = [Vol[0,:,:], Vol[1,:,:], Vol[2,:,:], Vol[3,:,:], Vol[4,:,:]] subslicestime = [] for k in range(1,5): # substract volume subVol = np.asarray(imgslicestime[k]) - np.asarray(imgslicestime[0]) subslicestime.append(subVol) #append subsnp_test_x.append( np.asarray(subslicestime).reshape(4*10*10) ) train_set_x, train_set_y = self.shared_dataset(subsnp_train_x, np_train_y) valid_set_x, valid_set_y = self.shared_dataset(subsnp_valid_x, np_valid_y) test_set_x, test_set_y = self.shared_dataset(subsnp_test_x, np_test_y) subsdatasets = [(train_set_x, train_set_y), (valid_set_x, valid_set_y), (test_set_x, test_set_y) ] ################# # To normalize find mean and std, then convert to float in [0-1] range # ref on mininatch normalization: chrome-extension://oemmndcbldboiebfnladdacbdfmadadm/https://arxiv.org/pdf/1502.03167v3.pdf # stackedOver: http://stats.stackexchange.com/questions/211436/why-do-we-normalize-images-by-subtracting-the-datasets-image-mean-and-not-the-c?rq=1 ################# # Xtrain = np.asarray(np_train_x) # muXtrain = np.mean(Xtrain) # varXtrain = np.var(Xtrain) # print(muXtrain,varXtrain) # # Xvalid = np.asarray(np_valid_x) # muXvalid = np.mean(Xvalid) # varXvalid = np.var(Xvalid) # print(muXvalid,varXvalid) # # Xtest = np.asarray(np_test_x) # muXtest = np.mean(Xtest) # varXtest = np.var(Xtest) # print(muXtest,varXtest) # # fnp_train_x = [] # fnp_valid_x = [] # fnp_test_x = [] # for img in np_train_x: # #img=np_train_x[1] # fimg = (img - muXtrain)/np.sqrt(varXtrain+0.00001) # rVol = exposure.rescale_intensity(fimg.reshape(4,10,10), out_range=(0, 1)) # img = sitk.GetImageFromArray(rVol) # aimg = sitk.Cast(img,sitk.sitkFloat32) # sitkVol = sitk.GetArrayFromImage(aimg) # img = sitkVol.reshape(400) # fnp_train_x.append(img) # for img in np_valid_x: # fimg = (img - muXvalid)/np.sqrt(varXvalid+0.00001) # rVol = exposure.rescale_intensity(fimg.reshape(4,10,10), out_range=(0, 1)) # img = sitk.GetImageFromArray(rVol) # aimg = sitk.Cast(img,sitk.sitkFloat32) # sitkVol = sitk.GetArrayFromImage(aimg) # img = sitkVol.reshape(400) # fnp_valid_x.append(img) # for img in np_test_x: # fimg = (img - muXtest)/np.sqrt(varXtest+0.00001) # rVol = exposure.rescale_intensity(fimg.reshape(4,10,10), out_range=(0, 1)) # img = sitk.GetImageFromArray(rVol) # aimg = sitk.Cast(img,sitk.sitkFloat32) # sitkVol = sitk.GetArrayFromImage(aimg) # img = sitkVol.reshape(400) # fnp_test_x.append(img) # ################# ## plot ooptional ################# # fig, axes = plt.subplots(ncols=4, nrows=2) # axes[0,0].imshow(np_train_x[1].reshape(5,10,10)[0,:,:], cmap="Greys_r") # axes[0,1].imshow(np_train_x[1].reshape(5,10,10)[1,:,:], cmap="Greys_r") # axes[0,2].imshow(np_train_x[1].reshape(5,10,10)[2,:,:], cmap="Greys_r") # axes[0,3].imshow(np_train_x[1].reshape(5,10,10)[3,:,:], cmap="Greys_r") # axes[1,0].imshow( fnp_train_x[1].reshape(5,10,10)[0,:,:], cmap="Greys_r") # axes[1,1].imshow( fnp_train_x[1].reshape(5,10,10)[1,:,:], cmap="Greys_r") # axes[1,2].imshow( fnp_train_x[1].reshape(5,10,10)[2,:,:], cmap="Greys_r") # axes[1,3].imshow( fnp_train_x[1].reshape(5,10,10)[3,:,:], cmap="Greys_r") # train_set_x, train_set_y = self.shared_dataset(fnp_train_x, np_train_y) # valid_set_x, valid_set_y = self.shared_dataset(fnp_valid_x, np_valid_y) # test_set_x, test_set_y = self.shared_dataset(fnp_test_x, np_test_y) # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] n_train_batches //= batch_size # numpy random generator numpy_rng = np.random.RandomState(89677) print('... building the Stacked Autoenconder model') if not os.path.isdir(output_folder): os.makedirs(output_folder) # construct the stacked denoising autoencoder class sda = SdA( numpy_rng=numpy_rng, n_ins = 4*10*10, # before 30*30*4 hidden_layers_sizes=hidden_layers_sizes, corruption_levels=corruption_levels, n_outs=2 ) ######################### # PRETRAINING THE MODEL # ######################### dA_avg_costs = [] dA_iter = [] layer_i = [] print('... getting the pretraining functions') pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x, np_train_y=np_train_y, batch_size=batch_size) print('... pre-training the model') start_time = timeit.default_timer() ## Pre-train layer-wise for i in range(sda.n_layers): # go through pretraining epochs meanc = [] for epoch in range(pretraining_epochs): # go through the training set c = []; if( epoch % 100 == 0): pretrain_lr = pretrain_lr/10 for batch_index in range(n_train_batches): c.append(pretraining_fns[i](index=batch_index, corruption=corruption_levels[i], lr=pretrain_lr)) # append dA_avg_costs.append( np.mean(c) ) dA_iter.append(epoch) layer_i.append(i) print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, np.mean(c))) # check that Cost does not improve more than 1% on the training set after an epoch meanc.append( np.mean(c) ) if (epoch > 195): decrCost = abs(meanc[epoch-1] - meanc[epoch])/meanc[epoch]*100 if(decrCost <= 0.01): break end_time = timeit.default_timer() print(('The pretraining code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.))) ##################################### # Plot images in 2D ##################################### Xtmp = sda.dA_layers[0].W.get_value(borrow=True).T imagefilters = [] for k in range(4): imgX = Xtmp.reshape( Xtmp.shape[0], 4, 10, 10)[:,k,:,:] image = tile_images(X=imgX , img_shape=(10, 10), tile_shape=(10, 10), tile_spacing=(1, 1)) # append to all imagefilters.append( image ) ############## # Format ################# LLdata = [float(L) for L in dA_avg_costs] LLiter = [float(it) for it in dA_iter] LLilayer = [ilayer for ilayer in layer_i] dfpredata = pd.DataFrame( LLdata ) dfpredata.columns = ['LL_iter'] dfpredata['iter'] = LLiter dfpredata['layer'] = LLilayer ######################## # FINETUNING THE MODEL # ######################## # for fine tunning make minibatch size = 1 # compute number of minibatches for training, validation and testing #batch_size = 4 #n_train_batches = train_set_x.get_value(borrow=True).shape[0]//4 # get the training, validation and testing function for the model print('... getting the finetuning functions') train_fn, validate_model, test_model = sda.build_finetune_functions( datasets=subsdatasets, batch_size=batch_size, learning_rate=finetune_lr ) print('... finetunning the Stacked model') ############ ### for plotting likelihood or cost, accumulate returns of train_model ############ minibatch_avg_costs = [] minibatch_iter = [] minibatch_loss = [] # early-stopping parameters patience = 1000 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience // 2) # go through this many # minibatche before checking the network # on the validation set; in this case we # check every epoch best_validation_loss = np.inf test_score = 0. start_time = timeit.default_timer() done_looping = False epoch = 0 while (epoch < training_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_fn(minibatch_index) iter = (epoch - 1) * n_train_batches + minibatch_index if (iter + 1) % validation_frequency == 0: validation_losses = validate_model() this_validation_loss = np.mean(validation_losses) print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) ############## # append ################# minibatch_avg_costs.append(minibatch_avg_cost) minibatch_iter.append(iter) minibatch_loss.append(this_validation_loss*100) # if we got the best validation score until now if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough if ( this_validation_loss < best_validation_loss * improvement_threshold ): patience = max(patience, iter * patience_increase) # save best validation score and iteration number best_validation_loss = this_validation_loss best_iter = iter # test it on the test set test_losses = test_model() test_score = np.mean(test_losses) print((' epoch %i, minibatch %i/%i, test error of ' 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.)) if patience <= iter: done_looping = True break end_time = timeit.default_timer() print 'Optimization complete with best validation score of %f, on iteration %i, with test performance %f' % ((best_validation_loss * 100.), (best_iter + 1), (test_score * 100.)) print(('The training code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.))) ############## # Format ################# LLdata = [float(L) for L in minibatch_avg_costs] LLiter = [float(it) for it in minibatch_iter] LLoss = [float(l) for l in minibatch_loss] dfinedata = pd.DataFrame( LLdata ) dfinedata.columns = ['LL_iter'] dfinedata['iter'] = LLiter dfinedata['loss'] = LLoss ############### ## Predictions ############### # get training data in numpy format X,y = subsnp_train_x, np_train_y #datasets[3] Xtrain = np.asarray(X) ytrain = utils.makeMultiClass(y) # get valid data in numpy format X,y = subsnp_valid_x, np_valid_y # #datasets[4] Xvalid = np.asarray(X) yvalid = utils.makeMultiClass(y) # get test data in numpy format X,y = subsnp_test_x, np_test_y #datasets[5] Xtest = np.asarray(X) ytest = utils.makeMultiClass(y) ############### # predicting using the SDA ############### # in train predtrain = sda.predict_functions(Xtrain).argmax(1) # let's see how the network did y = ytrain.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predtrain[i] if(y[i] == 0): e0 += y[i]==predtrain[i] # printing the result, this structure should result in 80% accuracy Acutrain0=100*e0/y0 Acutrain1=100*e1/y1 print "Train Accuracy for class 0: %2.2f%%"%(Acutrain0) print "Train Accuracy for class 1: %2.2f%%"%(Acutrain1) # in Valid predvalid = sda.predict_functions(Xvalid).argmax(1) # let's see how the network did y = yvalid.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predvalid[i] if(y[i] == 0): e0 += y[i]==predvalid[i] # printing the result, this structure should result in 80% accuracy Acuvalid0=100*e0/y0 Acuvalid1=100*e1/y1 print "Valid Accuracy for class 0: %2.2f%%"%(Acuvalid0) print "Valid Accuracy for class 1: %2.2f%%"%(Acuvalid1) # in Xtest predtest = sda.predict_functions(Xtest).argmax(1) # let's see how the network did y = ytest.argmax(1) e0 = 0.0; y0 = len([0 for yi in range(len(y)) if y[yi]==0]) e1 = 0.0; y1 = len([1 for yi in range(len(y)) if y[yi]==1]) for i in range(len(y)): if(y[i] == 1): e1 += y[i]==predtest[i] if(y[i] == 0): e0 += y[i]==predtest[i] # printing the result, this structure should result in 80% accuracy Acutest0=100*e0/y0 Acutest1=100*e1/y1 print "Test Accuracy for class 0: %2.2f%%"%(Acutest0) print "Test Accuracy for class 1: %2.2f%%"%(Acutest1) return [dfpredata, dfinedata, imagefilters, sda, Acutrain0, Acutrain1, Acuvalid0, Acuvalid1, Acutest0, Acutest1]
Train[x][len(Train[x])-1] = int(Train[x][len(Train[x])-1]) ClassTrain += [Train[x][len(Train[x])-1]] del(Train[x][len(Train[x])-1]) emg = nk.emg_process(Train[x]) DataTrain += [emg["df"]["EMG_Envelope"]] with open("../../Apps/test.csv","r") as f: reader = csv.reader(f) Test = list(reader) for x in (range(0,len(Test))): Test[x] =[float(n.strip('"')) for n in Test[x]] Test[x][len(Test[x])-1] = int(Test[x][len(Test[x])-1]) ClassTest += [Test[x][len(Test[x])-1]] del(Test[x][len(Test[x])-1]) emg = nk.emg_process(Test[x]) DataTest += [emg["df"]["EMG_Envelope"]] ClassTest = makeMultiClass(ClassTest) ClassTrain = makeMultiClass(ClassTrain) print(ClassTrain[0]) my_df = pd.DataFrame(DataTrain) my_df.to_csv('../data/trainX.csv', index=False, header=False) my_df = pd.DataFrame(ClassTrain) my_df.to_csv('../data/trainY.csv', index=False, header=False) my_df = pd.DataFrame(DataTest) my_df.to_csv('../data/testX.csv', index=False, header=False) my_df = pd.DataFrame(ClassTest) my_df.to_csv('../data/testY.csv', index=False, header=False)