def createDBN(traindata, trainlabel, testdata, testlabel, parameter_file):
f = open(parameter_file, 'r')
n = int(f.readline())
for k in range(n):
s = f.readline()
args = s.split(';')
num_hid_layer = int(args[0])
#hidden layer sizes
s = args[1].split(',')
hid_layer_sizes = []
for i in range(num_hid_layers):
hid_layer_sizes.append(int(s[i]))
#momentums
s = args[2].split(',')
momentums = []
for i in range(num_hid_layers):
momentums.append(float(s[i]))
#sparsities
s = args[3].split(',')
sparsities = [None] * num_hid_layer
for i in range(num_hid_layers):
sparsities[i] = float(s[i])
dbn = DBN(num_hid_layer, hid_layer_sizes, momentums, sparsities, traindata, trainlabel)
trainingOption(dbn, num_hid_layers, traindata, trainlabel, testdata, testlabel, args[4], 'regRBM_'+parameter_file+'_')
f.close()
def DBNPretraining(self, dataDir):
modelDir = os.path.dirname(self.modelFile)
title = os.path.basename(self.modelFile).split('.')[0]
layerSizes = self.AELayerSizes[:len(self.AELayerSizes) / 2 + 1]
types = ['GB'] + ['BB'] * (len(layerSizes) - 2)
dbn = DBN(title, layerSizes, types, modelDir)
dbn.DBNTrain(dataDir)
def compare_dnn_init(
X_train,
y_train,
X_test,
y_test,
layers=[784, 100, 10],
):
print(f"Architecture : {layers}")
# 1. initialiser deux réseaux identiques;
dnn_random = DNN(layers)
dnn_pretrained = DNN(layers)
# 2. pré-apprendre un des deux réseau en le considérant comme un empilement de RBM (apprentissage non
# supervisé);
layers_dbn = layers[:-1]
dbn = DBN(layers_dbn)
print("Pretraining RBM...")
dbn.train(X_train, epochs=100, batch_size=32, learning_rate=0.1)
dnn_pretrained.init_DNN_with_DBN(dbn)
# 3. apprendre le réseau pré-appris préalablement avec l’algorithme de rétro-propagation;
dnn_pretrained.train(X_train,
y_train,
epochs=20,
batch_size=128,
learning_rate=0.1)
# 4. apprendre le second réseau qui a été initialisé aléatoirement avec l’algorithme de rétro-propagation;
dnn_random.train(X_train,
y_train,
epochs=20,
batch_size=128,
learning_rate=0.1)
# 5. Calculer les taux de mauvaises classifications avec le réseau 1 (pré-entrainé + entraîné) et le réseau 2
# (entraîné) à partir du jeu ’train’ et du jeu ’test’
error_random = dnn_random.error_rate(X_test, y_test)
error_pretrained = dnn_pretrained.error_rate(X_test, y_test)
print(f"Error rate random init : {error_random:.3f}")
print(f"Error rate RBM pretrained : {error_pretrained:.3f}")
return error_random, error_pretrained
error_pretrained_all = []
for training_size in training_sizes:
sub_X = X_train[:training_size]
sub_y = y_train[:training_size]
error_random, error_pretrained = compare_dnn_init(sub_X, sub_y, X_test,
y_test, layers)
error_random_all.append(error_random)
error_pretrained_all.append(error_pretrained)
print("Random:", error_random_all)
print("Pretrain:", error_pretrained_all)
plot_results(
error_random_all,
error_pretrained_all,
training_sizes,
"Training size",
"Comparing traing sizes",
)
# On cherchera enfin une configuration permettant d’obtenir le meilleur
# taux de classification possible (ne pas hésiter à utiliser les 60000 données
# et des réseaux de grande taille).
dnn = DNN([784, 600, 400, 200, 10])
dbn = DBN([784, 600, 400, 200])
print("Pretraining RBM...")
dbn.train(X_train[:10000], epochs=100, batch_size=128, learning_rate=0.1)
dnn.init_DNN_with_DBN(dbn)
dnn.train(X_train, y_train, epochs=100, batch_size=128, learning_rate=0.1)
error = dnn.error_rate(X_test, y_test)
print(f"Meilleure erreur: {error}")
label_test = np.squeeze(np.asarray(label_test)) - 1
#data_train=data_train.flatten()
print(data_train.shape)
n_train_batches = data_train.shape[0] // batch_size
print(n_train_batches)
data_train = theano.shared(data_train)
label_train = theano.shared(label_train)
data_test = theano.shared(data_test)
label_test = theano.shared(label_test)
numpy_rng = np.random.RandomState(0)
dbn = DBN.DBN(numpy_rng=numpy_rng,
n_ins=20,
hidden_layers_sizes=[10],
n_outs=4)
print('... getting the pretraining functions')
pretraining_fns = dbn.pretraining_functions(train_set_x=data_train,
batch_size=batch_size,
k=k)
print('... pre-training the model')
start_time = timeit.default_timer()
# Pre-train layer-wise
for i in range(dbn.n_layers):
# go through pretraining epochs
for epoch in range(pretraining_epochs):
# go through the training set
c = []
def main():
# Set parameters and print them
args = parse_args()
print_args(args)
whatLayer = args['whatLayer']
epochs = args['epochs']
visibleUnits = args['visibleUnits']
hiddenUnits = args['hiddenUnits']
secondLayerUnits = args['secondLayerUnits']
approxMethod = args['approxMethod']
approxSteps = args['approxSteps']
learnRate = args['learnRate']
persistStart = args['persistStart']
momentum = args['momentum']
decayMagnitude = args['decayMagnitude']
decayType = args['decayType']
sigma = args['sigma']
batchSize = args['batchSize']
binarization = args['binarization']
trace_file = args['trace_file'] # saving results
save_file = args['save_file'] # saving parameters
load_file = args['load_file'] # loading parameters
print('Loading data')
with gzip.open('../data/mnist.pkl.gz', 'r') as f:
# combine train and valid and leave test
(TrainSet,
y_train), (x_test, y_test), (ValidSet,
y_Valid) = pickle.load(f,
encoding='latin1')
TrainSet = np.concatenate((TrainSet, x_test), axis=0)
TrainSet = sklearn.preprocessing.binarize(
TrainSet, threshold=binarization).T # Create binary data
ValidSet = sklearn.preprocessing.binarize(
ValidSet, threshold=binarization).T # Create binary data
if len(load_file) == 0:
for ii in range(len(layers) - 1):
if ii == 0:
print('Initializing model')
model = DBN.DBN(
n_vis=layers[ii],
n_hid=layers[ii + 1],
momentum=momentum,
sigma=sigma,
trainData=TrainSet,
wiseStart=True,
)
else:
params = DBN.DBN.load("DBN")
model = DBN.DBN(layer=ii + 1,
params=params,
n_vis=layers[ii],
n_hid=layers[ii + 1])
print('Start of training')
Training.fit(model,
TrainSet,
ValidSet,
n_epochs=epochs,
weight_decay=decayType,
decay_magnitude=decayMagnitude,
lr=learnRate,
batch_size=batchSize,
NormalizationApproxIter=approxSteps,
approx=approxMethod,
persistent_start=persistStart,
trace_file=trace_file,
save_file=save_file)
else:
print('Initializing model')
params = DBN.DBN.load(load_file)
model = DBN.DBN(layer=whatLayer, params=params)
print('W1 shape', model.W1.shape, 'hbias', model.hbias1.shape, 'vbias',
model.vbias1.shape)
if model.layer >= 3:
print('W2 shape', model.W22.shape, 'hbias', model.hbias2.shape,
'vbias', model.vbias2.shape)
if model.layer == 4:
print('W3 shape', model.W33.shape, 'hbias', model.hbias3.shape,
'vbias', model.vbias3.shape)
print('Next W shape', model.W.shape, 'hbias', model.hbias.shape,
'vbias', model.vbias.shape)
def test_DBN(finetune_lr=0.01,
pretraining_epochs=100,
pretrain_lr=0.01,
k=1,
training_epochs=1000,
dataset="C:\Python27\Lib\data\dex.pkl.gz",
batch_size=10):
"""
Demonstrates how to train and test a Deep Belief Network.
This is demonstrated on MNIST.
:type finetune_lr: float
:param finetune_lr: learning rate used in the finetune stage
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type k: int
:param k: number of Gibbs steps in CD/PCD
:type training_epochs: int
:param training_epochs: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
:type batch_size: int
:param batch_size: the size of a minibatch
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# print train_set_y
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
# construct the Deep Belief Network
H_L_table = createLayerTable(num_of_layers, lsize)
#H_L_table = createFunnelTable(num_of_layers, lsize)
dbn = DBN(numpy_rng=numpy_rng,
n_ins=n_ins,
hidden_layers_sizes=H_L_table,
n_outs=2)
# ########################
# PRETRAINING THE MODEL #
# ########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = dbn.build_finetune_functions(
datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr)
print '... finetunning the model'
# early-stopping parameters
patience = 100 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_params = None
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' % \
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (this_validation_loss <
best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(('Optimization complete with best validation score of %f %%,'
'with test performance %f %%') %
(best_validation_loss * 100., test_score * 100.))
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
global res_val
global res_test
res_val = best_validation_loss
res_test = test_score
