def greedy_layerwise_train(self,
layer_no,
nb_epochs,
callbacks=[],
metrics=[]):
print ">>>", layer_no
# preparing input for each layer
if (layer_no == 0):
ins = self.X
else:
pre_model = Sequential()
for i, rbm in enumerate(self.rbms[0:layer_no]):
pre_model.add(rbm.get_h_given_x_layer((i == 0)))
if (self.hidden_unit_type == 'nrlu'):
pre_model.add(SampleBernoulli(mode='nrlu'))
else:
pre_model.add(SampleBernoulli(mode='random'))
pre_model.compile(SGD(), loss='mean_squared_error')
ins = pre_model.predict(self.X)
input_dim = self.rbms[layer_no].input_dim
# preparing model
#model = SingleLayerUnsupervised()
model = Sequential()
model.add(self.rbms[layer_no])
loss = self.get_layer_loss(self.rbms[layer_no], layer_no)
opt = self.get_layer_optimizer(layer_no)
model.compile(optimizer=opt, loss=loss, metrics=metrics)
model.summary()
model.fit(ins,
ins,
batch_size=self.batch_size,
nb_epoch=nb_epochs,
verbose=self.verbose,
shuffle=self.shuffle,
callbacks=callbacks)
def __init__(self, dbn, data, layer_no):
self.dbn = dbn
self.data = data
self.layer_no = layer_no
''' create a forward sampling model '''
self.pre_model = Sequential()
for i, rbm in enumerate(self.dbn.rbms[0:self.layer_no]):
self.pre_model.add(rbm.get_h_given_x_layer((i == 0)))
self.pre_model.add(SampleBernoulli(mode='random'))
if self.layer_no > 0:
self.pre_model.compile(SGD(), loss='mean_squared_error')
def main():
# generate dummy dataset
nframes = 10000
dataset = np.random.normal(loc=np.zeros(input_dim), scale=np.ones(input_dim), size=(nframes, input_dim))
# standardize (in this case superfluous)
#dataset, mean, stddev = standardize(dataset)
# split into train and test portion
ntest = 1000
X_train = dataset[:-ntest :] # all but last 1000 samples for training
X_test = dataset[-ntest:, :] # last 1000 samples for testing
X_trainsub = dataset[:ntest, :] # subset of training data with same number of samples as testset
assert X_train.shape[0] >= X_test.shape[0], 'Train set should be at least size of test set!'
# setup model structure
print('Creating training model...')
rbm = GBRBM(input_dim=input_dim, hidden_dim=hidden_dim, init=glorot_uniform_sigm)
rbm.srng = RandomStreams(seed=srng_seed)
train_model = SingleLayerUnsupervised()
train_model.add(rbm)
# setup optimizer, loss
momentum_schedule = make_stepped_schedule([(0, 0.5), (5, 0.9)])
momentum_scheduler = MomentumScheduler(momentum_schedule)
opt = SGD(lr, 0., decay=0.0, nesterov=False)
contrastive_divergence = rbm.contrastive_divergence_loss(nb_gibbs_steps=1)
# compile theano graph
print('Compiling Theano graph...')
train_model.compile(optimizer=opt, loss=contrastive_divergence)
# additional monitors
#rec_loss = rbm.reconstruction_loss(nb_gibbs_steps=1)
#rec_err_logger = UnsupervisedLoss1Logger(X_train, loss=rec_loss, label=' - input reconstruction loss', every_n_epochs=1)
#rec_err_logger.compile()
#free_energy_gap = rbm.free_energy_gap
#free_energy_gap_logger = UnsupervisedLoss2Logger(X_trainsub, X_test, loss=free_energy_gap, label=' - free energy gap', every_n_epochs=1)
#free_energy_gap_logger.compile()
# do training
print('Training...')
begin_time = time.time()
#callbacks = [momentum_scheduler, rec_err_logger, free_energy_gap_logger]
callbacks = [momentum_scheduler]
train_model.fit(X_train, batch_size, nb_epoch, verbose=1, shuffle=False, callbacks=callbacks)
end_time = time.time()
print('Training took %f minutes' % ((end_time - begin_time)/60.0))
# save model parameters
print('Saving model...')
rbm.save_weights('example.hdf5', overwrite=True)
# load model parameters
print('Loading model...')
rbm.load_weights('example.hdf5')
# generate hidden features from input data
print('Creating inference model...')
h_given_x = rbm.get_h_given_x_layer()
inference_model = Sequential([h_given_x, SampleBernoulli(mode='maximum_likelihood')])
print('Compiling Theano graph...')
inference_model.compile(opt, loss='mean_squared_error') # XXX: optimizer and loss are not used!
print('Doing inference...')
h = inference_model.predict(dataset)
print(h)
print('Done!')
def main():
# generate dummy dataset
nframes = 10000
dataset = np.random.normal(loc=np.zeros(input_dim),
scale=np.ones(input_dim),
size=(nframes, input_dim))
# split into train and test portion
ntest = 1000
X_train = dataset[:-ntest:] # all but last 1000 samples for training
X_test = dataset[-ntest:, :] # last 1000 samples for testing
assert X_train.shape[0] >= X_test.shape[
0], 'Train set should be at least size of test set!'
# setup model structure
print('Creating training model...')
rbm1 = RBM(hidden_dim[0],
input_dim=input_dim,
init=glorot_uniform_sigm,
visible_unit_type='gaussian',
hidden_unit_type='binary',
nb_gibbs_steps=nb_gibbs_steps,
persistent=True,
batch_size=batch_size,
dropout=dropouts[0])
rbm2 = RBM(hidden_dim[1],
input_dim=hidden_dim[0],
init=glorot_uniform_sigm,
visible_unit_type='binary',
hidden_unit_type='binary',
nb_gibbs_steps=nb_gibbs_steps,
persistent=True,
batch_size=batch_size,
dropout=dropouts[1])
#When using nrlu unit, nb_gibbs_steps and persistent param are ignored
rbm3 = RBM(hidden_dim[2],
input_dim=hidden_dim[1],
init=glorot_uniform_sigm,
visible_unit_type='binary',
hidden_unit_type='nrlu',
nb_gibbs_steps=1,
persistent=False,
batch_size=batch_size,
dropout=dropouts[2])
rbms = [rbm1, rbm2, rbm3]
dbn = DBN(rbms)
# setup optimizer, loss
def get_layer_loss(rbm, layer_no):
return rbm.contrastive_divergence_loss
def get_layer_optimizer(layer_no):
return SGD((layer_no + 1) * lr, 0., decay=0.0, nesterov=False)
metrics = []
for rbm in rbms:
metrics.append([rbm.reconstruction_loss])
dbn.compile(layer_optimizer=get_layer_optimizer,
layer_loss=get_layer_loss,
metrics=metrics)
# do training
print('Training...')
dbn.fit(X_train, batch_size, nb_epoch, verbose=1, shuffle=False)
# generate hidden features from input data
print('Creating inference model...')
F = dbn.get_forward_inference_layers()
B = dbn.get_backward_inference_layers()
inference_model = Sequential()
for f in F:
inference_model.add(f)
inference_model.add(SampleBernoulli(mode='random'))
for b in B[:-1]:
inference_model.add(b)
inference_model.add(SampleBernoulli(mode='random'))
# last layer is a gaussian layer
inference_model.add(B[-1])
print('Compiling Theano graph...')
opt = SGD()
inference_model.compile(opt, loss='mean_squared_error')
print('Doing inference...')
h = inference_model.predict(dataset)
print(h)
print('Done!')
def main():
#grab input data set and set up dataset here
X_train = []
X_test = []
print('Creating training model')
#start with a GBRBM and then followed by 5 more RBMs for 5*2 = 10 hidden layers
dbn = DBN([
GBRBM(input_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm)
])
def get_layer_loss(rbm, layer_no):
return rbm.contrastive_divergence_loss(nb_gibbs_steps=1)
def get_layer_optimizer(layer_no):
return SGD((layer_no + 1) * lr, 0., decay=0.0, nesterov=False)
dbn.compile(layer_optimizer=get_layer_optimizer, layer_loss=get_layer_loss)
#Train
#train off token vectors from early version of software
print('Training')
begin_time = time.time()
dbn.fit(X_train, batch_size, nb_epoch, verbose=1, shuffle=False)
end_time = time.time()
print('Training took %f minutes' % ((end_time - begin_time) / 60.0))
#save model parameters from training
print('Saving model')
dbn.save_weights('dbn_weights.hdf5', overwrite=True)
#load model from save
print('Loading model')
dbn.load_weights('dbn_weights.hdf5')
#generate hidden features from input data
print('Creating inference model')
F = dbn.get_forward_inference_layers()
B = dbn.get_backwards_inference_layers()
inference_model = Sequential()
for f in F:
inference_model.add(f)
inference_model.add(SampleBernoulli(mode='random'))
for b in B[:-1]:
inference_model.add(b)
inference_model.add(SampleBernoulli(mode='random'))
#last layer is a gaussian layer
inference_model.add(B[-1])
print('Compiling Theano graph')
opt = SGD()
inference_model.compile(opt, loss='mean_squared_error')
print('Doing inference')
h = inference_model.predict(X_test)
def main():
# generate dummy dataset
nframes = 10000
dataset = np.random.normal(loc=np.zeros(input_dim),
scale=np.ones(input_dim),
size=(nframes, input_dim))
# standardize (in this case superfluous)
#dataset, mean, stddev = standardize(dataset)
# split into train and test portion
ntest = 1000
X_train = dataset[:-ntest:] # all but last 1000 samples for training
X_test = dataset[-ntest:, :] # last 1000 samples for testing
X_trainsub = dataset[:
ntest, :] # subset of training data with same number of samples as testset
assert X_train.shape[0] >= X_test.shape[
0], 'Train set should be at least size of test set!'
# setup model structure
print('Creating training model...')
dbn = DBN([
GBRBM(input_dim, 200, init=glorot_uniform_sigm),
RBM(200, 400, init=glorot_uniform_sigm),
RBM(400, 300, init=glorot_uniform_sigm),
RBM(300, 50, init=glorot_uniform_sigm),
RBM(50, hidden_dim, init=glorot_uniform_sigm)
])
# setup optimizer, loss
def get_layer_loss(rbm, layer_no):
return rbm.contrastive_divergence_loss(nb_gibbs_steps=1)
def get_layer_optimizer(layer_no):
return SGD((layer_no + 1) * lr, 0., decay=0.0, nesterov=False)
dbn.compile(layer_optimizer=get_layer_optimizer, layer_loss=get_layer_loss)
# do training
print('Training...')
begin_time = time.time()
#callbacks = [momentum_scheduler, rec_err_logger, free_energy_gap_logger]
dbn.fit(X_train, batch_size, nb_epoch, verbose=1, shuffle=False)
end_time = time.time()
print('Training took %f minutes' % ((end_time - begin_time) / 60.0))
# save model parameters
print('Saving model...')
dbn.save_weights('example.hdf5', overwrite=True)
# load model parameters
print('Loading model...')
dbn.load_weights('example.hdf5')
# generate hidden features from input data
print('Creating inference model...')
F = dbn.get_forward_inference_layers()
B = dbn.get_backward_inference_layers()
inference_model = Sequential()
for f in F:
inference_model.add(f)
inference_model.add(SampleBernoulli(mode='random'))
for b in B[:-1]:
inference_model.add(b)
inference_model.add(SampleBernoulli(mode='random'))
# last layer is a gaussian layer
inference_model.add(B[-1])
print('Compiling Theano graph...')
opt = SGD()
inference_model.compile(
opt,
loss='mean_squared_error') # XXX: optimizer and loss are not used!
print('Doing inference...')
h = inference_model.predict(dataset)
print(h)
print('Done!')