def main():
# generate dummy dataset
nframes = 10000
dataset = np.random.normal(loc=np.zeros(input_dim),
scale=np.ones(input_dim),
size=(nframes, input_dim))
# split into train and test portion
ntest = 1000
X_train = dataset[:-ntest:] # all but last 1000 samples for training
X_test = dataset[-ntest:, :] # last 1000 samples for testing
assert X_train.shape[0] >= X_test.shape[
0], 'Train set should be at least size of test set!'
# setup model structure
print('Creating training model...')
rbm1 = RBM(hidden_dim[0],
input_dim=input_dim,
init=glorot_uniform_sigm,
visible_unit_type='gaussian',
hidden_unit_type='binary',
nb_gibbs_steps=nb_gibbs_steps,
persistent=True,
batch_size=batch_size,
dropout=dropouts[0])
rbm2 = RBM(hidden_dim[1],
input_dim=hidden_dim[0],
init=glorot_uniform_sigm,
visible_unit_type='binary',
hidden_unit_type='binary',
nb_gibbs_steps=nb_gibbs_steps,
persistent=True,
batch_size=batch_size,
dropout=dropouts[1])
#When using nrlu unit, nb_gibbs_steps and persistent param are ignored
rbm3 = RBM(hidden_dim[2],
input_dim=hidden_dim[1],
init=glorot_uniform_sigm,
visible_unit_type='binary',
hidden_unit_type='nrlu',
nb_gibbs_steps=1,
persistent=False,
batch_size=batch_size,
dropout=dropouts[2])
rbms = [rbm1, rbm2, rbm3]
dbn = DBN(rbms)
# setup optimizer, loss
def get_layer_loss(rbm, layer_no):
return rbm.contrastive_divergence_loss
def get_layer_optimizer(layer_no):
return SGD((layer_no + 1) * lr, 0., decay=0.0, nesterov=False)
metrics = []
for rbm in rbms:
metrics.append([rbm.reconstruction_loss])
dbn.compile(layer_optimizer=get_layer_optimizer,
layer_loss=get_layer_loss,
metrics=metrics)
# do training
print('Training...')
dbn.fit(X_train, batch_size, nb_epoch, verbose=1, shuffle=False)
# generate hidden features from input data
print('Creating inference model...')
F = dbn.get_forward_inference_layers()
B = dbn.get_backward_inference_layers()
inference_model = Sequential()
for f in F:
inference_model.add(f)
inference_model.add(SampleBernoulli(mode='random'))
for b in B[:-1]:
inference_model.add(b)
inference_model.add(SampleBernoulli(mode='random'))
# last layer is a gaussian layer
inference_model.add(B[-1])
print('Compiling Theano graph...')
opt = SGD()
inference_model.compile(opt, loss='mean_squared_error')
print('Doing inference...')
h = inference_model.predict(dataset)
print(h)
print('Done!')
def main():
# generate dummy dataset
nframes = 10000
dataset = np.random.normal(loc=np.zeros(input_dim), scale=np.ones(input_dim), size=(nframes, input_dim))
# split into train and test portion
ntest = 1000
X_train = dataset[:-ntest :] # all but last 1000 samples for training
X_test = dataset[-ntest:, :] # last 1000 samples for testing
assert X_train.shape[0] >= X_test.shape[0], 'Train set should be at least size of test set!'
print('Creating training model...')
#if persistent is True, you need tospecify batch_size
rbm = RBM(hidden_dim, input_dim=input_dim,
init=glorot_uniform_sigm,
hidden_unit_type='binary',
visible_unit_type='gaussian',
nb_gibbs_steps=nb_gibbs_steps,
persistent=True,
batch_size=batch_size,
dropout=0.5)
train_model = Sequential()
train_model.add(rbm)
opt = SGD(lr, 0., decay=0.0, nesterov=False)
loss = rbm.contrastive_divergence_loss
metrics = [rbm.reconstruction_loss]
logger = UnsupervisedLoss2Logger(X_train, X_test,
rbm.free_energy_gap,
verbose=1,
label='free_eng_gap',
batch_size=batch_size)
callbacks = [logger]
# compile theano graph
print('Compiling Theano graph...')
train_model.compile(optimizer=opt, loss=loss, metrics=metrics)
# do training
print('Training...')
train_model.fit(X_train, X_train, batch_size, nb_epoch,
verbose=1, shuffle=False, callbacks=callbacks)
# generate hidden features from input data
print('Creating inference model...')
h_given_x = rbm.get_h_given_x_layer(as_initial_layer=True)
inference_model = Sequential()
inference_model.add(h_given_x)
#inference_model.add(SampleBernoulli(mode='maximum_likelihood'))
print('Compiling Theano graph...')
inference_model.compile(opt, loss='mean_squared_error')
print('Doing inference...')
h = inference_model.predict(dataset)
print(h)
print('Done!')
input_dim = cos.shape[2]
print('Input shape:', cos.shape)
expected_output = np.zeros((len(cos), 1))
for i in range(len(cos) - lahead):
expected_output[i, 0] = np.mean(cos[i + 1:i + lahead + 1])
print('Output shape')
print(expected_output.shape)
print('Training Pretrain RBM')
rbm = RBM(hidden_dim,
init=glorot_uniform_sigm,
input_dim=input_dim,
hidden_unit_type='binary',
visible_unit_type='gaussian',
persistent=True,
batch_size=batch_size,
nb_gibbs_steps=10
)
model = Sequential()
model.add(Flatten(batch_input_shape=(batch_size, tsteps, input_dim)))
model.add(rbm)
opt = RMSprop(lr=lr)
model.compile(loss=rbm.contrastive_divergence_loss,
optimizer=opt,
metrics=[rbm.reconstruction_loss])
model.summary()
def main():
#grab input data set and set up dataset here
X_train = []
X_test = []
print('Creating training model')
#start with a GBRBM and then followed by 5 more RBMs for 5*2 = 10 hidden layers
dbn = DBN([
GBRBM(input_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm),
RBM(internal_dim, internal_dim, init=glorot_uniform_sigm)
])
def get_layer_loss(rbm, layer_no):
return rbm.contrastive_divergence_loss(nb_gibbs_steps=1)
def get_layer_optimizer(layer_no):
return SGD((layer_no + 1) * lr, 0., decay=0.0, nesterov=False)
dbn.compile(layer_optimizer=get_layer_optimizer, layer_loss=get_layer_loss)
#Train
#train off token vectors from early version of software
print('Training')
begin_time = time.time()
dbn.fit(X_train, batch_size, nb_epoch, verbose=1, shuffle=False)
end_time = time.time()
print('Training took %f minutes' % ((end_time - begin_time) / 60.0))
#save model parameters from training
print('Saving model')
dbn.save_weights('dbn_weights.hdf5', overwrite=True)
#load model from save
print('Loading model')
dbn.load_weights('dbn_weights.hdf5')
#generate hidden features from input data
print('Creating inference model')
F = dbn.get_forward_inference_layers()
B = dbn.get_backwards_inference_layers()
inference_model = Sequential()
for f in F:
inference_model.add(f)
inference_model.add(SampleBernoulli(mode='random'))
for b in B[:-1]:
inference_model.add(b)
inference_model.add(SampleBernoulli(mode='random'))
#last layer is a gaussian layer
inference_model.add(B[-1])
print('Compiling Theano graph')
opt = SGD()
inference_model.compile(opt, loss='mean_squared_error')
print('Doing inference')
h = inference_model.predict(X_test)
def call(self, x, mask=None):
input_shape = self.input_spec[0].shape
if self.unroll and input_shape[1] is None:
raise ValueError('Cannot unroll a RNN if the '
'time dimension is undefined. \n'
'- If using a Sequential model, '
'specify the time dimension by passing '
'an `input_shape` or `batch_input_shape` '
'argument to your first layer. If your '
'first layer is an Embedding, you can '
'also use the `input_length` argument.\n'
'- If using the functional API, specify '
'the time dimension by passing a `shape` '
'or `batch_shape` argument to your Input layer.')
if self.stateful:
initial_states = self.states
else:
initial_states = self.get_initial_states(x)
constants = self.get_constants(x)
preprocessed_input = self.preprocess_input(x)
last_output, outputs, states = K.rnn(self.step, preprocessed_input,
initial_states,
go_backwards=self.go_backwards,
mask=mask,
constants=constants,
unroll=self.unroll,
input_length=input_shape[1])
if self.stateful:
updates = []
for i in range(len(states)):
updates.append((self.states[i], states[i]))
u_t = states[0]
bv_t = states[1]
bh_t = states[2]
if(not self.finetune):
self.rbm_rnn = RBM(self.hidden_dim,init=glorot_uniform_sigm,
input_dim=self.input_dim,
hidden_unit_type='binary',
visible_unit_type='gaussian',
persistent=self.persistent,
batch_size=self.batch_input_shape[0],
nb_gibbs_steps=self.nb_gibbs_steps,
Wrbm=self.Wrbm, bx=bv_t, bh=bh_t,
dropout=self.dropout_RBM)
self.rbm_rnn.build([input_shape[0], self.input_dim])
self.loss = self.rbm_rnn.contrastive_divergence_loss
self.metrics = self.rbm_rnn.reconstruction_loss
x = K.reshape(x, (-1, self.input_dim))
if(not self.finetune):
return x
else:
#return K.sigmoid(K.dot(x, self.Wrbm) + bh_t)
return K.dot(x, self.Wrbm) + bh_t
def main():
# generate dummy dataset
nframes = 10000
dataset = np.random.normal(loc=np.zeros(input_dim),
scale=np.ones(input_dim),
size=(nframes, input_dim))
# standardize (in this case superfluous)
#dataset, mean, stddev = standardize(dataset)
# split into train and test portion
ntest = 1000
X_train = dataset[:-ntest:] # all but last 1000 samples for training
X_test = dataset[-ntest:, :] # last 1000 samples for testing
X_trainsub = dataset[:
ntest, :] # subset of training data with same number of samples as testset
assert X_train.shape[0] >= X_test.shape[
0], 'Train set should be at least size of test set!'
# setup model structure
print('Creating training model...')
dbn = DBN([
GBRBM(input_dim, 200, init=glorot_uniform_sigm),
RBM(200, 400, init=glorot_uniform_sigm),
RBM(400, 300, init=glorot_uniform_sigm),
RBM(300, 50, init=glorot_uniform_sigm),
RBM(50, hidden_dim, init=glorot_uniform_sigm)
])
# setup optimizer, loss
def get_layer_loss(rbm, layer_no):
return rbm.contrastive_divergence_loss(nb_gibbs_steps=1)
def get_layer_optimizer(layer_no):
return SGD((layer_no + 1) * lr, 0., decay=0.0, nesterov=False)
dbn.compile(layer_optimizer=get_layer_optimizer, layer_loss=get_layer_loss)
# do training
print('Training...')
begin_time = time.time()
#callbacks = [momentum_scheduler, rec_err_logger, free_energy_gap_logger]
dbn.fit(X_train, batch_size, nb_epoch, verbose=1, shuffle=False)
end_time = time.time()
print('Training took %f minutes' % ((end_time - begin_time) / 60.0))
# save model parameters
print('Saving model...')
dbn.save_weights('example.hdf5', overwrite=True)
# load model parameters
print('Loading model...')
dbn.load_weights('example.hdf5')
# generate hidden features from input data
print('Creating inference model...')
F = dbn.get_forward_inference_layers()
B = dbn.get_backward_inference_layers()
inference_model = Sequential()
for f in F:
inference_model.add(f)
inference_model.add(SampleBernoulli(mode='random'))
for b in B[:-1]:
inference_model.add(b)
inference_model.add(SampleBernoulli(mode='random'))
# last layer is a gaussian layer
inference_model.add(B[-1])
print('Compiling Theano graph...')
opt = SGD()
inference_model.compile(
opt,
loss='mean_squared_error') # XXX: optimizer and loss are not used!
print('Doing inference...')
h = inference_model.predict(dataset)
print(h)
print('Done!')