def main():
# Set parameters and print them
args = parse_args()
print_args(args)
epochs = args['epochs']
visibleUnits = args['visibleUnits']
hiddenUnits = args['hiddenUnits']
approxMethod = args['approxMethod']
approxSteps = args['approxSteps']
learnRate = args['learnRate']
persistStart = args['persistStart']
momentum = args['momentum']
decayMagnitude = args['decayMagnitude']
decayType = args['decayType']
sigma = args['sigma']
batchSize = args['batchSize']
binarization = args['binarization']
update = args['update']
trace_file = args['trace_file'] # saving results
save_file = args['save_file'] # saving parameters
load_file = args['load_file'] # loading parameters
print('Loading data')
with gzip.open('../data/mnist.pkl.gz', 'r') as f:
(TrainSet, y_train), (x_test, y_test), (ValidSet, y_Valid) = pickle.load(f, encoding='latin1')
# combine train and valid data
TrainSet = np.concatenate((TrainSet, x_test), axis=0)
# Create binary data
TrainSet = sklearn.preprocessing.binarize(TrainSet, threshold=binarization).T
ValidSet = sklearn.preprocessing.binarize(ValidSet, threshold=binarization).T
if len(load_file) == 0:
print('Initializing model')
model = RBM.RBM(n_vis=visibleUnits, n_hid=hiddenUnits,
momentum = momentum,
sigma = sigma,
trainData = TrainSet,
wiseStart = True,
)
print('Start of training')
Training.fit(model, TrainSet, ValidSet,
n_epochs = epochs,
weight_decay = decayType,
decay_magnitude = decayMagnitude,
lr = learnRate,
batch_size = batchSize,
NormalizationApproxIter = approxSteps,
approx = approxMethod,
update = update,
persistent_start = persistStart,
trace_file = trace_file,
save_file = save_file
)
else:
params = RBM.RBM.load(load_file)
model = RBM.RBM(params=params)
model.reconstructionArray(ValidSet[:, 0:10])
def __init__(self,
input_data=None,
label=None,
input_size=2,
hidden_layer_sizes=[3, 3],
output_size=1,
batch_size=10,
learning_rate=0.1,
epochs=50,
C=1,
rng=None):
self.input = input_data.astype(np.float32)
self.y = label.astype(np.float32)
self.rbm_layers = []
self.n_hidden_layers = len(
hidden_layer_sizes) # nombre de couches cachées du DBN
self.batch_size = batch_size
self.layer_svm = None
self.model_finetune = None
self.nepochs = epochs
self.learning_rate = learning_rate
self.C = C
self.rng = rng
#W,hbias,vbias = self.init_weights(input_size,hidden_layer_sizes[0])
# construct rbm_layer
rbm_layer = RBM(input=self.input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[0],
batch_size=10,
learning_rate=0.1,
epochs=1000,
rng=self.rng)
self.rbm_layers.append(rbm_layer)
sample_input = self.input
for rbm_index in range(self.n_hidden_layers - 1):
_, sample_input = self.rbm_layers[rbm_index].sample_h_given_v(
sample_input)
n_hid = hidden_layer_sizes[rbm_index + 1]
n_vis = hidden_layer_sizes[rbm_index]
#W,hbias,vbias = self.init_weights(n_vis,n_hid)
rbm_i = RBM(input=sample_input,
n_visible=n_vis,
n_hidden=n_hid,
batch_size=10,
learning_rate=0.09,
epochs=100,
rng=self.rng)
self.rbm_layers.append(rbm_i)
def __init__(self, architecture):
self.stack = []
previous_layer_size = architecture[0]
if len(architecture) > 1:
for layer_size in architecture[1:]:
self.stack.append(RBM(previous_layer_size, layer_size))
previous_layer_size = layer_size
def _init_coef(self, fan_in, fan_out):
if self.activation == 'logistic':
init_bound = np.sqrt(2. / (fan_in + fan_out))
elif self.activation in ('identity', 'tanh', 'relu'):
init_bound = np.sqrt(6. / (fan_in + fan_out))
else:
raise ValueError("Unknown activation function %s" %
self.activation)
if self.first_layer_init:
features, labels, sparse_encoder, int_encoder = Preprocessing.feature_extraction_sparse_train(
"project_training.txt")
#svd = TruncatedSVD(n_components=70)
#features = svd.fit_transform(features)
weights = RBM.train(features, 150)
weights = weights.reshape(-1, 1)
print fan_in, fan_out
coef_init = weights
self.first_layer_init = False
else:
coef_init = self._random_state.uniform(-init_bound, init_bound,
(fan_in, fan_out))
intercept_init = self._random_state.uniform(-init_bound, init_bound,
fan_out)
return coef_init, intercept_init
def __init__(self, title, layerSizes, types, modelDir):
#checking DBN configuration
if not len(types) == len(layerSizes) - 1:
print 'DBN initialize error: layerSizes types length mismatch'
exit()
for i in range(len(types) - 1):
if not (types[i] in ['BB', 'GB', 'BG']
and types[i + 1] in ['BB', 'GB', 'BG']):
print "DBN initialize error: RBM types error, RBM types must be ['BB','GB','BG']"
exit()
if not types[i][1] == types[i + 1][0]:
print 'DBN initialize error: RBM types sequence error'
exit()
print '\nInitializing DBN: %s, layerSizes: %s, layerTypes: %s' % (
title, layerSizes, types)
self.title = title
self.layerSizes = layerSizes
self.layerNum = len(self.layerSizes) - 1
self.modelDir = modelDir
self.rbms = []
self.rbmFiles = []
for i in range(self.layerNum):
self.rbms.append(
RBM(input=None,
n_visible=self.layerSizes[i],
n_hidden=self.layerSizes[i + 1],
type=types[i]))
self.rbmFiles.append(modelDir + os.sep +
'%s_layer%dRBM_%d-%d.rbm' %
(self.title, i + 1, self.layerSizes[i],
self.layerSizes[i + 1]))
'''
def pretrain(self, x, epochs, num_samples=50000):
'''
Greedy layer-wise training
The last layer is a RBM with linear hidden units
shape(x) = (v_dim, number_of_examples)
'''
RBM_layers = []
for i in range(self.num_hidden_layers): # initialize RBM's
if (i < self.num_hidden_layers - 1):
RBM_layers.append(
RBM(self.layer_dims[i], self.layer_dims[i + 1]))
else:
RBM_layers.append(
RBM_with_linear_hidden_units(self.layer_dims[i],
self.layer_dims[i + 1]))
for i in range(self.num_hidden_layers): # train RBM's
print("Training RBM layer %i" % (i + 1))
RBM_layers[i].train(x, epochs) # train the ith RBM
if not (i == self.num_hidden_layers -
1): # generate samples to train next layer
_, x = RBM_layers[i].gibbs_sampling(2, num_samples)
self.W.append(RBM_layers[i].W) # save trained weights
self.b.append(RBM_layers[i].b)
self.a.append(RBM_layers[i].a)
self.pretrained = True
return
def pre_rbm_train():
#pre train by rbm between two layers
RBM.rbm(Config.FILE_ZERO_RBM_PATH, Config.HIDDEN_UNITS_NUM_FIRST_RBM, Config.FILE_FIRST_RBM_PATH)
RBM.rbm(Config.FILE_FIRST_RBM_PATH, Config.HIDDEN_UNITS_NUM_SECOND_RBM, Config.FILE_SECOND_RBM_PATH)
RBM.rbm(Config.FILE_SECOND_RBM_PATH, Config.HIDDEN_UNITS_NUM_THIRD_RBM, Config.FILE_THIRD_RBM_PATH)
RBM.rbm(Config.FILE_THIRD_RBM_PATH, Config.OUTPUT_UNITS_NUM_FORTH_RBM, Config.FILE_FORTH_RBM_PATH)
def recurrence(nest, v_t):
bv_t = tf.add(bv, tf.matmul(nest[0], Wuv))
bh_t = tf.add(bh, tf.matmul(nest[0], Wuh))
v_t_sample = RBM.gibbs_sample(tf.reshape(v_t, [1, n_visible]), W, bv_t , bh_t, k=1 )
v_t_sample = tf.reshape(v_t_sample, [1, n_visible])
u_t = tf.sigmoid(bu + tf.matmul(v_t_sample, Wvu) + tf.matmul(nest[0], Wuu))
return [u_t, bv_t, bh_t, v_t_sample]
def train(x=x, size_bt=size_bt, BV_t=BV_t, BH_t=BH_t):
bv_init = tf.zeros([1, n_visible], tf.float32)
bh_init = tf.zeros([1, n_hidden], tf.float32)
u_t = tf.scan(rnn_recurrence, x, initializer=u0)
BV_t = tf.reshape(tf.scan(visible_bias_recurrence, u_t, bv_init), [size_bt, n_visible])
BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, u_t, bh_init), [size_bt, n_hidden])
sample, cost, monitor = RBM.get_free_energy_cost(x, W, BV_t, BH_t, k=15)
return x, sample, cost, monitor, params, size_bt
def build_rbm(input_var=None,d_x=21,d_y=21):
l_in = lasagne.layers.InputLayer(shape=(None, 1, d_x, d_y),
input_var=input_var)
l_in_drop = lasagne.layers.DropoutLayer(l_in, p=0.2)
l_hid = RBM.RBM(l_in_drop,num_units=128,kGibbs=1)
return l_hid
def train(x=x, size_bt=size_bt, BV_t=BV_t, BH_t=BH_t):
bv_init = tf.zeros([1, n_visible], tf.float32)
bh_init = tf.zeros([1, n_hidden], tf.float32)
u_t = tf.scan(rnn_recurrence, x, initializer=u0)
BV_t = tf.reshape(tf.scan(visible_bias_recurrence, u_t, bv_init), [size_bt, n_visible])
BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, u_t, bh_init), [size_bt, n_hidden])
sample, cost = RBM.build_rbm(x, W, BV_t, BH_t, k=15)
return x, sample, cost, params, size_bt
def dbn_train(NN,SD,opt):
lAmt=len(NN['layer'])
if 'maxStep' not in opt:
opt['maxStep']=50
### Train the 1st hidden layer(connected by input layer).
# Construct RBM from current layer of network.
rbm={}
rbm['W']=NN['layer'][2]['W']
rbm['vB']=np.zeros(1,NN['sizes'](1))
rbm['hB']=NN['layer'][2]['B']
# Training of RBM.
rbm=RBM.rbm_train(rbm,SD,opt) # input data as training data for 1st RBM
# Update RBM parameters to network.
NN['layer'][2]['W']=rbm['W']
NN['layer'][1]['B']=rbm['vB']
NN['layer'][2]['B']=rbm['hB']
# Propagate input to next layer.
X=RBM.rbm_up(rbm,SD) #stochastic propagation.
### Train other hidden layers.
for li in range(2,lAmt):
# Construct RBM from current layer of network.
rbm={}
rbm['W']=NN['layer'][li]['W']
rbm['vB']=NN['layer'][li-1]['B']
rbm['hB']=NN['layer'][li]['B']
# Training of RBM.
rbm=RBM.rbm_train(rbm,X,opt)
# Update RBM parameters to network.
NN['layer'][li]['W']=rbm['W']
NN['layer'][li-1]['B']=rbm['vB']
NN['layer'][li]['B']=rbm['hB']
# Propagate input to next layer.
X=RBM.rbm_up(rbm,X,True) #stochastic propagation.
return NN
def get_RBM():
r = RBM(6, 2)
k = 9
dataset = [[1, 1, 1, 0, 0, 0], [1, 0, 1, 0, 0, 0], [1, 1, 1, 0, 0, 0],
[0, 0, 1, 1, 1, 0], [0, 0, 1, 1, 1, 0], [0, 0, 1, 1, 1, 0],
[0, 1, 0, 0, 0, 1], [0, 1, 0, 0, 0, 1], [0, 1, 1, 0, 0, 1]]
# Train at the following learning rates:
print "Training..."
learning_rates = [0.1, 0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001]
# learning_rates = [0.1]
for rate in learning_rates:
print "Learning rate =", rate
for i in range(100):
print ' ' + str(i) + '...',
r.train_epoch(dataset, rate, k)
err = np.sum([r.free_energy(data) for data in dataset])
print 'Energy = ' + str(err) + '...',
PL = r.pseudolikelihood(dataset)
print 'Pseudolikelihood = ' + str(PL) + '...',
L = r.likelihood(dataset)
print 'Likelihood = ' + str(L) + '...',
print 'Done'
return r, dataset
def train_rbm(nv,
nh,
batch_size,
epochs,
val_set,
gibbs_sampling_nb,
rbm=None):
if (rbm == None):
rbm = RBM(nv, nh)
for epoch in range(1, epochs + 1):
training_loss = 0
s = 0.0
for user in range(0, nb_users - batch_size, batch_size):
vk = val_set[user:user + batch_size]
v0 = val_set[user:user + batch_size]
for sample in range(gibbs_sampling_nb):
_, hk = rbm.sample_h(vk)
_, vk = rbm.sample_v(hk)
vk[v0 < 0] = v0[v0 < 0]
phk, _ = rbm.sample_h(vk)
ph0, _ = rbm.sample_h(v0)
rbm.train(v0, vk, ph0, phk)
training_loss += torch.mean(torch.abs(v0[v0 >= 0] - vk[v0 >= 0]))
s += 1
# print("Epoch:", epoch, "Training Loss:", training_loss/s)
return float(training_loss / s), rbm
def main():
print("reading data...")
data = ngsim_manipulation.Data()
x = tf.placeholder(tf.float32, [None, rnn_rbm.n_visible],
name="x") #The placeholder variable that holds our data
W = tf.Variable(tf.random_normal([rnn_rbm.n_visible, rnn_rbm.n_hidden],
0.01),
name="W") #The weight matrix of the RBM
Wuh = tf.Variable(tf.random_normal(
[rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden], 0.0001),
name="Wuh") #The RNN -> RBM hidden weight matrix
bh = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden], tf.float32),
name="bh") #The RNN -> RBM hidden bias vector
Wuv = tf.Variable(tf.random_normal(
[rnn_rbm.n_hidden_recurrent, rnn_rbm.n_visible], 0.0001),
name="Wuv") #The RNN -> RBM visible weight matrix
bv = tf.Variable(tf.zeros([1, rnn_rbm.n_visible], tf.float32),
name="bv") #The RNN -> RBM visible bias vector
Wvu = tf.Variable(tf.random_normal(
[rnn_rbm.n_visible, rnn_rbm.n_hidden_recurrent], 0.0001),
name="Wvu") #The data -> RNN weight matrix
Wuu = tf.Variable(tf.random_normal(
[rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden_recurrent], 0.0001),
name="Wuu") #The RNN hidden unit weight matrix
bu = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32),
name="bu") #The RNN hidden unit bias vector
u0 = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32),
name="u0") #The initial state of the RNN
#The RBM bias vectors. These matrices will get populated during rnn-rbm training and generation
BH_t = tf.Variable(tf.ones([1, rnn_rbm.n_hidden], tf.float32), name="BH_t")
BV_t = tf.Variable(tf.ones([1, rnn_rbm.n_visible], tf.float32),
name="BV_t")
#Build the RBM optimization
saver = tf.train.Saver()
#Note that we initialize the RNN->RBM bias vectors with the bias vectors of the trained RBM. These vectors will form the templates for the bv_t and
#bh_t of each RBM that we create when we run the RNN-RBM
updt = RBM.get_cd_update(x, W, bv, bh, k=1, lr=lr) # CD-k
print("training RBM for weight initialization...")
#Run the session
with tf.Session() as sess:
#Initialize the variables of the model
init = tf.global_variables_initializer()
sess.run(init)
#Run over each trajectory num_epoch times
for epoch in tqdm(range(num_epochs)):
#for idx in range(100):
for idx in range(data.num_trajectories):
sess.run(
updt, feed_dict={x: data.next_traj()}
) # even if traj is comprised of data in different time, we treat it like a batch of data in the same time, because we can only initialize one RBM.
#Save the initialized model here
save_path = saver.save(sess, "parameter_checkpoints/initialized.ckpt")
def main():
#Load the Songs
songs = input_manipulation.get_songs('Game_Music_Midi')
x = tf.placeholder(tf.float32, [None, rnn_rbm.n_visible],
name="x") #The placeholder variable that holds our data
#parameters of CRBM
W = tf.Variable(tf.random_normal([rnn_rbm.n_visible, rnn_rbm.n_hidden],
0.01),
name="W") #The weight matrix of the RBM
bh = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden], tf.float32),
name="bh") #The RNN -> RBM hidden bias vector
bv = tf.Variable(tf.zeros([1, rnn_rbm.n_visible], tf.float32),
name="bv") #The RNN -> RBM visible bias vector
#parameters related to RNN
Wuh = tf.Variable(tf.random_normal(
[rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden], 0.0001),
name="Wuh") #The RNN -> RBM hidden weight matrix
Wuv = tf.Variable(tf.random_normal(
[rnn_rbm.n_hidden_recurrent, rnn_rbm.n_visible], 0.0001),
name="Wuv") #The RNN -> RBM visible weight matrix
Wvu = tf.Variable(tf.random_normal(
[rnn_rbm.n_visible, rnn_rbm.n_hidden_recurrent], 0.0001),
name="Wvu") #The data -> RNN weight matrix
Wuu = tf.Variable(tf.random_normal(
[rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden_recurrent], 0.0001),
name="Wuu") #The RNN hidden unit weight matrix
bu = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32),
name="bu") #The RNN hidden unit bias vector
u0 = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32),
name="u0") #The initial state of the RNN
#The RBM bias vectors. These matrices will get populated during rnn-rbm training and generation
BH_t = tf.Variable(tf.ones([1, rnn_rbm.n_hidden], tf.float32), name="BH_t")
BV_t = tf.Variable(tf.ones([1, rnn_rbm.n_visible], tf.float32),
name="BV_t")
#Build the RBM optimization
saver = tf.train.Saver()
#Note that we initialize the RNN->RBM bias vectors with the bias vectors of the trained RBM. These vectors will form the templates for the bv_t and
#bh_t of each RBM that we create when we run the RNN-RBM
updt = RBM.get_cd_update(x, W, bv, bh, 1, lr)
#Run the session
with tf.Session() as sess:
#Initialize the variables of the model
init = tf.initialize_all_variables()
sess.run(init)
#Run over each song num_epoch times
for epoch in tqdm(range(num_epochs)):
for song in songs:
sess.run(updt, feed_dict={x: song})
#Save the initialized model here
save_path = saver.save(sess, "parameter_checkpoints/initialized.ckpt")
def train_and_sample(T, lr):
temp_name = 'T = ' + format(T, '.2f') + '.npy'
file_name = 'T = ' + format(T, '.2f') + ', lr = ' + str(lr) + '.npy'
completeLoad = os.path.join(load_path, temp_name)
samples = (np.load(completeLoad) + 1) / 2 # convert to 0, 1
sz, N, N1 = samples.shape
samples_flat = np.reshape(samples, (sz, N * N1))
r = RBM(num_visible=64, num_hidden=nH)
r.train(samples_flat, max_epochs=me, learning_rate=lr, batch_size=bs)
print("Wights at T = " + format(T, '.2f') + ": ", r.weights)
RBM_data_flat = r.daydream(ns) * 2 - 1 # convert back to -1, 1
RBM_data = np.reshape(RBM_data_flat, (ns, N, N1))
completeSaveData = os.path.join(save_data_path, file_name)
np.save(completeSaveData, RBM_data)
completeSaveWeight = os.path.join(save_weight_path, file_name)
np.save(completeSaveWeight, r.weights)
completeSaveError = os.path.join(save_error_path, file_name)
np.save(completeSaveError, r.errors)
def generate_recurrence(count, k, u_tm1, primer, x, music):
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
primer = tf.zeros([1, n_visible], tf.float32)
# primer = tf.slice(x, [count, 0], [1, n_visible])
x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=25)
u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu)))
music = tf.concat(0, [music, x_out])
return count+1, k, u_t, x_out, x, music
def pre_rbm_train():
#pre train by rbm between two layers
RBM.rbm(Config.FILE_ZERO_RBM_PATH, Config.HIDDEN_UNITS_NUM_FIRST_RBM,
Config.FILE_FIRST_RBM_PATH)
RBM.rbm(Config.FILE_FIRST_RBM_PATH, Config.HIDDEN_UNITS_NUM_SECOND_RBM,
Config.FILE_SECOND_RBM_PATH)
RBM.rbm(Config.FILE_SECOND_RBM_PATH, Config.HIDDEN_UNITS_NUM_THIRD_RBM,
Config.FILE_THIRD_RBM_PATH)
RBM.rbm(Config.FILE_THIRD_RBM_PATH, Config.OUTPUT_UNITS_NUM_FORTH_RBM,
Config.FILE_FORTH_RBM_PATH)
def generate(num,
x=x,
size_bt=size_bt,
u0=u0,
n_visible=n_visible,
prime_length=100):
"""
This function handles generating music. This function is one of the outputs of the build_rnnrbm function
Args:
num (int): The number of timesteps to generate
x (tf.placeholder): The data vector. We can use feed_dict to set this to the music primer.
size_bt (tf.float32): The batch size
u0 (tf.Variable): The initial state of the RNN
n_visible (int): The size of the data vectors
prime_length (int): The number of timesteps into the primer song that we use befoe beginning to generate music
Returns:
The generated music, as a tf.Tensor
"""
Uarr = tf.scan(rnn_recurrence, x, initializer=u0)
U = Uarr[int(np.floor(prime_length /
midi_manipulation.num_timesteps)), :, :]
# def generate_recurrence(count, k, u_tm1, primer, x, music):
# return count + 1, k, u_t, x_out, x, music
count = 0
music = tf.zeros([1, n_visible], tf.float32)
u_tm1 = U
primer = tf.zeros([1, n_visible], tf.float32)
while count < num:
# This function builds and runs the gibbs steps for each RBM in the chain to generate music
# Get the bias vectors from the current state of the RNN
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
# Run the Gibbs step to get the music output. Prime the RBM with the previous musical output.
x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=25)
# Update the RNN hidden state based on the musical output and current hidden state.
u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu)))
# Add the new output to the musical piece
music = tf.concat([music, x_out], 0)
count += 1
u_tm1 = u_t
primer = x_out
# [_, _, _, _, _, music] = control_flow_ops.while_loop(lambda count, num_iter, *args: count < num_iter,
# generate_recurrence, [tf.constant(1, tf.int32), tf.constant(num), U,
# tf.zeros([1, n_visible], tf.float32), x,
# tf.zeros([1, n_visible], tf.float32)],
# shape_invariants=None)
return music
def generate_recurrence(count, k, u_tm1, primer, x, music):
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
# primer = tf.zeros([1, n_visible], tf.float32)
# primer = tf.slice(x, [count, 0], [1, n_visible])
x_out, _ = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=25, c_1=0.5, c_2=0.5)
u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu)))
music = tf.concat(0, [music, x_out])
return count+1, k, u_t, x_out, x, music
def initRBMs(self):
self.RBMList = []
for i in range(self.layers):
currVisDim = self.dimensionList[i]
currHidDim = self.dimensionList[i + 1]
currWR = self.trainInfoList[i][0]
groupWeights = False
currRBM = RBM(currVisDim, currHidDim, currWR, groupWeights)
self.RBMList.append(currRBM)
def gen_sample(self, k, x=None):
with tf.variable_scope('dbn_gen_sample'):
# Get input value for the top rbm's visible layer
if x is None:
x = tf.zeros([1, int(self.bs[-2].shape[-1])])
else:
# Propagate input value up to visible layer of top rbm
for i in range(len(self.ws) - 1):
x_prob = tf.sigmoid(tf.matmul(x, self.ws[i]) + self.bs[i+1])
x = RBM.sample(x_prob)
# Sample from the top layer
top_rbm = RBM.RBM(self.ws[-1], self.bs[-2], self.bs[-1])
_, x_sample = top_rbm.gibbs_sample(x, k)
# Propagate sample down to visible layer
for i in reversed(range(len(self.ws) - 1)):
x_prob = tf.sigmoid(tf.matmul(x_sample, tf.transpose(self.ws[i])) + self.bs[i])
x_sample = RBM.sample(x_prob)
return x_sample
def reconstruction():
#This function handles reconstructing de trajectory. This function is one of the outputs of the rnnrbm() function
primer=x
#Scan through the rnn and generate the value for each hidden node in the batch
Uarr = tf.scan(rnn_recurrence, primer, initializer=u0) # primer is of dimension 2, Uarr is of dimension 3
Uarr = tf.concat([tf.reshape(u0,[1, 1, n_hidden_recurrent]), Uarr[:-1,:,:]], 0)
#Scan through the rnn and generate the visible and hidden biases for each RBM in the batch
BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, bv), [size_bt, n_visible])
BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, bh), [size_bt, n_hidden])
traj = RBM.gibbs_sample(primer, W, BV_t, BH_t, k=1)
return traj[1:,:] #we discard the first time step because it has a huge deviation
def generate_recurrence(count, num, u_tm1, primer, x, music, k_in):
#This function builds and runs the gibbs steps for each RBM in the chain to generate music
#Get the bias vectors from the current state of the RNN
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
#Run the Gibbs step to get the music output. Prime the RBM with the previous musical output.
x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=k_in)
#Update the RNN hidden state based on the musical output and current hidden state.
u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu)))
#Add the new output to the musical piece
music = tf.concat([music, x_out], 0)
return count + 1, num, u_t, x_out, x, music, k_in
def recurrence(nest, v_t):
# This function do all the procedures in one RBM.
bv_t = tf.add(bv,
tf.matmul(nest[0],
Wuv)) # generate current bv_t based on u_{t-1}
bh_t = tf.add(bh,
tf.matmul(nest[0],
Wuh)) # generate current bh_t based on u_{t-1}
v_t_sample = RBM.gibbs_sample(
tf.reshape(v_t, [1, n_visible]), W, bv_t, bh_t,
k=1) # sample v_t according generated bias
v_t_sample = tf.reshape(v_t_sample, [1, n_visible])
u_t = tf.sigmoid(bu + tf.matmul(v_t_sample, Wvu) +
tf.matmul(nest[0], Wuu)) # calculate current u_t
return [u_t, bv_t, bh_t, v_t_sample]
def fit(self, trainset):
AlgoBase.fit(self, trainset)
numUsers = trainset.n_users
numItems = trainset.n_items
trainingMatrix = np.zeros([numUsers, numItems, 10], dtype=np.float32)
for (uid, iid, rating) in trainset.all_ratings():
adjustedRating = int(float(rating) * 2.0) - 1
trainingMatrix[int(uid), int(iid), adjustedRating] = 1
# Flatten to a 2D array, with nodes for each possible rating type on each possible item, for every user.
trainingMatrix = np.reshape(trainingMatrix,
[trainingMatrix.shape[0], -1])
# Create an RBM with (num items * rating values) visible nodes
rbm = RBM.RBM(trainingMatrix.shape[1],
hiddenDimensions=self.hiddenDim,
learningRate=self.learningRate,
batchSize=self.batchSize,
epochs=self.epochs)
rbm.Train(trainingMatrix)
self.predictedRatings = np.zeros([numUsers, numItems],
dtype=np.float32)
for uiid in range(trainset.n_users):
if (uiid % 50 == 0):
print("Processing user ", uiid)
recs = rbm.GetRecommendations([trainingMatrix[uiid]])
recs = np.reshape(recs, [numItems, 10])
for itemID, rec in enumerate(recs):
# The obvious thing would be to just take the rating with the highest score:
#rating = rec.argmax()
# ... but this just leads to a huge multi-way tie for 5-star predictions.
# The paper suggests performing normalization over K values to get probabilities
# and take the expectation as your prediction, so we'll do that instead:
normalized = self.softmax(rec)
rating = np.average(np.arange(10), weights=normalized)
self.predictedRatings[uiid, itemID] = (rating + 1) * 0.5
return self
def generate_recurrence(step, u_tm1, music):
#To generate music: This function builds and runs the gibbs steps for each RBM in the chain
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
#Run the Gibbs step to get the music output. Prime the RBM with the previous musical output.
x_out = RBM.gibbs_sample(tf.reshape(music[-1, :], [1, n_visible]),
W,
bv_t,
bh_t,
k=25)
#Update the RNN hidden state based on the musical output and current hidden state.
u_t = (tf.sigmoid(bu + tf.matmul(x_out, Wvu) +
tf.matmul(u_tm1, Wuu)))
#Add the new output to the musical piece
music = tf.concat([music, x_out], axis=0)
return step + 1, u_t, music
def pretrain(self, x, epochs, num_samples=50000):
'''
Greedy layer-wise training
The last layer is a RBM with linear hidden units
shape(x) = (v_dim, number_of_examples)
'''
RBM_layers = []
# initialize RBMs
for i in range(self.num_hidden_layers):
if i == 0:
RBM_layers.append(
RBM_with_linear_visible_units(
self.layer_dims[i],
self.layer_dims[i + 1])) # linear input
elif (i < self.num_hidden_layers - 1):
RBM_layers.append(
RBM(self.layer_dims[i], self.layer_dims[i + 1])) #
else:
RBM_layers.append(
RBM_with_linear_hidden_units(
self.layer_dims[i],
self.layer_dims[i + 1])) # linear output
# train stack of RBMs
for i in range(self.num_hidden_layers): # train RBM's
print("Training RBM layer %i" % (i + 1))
x = RBM_layers[i].train(x, epochs) # train the ith RBM
self.W.append(RBM_layers[i].W) # save trained weights
self.b.append(RBM_layers[i].b)
self.a.append(RBM_layers[i].a)
self.pretrained = True
return
def get_RBM():
r = RBM(6,2)
k = 9
dataset = [[1,1,1,0,0,0],
[1,0,1,0,0,0],
[1,1,1,0,0,0],
[0,0,1,1,1,0],
[0,0,1,1,1,0],
[0,0,1,1,1,0],
[0,1,0,0,0,1],
[0,1,0,0,0,1],
[0,1,1,0,0,1]]
# Train at the following learning rates:
print "Training..."
learning_rates = [0.1, 0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001]
# learning_rates = [0.1]
for rate in learning_rates:
print "Learning rate =", rate
for i in range(100):
print ' ' + str(i) + '...',
r.train_epoch(dataset, rate, k)
err = np.sum([r.free_energy(data) for data in dataset])
print 'Energy = ' + str(err) + '...',
PL = r.pseudolikelihood(dataset)
print 'Pseudolikelihood = ' + str(PL) + '...',
L = r.likelihood(dataset)
print 'Likelihood = ' + str(L) + '...',
print 'Done'
return r, dataset
def get_RBM():
print "Loading digits...."
digits, classes = load_digits('digits_train.txt')
digits = [digit_gray_to_binary(d) for d in digits]
r = RBM(196,25)
# Train at the following learning rates:
print "Training..."
learning_rates = [0.1, 0.05] #, 0.01, 0.005, 0.001]
for rate in learning_rates:
print "Learning rate =", rate
for i in range(20):
print ' ' + str(i) + '...',
r.train_epoch(digits, rate, 5)
err = np.sum([r.free_energy(digit) for digit in digits])
print 'Energy = ' + str(err) + '...',
PL = r.pseudolikelihood(digits)
print 'Pseudolikelihood = ' + str(PL) + '...',
print 'Done'
return r, digits
import matplotlib.pyplot as plt
from scipy.ndimage import convolve
from sklearn import linear_model, datasets, metrics
from sklearn.cross_validation import train_test_split
from sklearn.neural_network import BernoulliRBM
from sklearn.pipeline import Pipeline
from sklearn.metrics import confusion_matrix
import scipy.io
from RBM import *
mat = scipy.io.loadmat('input.mat')
train_data=mat['train_x']
test_data=mat['test_x']
train_label=mat['train_y']
test_label=mat['test_y']
r1 = RBM(num_visible = 256, num_hidden = 250)
train_data1 = train_data
test_data1 = test_data
r1.train(train_data1, max_epochs = 5000)
output_train1,prob_train1=r1.run_visible(train_data1)
output_test1,prob_test1=r1.run_visible(test_data1)
r2 = RBM(num_visible = 250, num_hidden = 220)
train_data2 = output_train1
test_data2 = output_test1
r2.train(train_data2, max_epochs = 5000)
def rnnrbm():
#This function builds the RNN-RBM and returns the parameters of the model
x = tf.placeholder(
tf.float32,
[None, n_visible]) #The placeholder variable that holds our data
lr = tf.placeholder(
tf.float32
) #The learning rate. We set and change this value during training.
size_bt = tf.shape(x)[0] #the batch size
# parameters
W = tf.Variable(tf.zeros([n_visible, n_hidden]), name="W")
Wuh = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden]), name="Wuh")
Wuv = tf.Variable(tf.zeros([n_hidden_recurrent, n_visible]), name="Wuv")
Wvu = tf.Variable(tf.zeros([n_visible, n_hidden_recurrent]), name="Wvu")
Wuu = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden_recurrent]),
name="Wuu")
bh = tf.Variable(tf.zeros([1, n_hidden]), name="bh")
bv = tf.Variable(tf.zeros([1, n_visible]), name="bv")
bu = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="bu")
u0 = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="u0")
BH_t = tf.Variable(tf.zeros([1, n_hidden]), name="BH_t")
BV_t = tf.Variable(tf.zeros([1, n_visible]), name="BV_t")
def rnn_recurrence(u_tm1, v_t):
#Iterate through the data in the batch and generate the values of the RNN hidden nodes
v_t = tf.reshape(v_t, [1, n_visible])
u_t = (tf.sigmoid(bu + tf.matmul(v_t, Wvu) + tf.matmul(u_tm1, Wuu)))
return u_t
def visible_bias_recurrence(bv_t, u_tm1):
#Iterate through the values of the RNN hidden nodes and generate the values of the visible bias vectors
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
return bv_t
def hidden_bias_recurrence(bh_t, u_tm1):
#Iterate through the values of the RNN hidden nodes and generate the values of the hidden bias vectors
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
return bh_t
####################
#Below is two functions for generation and construction, but not used in training
####################
def generate(timesteps, primer=x):
"""
This function handles generating music. This function is one of the outputs of the build_rnnrbm function
Args:
timesteps (int): The number of timesteps to generate
primer (tf.placeholder): The primer song
Returns:
The generated music, as a tf.Tensor
"""
def generate_recurrence(step, u_tm1, music):
#To generate music: This function builds and runs the gibbs steps for each RBM in the chain
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
#Run the Gibbs step to get the music output. Prime the RBM with the previous musical output.
x_out = RBM.gibbs_sample(tf.reshape(music[-1, :], [1, n_visible]),
W,
bv_t,
bh_t,
k=25)
#Update the RNN hidden state based on the musical output and current hidden state.
u_t = (tf.sigmoid(bu + tf.matmul(x_out, Wvu) +
tf.matmul(u_tm1, Wuu)))
#Add the new output to the musical piece
music = tf.concat([music, x_out], axis=0)
return step + 1, u_t, music
Uarr = tf.scan(
rnn_recurrence, primer,
initializer=u0) # x is of dimension 2, Uarr is of dimension 3
u_tm1 = Uarr[
-1, :, :] # the beginning hidden recurrent unit of our generation
music = tf.zeros([1, n_visible])
loop_vars = [tf.constant(0), u_tm1, music]
cond = lambda count, *args: count < timesteps
[_, _, music] = tf.while_loop(cond,
generate_recurrence,
loop_vars,
shape_invariants=[
tf.constant(0).get_shape(),
u_tm1.get_shape(),
tf.TensorShape([None, None])
])
return music
def reconstruction():
#This function handles reconstructing music. This function is one of the outputs of the rnnrbm() function
primer = x
#Scan through the rnn and generate the value for each hidden node in the batch
Uarr = tf.scan(
rnn_recurrence, primer,
initializer=u0) # primer is of dimension 2, Uarr is of dimension 3
Uarr = tf.concat(
[tf.reshape(u0, [1, 1, n_hidden_recurrent]), Uarr[:-1, :, :]], 0)
#Scan through the rnn and generate the visible and hidden biases for each RBM in the batch
BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, bv),
[size_bt, n_visible])
BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, bh),
[size_bt, n_hidden])
music = RBM.gibbs_sample(primer, W, BV_t, BH_t, k=1)
return music
##########################
# Below is for training
##########################
#Reshape our bias matrices to be the same size as the batch.
tf.assign(BH_t, tf.tile(BH_t, [size_bt, 1]))
tf.assign(BV_t, tf.tile(BV_t, [size_bt, 1]))
#Scan through the rnn and generate the value for each hidden node in the batch
u_t = tf.scan(rnn_recurrence, x, initializer=u0)
#Scan through the rnn and generate the visible and hidden biases for each RBM in the batch
BV_t = tf.reshape(
tf.scan(visible_bias_recurrence, u_t,
tf.zeros([1, n_visible], tf.float32)), [size_bt, n_visible])
BH_t = tf.reshape(
tf.scan(hidden_bias_recurrence, u_t,
tf.zeros([1, n_hidden], tf.float32)), [size_bt, n_hidden])
#Get the free energy cost from each of the RBMs in the batch
cost = RBM.get_free_energy_cost(x, W, BV_t, BH_t, k=15)
return x, cost, generate, reconstruction, W, Wuh, Wuv, Wvu, Wuu, bh, bv, bu, lr, u0
def rnnrbm(k_input=15):
#This function builds the RNN-RBM and returns the parameters of the model
x = tf.placeholder(
tf.float32,
[None, n_visible]) #The placeholder variable that holds our data
lr = tf.placeholder(
tf.float32
) #The learning rate. We set and change this value during training.
size_bt = tf.shape(x)[0] #the batch size
#Here we set aside the space for each of the variables.
#We intialize these variables when we load saved parameters in rnn_rbm_train.py or rnn_rbm_generate.py
W = tf.Variable(tf.zeros([n_visible, n_hidden]), name="W")
Wuh = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden]), name="Wuh")
Wuv = tf.Variable(tf.zeros([n_hidden_recurrent, n_visible]), name="Wuv")
Wvu = tf.Variable(tf.zeros([n_visible, n_hidden_recurrent]), name="Wvu")
Wuu = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden_recurrent]),
name="Wuu")
bh = tf.Variable(tf.zeros([1, n_hidden]), name="bh")
bv = tf.Variable(tf.zeros([1, n_visible]), name="bv")
bu = tf.Variable(
tf.zeros([1, n_hidden_recurrent]), name="bu"
) # Bias for 'hidden_recurrent' -> next 'hidden_recurrent' [RNN]
u0 = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="u0")
BH_t = tf.Variable(tf.zeros([1, n_hidden]), name="BH_t")
BV_t = tf.Variable(tf.zeros([1, n_visible]), name="BV_t")
def rnn_recurrence(u_tm1, sl):
#Iterate through the data in the batch and generate the values of the RNN hidden nodes
sl = tf.reshape(sl, [1, n_visible])
u_t = (tf.tanh(bu + tf.matmul(sl, Wvu) + tf.matmul(u_tm1, Wuu)))
return u_t
def visible_bias_recurrence(bv_t, u_tm1):
#Iterate through the values of the RNN hidden nodes and generate the values of the visible bias vectors
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
return bv_t
def hidden_bias_recurrence(bh_t, u_tm1):
#Iterate through the values of the RNN hidden nodes and generate the values of the hidden bias vectors
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
return bh_t
def generate_recurrence(count, num, u_tm1, primer, x, music, k_in):
#This function builds and runs the gibbs steps for each RBM in the chain to generate music
#Get the bias vectors from the current state of the RNN
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
#Run the Gibbs step to get the music output. Prime the RBM with the previous musical output.
x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=k_in)
#Update the RNN hidden state based on the musical output and current hidden state.
u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu)))
#Add the new output to the musical piece
music = tf.concat([music, x_out], 0)
return count + 1, num, u_t, x_out, x, music, k_in
def generate(num,
x=x,
size_bt=size_bt,
u0=u0,
n_visible=n_visible,
prime_length=100,
k_in=15):
"""
This function handles generating music. This function is one of the outputs of the build_rnnrbm function
Args:
num (int): The number of timesteps to generate
x (tf.placeholder): The data vector. We can use feed_dict to set this to the music primer.
size_bt (tf.float32): The batch size
u0 (tf.Variable): The initial state of the RNN
n_visible (int): The size of the data vectors
prime_length (int): The number of timesteps into the primer song that we use befoe beginning to generate music
Returns:
The generated music, as a tf.Tensor
"""
Uarr = tf.scan(
rnn_recurrence, x, initializer=u0
) # Generate: Hidden state Array from 'x(input)' and initializer 'u0'
U = Uarr[np.int32(
np.floor(prime_length / midi_manipulation.num_timesteps)
), :, :] # acts as initial-hiddenstate....
[_, _, _, _, _, music,
_] = tf.while_loop(lambda count, num_iter, *args: count < num_iter,
generate_recurrence, [
tf.constant(1, tf.int32),
tf.constant(num), U,
tf.zeros([1, n_visible], tf.float32), x,
tf.zeros([1, n_visible], tf.float32),
tf.constant(k_in)
], [
tf.TensorShape([]),
tf.TensorShape([]), U.shape,
tf.TensorShape([1, n_visible]), x.shape,
tf.TensorShape([None, n_visible]),
tf.TensorShape([])
])
return music
#Reshape our bias matrices to be the same size as the batch.
tf.assign(BH_t, tf.tile(BH_t, [size_bt, 1]))
tf.assign(BV_t, tf.tile(BV_t, [size_bt, 1]))
#Scan through the rnn and generate the value for each hidden node in the batch
u_t = tf.scan(rnn_recurrence, x, initializer=u0)
#Scan through the rnn and generate the visible and hidden biases for each RBM in the batch
BV_t = tf.reshape(
tf.scan(visible_bias_recurrence, u_t,
tf.zeros([1, n_visible], tf.float32)), [size_bt, n_visible])
BH_t = tf.reshape(
tf.scan(hidden_bias_recurrence, u_t,
tf.zeros([1, n_hidden], tf.float32)), [size_bt, n_hidden])
#Get the free energy cost from each of the RBMs in the batch
out1, out2, cost = RBM.get_free_energy_cost(x, W, BV_t, BH_t,
k=k_input) #k 15...
return x, out1, out2, cost, generate, W, bh, bv, lr, Wuh, Wuv, Wvu, Wuu, bu, u0
def rnnrbm():
#This function builds the RNN-RBM and returns the parameters of the model
x = tf.placeholder(tf.float32, [None, n_visible]) #The placeholder variable that holds our data
lr = tf.placeholder(tf.float32) #The learning rate. We set and change this value during training.
size_bt = tf.shape(x)[0] # Not really batch, but the longeur of time series
#parameters to learn, we find that except W, the other four Weight matrices aren't well learned,
#So there isn't a good evolution of RBM in time. We can train a single RBM and produce a similar result.
W = tf.Variable(tf.zeros([n_visible, n_hidden]), name="W")
Wuh = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden]), name="Wuh")
Wuv = tf.Variable(tf.zeros([n_hidden_recurrent, n_visible]), name="Wuv")
Wvu = tf.Variable(tf.zeros([n_visible, n_hidden_recurrent]), name="Wvu")
Wuu = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden_recurrent]), name="Wuu")
bh = tf.Variable(tf.zeros([1, n_hidden]), name="bh")
bv = tf.Variable(tf.zeros([1, n_visible]), name="bv")
bu = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="bu")
u0 = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="u0")
BH_t = tf.Variable(tf.zeros([1, n_hidden]), name="BH_t")
BV_t = tf.Variable(tf.zeros([1, n_visible]), name="BV_t")
def rnn_recurrence(u_tm1, v_t):
#Iterate through the data in the batch and generate the values of the RNN hidden nodes
v_t = tf.reshape(v_t, [1, n_visible])
u_t = tf.sigmoid(bu + tf.matmul(v_t, Wvu) + tf.matmul(u_tm1, Wuu))
return u_t
def visible_bias_recurrence(bv_t, u_tm1):
#Iterate through the values of the RNN hidden nodes and generate the values of the visible bias vectors
bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv))
return bv_t
def hidden_bias_recurrence(bh_t, u_tm1):
#Iterate through the values of the RNN hidden nodes and generate the values of the hidden bias vectors
bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh))
return bh_t
#Reshape our bias matrices to be the same size as the batch.
tf.assign(BH_t, tf.tile(BH_t, [size_bt, 1]))
tf.assign(BV_t, tf.tile(BV_t, [size_bt, 1]))
#Scan through the rnn and generate the value for each hidden node in the batch
Uarr = tf.scan(rnn_recurrence, x, initializer=u0)
Uarr = tf.concat([tf.reshape(u0, [1, 1, n_hidden_recurrent]), Uarr[:-1,:,:]], 0)
#Scan through the rnn and generate the visible and hidden biases for each RBM in the batch
BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, tf.zeros([1, n_visible])), [size_bt, n_visible])
BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, tf.zeros([1, n_hidden])), [size_bt, n_hidden])
#Get the free energy cost from each of the RBMs in the batch
cost = RBM.get_free_energy_cost(x, W, BV_t, BH_t, k=1)
def reconstruction():
#This function handles reconstructing de trajectory. This function is one of the outputs of the rnnrbm() function
primer=x
#Scan through the rnn and generate the value for each hidden node in the batch
Uarr = tf.scan(rnn_recurrence, primer, initializer=u0) # primer is of dimension 2, Uarr is of dimension 3
Uarr = tf.concat([tf.reshape(u0,[1, 1, n_hidden_recurrent]), Uarr[:-1,:,:]], 0)
#Scan through the rnn and generate the visible and hidden biases for each RBM in the batch
BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, bv), [size_bt, n_visible])
BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, bh), [size_bt, n_hidden])
traj = RBM.gibbs_sample(primer, W, BV_t, BH_t, k=1)
return traj[1:,:] #we discard the first time step because it has a huge deviation
return x, cost, reconstruction, W, Wuh, Wuv, Wvu, Wuu, bh, bv, bu, lr, u0
from RBM import *
mnist = input_data.read_data_sets('../MNIST_data', one_hot=True)
batch_size = 10
batch = mnist.train.next_batch(batch_size)
rbm_input = np.concatenate((to_binary_2D(batch[0]), batch[1]), axis = 1)
print(rbm_input.shape)
r = RBM(data_num = batch_size, m = 28*28+10, n = 20, k = 100)
r.print_tensors()
Wuu = tf.Variable(tf.random_normal([rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden_recurrent], 0.0001), name="Wuu")
bh = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden], tf.float32), name="bh")
bv = tf.Variable(tf.zeros([1, rnn_rbm.n_visible], tf.float32), name="bv")
bu = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="bu")
u0 = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="u0")
BH_t = tf.Variable(tf.ones([1, rnn_rbm.n_hidden], tf.float32), name="BH_t")
BV_t = tf.Variable(tf.ones([1, rnn_rbm.n_visible], tf.float32), name="BV_t")
size_bt = tf.shape(x_rbm)[0]
#Build the RBM optimization
saver = tf.train.Saver()
xk1, cost = RBM.build_rbm(x_rbm, W, bv, bh, 1)
rbm_train_var = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)
#Run the session
with tf.Session() as sess:
init = tf.initialize_all_variables()
sess.run(init)
# loop with batch
for song in tqdm(songs):
for i in range(1, len(song), batch_size):
tr_x = song[i:i + batch_size]
sess.run(rbm_train_var, feed_dict={x_rbm: tr_x})
save_path = saver.save(sess, "initializations/rbm.ckpt")
