def main(): # Set parameters and print them args = parse_args() print_args(args) epochs = args['epochs'] visibleUnits = args['visibleUnits'] hiddenUnits = args['hiddenUnits'] approxMethod = args['approxMethod'] approxSteps = args['approxSteps'] learnRate = args['learnRate'] persistStart = args['persistStart'] momentum = args['momentum'] decayMagnitude = args['decayMagnitude'] decayType = args['decayType'] sigma = args['sigma'] batchSize = args['batchSize'] binarization = args['binarization'] update = args['update'] trace_file = args['trace_file'] # saving results save_file = args['save_file'] # saving parameters load_file = args['load_file'] # loading parameters print('Loading data') with gzip.open('../data/mnist.pkl.gz', 'r') as f: (TrainSet, y_train), (x_test, y_test), (ValidSet, y_Valid) = pickle.load(f, encoding='latin1') # combine train and valid data TrainSet = np.concatenate((TrainSet, x_test), axis=0) # Create binary data TrainSet = sklearn.preprocessing.binarize(TrainSet, threshold=binarization).T ValidSet = sklearn.preprocessing.binarize(ValidSet, threshold=binarization).T if len(load_file) == 0: print('Initializing model') model = RBM.RBM(n_vis=visibleUnits, n_hid=hiddenUnits, momentum = momentum, sigma = sigma, trainData = TrainSet, wiseStart = True, ) print('Start of training') Training.fit(model, TrainSet, ValidSet, n_epochs = epochs, weight_decay = decayType, decay_magnitude = decayMagnitude, lr = learnRate, batch_size = batchSize, NormalizationApproxIter = approxSteps, approx = approxMethod, update = update, persistent_start = persistStart, trace_file = trace_file, save_file = save_file ) else: params = RBM.RBM.load(load_file) model = RBM.RBM(params=params) model.reconstructionArray(ValidSet[:, 0:10])
def __init__(self, input_data=None, label=None, input_size=2, hidden_layer_sizes=[3, 3], output_size=1, batch_size=10, learning_rate=0.1, epochs=50, C=1, rng=None): self.input = input_data.astype(np.float32) self.y = label.astype(np.float32) self.rbm_layers = [] self.n_hidden_layers = len( hidden_layer_sizes) # nombre de couches cachées du DBN self.batch_size = batch_size self.layer_svm = None self.model_finetune = None self.nepochs = epochs self.learning_rate = learning_rate self.C = C self.rng = rng #W,hbias,vbias = self.init_weights(input_size,hidden_layer_sizes[0]) # construct rbm_layer rbm_layer = RBM(input=self.input, n_visible=input_size, n_hidden=hidden_layer_sizes[0], batch_size=10, learning_rate=0.1, epochs=1000, rng=self.rng) self.rbm_layers.append(rbm_layer) sample_input = self.input for rbm_index in range(self.n_hidden_layers - 1): _, sample_input = self.rbm_layers[rbm_index].sample_h_given_v( sample_input) n_hid = hidden_layer_sizes[rbm_index + 1] n_vis = hidden_layer_sizes[rbm_index] #W,hbias,vbias = self.init_weights(n_vis,n_hid) rbm_i = RBM(input=sample_input, n_visible=n_vis, n_hidden=n_hid, batch_size=10, learning_rate=0.09, epochs=100, rng=self.rng) self.rbm_layers.append(rbm_i)
def __init__(self, architecture): self.stack = [] previous_layer_size = architecture[0] if len(architecture) > 1: for layer_size in architecture[1:]: self.stack.append(RBM(previous_layer_size, layer_size)) previous_layer_size = layer_size
def _init_coef(self, fan_in, fan_out): if self.activation == 'logistic': init_bound = np.sqrt(2. / (fan_in + fan_out)) elif self.activation in ('identity', 'tanh', 'relu'): init_bound = np.sqrt(6. / (fan_in + fan_out)) else: raise ValueError("Unknown activation function %s" % self.activation) if self.first_layer_init: features, labels, sparse_encoder, int_encoder = Preprocessing.feature_extraction_sparse_train( "project_training.txt") #svd = TruncatedSVD(n_components=70) #features = svd.fit_transform(features) weights = RBM.train(features, 150) weights = weights.reshape(-1, 1) print fan_in, fan_out coef_init = weights self.first_layer_init = False else: coef_init = self._random_state.uniform(-init_bound, init_bound, (fan_in, fan_out)) intercept_init = self._random_state.uniform(-init_bound, init_bound, fan_out) return coef_init, intercept_init
def __init__(self, title, layerSizes, types, modelDir): #checking DBN configuration if not len(types) == len(layerSizes) - 1: print 'DBN initialize error: layerSizes types length mismatch' exit() for i in range(len(types) - 1): if not (types[i] in ['BB', 'GB', 'BG'] and types[i + 1] in ['BB', 'GB', 'BG']): print "DBN initialize error: RBM types error, RBM types must be ['BB','GB','BG']" exit() if not types[i][1] == types[i + 1][0]: print 'DBN initialize error: RBM types sequence error' exit() print '\nInitializing DBN: %s, layerSizes: %s, layerTypes: %s' % ( title, layerSizes, types) self.title = title self.layerSizes = layerSizes self.layerNum = len(self.layerSizes) - 1 self.modelDir = modelDir self.rbms = [] self.rbmFiles = [] for i in range(self.layerNum): self.rbms.append( RBM(input=None, n_visible=self.layerSizes[i], n_hidden=self.layerSizes[i + 1], type=types[i])) self.rbmFiles.append(modelDir + os.sep + '%s_layer%dRBM_%d-%d.rbm' % (self.title, i + 1, self.layerSizes[i], self.layerSizes[i + 1])) '''
def pretrain(self, x, epochs, num_samples=50000): ''' Greedy layer-wise training The last layer is a RBM with linear hidden units shape(x) = (v_dim, number_of_examples) ''' RBM_layers = [] for i in range(self.num_hidden_layers): # initialize RBM's if (i < self.num_hidden_layers - 1): RBM_layers.append( RBM(self.layer_dims[i], self.layer_dims[i + 1])) else: RBM_layers.append( RBM_with_linear_hidden_units(self.layer_dims[i], self.layer_dims[i + 1])) for i in range(self.num_hidden_layers): # train RBM's print("Training RBM layer %i" % (i + 1)) RBM_layers[i].train(x, epochs) # train the ith RBM if not (i == self.num_hidden_layers - 1): # generate samples to train next layer _, x = RBM_layers[i].gibbs_sampling(2, num_samples) self.W.append(RBM_layers[i].W) # save trained weights self.b.append(RBM_layers[i].b) self.a.append(RBM_layers[i].a) self.pretrained = True return
def pre_rbm_train(): #pre train by rbm between two layers RBM.rbm(Config.FILE_ZERO_RBM_PATH, Config.HIDDEN_UNITS_NUM_FIRST_RBM, Config.FILE_FIRST_RBM_PATH) RBM.rbm(Config.FILE_FIRST_RBM_PATH, Config.HIDDEN_UNITS_NUM_SECOND_RBM, Config.FILE_SECOND_RBM_PATH) RBM.rbm(Config.FILE_SECOND_RBM_PATH, Config.HIDDEN_UNITS_NUM_THIRD_RBM, Config.FILE_THIRD_RBM_PATH) RBM.rbm(Config.FILE_THIRD_RBM_PATH, Config.OUTPUT_UNITS_NUM_FORTH_RBM, Config.FILE_FORTH_RBM_PATH)
def recurrence(nest, v_t): bv_t = tf.add(bv, tf.matmul(nest[0], Wuv)) bh_t = tf.add(bh, tf.matmul(nest[0], Wuh)) v_t_sample = RBM.gibbs_sample(tf.reshape(v_t, [1, n_visible]), W, bv_t , bh_t, k=1 ) v_t_sample = tf.reshape(v_t_sample, [1, n_visible]) u_t = tf.sigmoid(bu + tf.matmul(v_t_sample, Wvu) + tf.matmul(nest[0], Wuu)) return [u_t, bv_t, bh_t, v_t_sample]
def train(x=x, size_bt=size_bt, BV_t=BV_t, BH_t=BH_t): bv_init = tf.zeros([1, n_visible], tf.float32) bh_init = tf.zeros([1, n_hidden], tf.float32) u_t = tf.scan(rnn_recurrence, x, initializer=u0) BV_t = tf.reshape(tf.scan(visible_bias_recurrence, u_t, bv_init), [size_bt, n_visible]) BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, u_t, bh_init), [size_bt, n_hidden]) sample, cost, monitor = RBM.get_free_energy_cost(x, W, BV_t, BH_t, k=15) return x, sample, cost, monitor, params, size_bt
def build_rbm(input_var=None,d_x=21,d_y=21): l_in = lasagne.layers.InputLayer(shape=(None, 1, d_x, d_y), input_var=input_var) l_in_drop = lasagne.layers.DropoutLayer(l_in, p=0.2) l_hid = RBM.RBM(l_in_drop,num_units=128,kGibbs=1) return l_hid
def train(x=x, size_bt=size_bt, BV_t=BV_t, BH_t=BH_t): bv_init = tf.zeros([1, n_visible], tf.float32) bh_init = tf.zeros([1, n_hidden], tf.float32) u_t = tf.scan(rnn_recurrence, x, initializer=u0) BV_t = tf.reshape(tf.scan(visible_bias_recurrence, u_t, bv_init), [size_bt, n_visible]) BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, u_t, bh_init), [size_bt, n_hidden]) sample, cost = RBM.build_rbm(x, W, BV_t, BH_t, k=15) return x, sample, cost, params, size_bt
def dbn_train(NN,SD,opt): lAmt=len(NN['layer']) if 'maxStep' not in opt: opt['maxStep']=50 ### Train the 1st hidden layer(connected by input layer). # Construct RBM from current layer of network. rbm={} rbm['W']=NN['layer'][2]['W'] rbm['vB']=np.zeros(1,NN['sizes'](1)) rbm['hB']=NN['layer'][2]['B'] # Training of RBM. rbm=RBM.rbm_train(rbm,SD,opt) # input data as training data for 1st RBM # Update RBM parameters to network. NN['layer'][2]['W']=rbm['W'] NN['layer'][1]['B']=rbm['vB'] NN['layer'][2]['B']=rbm['hB'] # Propagate input to next layer. X=RBM.rbm_up(rbm,SD) #stochastic propagation. ### Train other hidden layers. for li in range(2,lAmt): # Construct RBM from current layer of network. rbm={} rbm['W']=NN['layer'][li]['W'] rbm['vB']=NN['layer'][li-1]['B'] rbm['hB']=NN['layer'][li]['B'] # Training of RBM. rbm=RBM.rbm_train(rbm,X,opt) # Update RBM parameters to network. NN['layer'][li]['W']=rbm['W'] NN['layer'][li-1]['B']=rbm['vB'] NN['layer'][li]['B']=rbm['hB'] # Propagate input to next layer. X=RBM.rbm_up(rbm,X,True) #stochastic propagation. return NN
def get_RBM(): r = RBM(6, 2) k = 9 dataset = [[1, 1, 1, 0, 0, 0], [1, 0, 1, 0, 0, 0], [1, 1, 1, 0, 0, 0], [0, 0, 1, 1, 1, 0], [0, 0, 1, 1, 1, 0], [0, 0, 1, 1, 1, 0], [0, 1, 0, 0, 0, 1], [0, 1, 0, 0, 0, 1], [0, 1, 1, 0, 0, 1]] # Train at the following learning rates: print "Training..." learning_rates = [0.1, 0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001] # learning_rates = [0.1] for rate in learning_rates: print "Learning rate =", rate for i in range(100): print ' ' + str(i) + '...', r.train_epoch(dataset, rate, k) err = np.sum([r.free_energy(data) for data in dataset]) print 'Energy = ' + str(err) + '...', PL = r.pseudolikelihood(dataset) print 'Pseudolikelihood = ' + str(PL) + '...', L = r.likelihood(dataset) print 'Likelihood = ' + str(L) + '...', print 'Done' return r, dataset
def train_rbm(nv, nh, batch_size, epochs, val_set, gibbs_sampling_nb, rbm=None): if (rbm == None): rbm = RBM(nv, nh) for epoch in range(1, epochs + 1): training_loss = 0 s = 0.0 for user in range(0, nb_users - batch_size, batch_size): vk = val_set[user:user + batch_size] v0 = val_set[user:user + batch_size] for sample in range(gibbs_sampling_nb): _, hk = rbm.sample_h(vk) _, vk = rbm.sample_v(hk) vk[v0 < 0] = v0[v0 < 0] phk, _ = rbm.sample_h(vk) ph0, _ = rbm.sample_h(v0) rbm.train(v0, vk, ph0, phk) training_loss += torch.mean(torch.abs(v0[v0 >= 0] - vk[v0 >= 0])) s += 1 # print("Epoch:", epoch, "Training Loss:", training_loss/s) return float(training_loss / s), rbm
def main(): print("reading data...") data = ngsim_manipulation.Data() x = tf.placeholder(tf.float32, [None, rnn_rbm.n_visible], name="x") #The placeholder variable that holds our data W = tf.Variable(tf.random_normal([rnn_rbm.n_visible, rnn_rbm.n_hidden], 0.01), name="W") #The weight matrix of the RBM Wuh = tf.Variable(tf.random_normal( [rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden], 0.0001), name="Wuh") #The RNN -> RBM hidden weight matrix bh = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden], tf.float32), name="bh") #The RNN -> RBM hidden bias vector Wuv = tf.Variable(tf.random_normal( [rnn_rbm.n_hidden_recurrent, rnn_rbm.n_visible], 0.0001), name="Wuv") #The RNN -> RBM visible weight matrix bv = tf.Variable(tf.zeros([1, rnn_rbm.n_visible], tf.float32), name="bv") #The RNN -> RBM visible bias vector Wvu = tf.Variable(tf.random_normal( [rnn_rbm.n_visible, rnn_rbm.n_hidden_recurrent], 0.0001), name="Wvu") #The data -> RNN weight matrix Wuu = tf.Variable(tf.random_normal( [rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden_recurrent], 0.0001), name="Wuu") #The RNN hidden unit weight matrix bu = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="bu") #The RNN hidden unit bias vector u0 = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="u0") #The initial state of the RNN #The RBM bias vectors. These matrices will get populated during rnn-rbm training and generation BH_t = tf.Variable(tf.ones([1, rnn_rbm.n_hidden], tf.float32), name="BH_t") BV_t = tf.Variable(tf.ones([1, rnn_rbm.n_visible], tf.float32), name="BV_t") #Build the RBM optimization saver = tf.train.Saver() #Note that we initialize the RNN->RBM bias vectors with the bias vectors of the trained RBM. These vectors will form the templates for the bv_t and #bh_t of each RBM that we create when we run the RNN-RBM updt = RBM.get_cd_update(x, W, bv, bh, k=1, lr=lr) # CD-k print("training RBM for weight initialization...") #Run the session with tf.Session() as sess: #Initialize the variables of the model init = tf.global_variables_initializer() sess.run(init) #Run over each trajectory num_epoch times for epoch in tqdm(range(num_epochs)): #for idx in range(100): for idx in range(data.num_trajectories): sess.run( updt, feed_dict={x: data.next_traj()} ) # even if traj is comprised of data in different time, we treat it like a batch of data in the same time, because we can only initialize one RBM. #Save the initialized model here save_path = saver.save(sess, "parameter_checkpoints/initialized.ckpt")
def main(): #Load the Songs songs = input_manipulation.get_songs('Game_Music_Midi') x = tf.placeholder(tf.float32, [None, rnn_rbm.n_visible], name="x") #The placeholder variable that holds our data #parameters of CRBM W = tf.Variable(tf.random_normal([rnn_rbm.n_visible, rnn_rbm.n_hidden], 0.01), name="W") #The weight matrix of the RBM bh = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden], tf.float32), name="bh") #The RNN -> RBM hidden bias vector bv = tf.Variable(tf.zeros([1, rnn_rbm.n_visible], tf.float32), name="bv") #The RNN -> RBM visible bias vector #parameters related to RNN Wuh = tf.Variable(tf.random_normal( [rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden], 0.0001), name="Wuh") #The RNN -> RBM hidden weight matrix Wuv = tf.Variable(tf.random_normal( [rnn_rbm.n_hidden_recurrent, rnn_rbm.n_visible], 0.0001), name="Wuv") #The RNN -> RBM visible weight matrix Wvu = tf.Variable(tf.random_normal( [rnn_rbm.n_visible, rnn_rbm.n_hidden_recurrent], 0.0001), name="Wvu") #The data -> RNN weight matrix Wuu = tf.Variable(tf.random_normal( [rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden_recurrent], 0.0001), name="Wuu") #The RNN hidden unit weight matrix bu = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="bu") #The RNN hidden unit bias vector u0 = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="u0") #The initial state of the RNN #The RBM bias vectors. These matrices will get populated during rnn-rbm training and generation BH_t = tf.Variable(tf.ones([1, rnn_rbm.n_hidden], tf.float32), name="BH_t") BV_t = tf.Variable(tf.ones([1, rnn_rbm.n_visible], tf.float32), name="BV_t") #Build the RBM optimization saver = tf.train.Saver() #Note that we initialize the RNN->RBM bias vectors with the bias vectors of the trained RBM. These vectors will form the templates for the bv_t and #bh_t of each RBM that we create when we run the RNN-RBM updt = RBM.get_cd_update(x, W, bv, bh, 1, lr) #Run the session with tf.Session() as sess: #Initialize the variables of the model init = tf.initialize_all_variables() sess.run(init) #Run over each song num_epoch times for epoch in tqdm(range(num_epochs)): for song in songs: sess.run(updt, feed_dict={x: song}) #Save the initialized model here save_path = saver.save(sess, "parameter_checkpoints/initialized.ckpt")
def train_and_sample(T, lr): temp_name = 'T = ' + format(T, '.2f') + '.npy' file_name = 'T = ' + format(T, '.2f') + ', lr = ' + str(lr) + '.npy' completeLoad = os.path.join(load_path, temp_name) samples = (np.load(completeLoad) + 1) / 2 # convert to 0, 1 sz, N, N1 = samples.shape samples_flat = np.reshape(samples, (sz, N * N1)) r = RBM(num_visible=64, num_hidden=nH) r.train(samples_flat, max_epochs=me, learning_rate=lr, batch_size=bs) print("Wights at T = " + format(T, '.2f') + ": ", r.weights) RBM_data_flat = r.daydream(ns) * 2 - 1 # convert back to -1, 1 RBM_data = np.reshape(RBM_data_flat, (ns, N, N1)) completeSaveData = os.path.join(save_data_path, file_name) np.save(completeSaveData, RBM_data) completeSaveWeight = os.path.join(save_weight_path, file_name) np.save(completeSaveWeight, r.weights) completeSaveError = os.path.join(save_error_path, file_name) np.save(completeSaveError, r.errors)
def generate_recurrence(count, k, u_tm1, primer, x, music): bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) primer = tf.zeros([1, n_visible], tf.float32) # primer = tf.slice(x, [count, 0], [1, n_visible]) x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=25) u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu))) music = tf.concat(0, [music, x_out]) return count+1, k, u_t, x_out, x, music
def generate(num, x=x, size_bt=size_bt, u0=u0, n_visible=n_visible, prime_length=100): """ This function handles generating music. This function is one of the outputs of the build_rnnrbm function Args: num (int): The number of timesteps to generate x (tf.placeholder): The data vector. We can use feed_dict to set this to the music primer. size_bt (tf.float32): The batch size u0 (tf.Variable): The initial state of the RNN n_visible (int): The size of the data vectors prime_length (int): The number of timesteps into the primer song that we use befoe beginning to generate music Returns: The generated music, as a tf.Tensor """ Uarr = tf.scan(rnn_recurrence, x, initializer=u0) U = Uarr[int(np.floor(prime_length / midi_manipulation.num_timesteps)), :, :] # def generate_recurrence(count, k, u_tm1, primer, x, music): # return count + 1, k, u_t, x_out, x, music count = 0 music = tf.zeros([1, n_visible], tf.float32) u_tm1 = U primer = tf.zeros([1, n_visible], tf.float32) while count < num: # This function builds and runs the gibbs steps for each RBM in the chain to generate music # Get the bias vectors from the current state of the RNN bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) # Run the Gibbs step to get the music output. Prime the RBM with the previous musical output. x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=25) # Update the RNN hidden state based on the musical output and current hidden state. u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu))) # Add the new output to the musical piece music = tf.concat([music, x_out], 0) count += 1 u_tm1 = u_t primer = x_out # [_, _, _, _, _, music] = control_flow_ops.while_loop(lambda count, num_iter, *args: count < num_iter, # generate_recurrence, [tf.constant(1, tf.int32), tf.constant(num), U, # tf.zeros([1, n_visible], tf.float32), x, # tf.zeros([1, n_visible], tf.float32)], # shape_invariants=None) return music
def generate_recurrence(count, k, u_tm1, primer, x, music): bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) # primer = tf.zeros([1, n_visible], tf.float32) # primer = tf.slice(x, [count, 0], [1, n_visible]) x_out, _ = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=25, c_1=0.5, c_2=0.5) u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu))) music = tf.concat(0, [music, x_out]) return count+1, k, u_t, x_out, x, music
def initRBMs(self): self.RBMList = [] for i in range(self.layers): currVisDim = self.dimensionList[i] currHidDim = self.dimensionList[i + 1] currWR = self.trainInfoList[i][0] groupWeights = False currRBM = RBM(currVisDim, currHidDim, currWR, groupWeights) self.RBMList.append(currRBM)
def gen_sample(self, k, x=None): with tf.variable_scope('dbn_gen_sample'): # Get input value for the top rbm's visible layer if x is None: x = tf.zeros([1, int(self.bs[-2].shape[-1])]) else: # Propagate input value up to visible layer of top rbm for i in range(len(self.ws) - 1): x_prob = tf.sigmoid(tf.matmul(x, self.ws[i]) + self.bs[i+1]) x = RBM.sample(x_prob) # Sample from the top layer top_rbm = RBM.RBM(self.ws[-1], self.bs[-2], self.bs[-1]) _, x_sample = top_rbm.gibbs_sample(x, k) # Propagate sample down to visible layer for i in reversed(range(len(self.ws) - 1)): x_prob = tf.sigmoid(tf.matmul(x_sample, tf.transpose(self.ws[i])) + self.bs[i]) x_sample = RBM.sample(x_prob) return x_sample
def reconstruction(): #This function handles reconstructing de trajectory. This function is one of the outputs of the rnnrbm() function primer=x #Scan through the rnn and generate the value for each hidden node in the batch Uarr = tf.scan(rnn_recurrence, primer, initializer=u0) # primer is of dimension 2, Uarr is of dimension 3 Uarr = tf.concat([tf.reshape(u0,[1, 1, n_hidden_recurrent]), Uarr[:-1,:,:]], 0) #Scan through the rnn and generate the visible and hidden biases for each RBM in the batch BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, bv), [size_bt, n_visible]) BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, bh), [size_bt, n_hidden]) traj = RBM.gibbs_sample(primer, W, BV_t, BH_t, k=1) return traj[1:,:] #we discard the first time step because it has a huge deviation
def generate_recurrence(count, num, u_tm1, primer, x, music, k_in): #This function builds and runs the gibbs steps for each RBM in the chain to generate music #Get the bias vectors from the current state of the RNN bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) #Run the Gibbs step to get the music output. Prime the RBM with the previous musical output. x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=k_in) #Update the RNN hidden state based on the musical output and current hidden state. u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu))) #Add the new output to the musical piece music = tf.concat([music, x_out], 0) return count + 1, num, u_t, x_out, x, music, k_in
def recurrence(nest, v_t): # This function do all the procedures in one RBM. bv_t = tf.add(bv, tf.matmul(nest[0], Wuv)) # generate current bv_t based on u_{t-1} bh_t = tf.add(bh, tf.matmul(nest[0], Wuh)) # generate current bh_t based on u_{t-1} v_t_sample = RBM.gibbs_sample( tf.reshape(v_t, [1, n_visible]), W, bv_t, bh_t, k=1) # sample v_t according generated bias v_t_sample = tf.reshape(v_t_sample, [1, n_visible]) u_t = tf.sigmoid(bu + tf.matmul(v_t_sample, Wvu) + tf.matmul(nest[0], Wuu)) # calculate current u_t return [u_t, bv_t, bh_t, v_t_sample]
def fit(self, trainset): AlgoBase.fit(self, trainset) numUsers = trainset.n_users numItems = trainset.n_items trainingMatrix = np.zeros([numUsers, numItems, 10], dtype=np.float32) for (uid, iid, rating) in trainset.all_ratings(): adjustedRating = int(float(rating) * 2.0) - 1 trainingMatrix[int(uid), int(iid), adjustedRating] = 1 # Flatten to a 2D array, with nodes for each possible rating type on each possible item, for every user. trainingMatrix = np.reshape(trainingMatrix, [trainingMatrix.shape[0], -1]) # Create an RBM with (num items * rating values) visible nodes rbm = RBM.RBM(trainingMatrix.shape[1], hiddenDimensions=self.hiddenDim, learningRate=self.learningRate, batchSize=self.batchSize, epochs=self.epochs) rbm.Train(trainingMatrix) self.predictedRatings = np.zeros([numUsers, numItems], dtype=np.float32) for uiid in range(trainset.n_users): if (uiid % 50 == 0): print("Processing user ", uiid) recs = rbm.GetRecommendations([trainingMatrix[uiid]]) recs = np.reshape(recs, [numItems, 10]) for itemID, rec in enumerate(recs): # The obvious thing would be to just take the rating with the highest score: #rating = rec.argmax() # ... but this just leads to a huge multi-way tie for 5-star predictions. # The paper suggests performing normalization over K values to get probabilities # and take the expectation as your prediction, so we'll do that instead: normalized = self.softmax(rec) rating = np.average(np.arange(10), weights=normalized) self.predictedRatings[uiid, itemID] = (rating + 1) * 0.5 return self
def generate_recurrence(step, u_tm1, music): #To generate music: This function builds and runs the gibbs steps for each RBM in the chain bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) #Run the Gibbs step to get the music output. Prime the RBM with the previous musical output. x_out = RBM.gibbs_sample(tf.reshape(music[-1, :], [1, n_visible]), W, bv_t, bh_t, k=25) #Update the RNN hidden state based on the musical output and current hidden state. u_t = (tf.sigmoid(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu))) #Add the new output to the musical piece music = tf.concat([music, x_out], axis=0) return step + 1, u_t, music
def pretrain(self, x, epochs, num_samples=50000): ''' Greedy layer-wise training The last layer is a RBM with linear hidden units shape(x) = (v_dim, number_of_examples) ''' RBM_layers = [] # initialize RBMs for i in range(self.num_hidden_layers): if i == 0: RBM_layers.append( RBM_with_linear_visible_units( self.layer_dims[i], self.layer_dims[i + 1])) # linear input elif (i < self.num_hidden_layers - 1): RBM_layers.append( RBM(self.layer_dims[i], self.layer_dims[i + 1])) # else: RBM_layers.append( RBM_with_linear_hidden_units( self.layer_dims[i], self.layer_dims[i + 1])) # linear output # train stack of RBMs for i in range(self.num_hidden_layers): # train RBM's print("Training RBM layer %i" % (i + 1)) x = RBM_layers[i].train(x, epochs) # train the ith RBM self.W.append(RBM_layers[i].W) # save trained weights self.b.append(RBM_layers[i].b) self.a.append(RBM_layers[i].a) self.pretrained = True return
def get_RBM(): r = RBM(6,2) k = 9 dataset = [[1,1,1,0,0,0], [1,0,1,0,0,0], [1,1,1,0,0,0], [0,0,1,1,1,0], [0,0,1,1,1,0], [0,0,1,1,1,0], [0,1,0,0,0,1], [0,1,0,0,0,1], [0,1,1,0,0,1]] # Train at the following learning rates: print "Training..." learning_rates = [0.1, 0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001] # learning_rates = [0.1] for rate in learning_rates: print "Learning rate =", rate for i in range(100): print ' ' + str(i) + '...', r.train_epoch(dataset, rate, k) err = np.sum([r.free_energy(data) for data in dataset]) print 'Energy = ' + str(err) + '...', PL = r.pseudolikelihood(dataset) print 'Pseudolikelihood = ' + str(PL) + '...', L = r.likelihood(dataset) print 'Likelihood = ' + str(L) + '...', print 'Done' return r, dataset
def get_RBM(): print "Loading digits...." digits, classes = load_digits('digits_train.txt') digits = [digit_gray_to_binary(d) for d in digits] r = RBM(196,25) # Train at the following learning rates: print "Training..." learning_rates = [0.1, 0.05] #, 0.01, 0.005, 0.001] for rate in learning_rates: print "Learning rate =", rate for i in range(20): print ' ' + str(i) + '...', r.train_epoch(digits, rate, 5) err = np.sum([r.free_energy(digit) for digit in digits]) print 'Energy = ' + str(err) + '...', PL = r.pseudolikelihood(digits) print 'Pseudolikelihood = ' + str(PL) + '...', print 'Done' return r, digits
import matplotlib.pyplot as plt from scipy.ndimage import convolve from sklearn import linear_model, datasets, metrics from sklearn.cross_validation import train_test_split from sklearn.neural_network import BernoulliRBM from sklearn.pipeline import Pipeline from sklearn.metrics import confusion_matrix import scipy.io from RBM import * mat = scipy.io.loadmat('input.mat') train_data=mat['train_x'] test_data=mat['test_x'] train_label=mat['train_y'] test_label=mat['test_y'] r1 = RBM(num_visible = 256, num_hidden = 250) train_data1 = train_data test_data1 = test_data r1.train(train_data1, max_epochs = 5000) output_train1,prob_train1=r1.run_visible(train_data1) output_test1,prob_test1=r1.run_visible(test_data1) r2 = RBM(num_visible = 250, num_hidden = 220) train_data2 = output_train1 test_data2 = output_test1 r2.train(train_data2, max_epochs = 5000)
def rnnrbm(): #This function builds the RNN-RBM and returns the parameters of the model x = tf.placeholder( tf.float32, [None, n_visible]) #The placeholder variable that holds our data lr = tf.placeholder( tf.float32 ) #The learning rate. We set and change this value during training. size_bt = tf.shape(x)[0] #the batch size # parameters W = tf.Variable(tf.zeros([n_visible, n_hidden]), name="W") Wuh = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden]), name="Wuh") Wuv = tf.Variable(tf.zeros([n_hidden_recurrent, n_visible]), name="Wuv") Wvu = tf.Variable(tf.zeros([n_visible, n_hidden_recurrent]), name="Wvu") Wuu = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden_recurrent]), name="Wuu") bh = tf.Variable(tf.zeros([1, n_hidden]), name="bh") bv = tf.Variable(tf.zeros([1, n_visible]), name="bv") bu = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="bu") u0 = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="u0") BH_t = tf.Variable(tf.zeros([1, n_hidden]), name="BH_t") BV_t = tf.Variable(tf.zeros([1, n_visible]), name="BV_t") def rnn_recurrence(u_tm1, v_t): #Iterate through the data in the batch and generate the values of the RNN hidden nodes v_t = tf.reshape(v_t, [1, n_visible]) u_t = (tf.sigmoid(bu + tf.matmul(v_t, Wvu) + tf.matmul(u_tm1, Wuu))) return u_t def visible_bias_recurrence(bv_t, u_tm1): #Iterate through the values of the RNN hidden nodes and generate the values of the visible bias vectors bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) return bv_t def hidden_bias_recurrence(bh_t, u_tm1): #Iterate through the values of the RNN hidden nodes and generate the values of the hidden bias vectors bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) return bh_t #################### #Below is two functions for generation and construction, but not used in training #################### def generate(timesteps, primer=x): """ This function handles generating music. This function is one of the outputs of the build_rnnrbm function Args: timesteps (int): The number of timesteps to generate primer (tf.placeholder): The primer song Returns: The generated music, as a tf.Tensor """ def generate_recurrence(step, u_tm1, music): #To generate music: This function builds and runs the gibbs steps for each RBM in the chain bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) #Run the Gibbs step to get the music output. Prime the RBM with the previous musical output. x_out = RBM.gibbs_sample(tf.reshape(music[-1, :], [1, n_visible]), W, bv_t, bh_t, k=25) #Update the RNN hidden state based on the musical output and current hidden state. u_t = (tf.sigmoid(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu))) #Add the new output to the musical piece music = tf.concat([music, x_out], axis=0) return step + 1, u_t, music Uarr = tf.scan( rnn_recurrence, primer, initializer=u0) # x is of dimension 2, Uarr is of dimension 3 u_tm1 = Uarr[ -1, :, :] # the beginning hidden recurrent unit of our generation music = tf.zeros([1, n_visible]) loop_vars = [tf.constant(0), u_tm1, music] cond = lambda count, *args: count < timesteps [_, _, music] = tf.while_loop(cond, generate_recurrence, loop_vars, shape_invariants=[ tf.constant(0).get_shape(), u_tm1.get_shape(), tf.TensorShape([None, None]) ]) return music def reconstruction(): #This function handles reconstructing music. This function is one of the outputs of the rnnrbm() function primer = x #Scan through the rnn and generate the value for each hidden node in the batch Uarr = tf.scan( rnn_recurrence, primer, initializer=u0) # primer is of dimension 2, Uarr is of dimension 3 Uarr = tf.concat( [tf.reshape(u0, [1, 1, n_hidden_recurrent]), Uarr[:-1, :, :]], 0) #Scan through the rnn and generate the visible and hidden biases for each RBM in the batch BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, bv), [size_bt, n_visible]) BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, bh), [size_bt, n_hidden]) music = RBM.gibbs_sample(primer, W, BV_t, BH_t, k=1) return music ########################## # Below is for training ########################## #Reshape our bias matrices to be the same size as the batch. tf.assign(BH_t, tf.tile(BH_t, [size_bt, 1])) tf.assign(BV_t, tf.tile(BV_t, [size_bt, 1])) #Scan through the rnn and generate the value for each hidden node in the batch u_t = tf.scan(rnn_recurrence, x, initializer=u0) #Scan through the rnn and generate the visible and hidden biases for each RBM in the batch BV_t = tf.reshape( tf.scan(visible_bias_recurrence, u_t, tf.zeros([1, n_visible], tf.float32)), [size_bt, n_visible]) BH_t = tf.reshape( tf.scan(hidden_bias_recurrence, u_t, tf.zeros([1, n_hidden], tf.float32)), [size_bt, n_hidden]) #Get the free energy cost from each of the RBMs in the batch cost = RBM.get_free_energy_cost(x, W, BV_t, BH_t, k=15) return x, cost, generate, reconstruction, W, Wuh, Wuv, Wvu, Wuu, bh, bv, bu, lr, u0
def rnnrbm(k_input=15): #This function builds the RNN-RBM and returns the parameters of the model x = tf.placeholder( tf.float32, [None, n_visible]) #The placeholder variable that holds our data lr = tf.placeholder( tf.float32 ) #The learning rate. We set and change this value during training. size_bt = tf.shape(x)[0] #the batch size #Here we set aside the space for each of the variables. #We intialize these variables when we load saved parameters in rnn_rbm_train.py or rnn_rbm_generate.py W = tf.Variable(tf.zeros([n_visible, n_hidden]), name="W") Wuh = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden]), name="Wuh") Wuv = tf.Variable(tf.zeros([n_hidden_recurrent, n_visible]), name="Wuv") Wvu = tf.Variable(tf.zeros([n_visible, n_hidden_recurrent]), name="Wvu") Wuu = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden_recurrent]), name="Wuu") bh = tf.Variable(tf.zeros([1, n_hidden]), name="bh") bv = tf.Variable(tf.zeros([1, n_visible]), name="bv") bu = tf.Variable( tf.zeros([1, n_hidden_recurrent]), name="bu" ) # Bias for 'hidden_recurrent' -> next 'hidden_recurrent' [RNN] u0 = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="u0") BH_t = tf.Variable(tf.zeros([1, n_hidden]), name="BH_t") BV_t = tf.Variable(tf.zeros([1, n_visible]), name="BV_t") def rnn_recurrence(u_tm1, sl): #Iterate through the data in the batch and generate the values of the RNN hidden nodes sl = tf.reshape(sl, [1, n_visible]) u_t = (tf.tanh(bu + tf.matmul(sl, Wvu) + tf.matmul(u_tm1, Wuu))) return u_t def visible_bias_recurrence(bv_t, u_tm1): #Iterate through the values of the RNN hidden nodes and generate the values of the visible bias vectors bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) return bv_t def hidden_bias_recurrence(bh_t, u_tm1): #Iterate through the values of the RNN hidden nodes and generate the values of the hidden bias vectors bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) return bh_t def generate_recurrence(count, num, u_tm1, primer, x, music, k_in): #This function builds and runs the gibbs steps for each RBM in the chain to generate music #Get the bias vectors from the current state of the RNN bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) #Run the Gibbs step to get the music output. Prime the RBM with the previous musical output. x_out = RBM.gibbs_sample(primer, W, bv_t, bh_t, k=k_in) #Update the RNN hidden state based on the musical output and current hidden state. u_t = (tf.tanh(bu + tf.matmul(x_out, Wvu) + tf.matmul(u_tm1, Wuu))) #Add the new output to the musical piece music = tf.concat([music, x_out], 0) return count + 1, num, u_t, x_out, x, music, k_in def generate(num, x=x, size_bt=size_bt, u0=u0, n_visible=n_visible, prime_length=100, k_in=15): """ This function handles generating music. This function is one of the outputs of the build_rnnrbm function Args: num (int): The number of timesteps to generate x (tf.placeholder): The data vector. We can use feed_dict to set this to the music primer. size_bt (tf.float32): The batch size u0 (tf.Variable): The initial state of the RNN n_visible (int): The size of the data vectors prime_length (int): The number of timesteps into the primer song that we use befoe beginning to generate music Returns: The generated music, as a tf.Tensor """ Uarr = tf.scan( rnn_recurrence, x, initializer=u0 ) # Generate: Hidden state Array from 'x(input)' and initializer 'u0' U = Uarr[np.int32( np.floor(prime_length / midi_manipulation.num_timesteps) ), :, :] # acts as initial-hiddenstate.... [_, _, _, _, _, music, _] = tf.while_loop(lambda count, num_iter, *args: count < num_iter, generate_recurrence, [ tf.constant(1, tf.int32), tf.constant(num), U, tf.zeros([1, n_visible], tf.float32), x, tf.zeros([1, n_visible], tf.float32), tf.constant(k_in) ], [ tf.TensorShape([]), tf.TensorShape([]), U.shape, tf.TensorShape([1, n_visible]), x.shape, tf.TensorShape([None, n_visible]), tf.TensorShape([]) ]) return music #Reshape our bias matrices to be the same size as the batch. tf.assign(BH_t, tf.tile(BH_t, [size_bt, 1])) tf.assign(BV_t, tf.tile(BV_t, [size_bt, 1])) #Scan through the rnn and generate the value for each hidden node in the batch u_t = tf.scan(rnn_recurrence, x, initializer=u0) #Scan through the rnn and generate the visible and hidden biases for each RBM in the batch BV_t = tf.reshape( tf.scan(visible_bias_recurrence, u_t, tf.zeros([1, n_visible], tf.float32)), [size_bt, n_visible]) BH_t = tf.reshape( tf.scan(hidden_bias_recurrence, u_t, tf.zeros([1, n_hidden], tf.float32)), [size_bt, n_hidden]) #Get the free energy cost from each of the RBMs in the batch out1, out2, cost = RBM.get_free_energy_cost(x, W, BV_t, BH_t, k=k_input) #k 15... return x, out1, out2, cost, generate, W, bh, bv, lr, Wuh, Wuv, Wvu, Wuu, bu, u0
def rnnrbm(): #This function builds the RNN-RBM and returns the parameters of the model x = tf.placeholder(tf.float32, [None, n_visible]) #The placeholder variable that holds our data lr = tf.placeholder(tf.float32) #The learning rate. We set and change this value during training. size_bt = tf.shape(x)[0] # Not really batch, but the longeur of time series #parameters to learn, we find that except W, the other four Weight matrices aren't well learned, #So there isn't a good evolution of RBM in time. We can train a single RBM and produce a similar result. W = tf.Variable(tf.zeros([n_visible, n_hidden]), name="W") Wuh = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden]), name="Wuh") Wuv = tf.Variable(tf.zeros([n_hidden_recurrent, n_visible]), name="Wuv") Wvu = tf.Variable(tf.zeros([n_visible, n_hidden_recurrent]), name="Wvu") Wuu = tf.Variable(tf.zeros([n_hidden_recurrent, n_hidden_recurrent]), name="Wuu") bh = tf.Variable(tf.zeros([1, n_hidden]), name="bh") bv = tf.Variable(tf.zeros([1, n_visible]), name="bv") bu = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="bu") u0 = tf.Variable(tf.zeros([1, n_hidden_recurrent]), name="u0") BH_t = tf.Variable(tf.zeros([1, n_hidden]), name="BH_t") BV_t = tf.Variable(tf.zeros([1, n_visible]), name="BV_t") def rnn_recurrence(u_tm1, v_t): #Iterate through the data in the batch and generate the values of the RNN hidden nodes v_t = tf.reshape(v_t, [1, n_visible]) u_t = tf.sigmoid(bu + tf.matmul(v_t, Wvu) + tf.matmul(u_tm1, Wuu)) return u_t def visible_bias_recurrence(bv_t, u_tm1): #Iterate through the values of the RNN hidden nodes and generate the values of the visible bias vectors bv_t = tf.add(bv, tf.matmul(u_tm1, Wuv)) return bv_t def hidden_bias_recurrence(bh_t, u_tm1): #Iterate through the values of the RNN hidden nodes and generate the values of the hidden bias vectors bh_t = tf.add(bh, tf.matmul(u_tm1, Wuh)) return bh_t #Reshape our bias matrices to be the same size as the batch. tf.assign(BH_t, tf.tile(BH_t, [size_bt, 1])) tf.assign(BV_t, tf.tile(BV_t, [size_bt, 1])) #Scan through the rnn and generate the value for each hidden node in the batch Uarr = tf.scan(rnn_recurrence, x, initializer=u0) Uarr = tf.concat([tf.reshape(u0, [1, 1, n_hidden_recurrent]), Uarr[:-1,:,:]], 0) #Scan through the rnn and generate the visible and hidden biases for each RBM in the batch BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, tf.zeros([1, n_visible])), [size_bt, n_visible]) BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, tf.zeros([1, n_hidden])), [size_bt, n_hidden]) #Get the free energy cost from each of the RBMs in the batch cost = RBM.get_free_energy_cost(x, W, BV_t, BH_t, k=1) def reconstruction(): #This function handles reconstructing de trajectory. This function is one of the outputs of the rnnrbm() function primer=x #Scan through the rnn and generate the value for each hidden node in the batch Uarr = tf.scan(rnn_recurrence, primer, initializer=u0) # primer is of dimension 2, Uarr is of dimension 3 Uarr = tf.concat([tf.reshape(u0,[1, 1, n_hidden_recurrent]), Uarr[:-1,:,:]], 0) #Scan through the rnn and generate the visible and hidden biases for each RBM in the batch BV_t = tf.reshape(tf.scan(visible_bias_recurrence, Uarr, bv), [size_bt, n_visible]) BH_t = tf.reshape(tf.scan(hidden_bias_recurrence, Uarr, bh), [size_bt, n_hidden]) traj = RBM.gibbs_sample(primer, W, BV_t, BH_t, k=1) return traj[1:,:] #we discard the first time step because it has a huge deviation return x, cost, reconstruction, W, Wuh, Wuv, Wvu, Wuu, bh, bv, bu, lr, u0
from RBM import * mnist = input_data.read_data_sets('../MNIST_data', one_hot=True) batch_size = 10 batch = mnist.train.next_batch(batch_size) rbm_input = np.concatenate((to_binary_2D(batch[0]), batch[1]), axis = 1) print(rbm_input.shape) r = RBM(data_num = batch_size, m = 28*28+10, n = 20, k = 100) r.print_tensors()
Wuu = tf.Variable(tf.random_normal([rnn_rbm.n_hidden_recurrent, rnn_rbm.n_hidden_recurrent], 0.0001), name="Wuu") bh = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden], tf.float32), name="bh") bv = tf.Variable(tf.zeros([1, rnn_rbm.n_visible], tf.float32), name="bv") bu = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="bu") u0 = tf.Variable(tf.zeros([1, rnn_rbm.n_hidden_recurrent], tf.float32), name="u0") BH_t = tf.Variable(tf.ones([1, rnn_rbm.n_hidden], tf.float32), name="BH_t") BV_t = tf.Variable(tf.ones([1, rnn_rbm.n_visible], tf.float32), name="BV_t") size_bt = tf.shape(x_rbm)[0] #Build the RBM optimization saver = tf.train.Saver() xk1, cost = RBM.build_rbm(x_rbm, W, bv, bh, 1) rbm_train_var = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost) #Run the session with tf.Session() as sess: init = tf.initialize_all_variables() sess.run(init) # loop with batch for song in tqdm(songs): for i in range(1, len(song), batch_size): tr_x = song[i:i + batch_size] sess.run(rbm_train_var, feed_dict={x_rbm: tr_x}) save_path = saver.save(sess, "initializations/rbm.ckpt")