def __init__( self, input, n_visible=784, n_hidden=500, W=None, hbias=None, vbias=None, numpy_rng=None, transpose=False, theano_rng=None, weight_decay=0.0002, ): RBM.__init__( self, input=input, n_visible=n_visible, n_hidden=n_hidden, W=W, hbias=hbias, vbias=vbias, numpy_rng=numpy_rng, theano_rng=theano_rng, weight_decay=weight_decay, )
class SFG: def __init__(self): self.image_width = self.image_height = 28 self.visible_units = self.image_width * self.image_height self.hidden_units = self.visible_units / 10 self.rbm = RBM(self.visible_units, self.hidden_units) #assumes there are only training images in the training_folder def train(self, training_folder, epochs = 500): data = [] for training_image in os.listdir(training_folder): image = pil.open(training_folder + '/' + training_image) image = self.array_for_image(image) data.append(image) self.rbm.train(data, epochs) #takes a pil Image and returns an arary of 1s and 0s def array_for_image(self, image): return np.array(image.convert("L")).flatten() / 255 def regen_image(self, image, samples): data = self.array_for_image(image) (v, _) = self.rbm.regenerate([data],samples) return self.image_for_array(v[0]) def image_for_array(self, array): img_array = [] for row in range(0, self.image_height): img_array.append(array[row * self.image_width : (row+1) * self.image_width]) img_array = np.asarray(img_array, np.uint8) * 255 return pil.fromarray(img_array)
def __init__( self, input, n_in=784, n_hidden=500, W=None, hbias=None, vbias=None, numpy_rng=None, transpose=False, activation=T.nnet.sigmoid, theano_rng=None, name="grbm", W_r=None, dropout=0, dropconnect=0, ): # initialize parent class (RBM) RBM.__init__( self, input=input, n_visible=n_in, n_hidden=n_hidden, W=W, hbias=hbias, vbias=vbias, numpy_rng=numpy_rng, theano_rng=theano_rng, )
def pretrain_rbm_layers(v, validation_v=None, n_hidden=[], gibbs_steps=[], batch_size=[], num_epochs=[], learning_rate=[], probe_epochs=[]): rbm_layers = [] n_rbm = len(n_hidden) # create rbm layers for i in range(n_rbm): rbm = RBM(n_hidden=n_hidden[i], gibbs_steps=gibbs_steps[i], batch_size=batch_size[i], num_epochs=num_epochs[i], learning_rate=learning_rate[i], probe_epochs=probe_epochs[i]) rbm_layers.append(rbm) # pretrain rbm layers input = v validation_input = validation_v for rbm, i in zip(rbm_layers, range(len(rbm_layers))): print '### pretraining RBM Layer {i}'.format(i=i) rbm.fit(input, validation_input) output = rbm.sample_h_given_v(input, rbm.params['W'], rbm.params['c']) if validation_input is not None: validation_output = rbm.sample_h_given_v(validation_input, rbm.params['W'], rbm.params['c']) else: validation_output = None input = output validation_input = validation_output return rbm_layers
def pretrainRBM(self,trainset): trainv = np.mat(trainset[1]) # 1xn vlen = trainv.shape[1] trainnum = len(trainset) hlen = 500 weights = [] print "vlen = %d" %(vlen) print "Trainnum = %d" %(trainnum) for i in range(self.nlayers): rbm = RBM(vlen,hlen) T,e = 3,0.05 if i == 0: traindata = trainset else: traindata = outdata outdata = np.zeros((trainnum,hlen)) for j in range(trainnum): print "layer:%d CD sample %d..." %(i,j) trainv = np.mat(traindata[j]) rbm.train_CD(trainv,T,e) outdata[j] = np.mat(rbm.sample(rbm.calc_forward(trainv))) # 1xhlen self.rbm_layers.append(rbm) weights.append(rbm.W) vlen = hlen # hlen -= 100 dump_data("data/dbn.pkl",weights) print "========= pretrainRBM complete ==========="
def __init__(self, input=None, n_visible=784, n_hidden=500, W=None, h_bias=None, v_bias=None, numpy_rng=None, theano_rng=None): """ GBRBM constructor. Defines the parameters of the model along with basic operations for inferring hidden from visible (and vice-versa). It initialize parent class (RBM). :param input: None for standalone RBMs or symbolic variable if RBM is part of a larger graph. :param n_visible: number of visible units :param n_hidden: number of hidden units :param W: None for standalone RBMs or symbolic variable pointing to a shared weight matrix in case RBM is part of a DBN network; in a DBN, the weights are shared between RBMs and layers of a MLP :param h_bias: None for standalone RBMs or symbolic variable pointing to a shared hidden units bias vector in case RBM is part of a different network :param v_bias: None for standalone RBMs or a symbolic variable pointing to a shared visible units bias """ RBM.__init__( self, input=input, n_visible=n_visible, n_hidden=n_hidden, W=W, h_bias=h_bias, v_bias=v_bias, numpy_rng=numpy_rng, theano_rng=theano_rng)
def __init__(self, input, n_visible=16, n_hidden=20, W=None, hbias=None, vbias=None, numpy_rng=None, theano_rng=None): # initialize parent class (RBM) RBM.__init__(self, input=input, n_visible=n_visible, n_hidden=n_hidden, W=W, hbias=hbias, vbias=vbias, numpy_rng=numpy_rng, theano_rng=theano_rng)
def load_dbn_param(self,dbnpath,softmaxpath): weights = cPickle.load(open(dbnpath,'rb')) vlen,hlen = 0,0 self.nlayers = len(weights) for i in range(self.nlayers): weight = weights[i] vlen,hlen = weight.shape[0],weight.shape[1] rbm = RBM(vlen,hlen) rbm.W = weight self.rbm_layers.append(rbm) print "RBM layer%d shape:%s" %(i,str(rbm.W.shape)) self.softmax = SoftMax() self.softmax.load_theta(softmaxpath) print "softmax parameter: "+str(self.softmax.theta.shape)
def load_from_matfile(cls, matfilename): data = loadmat(matfilename) stack_data = data.get('stack_data') numrbms = data.get('numrbms') rbms = [] for mac_i in range(numrbms): vbias = data.get(str(mac_i)+"_visbias") hbias = data.get(str(mac_i)+"_hidbias") vishid = data.get(str(mac_i)+"_vishid") rbm = RBM(vbias.size, hbias.size) rbm.get_vislayer().bias = vbias rbm.get_hidlayer().bias = hbias rbm.weights[0] = vishid rbms.append(rbm) return cls(stack_data, rbms)
class RBMTest(unittest.TestCase): def setUp(self): self.rbm = RBM(10,10) def can_make_rbm_test(self): rbm = RBM(10, 10) def logistic_function_test(self): self.assertEquals(self.rbm.logistic(0), 1) def train_throws_error_with_inconsistent_matrix_sizes_test(self): with self.assertRaises(TypeError): self.rbm.train([[1,0,1,1,1,1,0,1], [1,1,1,1,0], [1,1,1,1,1,1]]) def regenerate_throws_error_with_inconsistent_matrix_sizes_test(self): with self.assertRaises(TypeError): self.rbm.regenerate([[1,0,1,1,1,1,0,1], [1,1,1,1,0], [1,1,1,1,1,1]])
def _ulogprob_hid(self, Y, num_is_samples=100): """ Estimates the unnormalized marginal log-probabilities of hidden states. Use this method only if you know what you are doing. """ # approximate this SRBM with an RBM rbm = RBM(self.X.shape[0], self.Y.shape[0]) rbm.W = self.W rbm.b = self.b rbm.c = self.c # allocate memory Q = np.asmatrix(np.zeros([num_is_samples, Y.shape[1]])) for k in range(num_is_samples): # draw importance samples X = rbm.backward(Y) # store importance weights Q[k, :] = self._ulogprob(X, Y) - rbm._clogprob_vis_hid(X, Y) # average importance weights to get estimates return utils.logmeanexp(Q, 0)
def fit_network(self, X, labels=None): if labels is None: labels = numpy.zeros((X.shape[0], 2)) self.layers = [] temp_X = X for j in range(self.num_layers): print "\nTraining Layer %i" % (j + 1) print "components: %i" % self.components[j] print "batch_size: %i" % self.batch_size[j] print "learning_rate: %0.3f" % self.learning_rate[j] print "bias_learning_rate: %0.3f" % self.bias_learning_rate[j] print "epochs: %i" % self.epochs[j] print "Sparsity: %s" % str(self.sparsity_rate[j]) print "Sparsity Phi: %s" % str(self.phi) if j != 0: self.plot_weights = False model = RBM(n_components=self.components[j], batch_size=self.batch_size[j], learning_rate=self.learning_rate[j], regularization_mu=self.sparsity_rate[j], n_iter=self.epochs[j], verbose=True, learning_rate_bias=self.bias_learning_rate[j], plot_weights=self.plot_weights, plot_histograms=self.plot_histograms, phi=self.phi) if j + 1 == self.num_layers and labels is not None: model.fit(numpy.asarray(temp_X), numpy.asarray(labels)) else: model.fit(numpy.asarray(temp_X)) temp_X = model._mean_hiddens(temp_X) # hidden layer given visable units print "Trained Layer %i\n" % (j + 1) self.layers.append(model)
def pretrain_rbm_layers(v, validation_v=None, n_hidden=[], gibbs_steps=[], batch_size=[], num_epochs=[], learning_rate=[], probe_epochs=[]): """ Fake pre-training, just randomly initialising the weights of RBM layers :param v: :param validation_v: :param n_hidden: :param gibbs_steps: :param batch_size: :param num_epochs: :param learning_rate: :param probe_epochs: :return: """ rbm_layers = [] n_rbm = len(n_hidden) # create rbm layers for i in range(n_rbm): rbm = RBM(n_hidden=n_hidden[i], gibbs_steps=gibbs_steps[i], batch_size=batch_size[i], num_epochs=num_epochs[i], learning_rate=learning_rate[i], probe_epochs=probe_epochs[i]) rbm_layers.append(rbm) # pretrain rbm layers n_v = v.shape[1] for rbm, i in zip(rbm_layers, range(len(rbm_layers))): print '### pretraining RBM Layer {i}'.format(i=i) n_h = n_hidden[i] initial_W = np.float32(np.random.uniform( low=-4 * np.sqrt(6.0 / (n_h + n_v)), high=4 * np.sqrt(6.0 / (n_h + n_v)), size=(n_v, n_h) )) rbm.params['W'] = initial_W rbm.params['c'] = np.zeros((n_h, ), np.float32) n_v = n_h return rbm_layers
def __init__(self,knapsack_file="weing1.pkl"): super(ES, self).__init__() # GA stuff self.generations = 100 self.knapsack = pickle.load(open(knapsack_file)) print "k:",self.knapsack self.N = int(self.knapsack.items) # RMB stuff self.RBM = RBM(n_visible=self.N,n_hidden=50) self.sample_RBM() # Stats stuff self.population_snapshots = [] self.genotypes_history = Genotypes(min=False)
def test(learning_rate=0.1, k=1, training_epochs=15): print '... loading data' datasets = load_data('mnist.pkl.gz') train_set_x, train_set_y = datasets[0] test_set_x, test_set_y = datasets[2] print '... modeling' rbm = RBM(input=train_set_x, n_visible=28 * 28, n_hidden=500) print '... training' start_time = time.clock() for epoch in xrange(training_epochs): cost = rbm.get_cost_updates(lr=learning_rate, k=k) print 'Training epoch %d, cost is ' % epoch, cost end_time = time.clock() pretraining_time = (end_time - start_time) print ('Training took %f minutes' % (pretraining_time / 60.))
def get_representation(): # Load the dictionary and corresponding args. (W, b, hidden_size) = pickle.load(open("Models/RBM/model%d.pkl"%experiment_number,'rb')) # Set the constructor myObject = RBM(hidden_size=hidden_size) print "Loading dataset..." trainset,validset,testset = dataset_store.get_classification_problem('ocr_letters') encoded_trainset = [] encoded_validset = [] encoded_testset = [] print "Initializing..." myObject.initialize(W,b) print "Encoding the trainset..." counter = 0 #Inelegant, I know! I use this to only use the first 1000 values. for input,target in trainset: #Encode the sample. h = myObject.encode(input) encoded_trainset.append(h) # counter +=1 # if counter == 1000: # break # Save the datasets to files. filename = "Models/RBM/trainset%d.pkl"%(experiment_number) pickle.dump( np.asarray(encoded_trainset) , open(filename, 'wb')) counter = 0 print "Encoding the validset..." for input,target in validset: #Encode the sample. h = myObject.encode(input) encoded_validset.append(h) # counter +=1 # if counter == 1000: # break filename = "Models/RBM/validset%d.pkl"%(experiment_number) pickle.dump( np.asarray(encoded_validset) , open(filename, 'wb')) #Note: only need to do it for the best hyper-params at the end. print "Encoding the testset..." for input,target in testset: #Encode the sample. h = myObject.encode(input) encoded_testset.append(h) filename = "Models/RBM/testset%d.pkl"%(experiment_number) pickle.dump( np.asarray(encoded_testset), open(filename, 'wb'))
def train(dim_h, lrate, max_epoches, batch_size, train_ratio): """ traing a model and save to disk """ result_file = "mnist_%s.pkl" % strftime("%b_%d_%H_%M_%S", gmtime()) if os.access(result_file, os.R_OK): raise RuntimeError("%s already exists" % result_file) data = load_digits() learner = RBM.train( data, dim_h=dim_h, lrate=lrate, max_epoches=max_epoches, batch_size=batch_size, train_ratio=train_ratio ) with open(result_file, "wb") as handle: pickle.dump(learner, handle) print('learner sleeping at "%s"' % result_file)
def run_rbm(geral, teste, neuron, learning, epochs): path = '/home/jeferson/Desktop/horses' rbm = RBM(geral, neuron) rbm.set_learning_rate(learning) time_rbm = time.time() rbm.training_rbm(epochs, 0) time_rbm = abs(time.time() - time_rbm) rbm_output = rbm.predict(teste) image = result(rbm_output, size) rbm.save(rbm, path + 'rbm_teste_' + str(epochs)) cv2.imwrite(path + 'res_rbm_truco' + str(epochs) + '.png', image) print 'tempo treinamento RBM ', time_rbm, 'MSE', (( rbm_output - teste)**2).mean(axis=None), image.shape
def build_model(self): # feed 变量 self.input_data = tf.placeholder( tf.float32, [None, self.dbm_struct[0]]) # N等于_num_examples或batch_size # 构建rmbs self.rbm_list = list() for i in range(len(self.dbm_struct) - 1): n_v = self.dbm_struct[i] n_h = self.dbm_struct[i + 1] name = 'rbm-' + str(i + 1) rbm = RBM(name=name, rbm_v_type=self.rbm_v_type, rbm_struct=[n_v, n_h], rbm_epochs=self.rbm_epochs, batch_size=self.batch_size, cd_k=self.cd_k, rbm_lr=self.rbm_lr) self.rbm_list.append(rbm) # 加入list
def train_rbms(self): inpX = self._X rbm_list = [] input_size = inpX.shape[1] # for each RBM we want to generate for i, size in enumerate(self._sizes): rbm_list.append(RBM(input_size, size)) input_size = size # for each RBM in our RBM's list for i, rbm in enumerate(rbm_list): print('\n\nRBM {}: '.format(i)) rbm.train(inpX) inpX = rbm.rbm_output(inpX) rbm.save_weights(i) rbm.save_biases(i) return rbm_list
def __init__(self, n_in=784, n_out=10, hidden_layers_sizes=[500, 500]): """ :param n_in: int, the dimension of input :param n_out: int, the dimension of output :param hidden_layers_sizes: list or tuple, the hidden layer sizes """ # Number of layers assert len(hidden_layers_sizes) > 0 self.n_layers = len(hidden_layers_sizes) self.layers = [] # normal sigmoid layer self.rbm_layers = [] # RBM layer self.params = [] # keep track of params for training # Define the input and output self.x = tf.placeholder(tf.float32, shape=[None, n_in]) self.y = tf.placeholder(tf.float32, shape=[None, n_out]) # Contruct the layers of DBN for i in range(self.n_layers): if i == 0: layer_input = self.x input_size = n_in else: layer_input = self.layers[i-1].output input_size = hidden_layers_sizes[i-1] # Sigmoid layer sigmoid_layer = HiddenLayer(inpt=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=tf.nn.sigmoid) self.layers.append(sigmoid_layer) # Add the parameters for finetuning self.params.extend(sigmoid_layer.params) # Create the RBM layer self.rbm_layers.append(RBM(inpt=layer_input, n_visiable=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b)) # We use the LogisticRegression layer as the output layer self.output_layer = LogisticRegression(inpt=self.layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_out) self.params.extend(self.output_layer.params) # The finetuning cost self.cost = self.output_layer.cost(self.y) # The accuracy self.accuracy = self.output_layer.accuarcy(self.y)
def train_layers(self, layers_sizes): """ Method that iteratively trains the DBN layerwise """ for i in xrange(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden units of the # layer below or the input size if we are on the first layer if i == 0: input_size = layers_sizes[i - 1] # the input to this layer is either the activation of the hidden # layer below or the input of the DBN if you are on the first layer if i == 0: layer_input = self.x else: layer_input = self.sigmoid_layers[-1].output sigmoid_layer = HiddenLayer(rng=self.numpy_rng, _input=layer_input, n_in=input_size, n_out=layers_sizes[i], activation=T.nnet.sigmoid) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) # its arguably a philosophical question... but we are going to only declare that # the parameters of the sigmoid_layers are parameters of the DBN. The visible # biases in the RBM are parameters of those RBMs, but not of the DBN. self.params.extend(sigmoid_layer.params) # Construct an RBM that shared weights with this layer rbm_layer = RBM(numpy_rng=self.numpy_rng, theano_rng=self.theano_rng, _input=layer_input, n_visible=input_size, n_hidden=layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer)
def build_model(self): n_h = [] n_v = [self.n_nodes] n_hPercentage = [60, 20, 10] for i in n_hPercentage: nhid = int(np.round(self.n_nodes * (i / 100))) n_h.append(nhid) n_v.append(nhid) n_v.pop() self.bias_node = 0 #membangun model #layer RBM for n in range(3): rbm_layer = RBM(epoch=self.rbm_epoch, n_visible=n_v[n], n_hidden=n_h[n], alpha=0.01) self.rbm_layers.append(rbm_layer) #layer logistic regression self.lr_layer = logReg(self.max_epoch, self.alpha)
def build_dbn(self, _inputs): for i in xrange(0, len(self.hidden)): boltz = RBM(_inputs, self.hidden[i]) boltz.training_rbm(self.epochs[i],0) self.perceptron.set_hidden_layer(boltz.get_layer(),i+1) _inputs = boltz.predict_dbn(_inputs) print 'Camada ', i, 'ok' self.perceptron.layer[len(self.perceptron.layer)-1] = Layers(self.out.shape[1], self.perceptron.layer[len(self.perceptron.layer)-2].n_neurons)
def define_prior(self): """ Defines the prior distribution over z. The prior will be an RBM or Normal prior based on self.dist_type. Returns: a DistUtil object representing the prior distribution. """ # set up the rbm with tf.name_scope("rbm_prior"): num_var1 = self.config['num_latent_units'] * self.config[ 'num_latent_layers'] // 2 wd = self.config['weight_decay'] if self.dist_type == 'dvaes_gi': rbm_prior = GuassianIntRBM(num_var1=num_var1, num_var2=num_var1, num_samples=1000, weight_decay=wd, use_qupa=self.config['use_qupa'], minimum_lambda=self.config['beta']) elif self.dist_type in { 'dvaes_gauss', 'dvaes_exp', 'dvaes_power', 'dvaes_unexp' }: rbm_prior = MarginalRBMType1Generic( num_var1=num_var1, num_var2=num_var1, num_samples=1000, weight_decay=wd, use_qupa=self.config['use_qupa'], smoothing_dist=self.smoothing_dist) elif self.dist_type in { 'dvae_spike_exp', 'dvaepp_exp', 'dvaepp_power' }: rbm_prior = RBM(num_var1=num_var1, num_var2=num_var1, num_samples=1000, weight_decay=wd, kld_term=self.dist_type, use_qupa=self.config['use_qupa']) else: raise NotImplementedError return rbm_prior
def main(): args = parser.parse_args() pp.pprint(args) if not os.path.exists(args.checkpoint_dir): os.mkdir(args.checkpoint_dir) if not os.path.exists(args.sample_dir): os.mkdir(args.sample_dir) with tf.Session() as sess: rbm = RBM(sess) if args.is_train: rbm.train(args) else: rbm.load(args.checkpoint_dir)
def __init__(self, layer_sizes, class_count): """ Creates a DBN. :param layer_sizes: An iterable of integers indicating the desired sizes of all the layers of the DBN. Note that the fore-last layer will be augmented with #class_count neurons (which are placed on the start of the layer: lowest indices). :params class_count: The number of classes in the classification tast this DBN is trained for. """ log.info('Creating DBN, classes %d, layer sizes: %r', class_count, layer_sizes) self.class_count = class_count # create the RBMs rbms = [] for ind in range(1, len(layer_sizes)): # visible and hidden layer sizes for the RBM vis_size = layer_sizes[ind - 1] hid_size = layer_sizes[ind] # if making the last RBM, visible layer # get's extra neurons (for class indicators) if ind == (len(layer_sizes) - 1): vis_size += class_count # create the RBM rbms.append(RBM(vis_size, hid_size)) # these are the net's RBMs self.rbms = rbms
def train_rbms(self): inpX = self._X rbm_list = [] input_size = inpX.shape[1] # for each RBM we want to generate for i, size in enumerate(self._sizes): rbm_list.append(RBM(input_size, size)) input_size = size # for each RBM in our RBM's list for i, rbm in enumerate(rbm_list): print('\n\nRBM {}: '.format(i)) if (not 'rbm_vb_' + str(i) + '.npy' in os.listdir('./trained')): rbm.train(inpX) inpX = rbm.rbm_output(inpX) rbm.save_weights(i) rbm.save_biases(i) else: rbm.load_biases('./trained/rbm_vb_' + str(i) + '.npy', 'trained/rbm_hb_' + str(i) + '.npy') rbm.load_weights('./trained/rbm_weights_' + str(i) + '.npy') return rbm_list
def get_recc(att_df, cat_rating): util = Util() epochs = 50 rows = 40000 alpha = 0.01 H = 128 batch_size = 16 dir = 'etl/' ratings, attractions = util.read_data(dir) ratings = util.clean_subset(ratings, rows) rbm_att, train = util.preprocess(ratings) num_vis = len(ratings) rbm = RBM(alpha, H, num_vis) joined = ratings.set_index('attraction_id').join(attractions[[ "attraction_id", "category" ]].set_index("attraction_id")).reset_index('attraction_id') grouped = joined.groupby('user_id') category_df = grouped['category'].apply(list).reset_index() rating_df = grouped['rating'].apply(list).reset_index() cat_rat_df = category_df.set_index('user_id').join( rating_df.set_index('user_id')) cat_rat_df['cat_rat'] = cat_rat_df.apply(f, axis=1) cat_rat_df = cat_rat_df.reset_index()[['user_id', 'cat_rat']] cat_rat_df['user_data'] = [cat_rating for i in range(len(cat_rat_df))] cat_rat_df['sim_score'] = cat_rat_df.apply(sim_score, axis=1) user = cat_rat_df.sort_values(['sim_score']).values[0][0] print("Similar User: {u}".format(u=user)) filename = "e" + str(epochs) + "_r" + str(rows) + "_lr" + str( alpha) + "_hu" + str(H) + "_bs" + str(batch_size) reco, weights, vb, hb = rbm.load_predict(filename, train, user) unseen, seen = rbm.calculate_scores(ratings, attractions, reco, user) rbm.export(unseen, seen, 'rbm_models/' + filename, str(user)) return filename, user, rbm_att
class Ops(object): pass weights = None #weights None gets them initialized randomly visible_bias = None #visible bias None gets them initialized at zero hidden_bias = None #hidden bias None gets them initialized at zero # Load the MC configuration training data: trainFileName = 'data/train.txt' xtrain = np.loadtxt(trainFileName) ept = np.random.permutation(xtrain) # random permutation of training data iterations_per_epoch = xtrain.shape[0] / bsize # Initialize the RBM rbm = RBM(num_hidden=num_hidden, num_visible=num_visible,num_state_vis=num_state_vis, num_state_hid=num_state_hid, weights=weights, visible_bias=visible_bias,hidden_bias=hidden_bias, num_samples=num_samples) # Initialize operations and placeholders classes ops = Ops() placeholders = Placeholders() placeholders.visible_samples = tf.placeholder(tf.float32, shape=(None, num_visible*num_state_vis), name='v') # placeholder for training data total_iterations = 0 # starts at zero ops.global_step = tf.Variable(total_iterations, name='global_step_count', trainable=False) learning_rate = tf.train.exponential_decay( learning_rate_start, ops.global_step, 100 * xtrain.shape[0]/bsize, 1.0 # decay rate = 1 means no decay )
from rbm import RBM import numpy as np rbm = RBM(num_visible=6, num_hidden=2) base = np.array([[1, 1, 1, 0, 0, 0], [1, 0, 1, 0, 0, 0], [1, 1, 1, 0, 0, 0], [0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 0, 1], [0, 0, 1, 1, 0, 1]]) filmes = [ 'A bruxa', 'Invocação do mal', 'O chamado', 'Se beber não case', 'Gente grande', 'American pie' ] rbm.train(base, max_epochs=5000) rbm.weights '''com rbm.weights vc pode ver a matriz dos pesos obviamente, mas o que é interessante de se ver é que: a primeira linha representa o bias, as linhas seguintes são os filmes, onde o primeiro valor é o filme (id de certa forma) e os outros valores representam qual neurônio foi escolhido. achando assim um padrão''' usuario1 = np.array([[1, 1, 0, 1, 0, 0]]) usuario2 = np.array([[0, 0, 0, 1, 1, 0]]) resultado1 = rbm.run_visible(usuario1) resultado2 = rbm.run_visible(usuario2) recomendacao1 = rbm.run_hidden(resultado1) recomendacao2 = rbm.run_hidden(resultado2)
#!/usr/bin/env python2 # -*- coding: utf-8 -*- """ Created on Wed Mar 25 20:42:03 2020 @author: yurifarod """ from rbm import RBM import numpy as np rbm = RBM(num_visible=6, num_hidden=3) base = np.array([[0, 1, 1, 1, 0, 1], [1, 1, 0, 1, 1, 1], [0, 1, 0, 1, 0, 1], [0, 1, 1, 1, 0, 1], [1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 1, 1]]) filmes = [ "Freddy x Jason", "O Ultimato Bourne", "Star Trek", "Exterminador do Futuro", "Norbit", "Star Wars" ] rbm.train(base, max_epochs=5000) rbm.weights leonardo = np.array([[0, 1, 0, 1, 0, 0]]) rbm.run_visible(leonardo) camada_escondida = np.array([[1, 0, 1]]) recomendacao = rbm.run_hidden(camada_escondida)
def fit(self, X, y): self.build_net(X.shape[1], len(np.unique(y))) # Assign weights of layers as views of the big weights if self.coef_ is None: ws = list() for layer in self.layers: ws.append(layer.b.reshape(-1)) ws.append(layer.W.reshape(-1)) self.coef_ = np.concatenate(tuple(ws)) self.assign_weights() # Pretrain if self.pretrain_epochs > 0: if self.progress_bars: if self.pretrain_batches_per_epoch == -1: batches_per_epoch = int(X.shape[0] / self.pretrain_batch_size) else: batches_per_epoch = self.pretrain_batches_per_epoch maxiters = self.pretrain_epochs * batches_per_epoch * len( self.layers) pt_bar = ProgressBar(max=maxiters, desc='Pretrain') if self.pretrain_batch_size == -1: # Use full-batch self.pretrain_batch_size = X.shape[0] # Create RBM layers using the same weights self.rbm_layers = [] for i, layer in enumerate(self.layers): n_hid = layer.W.shape[1] new = RBM(layer) self.rbm_layers.append(new) # Actual pretrain for i, rbm_layer in enumerate(self.rbm_layers): for epoch in range(self.pretrain_epochs): mb = MBOpti.minibatches( X, batch_size=self.pretrain_batch_size, batches=self.pretrain_batches_per_epoch, random_state=self.rnd) for j, batch in enumerate(mb): if i == 0: input = batch else: # input = self.layers[i - 1].output(batcn) try: input = self.layers[i - 1].sample_h_given_v(input) except: print input.shape, self.layers[i - 1].W.shape raise Exception('1') rbm_layer.contrastive_divergence(input) if self.progress_bars: pt_bar.next() if self.pretrain_callback is not None: stop = self.pretrain_callback( self, layer, epoch + 1, j + 1) if stop == True: break if self.progress_bars: pt_bar.complete() # Finetune if self.finetune_epochs > 0: if self.progress_bars: if self.finetune_batches_per_epoch == -1: batches_per_epoch = int(X.shape[0] / self.finetune_batch_size) else: batches_per_epoch = self.finetune_batches_per_epoch maxiters = self.finetune_epochs * batches_per_epoch ft_bar = ProgressBar(max=maxiters, desc='Finetune') def _callback(epoch, i): if self.progress_bars: ft_bar.next() if self.finetune_callback is not None: return self.finetune_callback(self, epoch, i) self.finetune_options = self.finetune_options.copy() args = (self.layers, len(np.unique(y))) MBOpti.minimize(self.coef_, X, y, fun=cost, grad=cost_prime, weights=self.coef_, method=self.finetune_method, epochs=self.finetune_epochs, batch_size=self.finetune_batch_size, batches_per_epoch=self.finetune_batches_per_epoch, options=self.finetune_options, args=args, callback=_callback, random_state=self.rnd) if self.progress_bars: ft_bar.complete()
def __init__(self): self.image_width = self.image_height = 25 self.visible_units = self.image_width * self.image_height self.hidden_units = 400 self.rbm = RBM(self.visible_units, self.hidden_units, 0.05)
flags.DEFINE_integer('epochs', 50, 'The number of training epochs') flags.DEFINE_integer('batchsize', 30, 'The batch size') flags.DEFINE_boolean('restore_rbm', False, 'Whether to restore the RBM weights or not.') # ensure output dir exists if not os.path.isdir('out'): os.mkdir('out') mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels trX, teY = min_max_scale(trX, teX) # RBMs rbmobject1 = RBM(784, 900, ['rbmw1', 'rbvb1', 'rbmhb1'], 0.3) rbmobject2 = RBM(900, 500, ['rbmw2', 'rbvb2', 'rbmhb2'], 0.3) rbmobject3 = RBM(500, 250, ['rbmw3', 'rbvb3', 'rbmhb3'], 0.3) rbmobject4 = RBM(250, 2, ['rbmw4', 'rbvb4', 'rbmhb4'], 0.3) if FLAGS.restore_rbm: rbmobject1.restore_weights('./out/rbmw1.chp') rbmobject2.restore_weights('./out/rbmw2.chp') rbmobject3.restore_weights('./out/rbmw3.chp') rbmobject4.restore_weights('./out/rbmw4.chp') # Autoencoder autoencoder = AutoEncoder(784, [900, 500, 250, 2], [['rbmw1', 'rbmhb1'], ['rbmw2', 'rbmhb2'], ['rbmw3', 'rbmhb3'], ['rbmw4', 'rbmhb4']], tied_weights=False)
args = parser.parse_args() # Data pre-processing using pandas trainDataFrame = pd.read_csv(args.trainFile,delim_whitespace=True,header=None,usecols=[0,1,2]) trainDataFrame[2] = trainDataFrame[2].apply(lambda x: 1 if x>2 else 0) trainMatrix=trainDataFrame.as_matrix() trainArray = np.ndarray(shape=(totalUsers,totalMovies), dtype=int) row,column = trainMatrix.shape # Converting raw dataframe into training numpy array for i in range (0, row): userID = trainMatrix[i][0] -1 movieID = trainMatrix[i][1] -1 trainArray[userID][movieID]=trainMatrix[i][2] # training rbmObject = RBM(num_visible = totalMovies,num_hidden = args.hiddenUnits) rbmObject.train(trainArray, max_epochs = args.epochs) print(rbmObject.weights) # pickling modelName = "model-"+str(args.epochs)+"-"+str(args.hiddenUnits)+".p" print modelName fp = open( modelName, "wb" ) for obj in [infoMatrix, movieMatrix, rbmObject]: pickle.dump(obj, fp, protocol=pickle.HIGHEST_PROTOCOL) print 'Model trained and pickled'
# cnn1.pre_train() # output_list = cnn1.output() # saveImage(output_list, (74,46)) cnn2 = CNN(cnn1.output(), filter_shape, filter_shift_list[1], node_shape[1], node_shape[2], pre_train_lr, pre_train_epoch) output_list = cnn2.output() saveImage(output_list, node_shape[2], 'cnn2_before_train') cnn2.pre_train() output_list = cnn2.output() saveImage(output_list, node_shape[2], 'cnn2_after_train') rbm_size_list = (680, 340, 170, 85, 42, 21, 10, 3) # def __init__(self, W, input, data_size,input_size, output_size, isDropout): rbm1 = RBM(None, cnn2.output(), file_num, rbm_size_list[0], rbm_size_list[1], False) for i in xrange(pre_train_epoch): print 'rbm1 pre_train:' + str(i) rbm1.contrast_divergence() reinput = rbm1.reconstruct_from_input(rbm1.input) saveImage(reinput, node_shape[2], 'rbm1_after_train') saveW(rbm1.getW(), 'rbm1_after_train') rbm2 = RBM(None, rbm1.output(), file_num, rbm_size_list[1], rbm_size_list[2], False) for i in xrange(pre_train_epoch): print 'rbm2 pre_train:' + str(i) rbm2.contrast_divergence() reinput = rbm2.reconstruct_from_input(rbm2.input) reinput = rbm1.reconstruct_from_output(reinput) saveImage(reinput, node_shape[2], 'rbm2_after_train') saveW(rbm2.getW(), 'rbm2_after_train')
class ES(object): def __init__(self,knapsack_file="weing1.pkl"): super(ES, self).__init__() # GA stuff self.generations = 100 self.knapsack = pickle.load(open(knapsack_file)) print "k:",self.knapsack self.N = int(self.knapsack.items) # RMB stuff self.RBM = RBM(n_visible=self.N,n_hidden=50) self.sample_RBM() # Stats stuff self.population_snapshots = [] self.genotypes_history = Genotypes(min=False) def create_individual(self,N): I = Individual() I.genotype = [random.choice([0,1]) for i in range(N)] I.fitness = 0 return I def fitness_function(self,individual,knapsack=None): weights = [] for i,c in enumerate(knapsack.capacities): weights.append(np.sum(np.array(knapsack.constraints[i])*individual.genotype)) over = 0 for i,w in enumerate(weights): if w > knapsack.capacities[i]: over += (w - knapsack.capacities[i]) if over > 0: return -over else: return np.sum(np.array(knapsack.values)*individual.genotype) def evaluate_population(self,population,params=None): for p in population: p.fitness = self.fitness_function(p,params) def normalise_fitnesses(self,population): max_fitness = np.max([p.fitness for p in population]) min_fitness = np.min([p.fitness for p in population]) for p in population: p.normalised_fitness = (p.fitness + min_fitness)/(min_fitness+max_fitness) def offspring_from_sample(self,individual_to_copy): individual = copy.deepcopy(individual_to_copy) individual_genome = np.array(individual.genotype).reshape(1,-1) output = self.sample_from_RBM(np.array(individual_genome)) # print "output:",output individual.genotype[:] = output[0][:] return individual def train_RBM(self,k=20,lr=0.1): train_data = self.genotypes_history.top_x_percent() train_set = SequenceDataset(train_data,batch_size=20,number_batches=None) inputs,params,cost,monitor,updates,consider_constant = self.RBM.build_RBM(k=k) sgd_optimizer(params,[inputs],cost,train_set,updates_old=updates,monitor=monitor, consider_constant=[consider_constant],lr=0.1,num_epochs=10) def sample_RBM(self,k=20): v,v_sample,updates = self.RBM.sample_RBM(k=k) self.sample_from_RBM = theano.function([v],v_sample,updates=updates) def run_1_plus_1(self, path= "", experiment = 0): random.seed(random.uniform(0,1000000)) print("Start of evolution") parent = self.create_individual(self.N) # Evaluate the parent parent.fitness = self.fitness_function(parent,self.knapsack) self.genotypes_history.add_genotypes([parent]) self.genotypes_history.get_and_save_top_x(1.0) self.train_RBM() # Begin the evolution for g in range(self.generations): print("-- Generation %i --" % (g + 1)) offspring = self.offspring_from_sample(parent) offspring.fitness = self.fitness_function(parent,self.knapsack) print "parent_fitness:",parent.fitness print "offspring_fitness:",offspring.fitness if offspring.fitness > parent.fitness: print "offspring replacing parent" parent = offspring self.genotypes_history.add_genotypes([offspring]) self.genotypes_history.get_and_save_top_x(1.0) self.train_RBM() print("-- End of (successful) evolution --") return parent def run_mu_plus_lambda(self, path= "", experiment = 0): population_size = 50 random.seed(random.uniform(0,1000000)) print("Start of evolution") population = [self.create_individual(self.N) for i in range(population_size)] # Evaluate the population self.evaluate_population(population,self.knapsack) self.genotypes_history.add_genotypes(population) self.genotypes_history.get_and_save_top_x(1.0) self.train_RBM() # Begin the evolution for g in range(self.generations): print("-- Generation %i --" % (g + 1)) offspring = [] for ind in population: offspring.append(self.offspring_from_sample(ind)) self.evaluate_population(offspring,self.knapsack) self.genotypes_history.add_genotypes(offspring) self.genotypes_history.get_and_save_top_x(1.0) self.train_RBM() new_population = [] population = population + offspring while len(new_population) < population_size: # tournament selection on combined population a = int(len(population) * random.random()) b = int(len(population) * random.random()) while a == b: b = int(len(population) * random.random()) if population[a].fitness > population[b].fitness: new_population.append(population.pop(a)) else: new_population.append(population.pop(b)) population = new_population print "average fitness:",np.mean([p.fitness for p in population]) print "max fitness:",np.max([p.fitness for p in population]) print "min fitness:",np.min([p.fitness for p in population]) print("-- End of (successful) evolution --") return parent
'sparse_cost': 0.001, 'sparse_target': 0.01, 'persistant': False, 'kPCD': 1, 'use_fast_weights': False } n_epochs = 1 init = GlorotUniform() # it seems that the data have shape 30x30x30, though I think it should be 24 with padding=2 layers = [RBMConvolution3D([6, 6, 6, 48], strides=2, padding=0, init=init, name='l1_conv'), RBMConvolution3D([5, 5, 5, 160], strides=2, padding=0, init=init, name='l2_conv'), RBMConvolution3D([4, 4, 4, 512], strides=2, padding=0, init=init, name='l3_conv'), RBMLayer(1200, init=init, name='l4_rbm'), RBMLayerWithLabels(4000, n_classes, name='l4_rbm_with_labels')] rbm = RBM(layers=layers) # callbacks = Callbacks(rbm, data, output_file='./output.hdf5') callbacks = Callbacks(rbm, data) t = time.time() rbm.fit(data, optimizer=parameters, num_epochs=n_epochs, callbacks=callbacks) t = time.time() - t print "Training time: ", t
def test_rbm_mnist(learning_rate=0.01, training_epochs=10, batch_size=20, n_chains=30, n_samples=5, output_folder=None, isPCD=0, n_hidden=500): """ Demonstrate how to train and afterwards sample from it using Theano. This is demonstrated on MNIST. :param learning_rate: learning rate used for training the RBM :param training_epochs: number of epochs used for training :param dataset: path the the pickled dataset :param batch_size: size of a batch used to train the RBM :param n_chains: number of parallel Gibbs chains to be used for sampling :param n_samples: number of samples to plot for each chain e.g. test_rbm_mnist(output_folder='/home/eric/Desktop/rbm_plots') """ assert output_folder is not None from rbm_variants import RBM_Orthogonal as RBM # from rbm import RBM ################################# # Data Constructing # ################################# from sklearn.datasets import fetch_mldata mnist = fetch_mldata('MNIST original') from xylearn.utils.data_util import get_train_test from xylearn.utils.data_normalization import rescale data = get_train_test(rescale(mnist.data), mnist.target, useGPU=1, shuffle=True) train_x, train_y = data['train'] n_vis = train_x.get_value(borrow=True).shape[1] print numpy.linalg.matrix_rank(train_x.get_value(borrow=True)) n_train_batches = train_x.get_value(borrow=True).shape[0] / batch_size # construct the RBM class rbm = RBM(n_visible=n_vis, n_hidden=n_hidden, isPCD=isPCD) train_fn = rbm.get_train_fn(train_x, batch_size) ################################# # Training the RBM # ################################# if not os.path.isdir(output_folder): os.makedirs(output_folder) os.chdir(output_folder) plotting_time = 0. start_time = time.clock() import PIL.Image from visualizer import tile_raster_images # go through training epochs for epoch in xrange(training_epochs): # go through the training set mean_cost = [] for batch_index in xrange(n_train_batches): # for each batch, we extract the gibbs chain new_cost = train_fn(index=batch_index, lr=learning_rate) mean_cost += [new_cost] print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost) # monitor projected rank projection = rbm.project(train_x) print 'rank: ' + str(numpy.linalg.matrix_rank(projection)) # W shape is [784 500] # Plot filters after each training epoch plotting_start = time.clock() # Construct image from the weight matrix image = PIL.Image.fromarray(tile_raster_images( X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1))) image.save('filters_at_epoch_%i.png' % epoch) plotting_stop = time.clock() plotting_time += (plotting_stop - plotting_start) end_time = time.clock() pretraining_time = (end_time - start_time) - plotting_time print ('Training took %f minutes' % (pretraining_time / 60.)) ################################# # Sampling from the RBM # ################################# test_idx = 1 test_x, test_y = data['test'] sample_fn = rbm.get_sampling_fn(test_x, test_idx, n_chains) print '... begin sampling' # plot initial image first orig_img = test_x.get_value(borrow=True)[test_idx:test_idx + 1] image = PIL.Image.fromarray(tile_raster_images( X=orig_img, img_shape=(28, 28), tile_shape=(1, 1), tile_spacing=(1, 1))) image.save('orig_img.png') # create a space to store the image for plotting ( we need to leave # room for the tile_spacing as well) image_data = numpy.zeros((29 * n_samples + 1, 29 * n_chains - 1), dtype='uint8') for idx in xrange(n_samples): # generate `plot_every` intermediate samples that we discard, # because successive samples in the chain are too correlated vis_mf, vis_sample = sample_fn() print ' ... plotting sample ', idx image_data[29 * idx:29 * idx + 28, :] = tile_raster_images( X=vis_mf, img_shape=(28, 28), tile_shape=(1, n_chains), tile_spacing=(1, 1)) # construct image image = PIL.Image.fromarray(image_data) image.save('samples.png') os.chdir('../') ################################# # Projecting from the RBM # ################################# projection = rbm.project(train_x) print numpy.linalg.matrix_rank(projection)
return rbm.run_hidden(hidden_layer) def printRecommenders(recommender, news, user): print("-----------------------------------------------------") for i in range(len(user.listNews[0])): if user.listNews[0,i] == 0 and recommender[0,i] == 1: print("Nome: %s , Recomendação: %s" %(user.nome,news[i])) news = ["Sem reforma, déficit das previdências estaduais em 2060 deve ser 4 vezes maior que o de 2013, aponta estudo", "Relator da reforma da Previdência se reúne com Maia e líderes para debater parecer", "Lava Jato: 8 parlamentares esperam STF decidir se viram réus", "Revoltado com punição, Vettel reclama muito e coloca placa de 2º lugar à frente de carro de Hamilton", "Brasil goleia Honduras por 7 a 0 na maior vitória sob o comando de Tite", "Portugal bate Holanda e se sagra campeão da Liga das Nações"] userTrain = np.array([[1,1,1,0,0,0], [1,0,1,0,0,0], [1,1,1,0,0,0], [0,0,1,1,1,1], [0,0,1,1,0,1], [0,0,1,1,0,1]]) users = [] users.append(User("José", np.array([[1,1,0,1,0,0]]))) users.append(User("Maria", np.array([[0,0,0,1,1,0]]))) rbm = RBM(num_visible=6, num_hidden=2) rbm.train(userTrain, max_epochs=5000) rbm.weights for user in users: recommender = recommenderNews(rbm, user.listNews) printRecommenders(recommender, news, user)
import json import re import datetime import pandas as pd import numpy as np #ipdb.set_trace() print('reading ratings file') ratings = pd.read_csv('ratings_liked.csv',dtype = {'userId': np.int32, 'movieId': np.int32,'liked':np.bool}) movies = pd.read_csv('movies.csv') print('creating one hot encoding') ratings_wide = pd.pivot_table(ratings, values='liked', index=['userId'],columns=['movieId'], aggfunc=np.sum) from rbm import RBM vi_unit = ratings_wide.shape[1] rbm = RBM(num_visible = vi_unit, num_hidden = 500) ratings_wide = ratings_wide.fillna(0) ratings_wide = ratings_wide.astype(int) training_data = ratings_wide.as_matrix() print('starting training') rbm.train(training_data, max_epochs = 20) # Don't run the training for more than 5000 epochs. while(True): userInput = raw_input('enter "run" to predict: ') if userInput=="run": user_liked =[] with open(serverPath+'user_liked.json', 'r') as infile: user_liked = json.load(infile)
def __init__(self, n_in, n_out=4, hidden_layers_sizes=[2048, 2048, 50, 2048, 2048]): assert len(hidden_layers_sizes) > 0 self.rbm_layers = [] self.sess = tf.Session() self.x = tf.placeholder(tf.float32, shape=None) self.y = tf.placeholder(tf.float32, shape=None) #构筑DBN for i in range(len(hidden_layers_sizes)): if i == 0: layer_input = self.x input_size = n_in else: input_size = hidden_layers_sizes[i - 1] # 隐层 bound_val = 4.0 * np.sqrt(6.0 / (input_size + hidden_layers_sizes[i])) W = tf.Variable(tf.random_uniform( [input_size, hidden_layers_sizes[i]], minval=-bound_val, maxval=bound_val), dtype=tf.float32, name="W{}".format(i)) b = tf.Variable(tf.zeros([ hidden_layers_sizes[i], ]), dtype=tf.float32, name="b{}".format(i)) #sum_W = tf.matmul(layer_input, W) + b sum_W = tf.add(tf.matmul(layer_input, W), b, name="HiddenLayer{}".format(i)) t_layer_input = tf.nn.sigmoid(sum_W) if i > 0 and hidden_layers_sizes[i - 1] > hidden_layers_sizes[i]: self.DBF = t_layer_input # 创建RBM层 self.rbm_layers.append( RBM(inpt=layer_input, n_visiable=input_size, n_hidden=hidden_layers_sizes[i], W=W, hbias=b)) if i > 0 and hidden_layers_sizes[i] > hidden_layers_sizes[i - 1]: self.DBF = self.rbm_layers[i].input layer_input = t_layer_input W = tf.Variable( tf.zeros([hidden_layers_sizes[-1], n_out], dtype=tf.float32)) b = tf.Variable(tf.zeros([ n_out, ]), dtype=tf.float32) self.output = tf.nn.softmax(tf.matmul(layer_input, W) + b) self.y_pred = tf.argmax(self.output, axis=1) self.loss = -tf.reduce_mean( tf.reduce_sum(self.y * tf.log(self.output), axis=1)) #cross_entropy correct_pred = tf.equal(self.y_pred, tf.argmax(self.y, axis=1)) self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
layers = [# ConvolutionalRBMLayer(fshape=(1, fshape, fshape, n_filters), # init=init_norm, strides={'str_d': 1, 'str_h': 2, 'str_w': 2}, # sparse_cost=0.0, sparse_damping=0.0, sparse_target=0.0), RBMLayer(n_hidden=100, init=init_norm, sparse_cost=0.0, sparse_damping=0.0, sparse_target=0.0), RBMLayerWithLabels(n_hidden=50, init=init_norm, n_classes=nclass, use_fast_weights=True)] # setup optimizer optimizer = {'momentum': [0], 'step_config': 1, 'learning_rate': 0.1, 'weight_decay': 0} # initialize model object rbm = RBM(layers=layers) if args.model_file: assert os.path.exists(args.model_file), '%s not found' % args.model_file logger.info('loading initial model state from %s' % args.model_file) rbm.load_weights(args.model_file) # setup standard fit callbacks callbacks = Callbacks(rbm, train_set, output_file=args.output_file, progress_bar=args.progress_bar) # add a callback ot calculate if args.serialize > 0: # add callback for saving checkpoint file # every args.serialize epchs
"/Users/matthewzeitlin/Desktop/CS156b-Netflix/data/train.tf.dta") # test_set = tf.data.TextLineDataset("/home/CS156b-Netflix/data/probe.dta") dataset = dset.map(_user_parse_function) train_4_probe = train_4_probe.map(_user_parse_function) full_train = full_train.map(_user_parse_function) probe = probe.map(_test_parse_function) #test_set = test_set.map(_test_parse_function) #user_index_dataset = dset.map(_user_parse_function) # a = model.predict(test_set, user_index_dataset) # print(a) rbm = RBM() rbm.train(dataset, 50, probe, train_4_probe) ########### Submission ############### exit(0) saver = tf.contrib.eager.Saver(rbm.get_variables()) saver.restore("models522/rbm") print("Predicting") test_set = tf.data.TextLineDataset( "/home/ubuntu/CS156b-Netflix/deep_autoencoder/qual_edited.dta") test_set = test_set.map(_test_parse_function) rbm.pred_for_sub(test_set, dataset, True, "rbm_probe_qual.txt") # print("Created submission for test set") # rbm.pred_for_sub(full_train, full_train, True, "rbm_train.txt")
# -*- coding: utf-8 -*- from rbm import RBM from au import AutoEncoder import tensorflow as tf import matplotlib.pyplot as plt import numpy as np import PreprocessGenerative as i inputData = i.importData() # Train 4-Layer Deep Belief Network print('DBN') rbm1 = RBM(inputData[0].shape[0], 900, ['rbmw1', 'rbvb1', 'rbmhb1'], 0.3) rbm2 = RBM(900, 500, ['rbmw2', 'rbvb2', 'rbmhb2'], 0.3) rbm3 = RBM(500, 250, ['rbmw3', 'rbvb3', 'rbmhb3'], 0.3) rbm4 = RBM(250, 2, ['rbmw4', 'rbvb4', 'rbmhb4'], 0.3) epoch = 1 # Train First RBM print('first rbm') for g in range(epoch): for it in range(len(inputData)): trX = inputData[it][np.newaxis] rbm1.partial_fit(trX) print(rbm1.compute_cost(trX)) print(rbm1.compute_cost(trX))
import os import itertools import numpy as np import fcntl import copy from string import Template import mlpython.datasets.store as dataset_store import mlpython.mlproblems.generic as mlpb from rbm import RBM #from autoencoder import Autoencoder print "Loading dataset..." trainset,validset,testset = dataset_store.get_classification_problem('ocr_letters') print "Train RBM for 10 iterations... (this might take a few minutes)" rbm = RBM(n_epochs = 10, hidden_size = 200, lr = 0.01, CDk = 1, seed=1234 ) rbm.train(mlpb.SubsetFieldsProblem(trainset)) rbm.show_filters()
ncol = data.shape[1] data_target = data[...,0] data = data[...,1:ncol] num_cases = data.shape[0] num_dims = data.shape[1] num_vis = num_dims perm = np.random.permutation(num_cases) data = data[perm] data_target = data_target[perm] data_sh = theano.shared(np.asarray(data, dtype=theano.config.floatX), borrow=True) data_target_sh = theano.shared(np.asarray(data_target, dtype=theano.config.floatX), borrow=True) rbm = RBM(num_vis = num_dims, num_hid = 500) rbm_line = RBMBinLine(num_vis = 500, num_hid = 2) # hyper parameters train_params = { 'batch_size' : 100, 'learning_rate' : 0.1, 'cd_steps' : 2, 'max_epoch' : 25, 'persistent' : False} train_rbm(rbm, data_sh, train_params, False, False) fine_tune(rbm, data_sh, epochs = 10, batch_size=100) # collect statistics pre_sigm, hid_stat = theano.function([], rbm.prop_up(data_sh))() hid_stat_sh = theano.shared(np.asarray(hid_stat, dtype=theano.config.floatX), borrow=True) # hyper parameters train_params = { 'batch_size' : 100, 'learning_rate' : 0.05, 'cd_steps' : 1, 'max_epoch' : 20, 'persistent' : False } train_rbm(rbm_line, hid_stat_sh, train_params, False, False)
def rbm_instance(): rbmobject1 = RBM(17, 40, ['rbmw1', 'rbvb1', 'rbmhb1'], 0.001) rbmobject2 = RBM(40, 4, ['rbmw2', 'rbvb2', 'rbmhb2'], 0.001) return rbmobject1, rbmobject2
def setUp(self): self.rbm = RBM(10,10)
from rbm import RBM import numpy as np rbm = RBM(num_visible=6, num_hidden=2) base = np.array([[1, 0, 1, 0, 1, 0], [1, 0, 1, 1, 0, 0], [0, 1, 0, 1, 1, 0], [0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 0, 1], [1, 0, 0, 1, 0, 1]]) filmes = ["Blade Runner", "Logan", "O chamado", "It", "A freira", "Dunkirk"] rbm.train(base, max_epochs=5000) rbm.weights usuario1 = np.array([[1, 1, 0, 1, 0, 0]]) usuario2 = np.array([[0, 0, 0, 1, 1, 0]]) rbm.run_visible(usuario1) rbm.run_visible(usuario2) camada_escondida = np.array([[1, 0]]) recomendacao = rbm.run_hidden(camada_escondida) for i in range(len(usuario1[0])): #print(usuario1[0,i]) if usuario2[0, i] == 0 and recomendacao[0, i] == 1: print(filmes[i])
def __init__(self, numpy_rng, theano_rng=None, n_ins=DIMENSION * N_FRAMES, hidden_layers_sizes=[1024, 1024, 1024], n_outs=N_OUTS): """This class is made to support a variable number of layers. :type numpy_rng: numpy.random.RandomState :param numpy_rng: numpy random number generator used to draw initial weights numpy随机数生成器,用于初始化权重 :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams :param theano_rng: Theano random generator; if None is given one is generated based on a seed drawn from `rng` theano随机数生成器 :type n_ins: int :param n_ins: dimension of the input to the DBN DBN的输入样本维数 :type n_layers_sizes: list of ints :param n_layers_sizes: intermediate layers size, must contain at least one value 每个隐层大小,至少一个数 :type n_outs: int :param n_outs: dimension of the output of the network 输出维数 """ self.sigmoid_layers = [] self.rbm_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) assert self.n_layers > 0 if not theano_rng: theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) # allocate symbolic variables for the data # 为数据开辟符号变量 self.x = T.matrix('x') # the data is presented as rasterized images 数据表示为光栅图像 self.y = T.ivector('y') # the labels are presented as 1D vector 标签表示为一维整形数组 # of [int] labels # The DBN is an MLP, for which all weights of intermediate # layers are shared with a different RBM. We will first # construct the DBN as a deep multilayer perceptron, and when # constructing each sigmoidal layer we also construct an RBM # that shares weights with that layer. During pretraining we # will train these RBMs (which will lead to chainging the # weights of the MLP as well) During finetuning we will finish # training the DBN by doing stochastic gradient descent on the # MLP. #DBN是一个MLP,每个中间层的权重被一个RBM共享,且RBM互不相同.先建立一个MLP, #在建立MLP的若干sigmoid层时建立对应的RBM层,RBM层共享sigmoid层的权重. #预训练时训练RBM,并引发MLP的权重改变 #微调时通过在MLP中做随机梯度下降完成训练 for i in xrange(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden # units of the layer below or the input size if we are on # the first layer # 每层的输入大小:在第一层,为输入数据大小 # 不在第一层,为上一层输出大小 if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] # the input to this layer is either the activation of the # hidden layer below or the input of the DBN if you are on # the first layer # 每层的输入数据:在第一层,为输入数据 # 不在第一层,为上一层输出 if i == 0: layer_input = self.x else: layer_input = self.sigmoid_layers[-1].output sigmoid_layer = HiddenLayer(rng=numpy_rng, input=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=T.nnet.sigmoid) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) # its arguably a philosophical question... but we are # going to only declare that the parameters of the # sigmoid_layers are parameters of the DBN. The visible # biases in the RBM are parameters of those RBMs, but not # of the DBN. #sigmoid层的参数是DBN的参数 #但是RBM的可见偏置是RBM的参数,不是DBM的参数 self.params.extend(sigmoid_layer.params) # Construct an RBM that shared weights with this layer # 建立共享这一层权重的RBM rbm_layer = RBM(numpy_rng=numpy_rng, theano_rng=theano_rng, input=layer_input, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer) # We now need to add a logistic layer on top of the MLP # 在MLP上方加一个logistic层 self.logLayer = LogisticRegression( input=self.sigmoid_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) # compute the cost for second phase of training, defined as the # negative log likelihood of the logistic regression (output) layer # 计算训练第二阶段的代价,定义为logistic回归的负对数似然性 self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y # 根据模型参数计算梯度 # 指向有小批量数据产生的错误数的符号变量由self.x,self.y给出 self.errors = self.logLayer.errors(self.y)
#Read in the trained RBM parameters: path_to_params = 'data_ising2d/RBM_parameters_solutions/parameters_nH%d_L%d' % ( num_hidden, L) path_to_params += '_T' + str(T) + '.npz' params = np.load(path_to_params) weights = params['weights'] visible_bias = params['visible_bias'] hidden_bias = params['hidden_bias'] hidden_bias = np.reshape(hidden_bias, (hidden_bias.shape[0], 1)) visible_bias = np.reshape(visible_bias, (visible_bias.shape[0], 1)) # Initialize RBM class rbms.append( RBM(num_hidden=num_hidden, num_visible=num_visible, weights=weights, visible_bias=visible_bias, hidden_bias=hidden_bias, num_samples=num_samples)) rbm_samples.append(rbms[i].stochastic_maximum_likelihood(gibb_updates)) #end of loop over temperatures # Initialize tensorflow init = tf.group(tf.initialize_all_variables(), tf.initialize_local_variables()) # Sample thermodynamic observables: N = num_visible with tf.Session() as sess: sess.run(init) for i in range(nbins): print('bin %d' % i)
random_state=0) best_params = {'n_components': 50, 'learning_rate': 0.02, 'batch_size': 100} n_iter = 100 n_components = best_params['n_components'] learning_rate = best_params['learning_rate'] batch_size = best_params['batch_size'] verbose = True random_state = 0 room_temp = 0.8 n_temp = 10 # Models we will use rbm_pcd = RBM(random_state=random_state, verbose=verbose, learning_rate=learning_rate, n_iter=n_iter, n_components=n_components, batch_size=batch_size) rbm_cd = RBM_CD(random_state=random_state, verbose=verbose, learning_rate=learning_rate, n_iter=n_iter, n_components=n_components, batch_size=batch_size, cd_k=1) rbm_pt = RBM_PT(random_state=random_state, verbose=verbose, learning_rate=learning_rate, n_iter=n_iter, n_components=n_components, batch_size=batch_size,
def toy_test(learning_rate=0.01, training_epochs=100, batch_size=50, output_folder=None, isPCD=0, n_hidden=3): assert output_folder is not None # toy_data, word count vector, [num_terms, num_doc]. # each cell represents the number of times a term occurs # d1 d2 d3 d4 d5 toy_data = numpy.asarray([[0, 2, 0, 1, 0], [9, 0, 3, 1, 1], [4, 1, 1, 2, 1], [10, 10, 1, 1, 0], [1, 0, 8, 0, 10], [0, 1, 10, 1, 0], [1, 0, 2, 6, 1], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [1, 0, 1, 0, 0], [1, 1, 0, 0, 1], [10, 2, 0, 1, 0], [0, 0, 1, 0, 10], [1, 0, 0, 3, 0], [0, 0, 2, 0, 1], [10, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 1, 0], [1, 0, 1, 0, 0], [1, 0, 0, 0, 1], [1, 0, 1, 0, 0], [0, 0, 1, 0, 0]]) # from rbm import RBM from rbm_variants import RBM_Orthogonal as RBM # from rbm_variants import PoissonRBM as RBM train_x = toSharedX(toy_data, name="toy_data") n_vis = train_x.get_value(borrow=True).shape[1] n_samples = train_x.get_value(borrow=True).shape[0] if batch_size >= n_samples: batch_size = n_samples n_train_batches = n_samples / batch_size # construct the RBM class rbm = RBM(n_visible=n_vis, n_hidden=n_hidden, isPCD=isPCD) train_fn = rbm.get_train_fn(train_x, batch_size) print "... projecting" print rbm.project(train_x, hidSample=1) ################################# # Training the RBM # ################################# if not os.path.isdir(output_folder): os.makedirs(output_folder) os.chdir(output_folder) plotting_time = 0. start_time = time.clock() import PIL.Image from visualizer import tile_raster_images # go through training epochs for epoch in xrange(training_epochs): # go through the training set mean_cost = [] for batch_index in xrange(n_train_batches): # for each batch, we extract the gibbs chain new_cost = train_fn(index=batch_index, lr=learning_rate) mean_cost += [new_cost] print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost) if numpy.mean(mean_cost) >= 0: break # W shape is [784 500] # Plot filters after each training epoch plotting_start = time.clock() # Construct image from the weight matrix image = PIL.Image.fromarray(tile_raster_images( X=rbm.W.get_value(borrow=True).T, # weight is [n_vis, n_hidden] # so, among 'n_hidden' rows, # each row corresponds to propdown one hidden unit img_shape=(1, n_vis), tile_shape=(n_hidden, 1), tile_spacing=(1, 1))) image.save('filters_at_epoch_%i.png' % epoch) plotting_stop = time.clock() plotting_time += (plotting_stop - plotting_start) end_time = time.clock() pretraining_time = (end_time - start_time) - plotting_time print ('Training took %f minutes' % (pretraining_time / 60.)) print "... projecting" print rbm.project(train_x, hidSample=1) print "... reconstructing" print rbm.reconstruct(train_x, showSample=1) * train_x.get_value(borrow=True)
tf.flags.DEFINE_string("result_path", "result.txt","") tf.flags.DEFINE_string("sep", "\t", "") FLAGS = tf.flags.FLAGS ''' comments: train dataset/test dateset : "ml-100k/u1.base"/"ml-100k/u1.test"; columns: user_id, movie_id, ratings, timestamp profiles -- type: dict -- key: user_id, value: (movie_id, rating) :store ratings of movies from every user ''' if __name__ == "__main__": all_users, all_movies, tests = load_dataset(FLAGS.train_path, FLAGS.test_path, FLAGS.sep, user_based=True) rbm = RBM(len(all_movies) * 5, FLAGS.num_hidden) print("model created") init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) profiles = defaultdict(list) with open(FLAGS.train_path, 'rt') as data: for i, line in enumerate(data): uid, mid, rat, timstamp = line.strip().split(FLAGS.sep) profiles[uid].append((mid, float(rat))) print("Users and ratings loaded") for e in range(FLAGS.epochs): for batch_i, batch in enumerate(chunker(list(profiles.keys()), FLAGS.batch_size)): size = min(len(batch), FLAGS.batch_size)
_a[0]: self._X, y: self._Y })))) # 读取数据 mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels # 创建3个RBM模型 RBM_hidden_sizes = [500, 200, 50] inpX = trX # 保存模型 rbm_list = [] # 输入数量 input_size = inpX.shape[1] # 对RBM模型开始训练 print('Pre_train begins!') for i, size in enumerate(RBM_hidden_sizes): print('RBM: ', i, ' ', input_size, '->', size) rbm_list.append(RBM(input_size, size)) input_size = size for rbm in rbm_list: print('New RBM:') rbm.train(inpX) inpX = rbm.rbm_outpt(inpX) print('Train begins!') nNet = NN(RBM_hidden_sizes, trX, trY) nNet.load_from_rbms(RBM_hidden_sizes, rbm_list) nNet.train()
T) + '_train.txt' xtrain = np.loadtxt(trainFileName) testFileName = 'Data_ising2d/MC_results/ising2d_L' + str(L) + '_T' + str( T) + '_test.txt' xtest = np.loadtxt(testFileName) xtrain_randomized = np.random.permutation( xtrain) # random permutation of training data xtest_randomized = np.random.permutation( xtest) # random permutation of test data iterations_per_epoch = xtrain.shape[0] / bsize # Initialize the RBM class rbm = RBM(num_hidden=num_hidden, num_visible=num_visible, weights=weights, visible_bias=visible_bias, hidden_bias=hidden_bias, num_samples=num_samples) # Initialize operations and placeholders classes ops = Ops() placeholders = Placeholders() placeholders.visible_samples = tf.placeholder( tf.float32, shape=(None, num_visible), name='v') # placeholder for training data total_iterations = 0 # starts at zero ops.global_step = tf.Variable(total_iterations, name='global_step_count', trainable=False) learning_rate = tf.train.exponential_decay(
def __init__(self, numpy_rng, theano_rng=None, n_ins=784, hidden_layers_sizes=[500, 500], n_outs=10): """This class is made to support a variable number of layers. :type numpy_rng: numpy.random.RandomState :param numpy_rng: numpy random number generator used to draw initial weights :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams :param theano_rng: Theano random generator; if None is given one is generated based on a seed drawn from `rng` :type n_ins: int :param n_ins: dimension of the input to the DBN :type hidden_layers_sizes: list of ints :param hidden_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int :param n_outs: dimension of the output of the network """ self.sigmoid_layers = [] self.rbm_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) assert self.n_layers > 0 if not theano_rng: theano_rng = MRG_RandomStreams(numpy_rng.randint(2**30)) # allocate symbolic variables for the data self.x = T.matrix('x') # the data is presented as rasterized images self.y = T.ivector('y') # the labels are presented as 1D vector # of [int] labels # end-snippet-1 # The DBN is an MLP, for which all weights of intermediate # layers are shared with a different RBM. We will first # construct the DBN as a deep multilayer perceptron, and when # constructing each sigmoidal layer we also construct an RBM # that shares weights with that layer. During pretraining we # will train these RBMs (which will lead to chainging the # weights of the MLP as well) During finetuning we will finish # training the DBN by doing stochastic gradient descent on the # MLP. for i in xrange(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden # units of the layer below or the input size if we are on # the first layer if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] # the input to this layer is either the activation of the # hidden layer below or the input of the DBN if you are on # the first layer if i == 0: layer_input = self.x else: layer_input = self.sigmoid_layers[-1].output sigmoid_layer = HiddenLayer(rng=numpy_rng, input=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=T.nnet.sigmoid) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) # its arguably a philosophical question... but we are # going to only declare that the parameters of the # sigmoid_layers are parameters of the DBN. The visible # biases in the RBM are parameters of those RBMs, but not # of the DBN. self.params.extend(sigmoid_layer.params) # Construct an RBM that shared weights with this layer rbm_layer = RBM(numpy_rng=numpy_rng, theano_rng=theano_rng, input=layer_input, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer) # We now need to add a logistic layer on top of the MLP self.logLayer = LogisticRegression( input=self.sigmoid_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) # compute the cost for second phase of training, defined as the # negative log likelihood of the logistic regression (output) layer self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors = self.logLayer.errors(self.y)
# saveImage(output_list, node_shape[2], 'cnn2_after_train') # rbm_size_list = (680, 340, 170, 85, 42, 21, 10, 3) # rbm_size_list = (468, 234, 117, 58, 29, 14, 7, 7) rbm_size_list = (468, 234, 117, 58, 30, 14, 7, 7) result_path = 'data/kouryu_room/cnn2_after_training' result_data = load_result_image(result_path, file_num, isRGB) # result_path = 'data/4position_rumba/image7000/rbm1_train3434' # result_W = loadW(result_path) makeFolder() # def __init__(self, W, input, data_size,input_size, output_size, isDropout): rbm1 = RBM(None, result_data, file_num, rbm_size_list[0], rbm_size_list[1]) for i in xrange(pre_train_epoch): print 'rbm1 pre_train:' + str(i) rbm1.contrast_divergence(i) reinput = rbm1.reconstruct_from_input(rbm1.input) saveImage(reinput, node_shape[2], 'rbm1_after_train') saveW(rbm1.getW(), 'rbm1_after_train') rbm2 = RBM(None, rbm1.output(), file_num, rbm_size_list[1], rbm_size_list[2]) for i in xrange(pre_train_epoch): print 'rbm2 pre_train:' + str(i) rbm2.contrast_divergence(i) reinput = rbm2.reconstruct_from_input(rbm2.input) reinput = rbm1.reconstruct_from_output(reinput) saveImage(reinput, node_shape[2], 'rbm2_after_train') saveW(rbm2.getW(), 'rbm2_after_train')