def recons(): data = Mnist() data.MODE = "test" model = RBM() fig, ax = plt.subplots(nrows=2, ncols=10) c = 0 for i in range(0, 100, 10): x = data[i + 2][0].t()[0] * 255 x = x.view(784, 1) mask = (torch.rand(x.shape).cuda() > 0.2).float() x = x * mask x_new = x.clone() ax[0][c].imshow(x.view(28, 28).detach().cpu().numpy()) for i in range(6): x_new = model.reconstruction(x_new) x = x_new.view(28, 28).detach().cpu().numpy() ax[1][c].imshow(x) c += 1 plt.show()
def assess_rbm(clf, noisy_data, noisy_label, noisy_levels, tr_x,postfix=''): f_score = open('report{}.txt'.format(postfix), 'a') f_metric = open('metric{}.txt'.format(postfix), 'a') # Initialise architecture config = get_rbm_config(25, n_hidden=500, epochs=2) model = RBM(config) pred_table = {} for l in xrange(len(noisy_levels)): pred_table[l] = [] for i in xrange(50): # Train architecture model.train(tr_x) j = 0 for xs, ys in zip(noisy_data, noisy_label): recon_xs = model.reconstruct(xs, img_name='test_rbm') pred, metric = clf.get_score(recon_xs, ys, True) print pred print metric f_metric.write('{}25_25, Epoch:{}\n'.format(noisy_levels[j], i)) f_metric.write(metric) pred_table[j].append(pred) j += 1 for k in pred_table: f_score.write('{}:{}\n'.format(noisy_levels[k], pred_table[k])) f_score.close() f_metric.close()
def assess_rbm(clf, noisy_data, noisy_label, noisy_levels, tr_x, postfix=''): f_score = open('report{}.txt'.format(postfix), 'a') f_metric = open('metric{}.txt'.format(postfix), 'a') # Initialise architecture config = get_rbm_config(25, n_hidden=500, epochs=2) model = RBM(config) pred_table = {} for l in xrange(len(noisy_levels)): pred_table[l] = [] for i in xrange(50): # Train architecture model.train(tr_x) j = 0 for xs, ys in zip(noisy_data, noisy_label): recon_xs = model.reconstruct(xs, img_name='test_rbm') pred, metric = clf.get_score(recon_xs, ys, True) print pred print metric f_metric.write('{}25_25, Epoch:{}\n'.format(noisy_levels[j], i)) f_metric.write(metric) pred_table[j].append(pred) j += 1 for k in pred_table: f_score.write('{}:{}\n'.format(noisy_levels[k], pred_table[k])) f_score.close() f_metric.close()
def modeler(): data = Mnist() model = RBM() ks = [3, 1, 4, 2, 6, 4, 2, 7, 2, 6, 8, 11, 15, 2, 13, 1, 2, 6, 7, 4, 6] for k in ks: model.k = k model.CDk(data)
def recon(): x = torch.load("weights/rbm/test.pt") model = RBM() fig, ax = plt.subplots(nrows=1, ncols=2) x_new = x.clone() ax[0].imshow(x.view(28, 28).numpy()) x_new = model.reconstruction(x_new.cuda()) ax[1].imshow(x_new.view(28, 28).detach().cpu().numpy()) plt.show()
def test_rbm(): print "Testing RBM" data_manager = store.StorageManager('TestRBM') # Load Cohn Kanade dataset datasets = loader.load_kanade(pre={'scale': True}, n=100, set_name='sharp_equi25_25') train_set_x, train_set_y = datasets[0] test_set_x, test_set_y = datasets[2] # Initilise the RBM and training parameters tr = TrainParam(learning_rate=0.0001, momentum_type=NESTEROV, momentum=0.5, weight_decay=0.0001, sparsity_constraint=True, sparsity_target=0.01, sparsity_cost=0.1, sparsity_decay=0.9, dropout=True, dropout_rate=0.5, batch_size=10, epochs=10) n_visible = train_set_x.get_value(borrow=True).shape[1] n_hidden = 10 config = RBMConfig() config.v_n = n_visible config.h_n = n_hidden config.v_unit = rbm_units.GaussianVisibleUnit # config.h_unit = rbm_units.ReLUnit config.progress_logger = ProgressLogger(img_shape=(25, 25)) config.train_params = tr rbm = RBM(config) print "... initialised RBM" load = store.retrieve_object(str(rbm)) if load: rbm = load for i in xrange(0, 1): # Train RBM rbm.train(train_set_x) data_manager.persist(rbm) # Test RBM Reconstruction via Linear Classifier clf = SimpleClassifier(classifier='logistic', train_x=train_set_x, train_y=train_set_y) recon_te = rbm.reconstruct(test_set_x, k=1, plot_n=100, plot_every=1,img_name='recon_te_{}.png'.format(i)) print 'Original Score: {}'.format(clf.get_score(test_set_x, test_set_y)) print 'Recon Score: {}'.format(clf.get_score(recon_te, test_set_y.eval()))
def untie_weights(self, include_top=False): # Untie all the weights except for the top layer self.untied = True layers = self.rbm_layers if include_top else self.rbm_layers[:-1] for i, rbm in enumerate(layers): W = rbm.W.get_value(borrow=False) h_bias = rbm.h_bias.get_value(borrow=False) v_bias = rbm.v_bias.get_value(borrow=False) W1 = theano.shared(W, name=('inference_W_%d' % i)) v1 = theano.shared(v_bias, name=('inference_vbias_%d' % i)) h1 = theano.shared(h_bias, name=('inference_hbias_%d' % i)) W2 = theano.shared(copy.deepcopy(W), name=('generative_W_%d' % i)) v2 = theano.shared(copy.deepcopy(v_bias), name=('generative_vbias_%d' % i)) h2 = theano.shared(copy.deepcopy(h_bias), name=('generative_hbias_%d' % i)) self.inference_layers.append(RBM(config=rbm.config, W=W1, h_bias=h1, v_bias=v1)) self.generative_layers.append(RBM(config=rbm.config, W=W2, h_bias=h2, v_bias=v2))
def get_brain_model_RBM(shape): # Initialise RBM tr = TrainParam(learning_rate=0.0001, momentum_type=NESTEROV, momentum=0.9, weight_decay=0.0001, sparsity_constraint=True, sparsity_decay=0.9, sparsity_cost=100, sparsity_target=0.01, batch_size=10, epochs=10) n_visible = shape * shape * 2 n_hidden = 500 config = RBMConfig() config.v_n = n_visible config.h_n = n_hidden config.v_unit = rbm_units.GaussianVisibleUnit # config.h_unit = rbm_units.ReLUnit config.progress_logger = ProgressLogger(img_shape=(shape * 2, shape)) config.train_params = tr brain_c = RBM(config) print "... initialised RBM" return brain_c
def setUpAssociativeRBM(self, v=5, v2=5, h=10): tr = TrainParam(learning_rate=0.01, momentum_type=CLASSICAL, momentum=0.5, weight_decay=0.01, sparsity_constraint=False, batch_size=1, epochs=5) config = RBMConfig() config.associative = True config.v_n = v config.v2_n = v2 config.h_n = h config.train_params = tr config.progress_logger = AssociationProgressLogger() self.rbm = RBM(config) self.x = np.array([[1, 1, 1, 0, 0]], dtype=t_float_x) self.y = np.array([[0, 0, 0, 0, 1]], dtype=t_float_x) self.x2 = np.array([[1, 1, 1, 0, 0], [0, 0, 0, 0, 1]], dtype=t_float_x) self.y2 = np.array([[0, 0, 1, 1, 1], [0, 0, 0, 0, 1]], dtype=t_float_x) self.yl = np.array([[0, 0], [0, 1]], dtype=t_float_x) self.tx = theano.shared(np.array([[1, 1, 1, 0, 0]], dtype=t_float_x)) self.ty = theano.shared(np.array([[0, 0, 0, 0, 1]], dtype=t_float_x)) self.tz = theano.shared( np.array([[0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1]], dtype=t_float_x)) self.tl = theano.shared( np.array([[0, 1], [0, 1], [0, 1], [0, 1], [1, 0]], dtype=t_float_x))
def classify(): data = Mnist() rbm = RBM() model = RBMClassifier(rbm).to(device) loss_fn = torch.nn.CrossEntropyLoss(reduction='mean') optimizer = Adam(model.classifier.parameters(), lr=0.0001, weight_decay=0.00001) for e in range(100): tots = 0 for i, (x, y) in enumerate(data): if torch.Size([784, 0]) == x.shape: break out = model(x) optimizer.zero_grad() loss = loss_fn(out, y.long()) tots += loss.item() loss.backward() optimizer.step() if i % 50 == 0: print(e, i, tots / (i + 1)) torch.save(model.state_dict(), "weights/rbm/rbmclass.pth")
def train_kanade(): print "Testing RBM" data_manager = store.StorageManager('Kanade/SimpleRBMTest') # Load mnist hand digits datasets = loader.load_kanade(n=500, set_name='25_25', emotions=['happy', 'sadness'], pre={'scale2unit': True}) train_x, train_y = datasets[0] sparsity_constraint = True # Initialise the RBM and training parameters tr = rbm_config.TrainParam(learning_rate=0.0001, momentum_type=rbm_config.NESTEROV, momentum=0.9, weight_decay=0.001, sparsity_constraint=sparsity_constraint, sparsity_target=0.01, sparsity_cost=1, sparsity_decay=0.9, dropout=True, epochs=100) n_visible = train_x.get_value().shape[1] n_hidden = 500 config = rbm_config.RBMConfig(v_n=n_visible, v2_n=n_visible, h_n=n_hidden, v_unit=rbm_units.GaussianVisibleUnit, associative=False, cd_type=rbm_config.CLASSICAL, cd_steps=1, train_params=tr, progress_logger=rbm_logger.ProgressLogger(img_shape=(25, 25))) rbm = RBM(config) print "... initialised RBM" # Train RBM rbm.train(train_x) # Test RBM rbm.reconstruct(train_x, k=5, plot_n=10, plot_every=1) # Store Parameters data_manager.persist(rbm)
def __init__(self, visible_units=256, hidden_units=[64, 100], k=2, learning_rate=1e-5, learning_rate_decay=False, xavier_init=False, increase_to_cd_k=False, use_gpu=False): super(DBN, self).__init__() self.n_layers = len(hidden_units) self.rbm_layers = [] self.rbm_nodes = [] # Creating different RBM layers for i in range(self.n_layers): input_size = 0 if i == 0: input_size = visible_units else: input_size = hidden_units[i - 1] rbm = RBM(visible_units=input_size, hidden_units=hidden_units[i], k=k, learning_rate=learning_rate, learning_rate_decay=learning_rate_decay, xavier_init=xavier_init, increase_to_cd_k=increase_to_cd_k, use_gpu=use_gpu) self.rbm_layers.append(rbm) # rbm_layers = [RBM(rbn_nodes[i-1] , rbm_nodes[i],use_gpu=use_cuda) for i in range(1,len(rbm_nodes))] self.W_rec = [ nn.Parameter(self.rbm_layers[i].W.data.clone()) for i in range(self.n_layers - 1) ] self.W_gen = [ nn.Parameter(self.rbm_layers[i].W.data) for i in range(self.n_layers - 1) ] self.bias_rec = [ nn.Parameter(self.rbm_layers[i].h_bias.data.clone()) for i in range(self.n_layers - 1) ] self.bias_gen = [ nn.Parameter(self.rbm_layers[i].v_bias.data) for i in range(self.n_layers - 1) ] self.W_mem = nn.Parameter(self.rbm_layers[-1].W.data) self.v_bias_mem = nn.Parameter(self.rbm_layers[-1].v_bias.data) self.h_bias_mem = nn.Parameter(self.rbm_layers[-1].h_bias.data) for i in range(self.n_layers - 1): self.register_parameter('W_rec%i' % i, self.W_rec[i]) self.register_parameter('W_gen%i' % i, self.W_gen[i]) self.register_parameter('bias_rec%i' % i, self.bias_rec[i]) self.register_parameter('bias_gen%i' % i, self.bias_gen[i])
def associate_data2label(cache=False): cache=True print "Testing ClassRBM with generative target (i.e. AssociativeRBM with picture-label association)" # Load mnist hand digits, class label is already set to binary train, valid, test = m_loader.load_digits(n=[50000, 100, 30], pre={'label_vector': True}) train_x, train_y = train test_x, test_y = test train_y = T.cast(train_y, dtype=theano.config.floatX) test_y = T.cast(test_y, dtype=theano.config.floatX) # Initialise the RBM and training parameters tr = TrainParam(learning_rate=0.001, momentum_type=CLASSICAL, momentum=0.5, weight_decay=0.0001, sparsity_constraint=False, sparsity_target=0.1, sparsity_cost=0.01, sparsity_decay=0.9, dropout=True, dropout_rate=0.8, epochs=20) n_visible = train_x.get_value().shape[1] n_visible2 = 10 n_hidden = 500 config = RBMConfig(v_n=n_visible, v2_n=n_visible2, h_n=n_hidden, associative=True, cd_type=CLASSICAL, cd_steps=1, train_params=tr, progress_logger=ProgressLogger()) rbm = RBM(config) store.move_to('label_test/' + str(rbm)) loaded = store.retrieve_object(str(rbm)) if loaded and cache: rbm = loaded print "... loaded precomputed rbm" else: rbm.train(train_x, train_y) rbm.save() # rbm.associative = False # rbm.reconstruct(test_x, 10) # rbm.associative = True y = map(np.argmax, test_y.eval()) pred = rbm.classify(test_x) print y print pred score = np.sum(y == pred) * 1. / len(y) print "Classification Rate: {}".format(score)
def get_instance(model, n_hidden=64): settings = { "n_visible": 784, "n_hidden": n_hidden, "learning_rate": 0.01, "momentum": 0.95, "use_tqdm": True } if model == "rbm": return RBM(**settings) if model == "frbm": return FRBM(**settings) return None
def test(): data = Mnist() data.MODE = "test" rbm = RBM() model = RBMClassifier(rbm).to(device) model.load_state_dict(torch.load("weights/rbm/rbmclass.pth")) sm = torch.nn.Softmax() x, y = data[0] out = model(x) val, ind = out.max(1)
def __init__(self, config=DBNConfig()): numpy_rng = config.numpy_rng theano_rng = config.theano_rng topology = config.topology load_layer = config.load_layer n_outs = config.n_outs data_manager = config.data_manager out_dir = config.out_dir tr = config.training_parameters rbm_configs = config.rbm_configs n_ins = topology[1] hidden_layers_sizes = topology[1:] self.sigmoid_layers = [] self.rbm_layers = [] self.untied = False self.inference_layers = [] self.generative_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) self.topology = topology self.out_dir = out_dir self.data_manager = data_manager assert self.n_layers > 0 if not numpy_rng: numpy_rng = np.random.RandomState(123) else: numpy_rng = np.random.RandomState(123) if not theano_rng: theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) if type(tr) is not list: tr = [tr for i in xrange(self.n_layers)] if type(rbm_configs) is not list: rbm_configs = [rbm_configs for i in xrange(self.n_layers)] for i in xrange(self.n_layers): rbm_config = rbm_configs[i] rbm_config.v_n = topology[i] rbm_config.h_n = topology[i + 1] # rbm_config.training_parameters = tr[i] # Ensure it has parameters rbm_layer = RBM(rbm_config) self.rbm_layers.append(rbm_layer)
def train_kanade(): print "Testing RBM" data_manager = store.StorageManager('Kanade/SimpleRBMTest') # Load mnist hand digits datasets = loader.load_kanade(n=500, set_name='25_25', emotions=['happy', 'sadness'], pre={'scale2unit': True}) train_x, train_y = datasets[0] sparsity_constraint = True # Initialise the RBM and training parameters tr = rbm_config.TrainParam(learning_rate=0.0001, momentum_type=rbm_config.NESTEROV, momentum=0.9, weight_decay=0.001, sparsity_constraint=sparsity_constraint, sparsity_target=0.01, sparsity_cost=1, sparsity_decay=0.9, dropout=True, epochs=100) n_visible = train_x.get_value().shape[1] n_hidden = 500 config = rbm_config.RBMConfig( v_n=n_visible, v2_n=n_visible, h_n=n_hidden, v_unit=rbm_units.GaussianVisibleUnit, associative=False, cd_type=rbm_config.CLASSICAL, cd_steps=1, train_params=tr, progress_logger=rbm_logger.ProgressLogger(img_shape=(25, 25))) rbm = RBM(config) print "... initialised RBM" # Train RBM rbm.train(train_x) # Test RBM rbm.reconstruct(train_x, k=5, plot_n=10, plot_every=1) # Store Parameters data_manager.persist(rbm)
def setUpSimpleRBM(self): v = 2 v2 = 2 h = 10 tr = TrainParam(learning_rate=0.01, momentum_type=CLASSICAL, momentum=0.5, weight_decay=0.01, sparsity_constraint=False, batch_size=1, epochs=5) config = RBMConfig() config.associative = True config.v_n = v config.v2_n = v2 config.h_n = h config.train_params = tr config.progress_logger = ProgressLogger() self.rbm = RBM(config)
def setUpRBM(self): v = 5 v2 = 5 h = 10 tr = TrainParam(learning_rate=0.01, momentum_type=CLASSICAL, momentum=0.5, weight_decay=0.01, sparsity_constraint=False, batch_size=1, epochs=5) config = RBMConfig() config.associative = True config.v_n = v config.v2_n = v2 config.h_n = h config.train_params = tr config.progress_logger = ProgressLogger() self.rbm = RBM(config) self.rbmx1 = np.array([[2, 5, 5, 2, 1]], dtype=t_float_x) self.rbmx2 = np.array([[5, 2, 3, 8, 1]], dtype=t_float_x)
def setUpRBM(self): v = 5 v2 = 5 h = 10 tr = TrainParam(learning_rate=0.01, momentum_type=CLASSICAL, momentum=0.5, weight_decay=0.01, sparsity_constraint=False, batch_size=1, epochs=15) config = RBMConfig() config.v_n = v config.v2_n = v2 config.h_n = h config.train_params = tr config.progress_logger = ProgressLogger() self.rbm = RBM(config) self.x = np.array([[1, 1, 1, 0, 0]], dtype=t_float_x) self.tx = theano.shared(np.array([[1, 1, 1, 0, 0]], dtype=t_float_x)) self.x2 = np.array([[1, 1, 1, 0, 0], [0, 0, 0, 0, 1]], dtype=t_float_x) self.tx2 = theano.shared( np.array([ [1, 1, 1, 0, 0], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1], ], dtype=t_float_x))
biases_h, biases_v, weights, likelihoods = rbm1.train(); generate_animation(rbm1, 'rbm1', weights, biases_v, biases_h) generate_noise_animation(rbm1, 'rbm1', weights, biases_v, biases_h) generate_sample_animation(rbm1, 'rbm1', samples, weights, biases_v, biases_h) images, labels = import_mnist(); images = (images - np.min(images, 0)) / (np.max(images, 0) + 0.0001); rbm2 = RBM(images, num_iter=5, num_h=400); biases_h, biases_v, weights, likelihoods = rbm2.train(); generate_animation(rbm2, 'rbm2', weights, biases_v, biases_h) generate_noise_animation(rbm2, 'rbm2', weights, biases_v, biases_h) generate_sample_animation(rbm2, 'rbm2', samples, weights, biases_v, biases_h)""" rbm3 = RBM(images, num_iter=40, num_h=400) biases_h, biases_v, weights, likelihoods = rbm3.train() test_logistic_regression(weights, biases_v, biases_h) test_generator(rbm3, weights, biases_v, biases_h) generate_likelihood_plot(likelihoods) generate_filter_plots(weights) generate_sample_plot(rbm3, samples, weights, biases_v, biases_h) generate_noise_plot(rbm3, weights, biases_v, biases_h) # generate_animation(rbm3, 'rbm3', weights, biases_v, biases_h) # generate_noise_animation(rbm3, 'rbm3', weights, biases_v, biases_h) # generate_sample_animation(rbm3, 'rbm3', samples, weights, biases_v, biases_h) # test_logistic_regression(weights, biases_v, biases_h)
EPOCHS = 1 SIZE_HIDDEN = 500 SIZE_VISIBLE = 784 # load binary mnist sample dataset dataset = pandas.read_csv(MNIST_TRAIN_BINARY, delimiter=',', dtype=int, header=None) # leave the first column out since it contains the labels dataset = dataset.values[:, 1:] # compute batch set idx = prepare_batches(len(dataset), SIZE_BATCH) # load distribution bernoulli = Bernoulli(SIZE_VISIBLE, SIZE_HIDDEN) gibbs = BlockGibbsSampler(bernoulli, sampling_steps=1) sgd = SGD(bernoulli) rbm = RBM(bernoulli, gibbs, sgd) y_axis = list() # pyplot.plot(y_axis) # pyplot.ylabel('update') # pyplot.xlabel('Batch') # pyplot.ion() # pyplot.show() for epoch in range(EPOCHS): for b_idx in idx: batch = dataset[b_idx[0]:b_idx[1], :] d_weight_update, _, _ = rbm.train_batch(batch) rec_error = rbm.compute_reconstruction_error(batch)
def __init__(self, np_rng, theano_rng=None, n_ins=784, hidden_layers_sizes=[500, 500], n_outs=10): """This class is made to support a variable number of layers. :type np_rng: np.random.RandomState :param np_rng: np random number generator used to draw initial weights :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams :param theano_rng: Theano random generator; if None is given one is generated based on a seed drawn from `rng` :type n_ins: int :param n_ins: dimension of the inputs to the DBN :type hidden_layers_sizes: list of ints :param hidden_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int :param n_outs: dimension of the output of the network """ self.sigmoid_layers = [] self.rbm_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) assert self.n_layers > 0 if not theano_rng: theano_rng = MRG_RandomStreams(np_rng.randint(2**30)) # allocate symbolic variables for the data # the data is presented as rasterized images self.x = self.inputs = T.matrix('x') # the labels are presented as 1D vector of [int] labels self.y = T.ivector('y') # end-snippet-1 # The DBN is an MLP, for which all weights of intermediate # layers are shared with a different RBM. We will first # construct the DBN as a deep multilayer perceptron, and when # constructing each sigmoidal layer we also construct an RBM # that shares weights with that layer. During pretraining we # will train these RBMs (which will lead to chainging the # weights of the MLP as well) During finetuning we will finish # training the DBN by doing stochastic gradient descent on the # MLP. for i in range(self.n_layers): # construct the sigmoidal layer # the size of the inputs is either the number of hidden # units of the layer below or the inputs size if we are on # the first layer if i == 0: inputs_size = n_ins else: inputs_size = hidden_layers_sizes[i - 1] # the inputs to this layer is either the activation of the # hidden layer below or the inputs of the DBN if you are on # the first layer if i == 0: layer_inputs = self.x else: layer_inputs = self.sigmoid_layers[-1].output sigmoid_layer = Hidden_Layer(rng=np_rng, inputs=layer_inputs, n_in=inputs_size, n_out=hidden_layers_sizes[i], activation=activation) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) # its arguably a philosophical question... but we are # going to only declare that the parameters of the # sigmoid_layers are parameters of the DBN. The visible # biases in the RBM are parameters of those RBMs, but not # of the DBN. self.params.extend(sigmoid_layer.params) # Construct an RBM that shared weights with this layer rbm_layer = RBM(np_rng=np_rng, theano_rng=theano_rng, inputs=layer_inputs, n_visible=inputs_size, n_hidden=hidden_layers_sizes[i], w=sigmoid_layer.w, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer) # We now need to add a logistic layer on top of the MLP self.logitLayer = Logistic_Regression( inputs=self.sigmoid_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.logitLayer.params) # compute the cost for second phase of training, defined as the # negative log likelihood of the logistic regression (output) layer self.finetune_cost = self.logitLayer.negativeLogLikelihood(self.y) # compute the gradients with respect to the model parameters # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors = self.logitLayer.errors(self.y) # get prediction from logistic sgd self.y_pred = self.logitLayer.y_pred self.p_y_given_x = self.logitLayer.p_y_given_x pass
def KanadeAssociativeRBM(cache=False, train_further=False): print "Testing Associative RBM which tries to learn the ID map " # print "Testing Associative RBM which tries to learn the following mapping: {anger, saddness, disgust} -> {sadness}, {contempt, happy, surprise} -> {happy}" # project set-up data_manager = store.StorageManager('Kanade/OptMFSparse0.01RBMTest', log=True) # data_manager = store.StorageManager('Kanade/OptAssociativeRBMTest', log=True) shape = 25 dataset_name = 'sharp_equi{}_{}'.format(shape, shape) # Load kanade database mapping = None # id map # mapping = {'anger': 'sadness', 'contempt': 'happy', 'disgust': 'sadness', 'fear': 'sadness', 'happy': 'happy', # 'sadness': 'sadness', 'surprise': 'happy'} train, valid, test = loader.load_kanade(pre={'scale': True}, set_name=dataset_name) train_x, train_y = train test_x, test_y = test # Sample associated image train_x_mapped, train_y_mapped = loader.sample_image(train_y, mapping=mapping, pre={'scale': True}, set_name=dataset_name) test_x_mapped, test_y_mapped = loader.sample_image(test_y, mapping=mapping, pre={'scale': True}, set_name=dataset_name) # Concatenate images concat1 = T.concatenate([train_x, train_x_mapped], axis=1) # concat2 = T.concatenate([train_x_mapped, train_x], axis=1) # concat = T.concatenate([concat1, concat2], axis=0) # train_tX = theano.function([], concat)() train_tX = theano.function([], concat1)() train_X = theano.shared(train_tX) # Train classifier to be used for classifying reconstruction associated image layer # mapped_data = loader.load_kanade(#emotions=['sadness', 'happy'], # pre={'scale': True}, # set_name=dataset_name) # Target Image # clf_orig = SimpleClassifier('logistic', mapped_data[0][0], mapped_data[0][1]) clf_orig = SimpleClassifier('logistic', train_x, train_y) # Initialise RBM tr = rbm_config.TrainParam(learning_rate=0.0001, momentum_type=rbm_config.NESTEROV, momentum=0.9, weight_decay=0.0001, sparsity_constraint=True, sparsity_target=0.01, sparsity_cost=100, sparsity_decay=0.9, batch_size=10, epochs=10) n_visible = shape * shape * 2 n_hidden = 500 config = rbm_config.RBMConfig() config.v_n = n_visible config.h_n = n_hidden config.v_unit = rbm_units.GaussianVisibleUnit # config.h_unit = rbm_units.ReLUnit config.progress_logger = rbm_logger.ProgressLogger(img_shape=(shape * 2, shape)) config.train_params = tr rbm = RBM(config) print "... initialised RBM" # Load RBM (test) loaded = data_manager.retrieve(str(rbm)) if loaded: rbm = loaded else: rbm.set_initial_hidden_bias() rbm.set_hidden_mean_activity(train_X) # Train RBM - learn joint distribution # rbm.pretrain_lr(train_x, train_x01) for i in xrange(0, 10): if not cache or train_further: rbm.train(train_X) data_manager.persist(rbm) print "... reconstruction of associated images" # Get reconstruction with train data to get 'mapped' images to train classifiers on reconstruction = rbm.reconstruct(train_X, 1, plot_n=100, plot_every=1, img_name='recon_train') reconstruct_assoc_part = reconstruction[:, (shape ** 2):] # Get associated images of test data nsamples = np.random.normal(0, 1, test_x.get_value(True).shape).astype(np.float32) initial_y = theano.shared(nsamples, name='initial_y') utils.save_images(nsamples[0:100], 'initialisation.png', (10, 10), (25, 25)) test_x_associated = rbm.reconstruct_association_opt(test_x, initial_y, 5, 0., plot_n=100, plot_every=1, img_name='recon_test_gibbs') mf_recon = rbm.mean_field_inference_opt(test_x, y=initial_y, sample=False, k=10, img_name='recon_test_mf_raw') # Concatenate images test_MFX = theano.function([], T.concatenate([test_x, mf_recon], axis=1))() test_MF = theano.shared(test_MFX) reconstruction = rbm.reconstruct(test_MF, 1, plot_n=100, plot_every=1, img_name='recon_test_mf_recon') mf_recon = reconstruction[:, (shape ** 2):] print "... reconstructed" # Classify the reconstructions # 1. Train classifier on original images score_orig = clf_orig.get_score(test_x_associated, test_y_mapped.eval()) score_orig_mf = clf_orig.get_score(test_x_associated, test_y_mapped.eval()) # 2. Train classifier on reconstructed images clf_recon = SimpleClassifier('logistic', reconstruct_assoc_part, train_y_mapped.eval()) score_retrain = clf_recon.get_score(test_x_associated, test_y_mapped.eval()) score_retrain_mf = clf_recon.get_score(mf_recon, test_y_mapped.eval()) out_msg = '{} (orig, retrain):{},{}'.format(rbm, score_orig, score_retrain) out_msg2 = '{} (orig, retrain):{},{}'.format(rbm, score_orig_mf, score_retrain_mf) print out_msg print out_msg2
new_training_params = {'n_epoch': 20, 'lrate': 0.1, 'w_penalty': 0.01, 'verbose': True} # 'display_every': 1., # 'visualize': True, # 'vis_fun': vis_fun} # UPDATE TRAINING PARAMETERS train_params.update(new_training_params) # print '' # print '-'*5+'Training parameters'+'-'*5 # train_params.show() # INITIALIZE RBM MODEL AND TRAINER model = RBM(model_params) trainer = RBMTrainer(model, train_params) # print '' # print '-'*5+'RBM Object'+'-'*5 # trainer.rbm.show() # print '' # print '-'*5+'RBM Trainer Object'+'-'*5 # trainer.show() import h5py mnist_file = '/home/dustin/data/featureLearning/MNIST/mnistLarge.mat' mnist = h5py.File(mnist_file) data = mnist['testData'].value.T
images, labels = import_mnist() images = (images - np.min(images, 0)) / (np.max(images, 0) + 0.0001) labels = labels # transform labels into one-hot encoding labels_one_hot = [] for i in range(0, np.size(labels, axis=0)): one_hot = [0] * 10 one_hot[labels[i]] = 1 labels_one_hot.append(one_hot) train_data = np.concatenate((images, labels_one_hot), axis=1) rbm1 = RBM(train_data, num_iter=16, num_h=450) biases_h, biases_v, weights, likelihoods = rbm1.train() # generate_likelihood_plot(likelihoods) images_test, labels_test = import_test() images_test = (images_test - np.min(images_test, 0)) / (np.max(images_test, 0) + 0.0001) num_correct = 0 predictions = [] for i in range(0, np.size(images_test, axis=0)): model_ans = compute_largest_prob(images_test[i], biases_h, weights) predictions.append(model_ans) if (model_ans == labels_test[i]):
def associate_data2data(cache=False, train_further=True): print "Testing Associative RBM which tries to learn even-oddness of numbers" # project set-up data_manager = store.StorageManager('EvenOddP', log=True) train_n = 10000 test_n = 1000 # Load mnist hand digits, class label is already set to binary dataset = m_loader.load_digits(n=[train_n, 100, test_n], digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], pre={'binary_label': True}) tr_x, tr_y = dataset[0] te_x, te_y = dataset[2] tr_x01 = m_loader.sample_image(tr_y) te_x01 = m_loader.sample_image(te_y) ones = m_loader.load_digits(n=[test_n, 0, 0], digits=[1])[0][0] zeroes = m_loader.load_digits(n=[test_n, 0, 0], digits=[0])[0][0] concat1 = theano.function([], T.concatenate([tr_x, tr_x01], axis=1))() # concat2 = theano.function([], T.concatenate([tr_x01, tr_x], axis=1))() # c = np.concatenate([concat1, concat2], axis=0) # np.random.shuffle(c) # tr_concat_x = theano.shared(c, name='tr_concat_x') tr_concat_x = theano.shared(concat1, name='tr_concat_x') tr = TrainParam(learning_rate=0.001, momentum_type=NESTEROV, momentum=0.5, weight_decay=0.1, sparsity_constraint=True, sparsity_target=0.1, sparsity_decay=0.9, sparsity_cost=0.1, dropout=True, dropout_rate=0.5, epochs=1) # Even odd test k = 1 n_visible = 784 * 2 n_visible2 = 0 n_hidden = 300 print "number of hidden nodes: %d" % n_hidden config = RBMConfig(v_n=n_visible, v2_n=n_visible2, h_n=n_hidden, cd_type=CLASSICAL, cd_steps=k, train_params=tr, progress_logger=ProgressLogger(img_shape=(28 * 2, 28))) rbm = RBM(config=config) # Load RBM (test) loaded = store.retrieve_object(str(rbm)) if loaded and cache: rbm = loaded print "... loaded precomputed rbm" errors = [] for i in xrange(0, 10): # Train RBM if not loaded or train_further: rbm.train(tr_concat_x) # Save RBM data_manager.persist(rbm) # Reconstruct using RBM recon_x = rbm.reconstruct_association_opt(te_x, k=10, bit_p=0) clf = SimpleClassifier('logistic', te_x.get_value(), te_y.eval()) orig = te_y.eval() error = clf.get_score(recon_x, orig) print error errors.append(error) print errors
def associate_data2dataDBN(cache=False): print "Testing Joint DBN which tries to learn even-oddness of numbers" # project set-up data_manager = store.StorageManager('associative_dbn_test', log=True) # Load mnist hand digits, class label is already set to binary train, valid, test = m_loader.load_digits(n=[500, 100, 100], digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], pre={'binary_label': True}) train_x, train_y = train test_x, test_y = test train_x01 = m_loader.sample_image(train_y) dataset01 = m_loader.load_digits(n=[500, 100, 100], digits=[0, 1]) # Initialise RBM parameters # fixed base train param base_tr = RBM.TrainParam(learning_rate=0.01, momentum_type=RBM.CLASSICAL, momentum=0.5, weight_decay=0.0005, sparsity_constraint=False, epochs=20) # top layer parameters tr = RBM.TrainParam(learning_rate=0.1, find_learning_rate=True, momentum_type=RBM.NESTEROV, momentum=0.5, weight_decay=0.001, sparsity_constraint=False, epochs=20) tr_top = RBM.TrainParam(learning_rate=0.1, find_learning_rate=True, momentum_type=RBM.CLASSICAL, momentum=0.5, weight_decay=0.001, sparsity_constraint=False, epochs=20) # Layer 1 # Layer 2 # Layer 3 topology = [784, 500, 500, 100] config = associative_dbn.DefaultADBNConfig() config.topology_left = [784, 500, 500, 100] config.topology_right = [784, 500, 500, 100] config.reuse_dbn = False config.top_rbm_params = tr_top config.base_rbm_params = [base_tr, tr, tr] for cd_type in [RBM.CLASSICAL, RBM.PERSISTENT]: for n_ass in [100, 250, 500, 750, 1000]: config.n_association = n_ass config.top_cd_type = cd_type # Construct DBN assoc_dbn = associative_dbn.AssociativeDBN(config=config, data_manager=data_manager) # Train assoc_dbn.train(train_x, train_x01, cache=cache, optimise=True) for n_recall in [1, 3, 5, 7, 10]: for n_think in [0, 1, 3, 5, 7, 10]: # 1, 3, 5, 7, 10]: # Reconstruct sampled = assoc_dbn.recall(test_x, n_recall, n_think) # Sample from top layer to generate data sample_n = 100 utils.save_images(sampled, image_name='reconstruced_{}_{}_{}.png'.format(n_ass, n_recall, n_think), shape=(sample_n / 10, 10)) dataset01[2] = (theano.shared(sampled), test_y)
def _create_prior(self): logger.debug("_create_prior") num_rbm_nodes_per_layer = self._config.model.n_latent_hierarchy_lvls * self._latent_dimensions // 2 return RBM(n_visible=num_rbm_nodes_per_layer, n_hidden=num_rbm_nodes_per_layer)
SIZE_VISIBLE = 784 # load binary mnist sample dataset dataset = pandas.read_csv(MNIST_TRAIN, delimiter=',', dtype=numpy.float64, header=None) # leave the first column out since it contains the labels # dataset must be normalized to have unit variance by column (sigma_i == 1) dataset = pre.scale(dataset.values[:,1:], axis=0) # compute batch set idx = prepare_batches(len(dataset), SIZE_BATCH) # load distribution gaussian = GaussianBinary(SIZE_VISIBLE, SIZE_HIDDEN) gibbs = BlockGibbsSampler(gaussian, sampling_steps=1) sgd = SGD(gaussian, learning_rate=0.001, weight_decay=0, momentum=0) rbm = RBM(gaussian, gibbs, sgd) # pyplot.figure(1) # pyplot.ion() # pyplot.show() # vmin = numpy.min(dataset) # vmax = numpy.max(dataset) for epoch in range(EPOCHS): for b_idx in idx: batch = dataset[b_idx[0]:b_idx[1], :] d_weight_update, _, _ = rbm.train_batch(batch) rec_probs, rec_state = rbm.reconstruct(batch,steps=10) pyplot.clf() img = numpy.reshape(rec_state[-1,:], newshape=(28,28))
def KanadeAssociativeRBM(cache=False, train_further=False): print "Testing Associative RBM which tries to learn the ID map " # print "Testing Associative RBM which tries to learn the following mapping: {anger, saddness, disgust} -> {sadness}, {contempt, happy, surprise} -> {happy}" # project set-up data_manager = store.StorageManager('Kanade/OptMFSparse0.01RBMTest', log=True) # data_manager = store.StorageManager('Kanade/OptAssociativeRBMTest', log=True) shape = 25 dataset_name = 'sharp_equi{}_{}'.format(shape, shape) # Load kanade database mapping = None # id map # mapping = {'anger': 'sadness', 'contempt': 'happy', 'disgust': 'sadness', 'fear': 'sadness', 'happy': 'happy', # 'sadness': 'sadness', 'surprise': 'happy'} train, valid, test = loader.load_kanade(pre={'scale': True}, set_name=dataset_name) train_x, train_y = train test_x, test_y = test # Sample associated image train_x_mapped, train_y_mapped = loader.sample_image(train_y, mapping=mapping, pre={'scale': True}, set_name=dataset_name) test_x_mapped, test_y_mapped = loader.sample_image(test_y, mapping=mapping, pre={'scale': True}, set_name=dataset_name) # Concatenate images concat1 = T.concatenate([train_x, train_x_mapped], axis=1) # concat2 = T.concatenate([train_x_mapped, train_x], axis=1) # concat = T.concatenate([concat1, concat2], axis=0) # train_tX = theano.function([], concat)() train_tX = theano.function([], concat1)() train_X = theano.shared(train_tX) # Train classifier to be used for classifying reconstruction associated image layer # mapped_data = loader.load_kanade(#emotions=['sadness', 'happy'], # pre={'scale': True}, # set_name=dataset_name) # Target Image # clf_orig = SimpleClassifier('logistic', mapped_data[0][0], mapped_data[0][1]) clf_orig = SimpleClassifier('logistic', train_x, train_y) # Initialise RBM tr = rbm_config.TrainParam(learning_rate=0.0001, momentum_type=rbm_config.NESTEROV, momentum=0.9, weight_decay=0.0001, sparsity_constraint=True, sparsity_target=0.01, sparsity_cost=100, sparsity_decay=0.9, batch_size=10, epochs=10) n_visible = shape * shape * 2 n_hidden = 500 config = rbm_config.RBMConfig() config.v_n = n_visible config.h_n = n_hidden config.v_unit = rbm_units.GaussianVisibleUnit # config.h_unit = rbm_units.ReLUnit config.progress_logger = rbm_logger.ProgressLogger(img_shape=(shape * 2, shape)) config.train_params = tr rbm = RBM(config) print "... initialised RBM" # Load RBM (test) loaded = data_manager.retrieve(str(rbm)) if loaded: rbm = loaded else: rbm.set_initial_hidden_bias() rbm.set_hidden_mean_activity(train_X) # Train RBM - learn joint distribution # rbm.pretrain_lr(train_x, train_x01) for i in xrange(0, 10): if not cache or train_further: rbm.train(train_X) data_manager.persist(rbm) print "... reconstruction of associated images" # Get reconstruction with train data to get 'mapped' images to train classifiers on reconstruction = rbm.reconstruct(train_X, 1, plot_n=100, plot_every=1, img_name='recon_train') reconstruct_assoc_part = reconstruction[:, (shape**2):] # Get associated images of test data nsamples = np.random.normal(0, 1, test_x.get_value(True).shape).astype( np.float32) initial_y = theano.shared(nsamples, name='initial_y') utils.save_images(nsamples[0:100], 'initialisation.png', (10, 10), (25, 25)) test_x_associated = rbm.reconstruct_association_opt( test_x, initial_y, 5, 0., plot_n=100, plot_every=1, img_name='recon_test_gibbs') mf_recon = rbm.mean_field_inference_opt(test_x, y=initial_y, sample=False, k=10, img_name='recon_test_mf_raw') # Concatenate images test_MFX = theano.function([], T.concatenate([test_x, mf_recon], axis=1))() test_MF = theano.shared(test_MFX) reconstruction = rbm.reconstruct(test_MF, 1, plot_n=100, plot_every=1, img_name='recon_test_mf_recon') mf_recon = reconstruction[:, (shape**2):] print "... reconstructed" # Classify the reconstructions # 1. Train classifier on original images score_orig = clf_orig.get_score(test_x_associated, test_y_mapped.eval()) score_orig_mf = clf_orig.get_score(test_x_associated, test_y_mapped.eval()) # 2. Train classifier on reconstructed images clf_recon = SimpleClassifier('logistic', reconstruct_assoc_part, train_y_mapped.eval()) score_retrain = clf_recon.get_score(test_x_associated, test_y_mapped.eval()) score_retrain_mf = clf_recon.get_score(mf_recon, test_y_mapped.eval()) out_msg = '{} (orig, retrain):{},{}'.format(rbm, score_orig, score_retrain) out_msg2 = '{} (orig, retrain):{},{}'.format(rbm, score_orig_mf, score_retrain_mf) print out_msg print out_msg2
def associate_data2dataDBN(cache=False): print "Testing Associative DBN which tries to learn even-oddness of numbers" # project set-up data_manager = store.StorageManager('Kanade/associative_dbn_test', log=True) # Load mnist hand digits, class label is already set to binary dataset = loader.load_kanade(n=500, emotions=['anger', 'sadness', 'happy'], pre={'scale2unit': True}) train_x, train_y = dataset train_x01 = loader.sample_image(train_y) dataset01 = loader.load_kanade(n=500) # Initialise RBM parameters # fixed base train param base_tr = RBM.TrainParam(learning_rate=0.001, momentum_type=RBM.CLASSICAL, momentum=0.5, weight_decay=0.0005, sparsity_constraint=False, epochs=20) # top layer parameters tr = RBM.TrainParam( learning_rate=0.001, # find_learning_rate=True, momentum_type=RBM.NESTEROV, momentum=0.5, weight_decay=0.001, sparsity_constraint=False, epochs=20) tr_top = RBM.TrainParam( learning_rate=0.001, # find_learning_rate=True, momentum_type=RBM.CLASSICAL, momentum=0.5, weight_decay=0.001, sparsity_constraint=False, epochs=20) # Layer 1 # Layer 2 # Layer 3 # topology = [784, 500, 500, 100] config = associative_dbn.DefaultADBNConfig() config.topology_left = [625, 500, 500, 100] config.topology_right = [625, 500, 500, 100] config.reuse_dbn = False config.top_rbm_params = tr_top config.base_rbm_params = [base_tr, tr, tr] count = 0 for cd_type in [RBM.CLASSICAL, RBM.PERSISTENT]: for n_ass in [100, 250, 500, 750, 1000]: config.n_association = n_ass config.top_cd_type = cd_type # Construct DBN ass_dbn = associative_dbn.AssociativeDBN(config=config, data_manager=data_manager) # Train for trainN in xrange(0, 5): ass_dbn.train(train_x, train_x01, cache=cache) for n_recall in [1, 3, 10]: for n_think in [0, 1, 3, 5, 10]: # 1, 3, 5, 7, 10]: # Reconstruct sampled = ass_dbn.recall(train_x, n_recall, n_think) # Sample from top layer to generate data sample_n = 100 utils.save_images( sampled, image_name='{}_reconstruced_{}_{}_{}.png'.format( count, n_ass, n_recall, n_think), shape=(sample_n / 10, 10), img_shape=(25, 25)) count += 1