def __init__(self, layers, mean=None, rng_seed=None): ''' layers is a list specifying the dimension of each network layer: layers = [#n_visible, #n_hidden1, #n_hidden2, ...] The final layer is a softmax layer, with one node per class label E.g., layers = [20,10,2] implements binary classification with inputs of dimension 20, and 10 units in the first hidden layer The DNN is trained by first calling pre_train() to perform unsupervised pre-training, and then fine_tune() to perform discriminative (supervised) fine tuning ''' self.layers = layers self.n_hid = len(layers) - 1 self.mlp = mlp.mlp(layers, rng_seed) # initialize multi-layer perceptron self.rbm = (self.n_hid - 1) * [None] # initialize RBMs if 1: self.rbm[0] = rbm.rbm(layers[0], layers[1], input_type='binary', mean=mean) else: self.rbm[0] = rbm.rbm(layers[0], layers[1], input_type='gaussian') for k in xrange(1, self.n_hid - 1): # skip last layer which uses softmax instead of binary units self.rbm[k] = rbm.rbm(layers[k], layers[k+1], input_type='binary')
def __init__(self,nvisible,nhidden=[100,100,50],nlabels=0,eta=0.1,momentum=0.,nCDsteps=5,nepochs=1000,useprobstate=0): self.layers = [rbm.rbm(nvisible,nhidden[0],nlabels=None,eta=eta,momentum=momentum,nCDsteps=nCDsteps,nepochs=nepochs)] self.nRBMs = len(nhidden)-1 for i in range(1,self.nRBMs): self.layers.append(rbm.rbm(nhidden[i-1],nhidden[i],nlabels=None,eta=eta,momentum=momentum,nCDsteps=nCDsteps,nepochs=nepochs)) self.layers.append(rbm.rbm(nhidden[self.nRBMs-1],nhidden[self.nRBMs],nlabels=nlabels,eta=eta,momentum=momentum,nCDsteps=nCDsteps,nepochs=nepochs)) self.eta = eta self.momentum = momentum self.nCDsteps = nCDsteps self.nepochs = nepochs for i in range(self.nRBMs+1): print self.layers[i].nvisible, self.layers[i].nhidden, self.layers[i].nlabels
def pre_train_rbm(self,data,n_iters=100,learning_rate=0.001,adapt_learn=False,session=None): assert self.full_connected,"Pretraining can be done only with a full autoencoder (encoder+decoder). Use generate_decoder() first." params=[] b_dec = [] out = data for i in range(self.enc_length): print 'Pre-training with RBM layer ',i+1,'...' if(adapt_learn): learning_rate = learning_rate/((self.units[i]+self.units[i+1])**0.5) with rbm.rbm(str(i),self.units[i],self.units[i+1],learning_rate=learning_rate,activation=self.act_func[i],euris=self.use_euristic) as temp: r_s = temp.init_rbm() for _ in range(n_iters): r_s.run(temp.cd1(out)) out = r_s.run(temp.propup(out)) params.extend([r_s.run(temp.weights),r_s.run(temp.h_bias)]) b_dec.extend([r_s.run(temp.v_bias)]) print '...Done' for i in range(self.enc_length): W_trained = tf.Variable(tf.add(tf.zeros(self.units[i:i+2]),params[i*2])) b_trained = tf.Variable(tf.add(tf.zeros([self.units[i+1]]),params[i*2+1])) b_trained_dec = tf.Variable(tf.add(tf.zeros([self.units[i]]),b_dec[i])) self.layers[i].assign(W_trained,b_trained) self.layers[-1-i].assign(W_trained,b_trained_dec,T=True) self.session = self.init_network() return params
def __init__(self,layers,momentum=0,alpha=1,show=True): self.lrbm={} self.size=layers for ii in xrange(1,len(layers)): self.lrbm[ii]=rbm(n_in=layers[ii-1],n_out=layers[ii],momentum=momentum,alpha=alpha,show=show) self.n=len(layers) return None
def test_rbm3(): import rbm r = rbm.rbm(5, 2,nCDsteps=1,momentum=0.9,nepochs = 100) inputs = np.array([[0,1,0,1,1],[1,1,0,1,0],[1,1,0,0,1],[1,1,0,0,1], [1,0,1,0,1],[1,1,1,0,0]]) r.contrastive_divergence(inputs) #r.cddemo(inputs) print r.weights print r.visiblebias print r.hiddenbias print r.hiddenact print "---" test = np.array([[1,1,0,0,1]]) r.compute_hidden(test) print r.hiddenact test = np.array([[1,0]]) r.compute_visible(test) print r.visibleact test = np.array([[1,0]]) r.compute_visible(test) print r.visibleact test = np.array([[1,0]]) r.compute_visible(test) print r.visibleact
def test_labelled_rbm_learning(): import rbm rb = rbm.rbm(4,4,nlabels=2,eta=0.3,momentum=0.5,nCDsteps=1,nepochs=1000) N=2500 v = np.zeros((2*N,4)) l = np.zeros((2*N,2)) for n in range(N): r = np.random.rand() if r>0.666: v[n,:] = [0,1,0,0] l[n,:] = [1,0] elif r>0.333: v[n,:] = [1,0,0,0] l[n,:] = [1,0] for n in range(N): r = np.random.rand() if r>0.666: v[N+n,:] = [0,0,0,1] l[N+n,:] = [0,1] elif r>0.333: v[N+n,:] = [0,0,1,0] l[N+n,:] = [0,1] rb.contrastive_divergence(v,l) #, delta=delta, momentum=mom)
def learn_squares(): N=20 data = make_squares() print np.shape(data) show_squares(data) import rbm rb = rbm.rbm(N*N,50,nlabels=None,eta=0.3,momentum=0.5,nCDsteps=3,nepochs=2000) show_weights(rb) rb.contrastive_divergence(data) show_weights(rb) #hid,phid = rb.compute_hidden(v) #vis,pvis,lab = rb.compute_visible(hid) nr, nc = 5,8 newdata = np.where(np.random.random((nr*nc,N*N)) > 0.5, 1, 0) show_squares(newdata) print np.shape(newdata) for t in range(25): hid,phid = rb.compute_hidden(newdata) newdata,pvis,lab = rb.compute_visible(hid) print np.shape(newdata) show_squares(newdata)
def test_rbm3(): import rbm r = rbm.rbm(5, 2, nCDsteps=1, momentum=0.9, nepochs=100) inputs = np.array([[0, 1, 0, 1, 1], [1, 1, 0, 1, 0], [1, 1, 0, 0, 1], [1, 1, 0, 0, 1], [1, 0, 1, 0, 1], [1, 1, 1, 0, 0]]) r.contrastive_divergence(inputs) #r.cddemo(inputs) print r.weights print r.visiblebias print r.hiddenbias print r.hiddenact print "---" test = np.array([[1, 1, 0, 0, 1]]) r.compute_hidden(test) print r.hiddenact test = np.array([[1, 0]]) r.compute_visible(test) print r.visibleact test = np.array([[1, 0]]) r.compute_visible(test) print r.visibleact test = np.array([[1, 0]]) r.compute_visible(test) print r.visibleact
def test_rbm(): import rbm import cPickle, gzip # Load the dataset f = gzip.open('mnist.pkl.gz', 'rb') train_set, valid_set, test_set = cPickle.load(f) f.close() r = rbm.rbm(28*28,100,inputs = train_set[0][:100,:],momentum=0.4,nCDsteps=3,nepochs=5000) r.contrastive_divergence(train_set[0][:100,:])
def __init__(self, layers, momentum=0, alpha=1, show=True): self.lrbm = {} self.size = layers for ii in xrange(1, len(layers)): self.lrbm[ii] = rbm(n_in=layers[ii - 1], n_out=layers[ii], momentum=momentum, alpha=alpha, show=show) self.n = len(layers) return None
def pre_train_layers(self,lev,data,k=10,iters=10): if(lev==0): b_zero = np.ones(self.layers[lev].n_in,).astype(theano.config.floatX) r_layer = rbm.rbm(self.layers[lev].W.get_value(),b_zero,self.layers[lev].b.get_value(),k) for i in range(iters): r_layer.train(data) self.layers[lev].W.set_value(r_layer.W.get_value()) if(lev<self.n_layers-1): out = self.layers[lev].current_output(data) self.pre_train_layers(lev+1,out,k,iters) else: r_layer = rbm.rbm(self.layers[lev].W.get_value(),self.layers[lev-1].b.get_value(),self.layers[lev].b.get_value(),k) for i in range(iters): r_layer.train(data) self.layers[lev].W.set_value(r_layer.W.get_value()) if(lev<self.n_layers-1): out = self.layers[lev].current_output(data) self.pre_train_layers(lev+1,out,k,iters)
def test_rbm(): import rbm import gzip # Load the dataset with gzip.open('mnist.pkl.gz', 'rb') as f : uuu = cPickle._Unpickler(f) uuu.encoding = 'latin1' train_set, valid_set, test_set = uuu.load() f.close() r = rbm.rbm(28*28,100,inputs = train_set[0][:100,:],momentum=0.4,nCDsteps=3,nepochs=5000) r.contrastive_divergence(train_set[0][:100,:])
def test_rbm2(): import rbm r = rbm.rbm(6, 2,nCDsteps=1,momentum=0.9,nepochs = 8000) #r = rbm.rbm(6, 2,nCDsteps=1,nepochs = 2) inputs = np.array([[1,1,1,0,0,0],[1,0,1,0,0,0],[1,1,1,0,0,0],[0,0,1,1,1,0], [0,0,1,1,0,0],[0,0,1,1,1,0]]) r.contrastive_divergence(inputs) print (r.weights) print (r.visiblebias) print (r.hiddenbias) test = np.array([[0,0,0,1,1,0]]) r.compute_hidden(test) print (r.hiddenact)
def test_rbm2(): import rbm r = rbm.rbm(6, 2,nCDsteps=1,momentum=0.9,nepochs = 8000) #r = rbm.rbm(6, 2,nCDsteps=1,nepochs = 2) inputs = np.array([[1,1,1,0,0,0],[1,0,1,0,0,0],[1,1,1,0,0,0],[0,0,1,1,1,0], [0,0,1,1,0,0],[0,0,1,1,1,0]]) r.contrastive_divergence(inputs) print r.weights print r.visiblebias print r.hiddenbias test = np.array([[0,0,0,1,1,0]]) r.compute_hidden(test) print r.hiddenact
def test_rbm(): import rbm import cPickle, gzip # Load the dataset f = gzip.open('mnist.pkl.gz', 'rb') train_set, valid_set, test_set = cPickle.load(f) f.close() r = rbm.rbm(28 * 28, 100, inputs=train_set[0][:100, :], momentum=0.4, nCDsteps=3, nepochs=5000) r.contrastive_divergence(train_set[0][:100, :])
def test_rbm_learning(): import rbm import mdp rb = rbm.rbm(4,2,nlabels=None,eta=0.3,momentum=0.9,nCDsteps=1,nepochs=500) rw = mdp.utils.random_rot(max(4,2), dtype='d')[:4, :2] rb.weights=rw # the observations consist of two disjunct patterns that never appear together N=10000 v = np.zeros((N,4)) for n in range(N): r = np.random.rand() if r>0.666: v[n,:] = [0,1,0,1] elif r>0.333: v[n,:] = [1,0,1,0] rb.contrastive_divergence(v) #, delta=delta, momentum=mom)
def preTrain(self, lR, k, nE, method): # method: 'pCD' or 'CDk' print('----------------------------------') print('Training...') # training data for stack of rbms (layer by layer) tData = copy.deepcopy(self.data) for i in range(self.nHL): print('rbm ', i + 1, ' of ', self.nHL) # initialize stack of rbms (k=1 for all but last rbm) RBM = rbm(self.net[i:i + 2], self.T, lR, k, self.bS, nE, roll=False) if i == 0 and self.roll: RBM.roll = True # load training data for this layer RBM.loadData(tData) # train this layer's rbm RBM.train(method) # produce training data for next rbm tData = RBM.compressedData() # update weights of dbn self.w[i] = RBM.w[0] self.wR[i] = RBM.w[0] self.b[i] = RBM.b[0] self.bR[i + 1] = RBM.b[1] if i == 0: self.bR[i] = RBM.b[0] if i == (self.nHL - 1): self.b[i + 1] = RBM.b[1]
def do_variables_init(self, data): def assign(a, b): #将b的值赋给a op = a.assign(b) self.sess.run(op) init = tf.global_variables_initializer() self.sess.run(init) #如果配置文件的模型加载地址存在,就加载已经保存好了的模型 if self.config.restore_model: self.restore_model(self.config.restore_model) print ("restore model" + self.config.restore_model) elif self.config.DBN_init: #self.struct存储的是神经网络的结构,每一层多少个神经元 #例如3层神经网络就是2个rbm shape = self.struct myRBMs = [] #i代表第几个rbm for i in range(len(shape) - 1): myRBM = rbm.rbm([shape[i], shape[i+1]], {"batch_size": self.config.dbn_batch_size, "learning_rate":self.config.dbn_learning_rate}) myRBMs.append(myRBM) for epoch in range(self.config.dbn_epochs): error = 0 #训练一个批次的样本就更新一次参数 for batch in range(0, data.N, self.config.dbn_batch_size): mini_batch = data.sample(self.config.dbn_batch_size).X for k in range(len(myRBMs) - 1): mini_batch = myRBMs[k].getH(mini_batch) #一个批次计算一次错误也就是损失,然后更新一次参数 error += myRBM.fit(mini_batch) #记录错误是一次迭代记录一次错误 print ("rbm epochs:", epoch, "error : ", error) W, bv, bh = myRBM.getWb() name = "encoder" + str(i) assign(self.W[name], W) assign(self.b[name], bh) name = "decoder" + str(self.layers - i - 2) assign(self.W[name], W.transpose()) assign(self.b[name], bv) self.is_Init = True
def build(self): """Build the model""" rbms = {} nets = OrderedDict() datasets = self.config["datasets"] self.global_step = tf.train.get_or_create_global_step() """lr = tf.train.noisy_linear_cosine_decay(self.config['init_lr'], self.global_step, self.config['decay_steps_lr'], num_periods=self.config['cycles_lr'], alpha=0.0, beta=0.001, name="learning_rate")""" lr = tf.constant(self.config['init_lr']) """lr = tf.multiply( tf.multiply( 1e-10, tf.pow( 10e0, tf.multiply( tf.cast( tf.div( self.global_step, 10 ), tf.float32 ), 1e-1 ) ) ), tf.cast( tf.mod( tf.add( self.global_step, 1 ), 2 ), tf.float32 ) )""" nets["learning_rate"] = lr self.lr = lr self.losses[()] = [("learning_rate", self.lr)] for d in datasets: rbms[datasets[d]['id']] = {} nets[datasets[d]['id'] + "_h1"] = [] nets[datasets[d]['id'] + "_v2"] = [] nets[datasets[d]['id'] + "_h2"] = [] self.optimizers[(datasets[d]['id'], )] = [] self.losses[(datasets[d]['id'], )] = [] diff = [] with tf.variable_scope("t_input_" + datasets[d]['id']): t_input = tf.placeholder(tf.float32, [ datasets[d]['sizes']['batch_size'], None, datasets[d]['sizes']['num_timesteps'], datasets[d]['sizes']['num_pitch'], datasets[d]['sizes']['num_track'] ], name='in_ph_' + datasets[d]['id']) t_seqlen = tf.placeholder(tf.float32, [datasets[d]['sizes']['batch_size']], name='len_ph_' + datasets[d]['id']) self.input[datasets[d]['id']] = (t_input, t_seqlen) t_input = tf.floor(t_input + tf.constant(1 - 0.1)) #binarize nets["t_input"] = tf.transpose( tf.reshape(t_input, (-1, datasets[d]['sizes']['num_timesteps'] * datasets[d]['sizes']['num_pitch'], datasets[d]['sizes']['num_track'])), [2, 0, 1]) with tf.variable_scope("RBM_" + datasets[d]['id']): for t in range(datasets[d]['sizes']['num_track']): rbms[datasets[d]['id']][t + 1] = rbm.rbm( datasets[d]['id'] + "_" + str(t), datasets[d]['sizes']['num_timesteps'] * datasets[d]['sizes']['num_pitch'], self.config['n_features'], nets["t_input"][t]) rbm.up_nodes(datasets[d]['id'] + "_" + str(t), rbms[datasets[d]['id']][t + 1], 1) nets[datasets[d]['id'] + "_h1"] += [rbms[datasets[d]['id']][t + 1]["h1"]] with tf.variable_scope("concat_" + datasets[d]['id']): nets[datasets[d]['id'] + "gen-v0"] = tf.concat( nets[datasets[d]['id'] + "_h1"], 1) rbms[datasets[d]['id']][0] = rbm.rbm( datasets[d]['id'] + "_gen", datasets[d]['sizes']['num_track'] * self.config['n_features'], self.config['n_features'], nets[datasets[d]['id'] + "gen-v0"]) rbm.up_nodes(datasets[d]['id'] + "_gen", rbms[datasets[d]['id']][0], 1) nets[datasets[d]['id'] + "gen-v1"] = tf.transpose( tf.reshape( rbm.down_nodes(datasets[d]['id'] + "_gen", rbms[datasets[d]['id']][0], 1), (-1, datasets[d]['sizes']['num_track'], self.config['n_features'])), [1, 0, 2]) for t in range(datasets[d]['sizes']['num_track']): rbm.down_nodes(datasets[d]['id'] + "_" + str(t), rbms[datasets[d]['id']][t + 1], 1, nets[datasets[d]['id'] + "gen-v1"][t]) rbm.up_nodes(datasets[d]['id'] + "_" + str(t), rbms[datasets[d]['id']][t + 1], 2) nets[datasets[d]['id'] + "_v2"] += [rbms[datasets[d]['id']][t + 1]["v2"]] nets[datasets[d]['id'] + "_h2"] += [rbms[datasets[d]['id']][t + 1]["h2"]] with tf.variable_scope("concat_" + datasets[d]['id']): nets[datasets[d]['id'] + "gen_v2"] = tf.concat( nets[datasets[d]['id'] + "_h2"], 1) rbm.up_nodes(datasets[d]['id'] + "_gen", rbms[datasets[d]['id']][0], 2, nets[datasets[d]['id'] + "gen_v2"]) for t in range(datasets[d]['sizes']['num_track'] + 1): with tf.variable_scope("losses_" + datasets[d]['id'] + "_" + str(t)): if t > 0: v = tf.reshape(rbms[datasets[d]['id']][t]["v1"], [ -1, datasets[d]['sizes']['num_timesteps'] * datasets[d]['sizes']['num_pitch'], 1 ]) v_sample = tf.reshape( rbms[datasets[d]['id']][t]["v2"], [ -1, datasets[d]['sizes']['num_timesteps'] * datasets[d]['sizes']['num_pitch'], 1 ]) else: v = tf.reshape(rbms[datasets[d]['id']][t]["v1"], [ -1, datasets[d]['sizes']['num_track'] * self.config['n_features'], 1 ]) v_sample = tf.reshape( rbms[datasets[d]['id']][t]["v2"], [ -1, datasets[d]['sizes']['num_track'] * self.config['n_features'], 1 ]) h = tf.reshape(rbms[datasets[d]['id']][t]["h1"], [-1, 1, self.config['n_features']]) h_sample = tf.reshape(rbms[datasets[d]['id']][t]["h2"], [-1, 1, self.config['n_features']]) W_adder = tf.reduce_mean( tf.subtract(tf.matmul(v, h), tf.matmul(v_sample, h_sample)), 0) bv_a = tf.reduce_mean( tf.reduce_sum(tf.subtract(v, v_sample), 2, True), 0) if t > 0: bv_adder = tf.reshape(bv_a, [ 1, datasets[d]['sizes']['num_timesteps'] * datasets[d]['sizes']['num_pitch'] ]) else: bv_adder = tf.reshape(bv_a, [ 1, datasets[d]['sizes']['num_track'] * self.config['n_features'] ]) bh_adder = tf.reshape( tf.reduce_mean( tf.reduce_sum(tf.subtract(h, h_sample), 1, True), 0), [1, self.config['n_features']]) if t > 0: diff += [bv_adder] self.losses[(datasets[d]['id'], )] += [ (datasets[d]['id'] + "_" + str(t) + "_W", tf.reduce_mean(W_adder)) ] self.losses[(datasets[d]['id'], )] += [ (datasets[d]['id'] + "_" + str(t) + "_bv", tf.reduce_mean(bv_adder)) ] self.losses[(datasets[d]['id'], )] += [ (datasets[d]['id'] + "_" + str(t) + "_bh", tf.reduce_mean(bh_adder)) ] with tf.variable_scope("optimizers_" + datasets[d]['id']): self.optimizers[(datasets[d]['id'], )] += [ (0, rbms[datasets[d]['id']][t]["W"].assign_add( tf.multiply(self.lr, W_adder))), (0, rbms[datasets[d]['id']][t]["bv"].assign_add( tf.multiply(self.lr, bv_adder))), (0, rbms[datasets[d]['id']][t]["bh"].assign_add( tf.multiply(self.lr, bh_adder))) ] self.losses[(datasets[d]['id'], )] += [ (datasets[d]['id'] + "_diff", tf.reduce_mean(tf.concat(diff, 0))) ] with tf.variable_scope("sample_" + datasets[d]['id']): sample = tf.concat(nets[datasets[d]['id'] + "_v2"], 1) sample = tf.reshape(sample, [ datasets[d]['sizes']['batch_size'], -1, datasets[d]['sizes']['num_track'], datasets[d]['sizes']['num_timesteps'], datasets[d]['sizes']['num_pitch'] ]) sample = tf.transpose(sample, [0, 1, 3, 4, 2]) nets["sample"] = sample self.output[(datasets[d]['id'], )] = [ (datasets[d]['id'], "input", t_input), (datasets[d]['id'], "train_sample", sample) ] self.savers["RBM_" + datasets[d]['id']] = tf.train.Saver( tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES, tf.get_default_graph().get_name_scope() + "/RBM_" + datasets[d]['id']), max_to_keep=2) self.components = nets self.reset_weights = tf.variables_initializer( var_list=tf.trainable_variables())
Xtest = np.load("../data/Xtest.npy") #Load parameters global WP, WB, WC, d, k WP = np.load("../models/MNISTWP.npy") WB = np.load("../models/MNISTWB.npy") WC = np.load("../models/MNISTWC.npy") # WP = np.load("../data/WP_learnt.npy") # WB = np.load("../data/WB_learnt.npy") # WC = np.load("../data/WC_learnt.npy") d = len(WC[0]) k = len(WB[0]) #Create RBM mnistRBM = rbm.rbm() mnistRBM.set_params(WP, WB, WC) #Number of Iterations iterations = 500 num_chains = 100 #Gibbs sampler with single chain x_samples, _, h_samples = mnistRBM.single_gibbs_sampler(X=Xtrain[0].reshape( (1, -1)), iterations=iterations, WP=WP, WB=WB, WC=WC)
def main(): try: original = mrcfile.open(sys.argv[1], mode='r') except IOError: print( "Could not open the input \nUsage tick_mrc inputfile outputfile.") sys.exit() # create list of layers. layers = [] for layer in original.data: layers.append(np.float32(layer)) #layers are the arrays containing the data. the_image = np.zeros_like(layers[0]) the_image = np.add(the_image, layers[40]) # nvis,nhid hw = 2 nb = 3 the_rbm = rbm.rbm((hw * 2 + 1) * (hw * 2 + 1) * nb, 1000) the_rbm.its_symmetric() the_rbm.add_fuzzy(-1., 1., 21) the_adapted_image = adapt.adaptor(hw, 3., the_image) the_adapted_image.make_nbit_image(nb) # for l in the_rbm.layers: # l = np.add(l, the_adapted_image.next(15,15)[2]) for l in range(0, len(the_rbm.layers)): the_rbm.layers[l] = np.add(the_rbm.layers[l], the_adapted_image.random_bits()[2]) the_adapted_image.reset() print("training starts") sys.stdout.flush() # a = the_adapted_image.next(1) for i in xrange(0, 2000): a = the_adapted_image.random_bits() print(i, a[1], the_rbm.train_fuzzy(a[2], 0.1, 1.0, a[1])) # if i > 1000: # print a[2] # print the_rbm.reconstruct(a[2]) # print(i, a[1]) print("training stops") sys.stdout.flush() new_image = np.float32(np.zeros_like(the_image)) # for i in xrange(0, the_image.shape[0]-2*hw-1): for i in xrange(0, 20): print("layer", i) sys.stdout.flush() for j in range(0, the_image.shape[1] - 2 * hw - 1): # for j in range(0,20): a = the_adapted_image.at_bits(i, j) b = the_rbm.reconstruct(a[2]) new_image[i + hw + 1][j + hw + 1] = the_rbm.estimate_EV(a[2]) the_image = Image.fromarray( np.uint8(rescale(the_adapted_image.the_image, 255.))) the_image.save('raw.jpg') the_image = Image.fromarray(np.uint8(rescale(new_image, 255.))) the_image.save('rbm.jpg')
from load_numpy import getNumpy from test_classifier import svm1 from preprocesses import pca from rbm import rbm from sklearn.cross_validation import train_test_split train_x,train_y,test_x = getNumpy() train_x_transform = pca(train_x,100) X_train, X_valid, Y_train, Y_valid = train_test_split(train_x_transform, train_y,test_size=0.2,random_state=0) # svm1(train_x_transform,train_y) rbm(train_x,train_y)
batchsize = 100 n_h = 340 n_v = 784 eachNLog = 5000 eachNErr = 50 wDecay = 0.0001 hDecay = 0.005 Nepochs = 500 mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images,\ mnist.test.labels statCnt = [] rbm_m1 = rbm.rbm(batchsize, alpha, wDecay, hDecay, n_h, n_v) #rbm_m2=rbm.rbm(batchsize,alpha, n_h,n_v, 'scope2') #rbm_m2=rbm.rbm(batchsize,alpha,wDecay,hDecay, n_h,n_v, 'scope55') #print (rbm_m1.sess.run(rbm_m1.err_sum, feed_dict={rbm_m1.X: trX} )) #rbm_m1.X=trX print(rbm_m1.err_sum(trX)) #trX=(trX>0).astype(float) #rbm_m1.load_weights("C:\\Users\\Andrei\\Documents\\UoGuelphResearch\\MileStones_MRR\\Runs\\GA_ES_DiffOnly_guided_R16_100iterPerGA\\weights-25000") #print("Loaded")
import tensorflow as tf import numpy as np import matplotlib.pyplot as plt instruction = input("Train model? y/n\n") if instruction == "y": os.system("make train_model") #After training (if training is chosen), we initiate reconstruction of an arbitrary data point from the test set. weights_filename = "weights.txt" visiblebias_filename = "visiblebias.txt" hiddenbias_filename = "hiddenbias.txt" RBM = rbm(nvisible=28*28, nhidden=14*14, nCDsteps = 1, nepochs = 100) with open(weights_filename, "r") as infile: lines = infile.readlines() i = 0 for line in lines: vals = line.split() for j in range(len(vals)): RBM.weights[i][j] = float(vals[j]) i += 1 print(RBM.weights) with open(visiblebias_filename, "r") as infile: lines = infile.readlines() i = 0
def learn_letters(): import scipy.io as sio nperclass = 39 classes = [10, 11, 28] #classes = [10, 13, 28] # A, C, S nclasses = len(classes) # Read in the data and prepare it data = sio.loadmat('binaryalphadigs.mat') inputs = np.ones((nclasses, nperclass, 20*16)) labels = np.zeros((nclasses, nperclass, nclasses)) for k in range(nclasses): for m in range(nperclass): inputs[k,m,:] = (data['dat'][classes[k],m].ravel()).astype('float') labels[k,m,k] = 1. nexamples = 20 v = inputs[:,:nexamples,:].reshape(nclasses*nexamples, 20*16) l = labels[:,:nexamples,:].reshape(nclasses*nexamples, nclasses) import pylab as pl # This shows a set of examples from the training set pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(v[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') import algo_rbm as rbm rb = rbm.rbm(20*16,50,nlabels=None,eta=0.3,momentum=0.5,nCDsteps=3,nepochs=1000) rb.contrastive_divergence(v) hid,phid = rb.compute_hidden(v) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') newv = inputs[:,nexamples:,:].reshape(nclasses*(39-nexamples),20*16) newl = labels[:,nexamples:,:].reshape(nclasses*(39-nexamples),nclasses) hid,phid = rb.compute_hidden(newv) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') rb = rbm.rbm(20*16,50,nlabels=nclasses,eta=0.3,momentum=0.2,nCDsteps=3,nepochs=1500) rb.contrastive_divergence(v,l) rb.classify(v,l) rb.classify(newv,newl) hid,phid = rb.compute_hidden(v,l) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') hid,phid = rb.compute_hidden(newv,newl) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') vis = np.random.randn(np.shape(v)[0],np.shape(v)[1])*0.05 for i in range(1000): hid,phid = rb.compute_hidden(vis,l) vis,pvis, lab = rb.compute_visible(phid) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
def DBNFit(X, label, numHid, isSaveModels = True, name = "/home/cheng/DBN.npy", isSingleDBN = True, **kwargs) : """implement DBN fitting X :data(np.array) label : the label of each sample(np.array, in a row) numHid : the node of each hidden layer(list)""" H = len(numHid) m = list() nArg = len(kwargs) if H >= 2 : # train the first rbm model if nArg >= 1 : string = "layer" + str(1) model_ = rbm.rbm(X, numHid[0], **kwargs[string]) else : model_ = rbm.rbm(X, numHid[0]) m.append(model_) if isSingleDBN : for index in range(1, H-1) : if nArg >= index : string = "layer" +str(index + 1) model_ = rbm.rbm(m[index-1].top, numHid[index], **kwargs[string]) else : model_ = rbm.rbm(m[index-1].top, numHid[index]) m.append(model_) # train the last rbm model if nArg >= H : string = "layer" + str(H) model_ = rbmFit.rbmFit(m[H-2].top, numHid[H-1], label, **kwargs[string]) else : model_ = rbmFit.rbmFit(m[H-2].top, numHid[H-1], label) m.append(model_) else : for index in range(1, H) : if nArg >= index : string = "layer" +str(index + 1) model_ = rbm.rbm(m[index-1].top, numHid[index], **kwargs[string]) else : model_ = rbm.rbm(m[index-1].top, numHid[index]) m.append(model_) else : # only a single layer if isSingleDBN : if nArg >= 1 : string = "layer" + str(1) model_ = rbmFit.rbmFit(X, numHid[0], label, **kwargs[string]) else : model_ = rbmFit.rbmFit(X, numHid[0], label) m.append(model_) else : if nArg >= 1 : string = "layer" + str(1) model_ = rbm.rbm(X, numHid[0], **kwargs[string]) else : model_ = rbm.rbm(X, numHid[0]) m.append(model_) if isSaveModels : models = np.array(m) np.save(name, models) return m
RANDOMIN_ = sio.loadmat("randomin_poshidstates.mat", verify_compressed_data_integrity=False) RANDOMIN = RANDOMIN_['randomin'] VISHID_ = sio.loadmat("vis_try___.mat", verify_compressed_data_integrity=False) VISHID = VISHID_['vishid_'] BATCH_DATA_ = sio.loadmat("batchdata_py.mat", verify_compressed_data_integrity=False) BATCH_DATA = BATCH_DATA_['batchdata'] NUM_CASES, NUM_DIMS, NUM_BATCHES = BATCH_DATA.shape # print("BATCH_DATA.shape= ", BATCH_DATA.shape) #100, 784, 600 print('Pretraining Layer 1 with RBM: {}-{}'.format(NUM_DIMS, NUM_HID)) RESTART = 1 BATCHPOSHIDPROBS, VISHID, HIDBIASES, VISBIASES = rbm(BATCH_DATA, RESTART, NUM_HID, NUM_DIMS, MAX_EPOCH, RANDOMIN, VISHID) print("BATCHPOSHIDPROBS.shape= ", BATCHPOSHIDPROBS.shape) #100, 1000 HIDRECBIASES = HIDBIASES VISHID__ = VISHID VISBIASES__ = VISBIASES print('Pretraining Layer 2 with RBM: {}-{}'.format(NUM_HID, NUMPEN)) BATCH_DATA = BATCHPOSHIDPROBS NUM_HID = NUMPEN RESTART = 1 BATCHPOSHIDPROBS, VISHID, HIDBIASES, VISBIASES = rbm(BATCH_DATA, RESTART, NUM_HID, NUM_DIMS, MAX_EPOCH, RANDOMIN, VISHID) HIDPEN = VISHID
def main(): try: original = mrcfile.open(sys.argv[1], mode='r') except IOError: print( "Could not open the input \nUsage tick_mrc inputfile outputfile.") sys.exit() # create list of layers. layers = [] for layer in original.data: layers.append(np.float32(layer)) #layers are the arrays containing the data. the_image = np.zeros_like(layers[0]) the_image = np.add(the_image, layers[40]) # nvis,nhid hw = 1 the_rbm = rbm.rbm((hw * 2 + 1) * (hw * 2 + 1), 10) the_rbm.add_fuzzy(-1., 1., 21) the_rbm.reinitialize_fuzzy() the_adapted_image = adapt.adaptor(hw, 3., the_image) # the_adapted_image.make_nbit_image(3) # for l in the_rbm.layers: # l = np.add(l, the_adapted_image.next(15,15)[2]) # for l in range(0, len(the_rbm.layers)): # the_rbm.layers[l] = np.add( the_rbm.layers[l], the_adapted_image.next(1,1)[2]) the_adapted_image.reset() print("training starts") sys.stdout.flush() # a = the_adapted_image.next(1) for i in range(0, 1000): a = the_adapted_image.random() the_rbm.train(a[2], 0.1, 0.5, a[1]) print(i, a[1]) a = the_adapted_image.random() i = 0 j = 0 errs = np.float32(np.zeros(100)) while (a[0]): the_rbm.train_fuzzy(a[2], 0.1, 0.5, a[1]) if i == 1000: the_rbm.reinitialize_fuzzy() x = the_rbm.estimate_EV(a[2]) errs[j] = a[1] - x j = (j + 1) % 100 print(i, a[1], x, errs.std()) # print(i, a[1], the_rbm.estimate_EV(a[2])) # a = the_adapted_image.next(1,1) a = the_adapted_image.random() sys.stdout.flush() i += 1 print("training stops") sys.stdout.flush() # for layer in layers: # the_image = np.add(the_image,layer) the_image = Image.fromarray( np.uint8(rescale(the_adapted_image.the_image, 255.))) the_image.save('sum.jpg')
def main(): try: original = mrcfile.open(sys.argv[1],mode='r') except IOError: print("Could not open the input \nUsage tick_mrc inputfile outputfile.") sys.exit() # create list of layers. layers = [] for layer in original.data: layers.append(np.float32(layer)) #layers are the arrays containing the data. the_image = np.zeros_like(layers[0]) # the_image = np.add(the_image, layers[180]) the_image = np.add(the_image, layers[40]) # nvis,nhid set the size of the rbm hw = 2 hw = 1 hw = 2 # nb = 3 nb = 8 # the number of outer fuzzy sets nfuzz = 21 the_fuzz = [] delta = 2./(nfuzz-1) for i in range(0,nfuzz): cent = -1. + i*delta the_fuzz.append( triangle.triangle( cent-delta,cent, cent+delta)) the_fuzz[i].associate_rbm( rbm.rbm( (hw*2+1)*(hw*2+1)*nb, 10)) the_fuzz[i].rbm.its_symmetric() the_fuzz[i].rbm.add_fuzzy(-1.,1.,21) # the_rbm = rbm.rbm( (hw*2+1)*(hw*2+1)*nb, 1000) # the_rbm.its_symmetric() # the_rbm.add_fuzzy(-1.,1., 21) the_adapted_image = adapt.adaptor( hw, 3., the_image) the_adapted_image.make_nbitmap_image(nb) # for l in the_rbm.layers: # l = np.add(l, the_adapted_image.next(15,15)[2]) # for l in range(0, len(the_rbm.layers)): # the_rbm.layers[l] = np.add( the_rbm.layers[l], the_adapted_image.random_bits()[2]) # the_adapted_image.reset() print("training starts") sys.stdout.flush() print("crisp initialization pass") sys.stdout.flush() for i in xrange(0,2000): a = the_adapted_image.random_bits() for f in the_fuzz: rate = f.belief(a[1]) if rate > 0. : print(i,a[1],rate,f.rbm.train(a[2],0.1,rate,a[1])) print("fuzzy pass") sys.stdout.flush() for i in xrange(0,10000): a = the_adapted_image.random_bits() for f in the_fuzz: rate = f.belief(a[1]) if rate > 0. : print(i,a[1],rate,f.rbm.train_fuzzy(a[2],0.1,rate,a[1])) print("training stops") sys.stdout.flush() new_image = np.float32(np.zeros_like(the_image)) for i in xrange(0, the_image.shape[0]-2*hw-1): # for i in xrange(0, 20): print("layer", i) sys.stdout.flush() for j in range(0,the_image.shape[1]-2*hw-1): # for j in range(0,20): a = the_adapted_image.at_bits(i,j) r = the_fuzz[0].rbm b = 1. fb = the_fuzz[0] for f in the_fuzz: bt = f.rbm.the_best_built_layer( a[2]) # print( bt[1],b) # catch untrained examples # now done in best_built tb = bt[1] # if tb < -1.: # tb = 0. if tb < b: b = tb r = f.rbm fb = f x = r.estimate_EV( a[2]) new_image[i][j] = x # print( x, a[1], fb.centre) # sys.stdout.flush() an_image = Image.fromarray(np.uint8( rescale(new_image,255.))) an_image.save('rbm_bitmap.jpg') # the_image = Image.fromarray(np.uint8( rescale(the_adapted_image.the_image,255.))) the_image.save('raw.jpg') the_image = Image.fromarray(np.uint8( rescale(new_image,255.))) the_image.save('rbm_bitmap.jpg')
test0, test1, test2, test3, test4, test5, test6, test7, test8, test9 = read_mnist.get_each( test_images, test_labels) databaches, testbatches = makebatches.generate(train_images, test_images) ''' train 4 rbms 784->1000->500->250->2 ''' dim = databaches.shape[1] numlay1 = 1000 numlay2 = 500 numlay3 = 250 numlay4 = 2 rbm1 = rbm.rbm(input=databaches, n_visible=dim, n_hidden=numlay1) h1 = rbm1.contrastive_divergence(maxepoch=10, k=1) rbm2 = rbm.rbm(input=h1, n_visible=numlay1, n_hidden=numlay2) h2 = rbm2.contrastive_divergence(maxepoch=10, k=1) rbm3 = rbm.rbm(input=h2, n_visible=numlay2, n_hidden=numlay3) h3 = rbm3.contrastive_divergence(maxepoch=10, k=1) rbm4 = rbm.rbm(input=h3, n_visible=numlay3, n_hidden=numlay4) h4 = rbm4.rbmhiddenlinear(maxepoch=10, k=1) VV, Dim = backprop.fine_tuning(rbm1, rbm2, rbm3, rbm4,
from rbm import rbm, make_binary print("Downloading dataset...") mnist = tf.keras.datasets.mnist (trainX, trainY), (testX, testY) = mnist.load_data() shuffled_indices = np.random.permutation(60000) trainX, trainY = trainX[shuffled_indices], trainY[shuffled_indices] shuffled_indices = np.random.permutation(10000) testX, testY = testX[shuffled_indices], testY[shuffled_indices] size_of_dataset = 60000 #Initialize the model my_rbm = rbm(nvisible=28 * 28, nhidden=8 * 8, eta=0.08, momentum=0.9, nCDsteps=25, nepochs=500, batch_size=100, size_of_dataset=size_of_dataset) print("Preparing data...") training_data = np.zeros((size_of_dataset, 28 * 28)) trainX = trainX / 255.0 trainX = trainX > 0.5 for k in range(size_of_dataset): training_data[k] = trainX[k].flat[:] print("Training model...") start = time.time() my_rbm.train_model(training_data) end = time.time()
def learn_letters(): import scipy.io as sio nperclass = 39 classes = [10, 11, 28] #classes = [10, 13, 28] # A, C, S nclasses = len(classes) # Read in the data and prepare it data = sio.loadmat('binaryalphadigs.mat') inputs = np.ones((nclasses, nperclass, 20*16)) labels = np.zeros((nclasses, nperclass, nclasses)) for k in range(nclasses): for m in range(nperclass): inputs[k,m,:] = (data['dat'][classes[k],m].ravel()).astype('float') labels[k,m,k] = 1. nexamples = 20 v = inputs[:,:nexamples,:].reshape(nclasses*nexamples, 20*16) l = labels[:,:nexamples,:].reshape(nclasses*nexamples, nclasses) import pylab as pl # This shows a set of examples from the training set pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(v[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') import rbm rb = rbm.rbm(20*16,50,nlabels=None,eta=0.3,momentum=0.5,nCDsteps=3,nepochs=1000) rb.contrastive_divergence(v) hid,phid = rb.compute_hidden(v) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') newv = inputs[:,nexamples:,:].reshape(nclasses*(39-nexamples),20*16) newl = labels[:,nexamples:,:].reshape(nclasses*(39-nexamples),nclasses) hid,phid = rb.compute_hidden(newv) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') rb = rbm.rbm(20*16,50,nlabels=nclasses,eta=0.3,momentum=0.2,nCDsteps=3,nepochs=1500) rb.contrastive_divergence(v,l) rb.classify(v,l) rb.classify(newv,newl) hid,phid = rb.compute_hidden(v,l) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') hid,phid = rb.compute_hidden(newv,newl) vis,pvis,lab = rb.compute_visible(hid) pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') vis = np.random.randn(np.shape(v)[0],np.shape(v)[1])*0.05 for i in range(1000): hid,phid = rb.compute_hidden(vis,l) vis,pvis, lab = rb.compute_visible(phid) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')