def __init__(self, layers, mean=None, rng_seed=None): ''' layers is a list specifying the dimension of each network layer: layers = [#n_visible, #n_hidden1, #n_hidden2, ...] The final layer is a softmax layer, with one node per class label E.g., layers = [20,10,2] implements binary classification with inputs of dimension 20, and 10 units in the first hidden layer The DNN is trained by first calling pre_train() to perform unsupervised pre-training, and then fine_tune() to perform discriminative (supervised) fine tuning ''' self.layers = layers self.n_hid = len(layers) - 1 self.mlp = mlp.mlp(layers, rng_seed) # initialize multi-layer perceptron self.rbm = (self.n_hid - 1) * [None] # initialize RBMs if 1: self.rbm[0] = rbm.rbm(layers[0], layers[1], input_type='binary', mean=mean) else: self.rbm[0] = rbm.rbm(layers[0], layers[1], input_type='gaussian') for k in xrange(1, self.n_hid - 1): # skip last layer which uses softmax instead of binary units self.rbm[k] = rbm.rbm(layers[k], layers[k+1], input_type='binary')
def __init__(self,nvisible,nhidden=[100,100,50],nlabels=0,eta=0.1,momentum=0.,nCDsteps=5,nepochs=1000,useprobstate=0): self.layers = [rbm.rbm(nvisible,nhidden[0],nlabels=None,eta=eta,momentum=momentum,nCDsteps=nCDsteps,nepochs=nepochs)] self.nRBMs = len(nhidden)-1 for i in range(1,self.nRBMs): self.layers.append(rbm.rbm(nhidden[i-1],nhidden[i],nlabels=None,eta=eta,momentum=momentum,nCDsteps=nCDsteps,nepochs=nepochs)) self.layers.append(rbm.rbm(nhidden[self.nRBMs-1],nhidden[self.nRBMs],nlabels=nlabels,eta=eta,momentum=momentum,nCDsteps=nCDsteps,nepochs=nepochs)) self.eta = eta self.momentum = momentum self.nCDsteps = nCDsteps self.nepochs = nepochs for i in range(self.nRBMs+1): print self.layers[i].nvisible, self.layers[i].nhidden, self.layers[i].nlabels
def pre_train_rbm(self,data,n_iters=100,learning_rate=0.001,adapt_learn=False,session=None):
assert self.full_connected,"Pretraining can be done only with a full autoencoder (encoder+decoder). Use generate_decoder() first."
params=[]
b_dec = []
out = data
for i in range(self.enc_length):
print 'Pre-training with RBM layer ',i+1,'...'
if(adapt_learn):
learning_rate = learning_rate/((self.units[i]+self.units[i+1])**0.5)
with rbm.rbm(str(i),self.units[i],self.units[i+1],learning_rate=learning_rate,activation=self.act_func[i],euris=self.use_euristic) as temp:
r_s = temp.init_rbm()
for _ in range(n_iters):
r_s.run(temp.cd1(out))
out = r_s.run(temp.propup(out))
params.extend([r_s.run(temp.weights),r_s.run(temp.h_bias)])
b_dec.extend([r_s.run(temp.v_bias)])
print '...Done'
for i in range(self.enc_length):
W_trained = tf.Variable(tf.add(tf.zeros(self.units[i:i+2]),params[i*2]))
b_trained = tf.Variable(tf.add(tf.zeros([self.units[i+1]]),params[i*2+1]))
b_trained_dec = tf.Variable(tf.add(tf.zeros([self.units[i]]),b_dec[i]))
self.layers[i].assign(W_trained,b_trained)
self.layers[-1-i].assign(W_trained,b_trained_dec,T=True)
self.session = self.init_network()
return params
def __init__(self,layers,momentum=0,alpha=1,show=True):
self.lrbm={}
self.size=layers
for ii in xrange(1,len(layers)):
self.lrbm[ii]=rbm(n_in=layers[ii-1],n_out=layers[ii],momentum=momentum,alpha=alpha,show=show)
self.n=len(layers)
return None
def test_rbm3(): import rbm r = rbm.rbm(5, 2,nCDsteps=1,momentum=0.9,nepochs = 100) inputs = np.array([[0,1,0,1,1],[1,1,0,1,0],[1,1,0,0,1],[1,1,0,0,1], [1,0,1,0,1],[1,1,1,0,0]]) r.contrastive_divergence(inputs) #r.cddemo(inputs) print r.weights print r.visiblebias print r.hiddenbias print r.hiddenact print "---" test = np.array([[1,1,0,0,1]]) r.compute_hidden(test) print r.hiddenact test = np.array([[1,0]]) r.compute_visible(test) print r.visibleact test = np.array([[1,0]]) r.compute_visible(test) print r.visibleact test = np.array([[1,0]]) r.compute_visible(test) print r.visibleact
def test_labelled_rbm_learning(): import rbm rb = rbm.rbm(4,4,nlabels=2,eta=0.3,momentum=0.5,nCDsteps=1,nepochs=1000) N=2500 v = np.zeros((2*N,4)) l = np.zeros((2*N,2)) for n in range(N): r = np.random.rand() if r>0.666: v[n,:] = [0,1,0,0] l[n,:] = [1,0] elif r>0.333: v[n,:] = [1,0,0,0] l[n,:] = [1,0] for n in range(N): r = np.random.rand() if r>0.666: v[N+n,:] = [0,0,0,1] l[N+n,:] = [0,1] elif r>0.333: v[N+n,:] = [0,0,1,0] l[N+n,:] = [0,1] rb.contrastive_divergence(v,l) #, delta=delta, momentum=mom)
def learn_squares(): N=20 data = make_squares() print np.shape(data) show_squares(data) import rbm rb = rbm.rbm(N*N,50,nlabels=None,eta=0.3,momentum=0.5,nCDsteps=3,nepochs=2000) show_weights(rb) rb.contrastive_divergence(data) show_weights(rb) #hid,phid = rb.compute_hidden(v) #vis,pvis,lab = rb.compute_visible(hid) nr, nc = 5,8 newdata = np.where(np.random.random((nr*nc,N*N)) > 0.5, 1, 0) show_squares(newdata) print np.shape(newdata) for t in range(25): hid,phid = rb.compute_hidden(newdata) newdata,pvis,lab = rb.compute_visible(hid) print np.shape(newdata) show_squares(newdata)
def test_rbm3():
import rbm
r = rbm.rbm(5, 2, nCDsteps=1, momentum=0.9, nepochs=100)
inputs = np.array([[0, 1, 0, 1, 1], [1, 1, 0, 1, 0], [1, 1, 0, 0, 1],
[1, 1, 0, 0, 1], [1, 0, 1, 0, 1], [1, 1, 1, 0, 0]])
r.contrastive_divergence(inputs)
#r.cddemo(inputs)
print r.weights
print r.visiblebias
print r.hiddenbias
print r.hiddenact
print "---"
test = np.array([[1, 1, 0, 0, 1]])
r.compute_hidden(test)
print r.hiddenact
test = np.array([[1, 0]])
r.compute_visible(test)
print r.visibleact
test = np.array([[1, 0]])
r.compute_visible(test)
print r.visibleact
test = np.array([[1, 0]])
r.compute_visible(test)
print r.visibleact
def test_labelled_rbm_learning():
import rbm
rb = rbm.rbm(4,4,nlabels=2,eta=0.3,momentum=0.5,nCDsteps=1,nepochs=1000)
N=2500
v = np.zeros((2*N,4))
l = np.zeros((2*N,2))
for n in range(N):
r = np.random.rand()
if r>0.666:
v[n,:] = [0,1,0,0]
l[n,:] = [1,0]
elif r>0.333:
v[n,:] = [1,0,0,0]
l[n,:] = [1,0]
for n in range(N):
r = np.random.rand()
if r>0.666:
v[N+n,:] = [0,0,0,1]
l[N+n,:] = [0,1]
elif r>0.333:
v[N+n,:] = [0,0,1,0]
l[N+n,:] = [0,1]
rb.contrastive_divergence(v,l) #, delta=delta, momentum=mom)
def test_rbm():
import rbm
import cPickle, gzip
# Load the dataset
f = gzip.open('mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
r = rbm.rbm(28*28,100,inputs = train_set[0][:100,:],momentum=0.4,nCDsteps=3,nepochs=5000)
r.contrastive_divergence(train_set[0][:100,:])
def __init__(self, layers, momentum=0, alpha=1, show=True):
self.lrbm = {}
self.size = layers
for ii in xrange(1, len(layers)):
self.lrbm[ii] = rbm(n_in=layers[ii - 1],
n_out=layers[ii],
momentum=momentum,
alpha=alpha,
show=show)
self.n = len(layers)
return None
def pre_train_layers(self,lev,data,k=10,iters=10):
if(lev==0):
b_zero = np.ones(self.layers[lev].n_in,).astype(theano.config.floatX)
r_layer = rbm.rbm(self.layers[lev].W.get_value(),b_zero,self.layers[lev].b.get_value(),k)
for i in range(iters):
r_layer.train(data)
self.layers[lev].W.set_value(r_layer.W.get_value())
if(lev<self.n_layers-1):
out = self.layers[lev].current_output(data)
self.pre_train_layers(lev+1,out,k,iters)
else:
r_layer = rbm.rbm(self.layers[lev].W.get_value(),self.layers[lev-1].b.get_value(),self.layers[lev].b.get_value(),k)
for i in range(iters):
r_layer.train(data)
self.layers[lev].W.set_value(r_layer.W.get_value())
if(lev<self.n_layers-1):
out = self.layers[lev].current_output(data)
self.pre_train_layers(lev+1,out,k,iters)
def test_rbm():
import rbm
import gzip
# Load the dataset
with gzip.open('mnist.pkl.gz', 'rb') as f :
uuu = cPickle._Unpickler(f)
uuu.encoding = 'latin1'
train_set, valid_set, test_set = uuu.load()
f.close()
r = rbm.rbm(28*28,100,inputs = train_set[0][:100,:],momentum=0.4,nCDsteps=3,nepochs=5000)
r.contrastive_divergence(train_set[0][:100,:])
def test_rbm2(): import rbm r = rbm.rbm(6, 2,nCDsteps=1,momentum=0.9,nepochs = 8000) #r = rbm.rbm(6, 2,nCDsteps=1,nepochs = 2) inputs = np.array([[1,1,1,0,0,0],[1,0,1,0,0,0],[1,1,1,0,0,0],[0,0,1,1,1,0], [0,0,1,1,0,0],[0,0,1,1,1,0]]) r.contrastive_divergence(inputs) print (r.weights) print (r.visiblebias) print (r.hiddenbias) test = np.array([[0,0,0,1,1,0]]) r.compute_hidden(test) print (r.hiddenact)
def test_rbm2(): import rbm r = rbm.rbm(6, 2,nCDsteps=1,momentum=0.9,nepochs = 8000) #r = rbm.rbm(6, 2,nCDsteps=1,nepochs = 2) inputs = np.array([[1,1,1,0,0,0],[1,0,1,0,0,0],[1,1,1,0,0,0],[0,0,1,1,1,0], [0,0,1,1,0,0],[0,0,1,1,1,0]]) r.contrastive_divergence(inputs) print r.weights print r.visiblebias print r.hiddenbias test = np.array([[0,0,0,1,1,0]]) r.compute_hidden(test) print r.hiddenact
def test_rbm():
import rbm
import cPickle, gzip
# Load the dataset
f = gzip.open('mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
r = rbm.rbm(28 * 28,
100,
inputs=train_set[0][:100, :],
momentum=0.4,
nCDsteps=3,
nepochs=5000)
r.contrastive_divergence(train_set[0][:100, :])
def test_rbm_learning():
import rbm
import mdp
rb = rbm.rbm(4,2,nlabels=None,eta=0.3,momentum=0.9,nCDsteps=1,nepochs=500)
rw = mdp.utils.random_rot(max(4,2), dtype='d')[:4, :2]
rb.weights=rw
# the observations consist of two disjunct patterns that never appear together
N=10000
v = np.zeros((N,4))
for n in range(N):
r = np.random.rand()
if r>0.666: v[n,:] = [0,1,0,1]
elif r>0.333: v[n,:] = [1,0,1,0]
rb.contrastive_divergence(v) #, delta=delta, momentum=mom)
def test_rbm_learning(): import rbm import mdp rb = rbm.rbm(4,2,nlabels=None,eta=0.3,momentum=0.9,nCDsteps=1,nepochs=500) rw = mdp.utils.random_rot(max(4,2), dtype='d')[:4, :2] rb.weights=rw # the observations consist of two disjunct patterns that never appear together N=10000 v = np.zeros((N,4)) for n in range(N): r = np.random.rand() if r>0.666: v[n,:] = [0,1,0,1] elif r>0.333: v[n,:] = [1,0,1,0] rb.contrastive_divergence(v) #, delta=delta, momentum=mom)
def preTrain(self, lR, k, nE, method):
# method: 'pCD' or 'CDk'
print('----------------------------------')
print('Training...')
# training data for stack of rbms (layer by layer)
tData = copy.deepcopy(self.data)
for i in range(self.nHL):
print('rbm ', i + 1, ' of ', self.nHL)
# initialize stack of rbms (k=1 for all but last rbm)
RBM = rbm(self.net[i:i + 2],
self.T,
lR,
k,
self.bS,
nE,
roll=False)
if i == 0 and self.roll:
RBM.roll = True
# load training data for this layer
RBM.loadData(tData)
# train this layer's rbm
RBM.train(method)
# produce training data for next rbm
tData = RBM.compressedData()
# update weights of dbn
self.w[i] = RBM.w[0]
self.wR[i] = RBM.w[0]
self.b[i] = RBM.b[0]
self.bR[i + 1] = RBM.b[1]
if i == 0:
self.bR[i] = RBM.b[0]
if i == (self.nHL - 1):
self.b[i + 1] = RBM.b[1]
def do_variables_init(self, data):
def assign(a, b):
#将b的值赋给a
op = a.assign(b)
self.sess.run(op)
init = tf.global_variables_initializer()
self.sess.run(init)
#如果配置文件的模型加载地址存在，就加载已经保存好了的模型
if self.config.restore_model:
self.restore_model(self.config.restore_model)
print ("restore model" + self.config.restore_model)
elif self.config.DBN_init:
#self.struct存储的是神经网络的结构，每一层多少个神经元
#例如3层神经网络就是2个rbm
shape = self.struct
myRBMs = []
#i代表第几个rbm
for i in range(len(shape) - 1):
myRBM = rbm.rbm([shape[i], shape[i+1]], {"batch_size": self.config.dbn_batch_size, "learning_rate":self.config.dbn_learning_rate})
myRBMs.append(myRBM)
for epoch in range(self.config.dbn_epochs):
error = 0
#训练一个批次的样本就更新一次参数
for batch in range(0, data.N, self.config.dbn_batch_size):
mini_batch = data.sample(self.config.dbn_batch_size).X
for k in range(len(myRBMs) - 1):
mini_batch = myRBMs[k].getH(mini_batch)
#一个批次计算一次错误也就是损失，然后更新一次参数
error += myRBM.fit(mini_batch)
#记录错误是一次迭代记录一次错误
print ("rbm epochs:", epoch, "error : ", error)
W, bv, bh = myRBM.getWb()
name = "encoder" + str(i)
assign(self.W[name], W)
assign(self.b[name], bh)
name = "decoder" + str(self.layers - i - 2)
assign(self.W[name], W.transpose())
assign(self.b[name], bv)
self.is_Init = True
def build(self):
"""Build the model"""
rbms = {}
nets = OrderedDict()
datasets = self.config["datasets"]
self.global_step = tf.train.get_or_create_global_step()
"""lr = tf.train.noisy_linear_cosine_decay(self.config['init_lr'],
self.global_step,
self.config['decay_steps_lr'],
num_periods=self.config['cycles_lr'],
alpha=0.0,
beta=0.001,
name="learning_rate")"""
lr = tf.constant(self.config['init_lr'])
"""lr = tf.multiply(
tf.multiply(
1e-10,
tf.pow(
10e0,
tf.multiply(
tf.cast(
tf.div(
self.global_step,
10
),
tf.float32
),
1e-1
)
)
),
tf.cast(
tf.mod(
tf.add(
self.global_step,
1
),
2
),
tf.float32
)
)"""
nets["learning_rate"] = lr
self.lr = lr
self.losses[()] = [("learning_rate", self.lr)]
for d in datasets:
rbms[datasets[d]['id']] = {}
nets[datasets[d]['id'] + "_h1"] = []
nets[datasets[d]['id'] + "_v2"] = []
nets[datasets[d]['id'] + "_h2"] = []
self.optimizers[(datasets[d]['id'], )] = []
self.losses[(datasets[d]['id'], )] = []
diff = []
with tf.variable_scope("t_input_" + datasets[d]['id']):
t_input = tf.placeholder(tf.float32, [
datasets[d]['sizes']['batch_size'], None,
datasets[d]['sizes']['num_timesteps'],
datasets[d]['sizes']['num_pitch'],
datasets[d]['sizes']['num_track']
],
name='in_ph_' + datasets[d]['id'])
t_seqlen = tf.placeholder(tf.float32,
[datasets[d]['sizes']['batch_size']],
name='len_ph_' + datasets[d]['id'])
self.input[datasets[d]['id']] = (t_input, t_seqlen)
t_input = tf.floor(t_input + tf.constant(1 - 0.1)) #binarize
nets["t_input"] = tf.transpose(
tf.reshape(t_input,
(-1, datasets[d]['sizes']['num_timesteps'] *
datasets[d]['sizes']['num_pitch'],
datasets[d]['sizes']['num_track'])), [2, 0, 1])
with tf.variable_scope("RBM_" + datasets[d]['id']):
for t in range(datasets[d]['sizes']['num_track']):
rbms[datasets[d]['id']][t + 1] = rbm.rbm(
datasets[d]['id'] + "_" + str(t),
datasets[d]['sizes']['num_timesteps'] *
datasets[d]['sizes']['num_pitch'],
self.config['n_features'], nets["t_input"][t])
rbm.up_nodes(datasets[d]['id'] + "_" + str(t),
rbms[datasets[d]['id']][t + 1], 1)
nets[datasets[d]['id'] +
"_h1"] += [rbms[datasets[d]['id']][t + 1]["h1"]]
with tf.variable_scope("concat_" + datasets[d]['id']):
nets[datasets[d]['id'] + "gen-v0"] = tf.concat(
nets[datasets[d]['id'] + "_h1"], 1)
rbms[datasets[d]['id']][0] = rbm.rbm(
datasets[d]['id'] + "_gen",
datasets[d]['sizes']['num_track'] *
self.config['n_features'], self.config['n_features'],
nets[datasets[d]['id'] + "gen-v0"])
rbm.up_nodes(datasets[d]['id'] + "_gen",
rbms[datasets[d]['id']][0], 1)
nets[datasets[d]['id'] + "gen-v1"] = tf.transpose(
tf.reshape(
rbm.down_nodes(datasets[d]['id'] + "_gen",
rbms[datasets[d]['id']][0], 1),
(-1, datasets[d]['sizes']['num_track'],
self.config['n_features'])), [1, 0, 2])
for t in range(datasets[d]['sizes']['num_track']):
rbm.down_nodes(datasets[d]['id'] + "_" + str(t),
rbms[datasets[d]['id']][t + 1], 1,
nets[datasets[d]['id'] + "gen-v1"][t])
rbm.up_nodes(datasets[d]['id'] + "_" + str(t),
rbms[datasets[d]['id']][t + 1], 2)
nets[datasets[d]['id'] +
"_v2"] += [rbms[datasets[d]['id']][t + 1]["v2"]]
nets[datasets[d]['id'] +
"_h2"] += [rbms[datasets[d]['id']][t + 1]["h2"]]
with tf.variable_scope("concat_" + datasets[d]['id']):
nets[datasets[d]['id'] + "gen_v2"] = tf.concat(
nets[datasets[d]['id'] + "_h2"], 1)
rbm.up_nodes(datasets[d]['id'] + "_gen",
rbms[datasets[d]['id']][0], 2,
nets[datasets[d]['id'] + "gen_v2"])
for t in range(datasets[d]['sizes']['num_track'] + 1):
with tf.variable_scope("losses_" + datasets[d]['id'] + "_" +
str(t)):
if t > 0:
v = tf.reshape(rbms[datasets[d]['id']][t]["v1"], [
-1, datasets[d]['sizes']['num_timesteps'] *
datasets[d]['sizes']['num_pitch'], 1
])
v_sample = tf.reshape(
rbms[datasets[d]['id']][t]["v2"], [
-1, datasets[d]['sizes']['num_timesteps'] *
datasets[d]['sizes']['num_pitch'], 1
])
else:
v = tf.reshape(rbms[datasets[d]['id']][t]["v1"], [
-1, datasets[d]['sizes']['num_track'] *
self.config['n_features'], 1
])
v_sample = tf.reshape(
rbms[datasets[d]['id']][t]["v2"], [
-1, datasets[d]['sizes']['num_track'] *
self.config['n_features'], 1
])
h = tf.reshape(rbms[datasets[d]['id']][t]["h1"],
[-1, 1, self.config['n_features']])
h_sample = tf.reshape(rbms[datasets[d]['id']][t]["h2"],
[-1, 1, self.config['n_features']])
W_adder = tf.reduce_mean(
tf.subtract(tf.matmul(v, h),
tf.matmul(v_sample, h_sample)), 0)
bv_a = tf.reduce_mean(
tf.reduce_sum(tf.subtract(v, v_sample), 2, True), 0)
if t > 0:
bv_adder = tf.reshape(bv_a, [
1, datasets[d]['sizes']['num_timesteps'] *
datasets[d]['sizes']['num_pitch']
])
else:
bv_adder = tf.reshape(bv_a, [
1, datasets[d]['sizes']['num_track'] *
self.config['n_features']
])
bh_adder = tf.reshape(
tf.reduce_mean(
tf.reduce_sum(tf.subtract(h, h_sample), 1, True),
0), [1, self.config['n_features']])
if t > 0:
diff += [bv_adder]
self.losses[(datasets[d]['id'], )] += [
(datasets[d]['id'] + "_" + str(t) + "_W",
tf.reduce_mean(W_adder))
]
self.losses[(datasets[d]['id'], )] += [
(datasets[d]['id'] + "_" + str(t) + "_bv",
tf.reduce_mean(bv_adder))
]
self.losses[(datasets[d]['id'], )] += [
(datasets[d]['id'] + "_" + str(t) + "_bh",
tf.reduce_mean(bh_adder))
]
with tf.variable_scope("optimizers_" + datasets[d]['id']):
self.optimizers[(datasets[d]['id'], )] += [
(0, rbms[datasets[d]['id']][t]["W"].assign_add(
tf.multiply(self.lr, W_adder))),
(0, rbms[datasets[d]['id']][t]["bv"].assign_add(
tf.multiply(self.lr, bv_adder))),
(0, rbms[datasets[d]['id']][t]["bh"].assign_add(
tf.multiply(self.lr, bh_adder)))
]
self.losses[(datasets[d]['id'], )] += [
(datasets[d]['id'] + "_diff",
tf.reduce_mean(tf.concat(diff, 0)))
]
with tf.variable_scope("sample_" + datasets[d]['id']):
sample = tf.concat(nets[datasets[d]['id'] + "_v2"], 1)
sample = tf.reshape(sample, [
datasets[d]['sizes']['batch_size'], -1,
datasets[d]['sizes']['num_track'],
datasets[d]['sizes']['num_timesteps'],
datasets[d]['sizes']['num_pitch']
])
sample = tf.transpose(sample, [0, 1, 3, 4, 2])
nets["sample"] = sample
self.output[(datasets[d]['id'], )] = [
(datasets[d]['id'], "input", t_input),
(datasets[d]['id'], "train_sample", sample)
]
self.savers["RBM_" + datasets[d]['id']] = tf.train.Saver(
tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES,
tf.get_default_graph().get_name_scope() + "/RBM_" +
datasets[d]['id']),
max_to_keep=2)
self.components = nets
self.reset_weights = tf.variables_initializer(
var_list=tf.trainable_variables())
Xtest = np.load("../data/Xtest.npy")
#Load parameters
global WP, WB, WC, d, k
WP = np.load("../models/MNISTWP.npy")
WB = np.load("../models/MNISTWB.npy")
WC = np.load("../models/MNISTWC.npy")
# WP = np.load("../data/WP_learnt.npy")
# WB = np.load("../data/WB_learnt.npy")
# WC = np.load("../data/WC_learnt.npy")
d = len(WC[0])
k = len(WB[0])
#Create RBM
mnistRBM = rbm.rbm()
mnistRBM.set_params(WP, WB, WC)
#Number of Iterations
iterations = 500
num_chains = 100
#Gibbs sampler with single chain
x_samples, _, h_samples = mnistRBM.single_gibbs_sampler(X=Xtrain[0].reshape(
(1, -1)),
iterations=iterations,
WP=WP,
WB=WB,
WC=WC)
def main():
try:
original = mrcfile.open(sys.argv[1], mode='r')
except IOError:
print(
"Could not open the input \nUsage tick_mrc inputfile outputfile.")
sys.exit()
# create list of layers.
layers = []
for layer in original.data:
layers.append(np.float32(layer))
#layers are the arrays containing the data.
the_image = np.zeros_like(layers[0])
the_image = np.add(the_image, layers[40])
# nvis,nhid
hw = 2
nb = 3
the_rbm = rbm.rbm((hw * 2 + 1) * (hw * 2 + 1) * nb, 1000)
the_rbm.its_symmetric()
the_rbm.add_fuzzy(-1., 1., 21)
the_adapted_image = adapt.adaptor(hw, 3., the_image)
the_adapted_image.make_nbit_image(nb)
# for l in the_rbm.layers:
# l = np.add(l, the_adapted_image.next(15,15)[2])
for l in range(0, len(the_rbm.layers)):
the_rbm.layers[l] = np.add(the_rbm.layers[l],
the_adapted_image.random_bits()[2])
the_adapted_image.reset()
print("training starts")
sys.stdout.flush()
# a = the_adapted_image.next(1)
for i in xrange(0, 2000):
a = the_adapted_image.random_bits()
print(i, a[1], the_rbm.train_fuzzy(a[2], 0.1, 1.0, a[1]))
# if i > 1000:
# print a[2]
# print the_rbm.reconstruct(a[2])
# print(i, a[1])
print("training stops")
sys.stdout.flush()
new_image = np.float32(np.zeros_like(the_image))
# for i in xrange(0, the_image.shape[0]-2*hw-1):
for i in xrange(0, 20):
print("layer", i)
sys.stdout.flush()
for j in range(0, the_image.shape[1] - 2 * hw - 1):
# for j in range(0,20):
a = the_adapted_image.at_bits(i, j)
b = the_rbm.reconstruct(a[2])
new_image[i + hw + 1][j + hw + 1] = the_rbm.estimate_EV(a[2])
the_image = Image.fromarray(
np.uint8(rescale(the_adapted_image.the_image, 255.)))
the_image.save('raw.jpg')
the_image = Image.fromarray(np.uint8(rescale(new_image, 255.)))
the_image.save('rbm.jpg')
from load_numpy import getNumpy from test_classifier import svm1 from preprocesses import pca from rbm import rbm from sklearn.cross_validation import train_test_split train_x,train_y,test_x = getNumpy() train_x_transform = pca(train_x,100) X_train, X_valid, Y_train, Y_valid = train_test_split(train_x_transform, train_y,test_size=0.2,random_state=0) # svm1(train_x_transform,train_y) rbm(train_x,train_y)
batchsize = 100
n_h = 340
n_v = 784
eachNLog = 5000
eachNErr = 50
wDecay = 0.0001
hDecay = 0.005
Nepochs = 500
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images,\
mnist.test.labels
statCnt = []
rbm_m1 = rbm.rbm(batchsize, alpha, wDecay, hDecay, n_h, n_v)
#rbm_m2=rbm.rbm(batchsize,alpha, n_h,n_v, 'scope2')
#rbm_m2=rbm.rbm(batchsize,alpha,wDecay,hDecay, n_h,n_v, 'scope55')
#print (rbm_m1.sess.run(rbm_m1.err_sum, feed_dict={rbm_m1.X: trX} ))
#rbm_m1.X=trX
print(rbm_m1.err_sum(trX))
#trX=(trX>0).astype(float)
#rbm_m1.load_weights("C:\\Users\\Andrei\\Documents\\UoGuelphResearch\\MileStones_MRR\\Runs\\GA_ES_DiffOnly_guided_R16_100iterPerGA\\weights-25000")
#print("Loaded")
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
instruction = input("Train model? y/n\n")
if instruction == "y":
os.system("make train_model")
#After training (if training is chosen), we initiate reconstruction of an arbitrary data point from the test set.
weights_filename = "weights.txt"
visiblebias_filename = "visiblebias.txt"
hiddenbias_filename = "hiddenbias.txt"
RBM = rbm(nvisible=28*28, nhidden=14*14, nCDsteps = 1, nepochs = 100)
with open(weights_filename, "r") as infile:
lines = infile.readlines()
i = 0
for line in lines:
vals = line.split()
for j in range(len(vals)):
RBM.weights[i][j] = float(vals[j])
i += 1
print(RBM.weights)
with open(visiblebias_filename, "r") as infile:
lines = infile.readlines()
i = 0
def learn_letters():
import scipy.io as sio
nperclass = 39
classes = [10, 11, 28]
#classes = [10, 13, 28] # A, C, S
nclasses = len(classes)
# Read in the data and prepare it
data = sio.loadmat('binaryalphadigs.mat')
inputs = np.ones((nclasses, nperclass, 20*16))
labels = np.zeros((nclasses, nperclass, nclasses))
for k in range(nclasses):
for m in range(nperclass):
inputs[k,m,:] = (data['dat'][classes[k],m].ravel()).astype('float')
labels[k,m,k] = 1.
nexamples = 20
v = inputs[:,:nexamples,:].reshape(nclasses*nexamples, 20*16)
l = labels[:,:nexamples,:].reshape(nclasses*nexamples, nclasses)
import pylab as pl
# This shows a set of examples from the training set
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(v[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
import algo_rbm as rbm
rb = rbm.rbm(20*16,50,nlabels=None,eta=0.3,momentum=0.5,nCDsteps=3,nepochs=1000)
rb.contrastive_divergence(v)
hid,phid = rb.compute_hidden(v)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
newv = inputs[:,nexamples:,:].reshape(nclasses*(39-nexamples),20*16)
newl = labels[:,nexamples:,:].reshape(nclasses*(39-nexamples),nclasses)
hid,phid = rb.compute_hidden(newv)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
rb = rbm.rbm(20*16,50,nlabels=nclasses,eta=0.3,momentum=0.2,nCDsteps=3,nepochs=1500)
rb.contrastive_divergence(v,l)
rb.classify(v,l)
rb.classify(newv,newl)
hid,phid = rb.compute_hidden(v,l)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
hid,phid = rb.compute_hidden(newv,newl)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
vis = np.random.randn(np.shape(v)[0],np.shape(v)[1])*0.05
for i in range(1000):
hid,phid = rb.compute_hidden(vis,l)
vis,pvis, lab = rb.compute_visible(phid)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
def DBNFit(X, label, numHid, isSaveModels = True, name = "/home/cheng/DBN.npy", isSingleDBN = True, **kwargs) :
"""implement DBN fitting
X :data(np.array)
label : the label of each sample(np.array, in a row)
numHid : the node of each hidden layer(list)"""
H = len(numHid)
m = list()
nArg = len(kwargs)
if H >= 2 :
# train the first rbm model
if nArg >= 1 :
string = "layer" + str(1)
model_ = rbm.rbm(X, numHid[0], **kwargs[string])
else :
model_ = rbm.rbm(X, numHid[0])
m.append(model_)
if isSingleDBN :
for index in range(1, H-1) :
if nArg >= index :
string = "layer" +str(index + 1)
model_ = rbm.rbm(m[index-1].top, numHid[index], **kwargs[string])
else :
model_ = rbm.rbm(m[index-1].top, numHid[index])
m.append(model_)
# train the last rbm model
if nArg >= H :
string = "layer" + str(H)
model_ = rbmFit.rbmFit(m[H-2].top, numHid[H-1], label, **kwargs[string])
else :
model_ = rbmFit.rbmFit(m[H-2].top, numHid[H-1], label)
m.append(model_)
else :
for index in range(1, H) :
if nArg >= index :
string = "layer" +str(index + 1)
model_ = rbm.rbm(m[index-1].top, numHid[index], **kwargs[string])
else :
model_ = rbm.rbm(m[index-1].top, numHid[index])
m.append(model_)
else :
# only a single layer
if isSingleDBN :
if nArg >= 1 :
string = "layer" + str(1)
model_ = rbmFit.rbmFit(X, numHid[0], label, **kwargs[string])
else :
model_ = rbmFit.rbmFit(X, numHid[0], label)
m.append(model_)
else :
if nArg >= 1 :
string = "layer" + str(1)
model_ = rbm.rbm(X, numHid[0], **kwargs[string])
else :
model_ = rbm.rbm(X, numHid[0])
m.append(model_)
if isSaveModels :
models = np.array(m)
np.save(name, models)
return m
RANDOMIN_ = sio.loadmat("randomin_poshidstates.mat",
verify_compressed_data_integrity=False)
RANDOMIN = RANDOMIN_['randomin']
VISHID_ = sio.loadmat("vis_try___.mat", verify_compressed_data_integrity=False)
VISHID = VISHID_['vishid_']
BATCH_DATA_ = sio.loadmat("batchdata_py.mat",
verify_compressed_data_integrity=False)
BATCH_DATA = BATCH_DATA_['batchdata']
NUM_CASES, NUM_DIMS, NUM_BATCHES = BATCH_DATA.shape
# print("BATCH_DATA.shape= ", BATCH_DATA.shape) #100, 784, 600
print('Pretraining Layer 1 with RBM: {}-{}'.format(NUM_DIMS, NUM_HID))
RESTART = 1
BATCHPOSHIDPROBS, VISHID, HIDBIASES, VISBIASES = rbm(BATCH_DATA, RESTART,
NUM_HID, NUM_DIMS,
MAX_EPOCH, RANDOMIN,
VISHID)
print("BATCHPOSHIDPROBS.shape= ", BATCHPOSHIDPROBS.shape) #100, 1000
HIDRECBIASES = HIDBIASES
VISHID__ = VISHID
VISBIASES__ = VISBIASES
print('Pretraining Layer 2 with RBM: {}-{}'.format(NUM_HID, NUMPEN))
BATCH_DATA = BATCHPOSHIDPROBS
NUM_HID = NUMPEN
RESTART = 1
BATCHPOSHIDPROBS, VISHID, HIDBIASES, VISBIASES = rbm(BATCH_DATA, RESTART,
NUM_HID, NUM_DIMS,
MAX_EPOCH, RANDOMIN,
VISHID)
HIDPEN = VISHID
def main():
try:
original = mrcfile.open(sys.argv[1], mode='r')
except IOError:
print(
"Could not open the input \nUsage tick_mrc inputfile outputfile.")
sys.exit()
# create list of layers.
layers = []
for layer in original.data:
layers.append(np.float32(layer))
#layers are the arrays containing the data.
the_image = np.zeros_like(layers[0])
the_image = np.add(the_image, layers[40])
# nvis,nhid
hw = 1
the_rbm = rbm.rbm((hw * 2 + 1) * (hw * 2 + 1), 10)
the_rbm.add_fuzzy(-1., 1., 21)
the_rbm.reinitialize_fuzzy()
the_adapted_image = adapt.adaptor(hw, 3., the_image)
# the_adapted_image.make_nbit_image(3)
# for l in the_rbm.layers:
# l = np.add(l, the_adapted_image.next(15,15)[2])
# for l in range(0, len(the_rbm.layers)):
# the_rbm.layers[l] = np.add( the_rbm.layers[l], the_adapted_image.next(1,1)[2])
the_adapted_image.reset()
print("training starts")
sys.stdout.flush()
# a = the_adapted_image.next(1)
for i in range(0, 1000):
a = the_adapted_image.random()
the_rbm.train(a[2], 0.1, 0.5, a[1])
print(i, a[1])
a = the_adapted_image.random()
i = 0
j = 0
errs = np.float32(np.zeros(100))
while (a[0]):
the_rbm.train_fuzzy(a[2], 0.1, 0.5, a[1])
if i == 1000:
the_rbm.reinitialize_fuzzy()
x = the_rbm.estimate_EV(a[2])
errs[j] = a[1] - x
j = (j + 1) % 100
print(i, a[1], x, errs.std())
# print(i, a[1], the_rbm.estimate_EV(a[2]))
# a = the_adapted_image.next(1,1)
a = the_adapted_image.random()
sys.stdout.flush()
i += 1
print("training stops")
sys.stdout.flush()
# for layer in layers:
# the_image = np.add(the_image,layer)
the_image = Image.fromarray(
np.uint8(rescale(the_adapted_image.the_image, 255.)))
the_image.save('sum.jpg')
def main():
try:
original = mrcfile.open(sys.argv[1],mode='r')
except IOError:
print("Could not open the input \nUsage tick_mrc inputfile outputfile.")
sys.exit()
# create list of layers.
layers = []
for layer in original.data:
layers.append(np.float32(layer))
#layers are the arrays containing the data.
the_image = np.zeros_like(layers[0])
# the_image = np.add(the_image, layers[180])
the_image = np.add(the_image, layers[40])
# nvis,nhid set the size of the rbm
hw = 2
hw = 1
hw = 2
# nb = 3
nb = 8
# the number of outer fuzzy sets
nfuzz = 21
the_fuzz = []
delta = 2./(nfuzz-1)
for i in range(0,nfuzz):
cent = -1. + i*delta
the_fuzz.append( triangle.triangle( cent-delta,cent, cent+delta))
the_fuzz[i].associate_rbm( rbm.rbm( (hw*2+1)*(hw*2+1)*nb, 10))
the_fuzz[i].rbm.its_symmetric()
the_fuzz[i].rbm.add_fuzzy(-1.,1.,21)
# the_rbm = rbm.rbm( (hw*2+1)*(hw*2+1)*nb, 1000)
# the_rbm.its_symmetric()
# the_rbm.add_fuzzy(-1.,1., 21)
the_adapted_image = adapt.adaptor( hw, 3., the_image)
the_adapted_image.make_nbitmap_image(nb)
# for l in the_rbm.layers:
# l = np.add(l, the_adapted_image.next(15,15)[2])
# for l in range(0, len(the_rbm.layers)):
# the_rbm.layers[l] = np.add( the_rbm.layers[l], the_adapted_image.random_bits()[2])
# the_adapted_image.reset()
print("training starts")
sys.stdout.flush()
print("crisp initialization pass")
sys.stdout.flush()
for i in xrange(0,2000):
a = the_adapted_image.random_bits()
for f in the_fuzz:
rate = f.belief(a[1])
if rate > 0. :
print(i,a[1],rate,f.rbm.train(a[2],0.1,rate,a[1]))
print("fuzzy pass")
sys.stdout.flush()
for i in xrange(0,10000):
a = the_adapted_image.random_bits()
for f in the_fuzz:
rate = f.belief(a[1])
if rate > 0. :
print(i,a[1],rate,f.rbm.train_fuzzy(a[2],0.1,rate,a[1]))
print("training stops")
sys.stdout.flush()
new_image = np.float32(np.zeros_like(the_image))
for i in xrange(0, the_image.shape[0]-2*hw-1):
# for i in xrange(0, 20):
print("layer", i)
sys.stdout.flush()
for j in range(0,the_image.shape[1]-2*hw-1):
# for j in range(0,20):
a = the_adapted_image.at_bits(i,j)
r = the_fuzz[0].rbm
b = 1.
fb = the_fuzz[0]
for f in the_fuzz:
bt = f.rbm.the_best_built_layer( a[2])
# print( bt[1],b)
# catch untrained examples
# now done in best_built
tb = bt[1]
# if tb < -1.:
# tb = 0.
if tb < b:
b = tb
r = f.rbm
fb = f
x = r.estimate_EV( a[2])
new_image[i][j] = x
# print( x, a[1], fb.centre)
# sys.stdout.flush()
an_image = Image.fromarray(np.uint8( rescale(new_image,255.)))
an_image.save('rbm_bitmap.jpg')
#
the_image = Image.fromarray(np.uint8( rescale(the_adapted_image.the_image,255.)))
the_image.save('raw.jpg')
the_image = Image.fromarray(np.uint8( rescale(new_image,255.)))
the_image.save('rbm_bitmap.jpg')
test0, test1, test2, test3, test4, test5, test6, test7, test8, test9 = read_mnist.get_each(
test_images, test_labels)
databaches, testbatches = makebatches.generate(train_images, test_images)
'''
train 4 rbms
784->1000->500->250->2
'''
dim = databaches.shape[1]
numlay1 = 1000
numlay2 = 500
numlay3 = 250
numlay4 = 2
rbm1 = rbm.rbm(input=databaches, n_visible=dim, n_hidden=numlay1)
h1 = rbm1.contrastive_divergence(maxepoch=10, k=1)
rbm2 = rbm.rbm(input=h1, n_visible=numlay1, n_hidden=numlay2)
h2 = rbm2.contrastive_divergence(maxepoch=10, k=1)
rbm3 = rbm.rbm(input=h2, n_visible=numlay2, n_hidden=numlay3)
h3 = rbm3.contrastive_divergence(maxepoch=10, k=1)
rbm4 = rbm.rbm(input=h3, n_visible=numlay3, n_hidden=numlay4)
h4 = rbm4.rbmhiddenlinear(maxepoch=10, k=1)
VV, Dim = backprop.fine_tuning(rbm1,
rbm2,
rbm3,
rbm4,
from rbm import rbm, make_binary
print("Downloading dataset...")
mnist = tf.keras.datasets.mnist
(trainX, trainY), (testX, testY) = mnist.load_data()
shuffled_indices = np.random.permutation(60000)
trainX, trainY = trainX[shuffled_indices], trainY[shuffled_indices]
shuffled_indices = np.random.permutation(10000)
testX, testY = testX[shuffled_indices], testY[shuffled_indices]
size_of_dataset = 60000
#Initialize the model
my_rbm = rbm(nvisible=28 * 28,
nhidden=8 * 8,
eta=0.08,
momentum=0.9,
nCDsteps=25,
nepochs=500,
batch_size=100,
size_of_dataset=size_of_dataset)
print("Preparing data...")
training_data = np.zeros((size_of_dataset, 28 * 28))
trainX = trainX / 255.0
trainX = trainX > 0.5
for k in range(size_of_dataset):
training_data[k] = trainX[k].flat[:]
print("Training model...")
start = time.time()
my_rbm.train_model(training_data)
end = time.time()
def learn_letters():
import scipy.io as sio
nperclass = 39
classes = [10, 11, 28]
#classes = [10, 13, 28] # A, C, S
nclasses = len(classes)
# Read in the data and prepare it
data = sio.loadmat('binaryalphadigs.mat')
inputs = np.ones((nclasses, nperclass, 20*16))
labels = np.zeros((nclasses, nperclass, nclasses))
for k in range(nclasses):
for m in range(nperclass):
inputs[k,m,:] = (data['dat'][classes[k],m].ravel()).astype('float')
labels[k,m,k] = 1.
nexamples = 20
v = inputs[:,:nexamples,:].reshape(nclasses*nexamples, 20*16)
l = labels[:,:nexamples,:].reshape(nclasses*nexamples, nclasses)
import pylab as pl
# This shows a set of examples from the training set
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(v[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
import rbm
rb = rbm.rbm(20*16,50,nlabels=None,eta=0.3,momentum=0.5,nCDsteps=3,nepochs=1000)
rb.contrastive_divergence(v)
hid,phid = rb.compute_hidden(v)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
newv = inputs[:,nexamples:,:].reshape(nclasses*(39-nexamples),20*16)
newl = labels[:,nexamples:,:].reshape(nclasses*(39-nexamples),nclasses)
hid,phid = rb.compute_hidden(newv)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
rb = rbm.rbm(20*16,50,nlabels=nclasses,eta=0.3,momentum=0.2,nCDsteps=3,nepochs=1500)
rb.contrastive_divergence(v,l)
rb.classify(v,l)
rb.classify(newv,newl)
hid,phid = rb.compute_hidden(v,l)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
hid,phid = rb.compute_hidden(newv,newl)
vis,pvis,lab = rb.compute_visible(hid)
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(pvis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
vis = np.random.randn(np.shape(v)[0],np.shape(v)[1])*0.05
for i in range(1000):
hid,phid = rb.compute_hidden(vis,l)
vis,pvis, lab = rb.compute_visible(phid)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
