def run_dbn(geral, teste):
#epochs = np.array(([5,10,15,20,200]))
hidden = np.array(([150]))
epochs = np.array(([15, 4]))
ann = DBN(geral.shape[1], geral.shape[1], hidden, epochs)
ann.training_set(geral, geral)
ann.train_dbn()
#ann = DBN.load('dbn_teste.pk1')
dbn_output1 = result(ann.predict(geral), size)
dbn_output2 = result(ann.predict(teste), size)
print ann.predict(geral)
#ann.save(ann, 'dbn_teste')
cv2.imwrite('dbn_result1.png', dbn_output1)
cv2.imwrite('dbn_result2.png', dbn_output2)
def test_another_rbmtrain(self, n):
_dbn = DBN([784, 1000, 500, 250, 30], learning_rate=0.01, cd_k=1)
print(len(mnist.train.images))
for j in range(5):
for i in range(10):
_dbn.pretrain(mnist.train.images[i * 5500:i * 5500 + 5500],
128, 5)
_nnet = NN([784, 1000, 500, 250, 30, 250, 500, 1000, 784], 0.01, 128,
50)
_nnet.load_from_dbn_to_reconstructNN(_dbn)
_nnet.train(mnist.train.images, mnist.train.images)
_nnet.test_linear(mnist.test.images, mnist.test.images)
x_in = mnist.test.images[:30]
_predict = _nnet.predict(x_in)
_predict_img = np.concatenate(np.reshape(_predict, [-1, 28, 28]),
axis=1)
x_in = np.concatenate(np.reshape(x_in, [-1, 28, 28]), axis=1)
img = Image.fromarray((1.0 - np.concatenate(
(_predict_img, x_in), axis=0)) * 255.0)
img = img.convert('L')
img.save(str(n) + '_.jpg')
img2 = Image.fromarray((np.concatenate(
(_predict_img, x_in), axis=0)) * 255.0)
img2 = img2.convert('L')
img2.save(str(n) + '.jpg')
def _perform(self):
p = self.params
class_count = p[0]
layer_sizes = p[1]
# make a copy of rbm-param list because we might
# modify it for training, and we don't want to
# affect the original
rbm_params = list(p[2])
dbn = DBN(layer_sizes, class_count)
# train the first RBM as an RbmJob (to be able to reuse)
# assuming the RBN is more then one layer deep
if len(layer_sizes) >= 2:
first_rbm_params = [class_count, layer_sizes[0], layer_sizes[1]]
first_rbm_params.extend(rbm_params[0])
first_rbm_job = RbmJob(first_rbm_params, self.X_train)
if not first_rbm_job.is_done():
first_rbm_job.perform()
# create the DBN, then replace the first
# RBM with the one already trained
dbn.rbms[0] = first_rbm_job.results()[0]
# make sure it's skipped in DBN training
rbm_params[0] = None
train_res = dbn.pretrain(self.X_train, self.y_train, rbm_params)
return (dbn, train_res)
def test_all(self, n):
_dbn=DBN([784,1000,500,250,30],learning_rate=0.01,cd_k=1)
_dbn.pretrain(mnist.train.images,128,50)
_nnet = NN([784, 1000, 500, 250, 30, 250, 500, 1000, 784], 0.01, 128, 50)
_nnet.load_from_dbn_to_reconstructNN(_dbn)
_nnet.train(mnist.train.images, mnist.train.images)
_nnet.test_linear(mnist.test.images, mnist.test.images)
x_in = mnist.test.images[:30]
_predict = _nnet.predict(x_in)
_predict_img = np.concatenate(np.reshape(_predict, [-1, 28, 28]), axis=1)
x_in = np.concatenate(np.reshape(x_in, [-1, 28, 28]), axis=1)
img = Image.fromarray(
(1.0-np.concatenate((_predict_img, x_in), axis=0))*255.0)
img = img.convert('L')
img.save(str(n)+'_.jpg')
img2 = Image.fromarray(
(np.concatenate((_predict_img, x_in), axis=0))*255.0)
img2 = img2.convert('L')
img2.save(str(n)+'.jpg')
nnet_encoder=NN()
nnet_encoder.load_layers_from_NN(_nnet,0,4)
# featrue=nnet_encoder.predict(mnist.test.images)
nnet_decoder=NN()
nnet_decoder.load_layers_from_NN(_nnet,5,8)
def train_classifier(X, y):
"""
Trains a classifier using best known
parameters on given data / labels.
:param X: Samples, a numpy array of
(N, n_vis) shape where N is number of
samples and n_vis number of visible
varliables (sample dimensionality).
:param y: Labels, a numpy array of
(N, 1) shape. Each lable should be
a label index.
"""
# split data into minibatches
X_mnb, y_mnb = util.create_minibatches(X, y, __CLASS_COUNT * 20)
# create a DBN and pretrain
dbn = DBN([32 * 24, 600, 600], __CLASS_COUNT)
pretrain_params = [[80, 0.05, True, 1, 0.085, 0.1],
[80, 0.05, True, 1, 0.000, 0.0]]
dbn.pretrain(X_mnb, y_mnb, pretrain_params)
# fine-tuning
mlp = dbn.to_mlp()
mlp.train(X_mnb, y_mnb, 1000, 0.1)
return mlp
def completion_by_prediction(t):
train_com_x, test_com_x, train_com_y, test_com_y, train_uncom_x, test_uncom_x = datasets[
t]
train_X = np.concatenate((train_com_x, test_com_x), axis=0)
train_Y = np.concatenate((train_com_y, test_com_y), axis=0)
if t == 0: train_Y, _, prep = preprocess(train_Y, None, 'mm')
x_dim = train_X.shape[1]
y_dim = train_Y.shape[1]
if t == 0:
use_for = 'prediction'
hidden_act_func = ['tanh', 'gauss']
output_act_func = 'affine'
lr = 1e-3
else:
use_for = 'classification'
hidden_act_func = ['tanh', 'gauss']
output_act_func = 'softmax'
lr = 1e-3
tf.reset_default_graph()
classifier = DBN(
hidden_act_func=hidden_act_func,
output_act_func=output_act_func,
loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, x_dim * 40, x_dim * 20, x_dim * 10, x_dim * 2, y_dim],
lr=lr,
use_for=use_for,
bp_algorithm='rmsp',
epochs=240,
batch_size=32,
dropout=0.08,
units_type=['gauss', 'gauss'],
rbm_lr=1e-4,
rbm_epochs=60,
cd_k=1,
pre_train=True)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
classifier.train_model(train_X=train_X, train_Y=train_Y, sess=sess)
train_uncom_y = sess.run(classifier.pred,
feed_dict={
classifier.input_data: train_uncom_x,
classifier.keep_prob: 1.0
})
test_uncom_y = sess.run(classifier.pred,
feed_dict={
classifier.input_data: test_uncom_x,
classifier.keep_prob: 1.0
})
if t == 0:
train_uncom_y = prep.inverse_transform(train_uncom_y.reshape(-1, 1))
test_uncom_y = prep.inverse_transform(test_uncom_y.reshape(-1, 1))
elif t == 1:
train_uncom_y = np.argmax(train_uncom_y, axis=1).reshape(-1, 1)
test_uncom_y = np.argmax(test_uncom_y, axis=1).reshape(-1, 1)
return [train_uncom_y, test_uncom_y]
def load(self, modelnamelist):
DBN.load(self,modelnamelist)
for i in range(self.H):
# print type(self.model[i].W)
self.W.append(self.model[i].W)
# self.W[i].copy_to_host()
# print self.W[i].numpy_array
self.dW.append(cm.CUDAMatrix(np.zeros(self.model[i].W.shape)))
self.W_inc.append(cm.CUDAMatrix(np.zeros(self.model[i].W.shape)))
self.b.append(self.model[i].hb)
self.db.append(cm.CUDAMatrix(np.zeros(self.model[i].hb.shape)))
self.b_inc.append(cm.CUDAMatrix(np.zeros(self.model[i].hb.shape)))
self.h.append(cm.empty((self.model[i].num_hid,self.batchsize)))
def train_top(batch_size, graph_output, joint_train_set, joint_val_set, rng):
top_DBN = DBN(numpy_rng=rng,
n_ins=joint_train_set.get_value().shape[1],
gauss=False,
hidden_layers_sizes=[24],
n_outs=3)
top_DBN.training(joint_train_set,
batch_size,
k=1,
pretraining_epochs=[800, 800],
pretrain_lr=[0.1, 0.1],
validation_set_x=joint_val_set,
graph_output=graph_output)
return top_DBN
def __init__(self,data, labels, testdata, testlabels, devdata, devlabels, featurefolder, postfeaturefolder, config):
DBN.__init__(self,data,labels,testdata,testlabels,config)
self.devdata = devdata
self.devlabels = devlabels
self.batchsize = 256
self.momentum = 0.5
self.featurefolder = featurefolder
self.postfeaturefolder = postfeaturefolder
self.eta = 0.01
self.W = []
self.b = []
self.dW = []
self.db = []
self.W_inc = []
self.b_inc = []
self.bestW = []
self.bestb = []
self.vis = []
self.num_class = self.getClassNum(labels)
self.h = []# a list of pre-allocated CUDAMatrix in order to save gpu RAM
def test_dbn():
log.info('Testing DBN')
# trainset loading
cls_count = 9
X, y, classes = get_data(cls_count=None)
X_mnb, y_mnb = util.create_minibatches(X, y, 20 * cls_count)
# lin_eps = util.lin_reducer(0.05, 0.002, 20)
dbn = DBN([32 * 24, 588, 588], cls_count)
dbn.train(X_mnb, y_mnb, [{
'epochs': 50,
'eps': 0.05,
'spars': 0.05,
'spars_cost': 0.3
}, {
'epochs': 1,
'eps': 0.05
}])
def ruleEncodingAlgorithm(knowledgeBase):
network = []
for l in len(knowledgeBase):
current = RBM(len(knowledgeBase[l][0][0].x), len(knowledgeBase[l][0]))
for i in range(len(knowledgeBase[l])):
for j in range(len(knowledgeBase[l][i].x)):
if knowledgeBase[l][i].x[j] == True:
current.w[i][j] = knowledgeBase[l][i].c
elif knowledgeBase[l][i].x[j] == False:
current.w[i][j] = -knowledgeBase[l][i].c
network.append(current)
dbn = DBN([network[0]._input_size] + [x._output_size for x in network], [],
[])
return dbn.load_from_rbms(len(network) - 1, network)
def train_bottom_layer(train_set,
validation_set,
batch_size=20,
k=1,
layers_sizes=[40],
pretraining_epochs=[800],
pretrain_lr=[0.005],
lambda_1=0.0,
lambda_2=0.1,
rng=None,
graph_output=False):
print('Visible nodes: %i' % train_set.get_value().shape[1])
print('Output nodes: %i' % layers_sizes[-1])
dbn = DBN(numpy_rng=rng,
n_ins=train_set.get_value().shape[1],
hidden_layers_sizes=layers_sizes[:-1],
n_outs=layers_sizes[-1])
dbn.training(train_set,
batch_size,
k=k,
pretraining_epochs=pretraining_epochs,
pretrain_lr=pretrain_lr,
lambda_1=lambda_1,
lambda_2=lambda_2,
validation_set_x=validation_set,
graph_output=graph_output)
output_train_set = dbn.get_output(train_set)
if validation_set is not None:
output_val_set = dbn.get_output(validation_set)
else:
output_val_set = None
return dbn, output_train_set, output_val_set
def track(self, num_bar_states=NUM_BEAT_STATES,
min_bpm=MIN_BPM, max_bpm=MAX_BPM, gmm_model=GMM_MODEL,
tempo_change_probability=TEMPO_CHANGE_PROBABILITY,
norm_observations=NORM_OBSERVATIONS):
"""
Track the conduction with a dynamic Bayesian network.
Parameters for the transition model:
:param num_bar_states: number of cells for one beat period
:param tempo_change_probability: probability of a tempo change between
two adjacent observations
:param min_bpm: minimum tempo used for beat tracking
:param max_bpm: maximum tempo used for beat tracking
Parameters for the observation model:
:param gmm_model: load the fitted GMM model from the
given file
:param norm_observations: normalise the observations
:return: detected beat positions
"""
# convert timing information to tempo spaces
max_tempo = int(np.ceil(max_bpm * num_bar_states / (60. * self.fps)))
min_tempo = int(np.floor(min_bpm * num_bar_states / (60. * self.fps)))
tempo_states = np.arange(min_tempo, max_tempo)
# transition model
tm = TransitionModel(num_bar_states=num_bar_states,
tempo_states=tempo_states,
tempo_change_probability=tempo_change_probability)
# observation model
om = ObservationModel(gmm_model, self.features,
num_states=tm.num_states,
num_bar_states=tm.num_bar_states,
norm_observations=norm_observations,
log_probability=True, norm_probability=True,
min_probability=0.3, max_probability=0.7)
# init the DBN
dbn = DBN(transition_model=tm, observation_model=om)
# save some information (mainly for visualisation)
self.densities = om.densities.astype(np.float)
self.path = dbn.bar_states_path.astype(np.int)
self.bar_start_positions = argrelmin(self.path, mode='wrap')[0] / \
float(self.fps)
# also return the bar start positions
return self.bar_start_positions
def build_(method=1, beta=None):
tf.reset_default_graph()
# Training
if method == 1:
classifier = DBN(
hidden_act_func=['tanh', 'gauss'],
output_act_func='affine',
loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数
struct=[
x_dim, x_dim * 40, x_dim * 20, x_dim * 10, x_dim * 2, y_dim
],
lr=1e-4,
use_for='prediction',
bp_algorithm='rmsp',
epochs=240,
batch_size=32,
dropout=0.08,
units_type=['gauss', 'gauss'],
rbm_lr=1e-4,
rbm_epochs=60,
cd_k=1,
pre_train=True)
elif method == 2:
classifier = supervised_sAE(
output_func='gauss',
hidden_func='affine',
loss_func='mse',
struct=[
x_dim, x_dim * 40, x_dim * 20, x_dim * 10, x_dim * 2, y_dim
],
lr=1e-4,
use_for='prediction',
epochs=180,
batch_size=32,
dropout=0.15,
ae_type='sae', # ae | dae | sae
act_type=[
'gauss', 'affine'
], # decoder：[sigmoid] with ‘cross_entropy’ | [affine] with ‘mse’
noise_type='mn', # Gaussian noise (gs) | Masking noise (mn)
beta=beta, # DAE：噪声损失系数 | SAE：稀疏损失系数 | YAE：Y系数比重
p=0.1, # DAE：样本该维作为噪声的概率 | SAE稀疏性参数：期望的隐层平均活跃度（在训练批次上取平均）
ae_lr=1e-4,
ae_epochs=60,
pre_train=True)
return classifier
def run_(method=1, beta=None):
tf.reset_default_graph()
# Training
if method == 1:
classifier = DBN(
hidden_act_func='gauss',
output_act_func='gauss',
loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, y_dim * 20, y_dim * 10, y_dim],
# struct=[x_dim, int(x_dim/2), int(x_dim/4), int(x_dim/8), y_dim],
lr=1e-4,
use_for='classification',
bp_algorithm='rmsp',
epochs=240,
batch_size=16,
dropout=0.05,
units_type=['gauss', 'gauss'],
rbm_lr=1e-4,
rbm_epochs=45,
cd_k=1,
pre_train=True)
elif method == 2:
classifier = supervised_sAE(
output_func='gauss',
hidden_func='gauss',
loss_func='mse',
struct=[x_dim, y_dim * 60, y_dim * 30, y_dim * 10, y_dim],
lr=1e-4,
use_for='classification',
epochs=240,
batch_size=32,
dropout=0.34,
ae_type='yae', # ae | dae | sae
act_type=[
'gauss', 'affine'
], # decoder：[sigmoid] with ‘cross_entropy’ | [affine] with ‘mse’
noise_type='mn', # Gaussian noise (gs) | Masking noise (mn)
beta=beta, # DAE：噪声损失系数 | SAE：稀疏损失系数 | YAE：Y系数比重
p=0.3, # DAE：样本该维作为噪声的概率 | SAE稀疏性参数：期望的隐层平均活跃度（在训练批次上取平均）
ae_lr=1e-4,
ae_epochs=30,
pre_train=True)
run_sess(classifier, datasets, filename, load_saver='')
return classifier
def __init__(self, dbn):
"""
Prepare the sampler.
"""
if not isinstance(dbn, DBN):
if isinstance(dbn, AbstractBM):
dbn = DBN(dbn)
else:
raise TypeError('DBN or RBM expected.')
self.dbn = dbn
for l in range(len(self.dbn)):
if not hasattr(self.dbn[l], '_ais_logz'):
self.dbn[l]._ais_logz = None
self.dbn[l]._ais_samples = None
self.dbn[l]._ais_logweights = None
DATA_FOLDER = './mnist'
train_dataset = torchvision.datasets.MNIST(root=DATA_FOLDER, train=True, download=False, transform=data_transform)
train_loader = data.DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=2)
test_dataset = torchvision.datasets.MNIST(root=DATA_FOLDER, train=False, download=False, transform=data_transform)
test_loader = data.DataLoader(test_dataset, batch_size=64, shuffle=True, num_workers=2)
#
# data_iter = iter(train_loader)
# images, labels = data_iter.next()
# imshow(torchvision.utils.make_grid(images))
# print(' '.join('%5s' % classes[labels[j]] for j in range(64)))
net = DBN()
print(net)
print(len(net.rbm_layers))
for i in range(len(net.rbm_layers)):
print("-------------No. {} layer's weights-------------".format(i+1))
print(net.rbm_layers[i].weights)
print(len(net.rbm_layers[i].weights))
print("-------------No. {} layer's visible_bias-------------".format(i+1))
print(net.rbm_layers[i].visible_bias)
print(len(net.rbm_layers[i].visible_bias))
print("-------------No. {} layer's hidden_bias-------------".format(i+1))
print(net.rbm_layers[i].hidden_bias)
print(len(net.rbm_layers[i].hidden_bias))
borrow=True)
data_t_sh = theano.shared(np.asarray(data_t, dtype=theano.config.floatX),
borrow=True)
return data_sh, T.cast(data_t_sh, 'int32')
train_data_sh, train_data_t_sh = make_theano_dataset(
(train_data, train_data_t))
valid_data_sh, valid_data_t_sh = make_theano_dataset(
(valid_data, valid_data_t))
test_data_sh, test_data_t_sh = make_theano_dataset((test_data, test_data_t))
datasets = [(train_data_sh, train_data_t_sh), (valid_data_sh, valid_data_t_sh),
(test_data_sh, test_data_t_sh)]
rbm_stack = RBMStack(num_dims, [500])
dbn = DBN(rbm_stack, 2)
batch_size = 100
max_epoch = 10
train_params = {
'batch_size': batch_size,
'learning_rate': 0.01,
'cd_steps': 2,
'max_epoch': max_epoch,
'persistent': True,
'finetune_learning_rate': 0.1
}
pre_fn = dbn.pretrain_fun(train_data_sh, train_params)
num_batches = train_data_sh.get_value(borrow=True).shape[0] / batch_size
if __name__ == '__main__':
hidden_units = 200
names = [ preprocess(line.strip()) for line in open(sys.argv[1],'r') ]
random.shuffle(names)
word_counter = CountVectorizer(
tokenizer=wordpunct_tokenize,
stop_words=stopwords,
binary=True,
dtype=np.byte
)
data = word_counter.fit_transform(names)
words = word_counter.get_feature_names()
data = data.toarray()
print data.shape
_,vocab = data.shape
n = DBN([
Sigmoid(data.shape[1]),
Sigmoid(hidden_units),
Sigmoid(hidden_units/2)
])
n.fit(data,None)
"""
visible = r.run_hidden(np.eye(hidden_units))
out = open('assoc_words','w')
for f in range(hidden_units):
out.write(' '.join( words[i] for i in range(len(words)) if visible[f,i] ) )
out.write('\n')
"""
def test_DBN(finetune_lr=0.01, pretraining_epochs=100,
pretrain_lr=0.1, k=1, training_epochs=100,
batch_size=5):
datasets = loadFeaturedData()
train_set_x, train_set_y = datasets[0]
test_set_x, test_set_y = datasets[1]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
dbn = DBN(numpy_rng=numpy_rng, n_ins=10,
hidden_layers_sizes=[100],
n_outs=2)
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
# Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
# end-snippet-2
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, test_model, test_confmatrix = dbn.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
index = T.lscalar('index') # index to a [mini]batch
# Custom function
stopcheck_model = theano.function(
inputs=[index],
outputs=dbn.finetune_cost,
givens={
dbn.x: test_set_x[index * batch_size:(index + 1) * batch_size],
dbn.y: test_set_y[index * batch_size:(index + 1) * batch_size]
}
)
train_check = theano.function(
inputs=[index],
outputs=dbn.errors,
givens={
dbn.x: train_set_x[index * batch_size: (index + 1) * batch_size],
dbn.y: train_set_y[index * batch_size: (index + 1) * batch_size]
}
)
print '... finetuning the model'
test_score = 0.
start_time = time.clock()
tr_cost = []
te_cost = []
epoch = 0
while (epoch < training_epochs):
epoch = epoch + 1
# go through the training set
d = []
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
d.append(minibatch_avg_cost)
print(('In epoch %d, ') % (epoch))
print 'Training cost = ', numpy.mean(d)
tr_cost.append(numpy.mean(d))
tc = []
for test_batch_index in xrange(n_test_batches):
testing_cost = stopcheck_model(test_batch_index)
tc.append(testing_cost)
print 'Testing cost = ', numpy.mean(tc)
te_cost.append(numpy.mean(tc))
train_losses = [train_check(i) for i in xrange(n_train_batches)]
print 'Training error = ', numpy.mean(train_losses)*100, ' %'
test_losses = test_model()
test_score = numpy.mean(test_losses)
test_confmatrices = test_confmatrix()
test_confelement = numpy.sum(test_confmatrices, axis=0)
true_pos = test_confelement[0]
true_neg = test_confelement[1]
false_pos = test_confelement[2]
false_neg = test_confelement[3]
f_score = (true_pos + true_neg)/float(true_pos + true_neg + false_pos + 5*false_neg)
print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))
end_time = time.clock()
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time)
/ 60.))
x_axis = numpy.arange(training_epochs)
plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost), '.')
plt.show()
y_dim = Y_train.shape[1]
p_dim = int(np.sqrt(x_dim))
sess = tf.Session()
# Training
select_case = 1
if select_case == 1:
classifier = DBN(
hidden_act_func='relu',
output_act_func='softmax',
loss_func='cross_entropy', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, 200, 100, y_dim],
lr=1e-3,
momentum=0.5,
use_for='classification',
bp_algorithm='rmsp',
epochs=100,
batch_size=32,
dropout=0.1,
units_type=['gauss', 'bin'],
rbm_lr=1e-3,
rbm_epochs=16,
cd_k=1)
if select_case == 2:
classifier = CNN(
output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
lr=1e-3,
epochs=30,
def test_DBN(finetune_lr=0.1, decay=False, training_epochs=100,
pretraining_epochs=10, pretrain_lr=0.01, k=1,
batch_size=10):
"""
Demonstrates how to train and test a Deep Belief Network.
This is demonstrated on CIFAR-10.
:type finetune_lr: float
:param finetune_lr: learning rate used in the finetune stage
:type decay: boolean
:param decay: whether use weight decay or not in the finetune stage
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type k: int
:param k: number of Gibbs steps in CD/PCD
:type training_epochs: int
:param training_epochs: maximal number of iterations ot run the optimizer
:type batch_size: int
:param batch_size: the size of a minibatch
"""
datasets = load_data()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
# construct the Deep Belief Network
dbn = DBN(numpy_rng=numpy_rng, n_ins=32 * 32 * 3,
hidden_layers_sizes=[1000, 1000, 1000],
n_outs=10)
# start-snippet-2
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
# end-snippet-2
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = dbn.build_finetune_functions(
datasets=datasets,
batch_size=batch_size
)
print '... finetuning the model'
# early-stopping parameters
patience = 4 * n_train_batches # look as this many examples regardless
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatches before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
epoch = 0
start_time = time.clock()
while (epoch < training_epochs):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
tlearning_rate = finetune_lr
if decay:
tlearning_rate = finetune_lr/(1.0 + epoch / 10.0)
minibatch_avg_cost = train_fn(minibatch_index, tlearning_rate)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print(
'epoch %i, minibatch %i/%i, validation error %f %%, learning rate %f'
% (
epoch,
minibatch_index + 1,
n_train_batches,
this_validation_loss * 100.,
tlearning_rate
)
)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
end_time = time.clock()
print(
(
'Optimization complete with best validation score of %f %%, '
'obtained at iteration %i, '
'with test performance %f %%'
) % (best_validation_loss * 100., best_iter + 1, test_score * 100.)
)
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time)
/ 60.))
def test_DBN(finetune_lr=0.001, pretraining_epochs=100,
pretrain_lr=0.001, k=1, training_epochs=100,
batch_size=25):
"""
Demonstrates how to train and main a Deep Belief Network.
This is demonstrated on MNIST.
:type finetune_lr: float
:param finetune_lr: learning rate used in the finetune stage
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type k: int
:param k: number of Gibbs steps in CD/PCD
:type training_epochs: int
:param training_epochs: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
:type batch_size: int
:param batch_size: the size of a minibatch
"""
# datasets = loadFeaturedData()
datasets = load10secData()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
# construct the Deep Belief Network
# Test for debugging
# dbn = DBN(numpy_rng=numpy_rng, n_ins=10,
# hidden_layers_sizes=[100, 100, 100],
# n_outs=2)
dbn = DBN(numpy_rng=numpy_rng, n_ins=7500,
hidden_layers_sizes=[5000, 5000, 1000],
n_outs=2)
# start-snippet-2
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
# end-snippet-2
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model, test_confmatrix = dbn.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
print '... finetuning the model'
# early-stopping parameters
patience = 4 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatches before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print(
'epoch %i, minibatch %i/%i, validation error %f %%'
% (
epoch,
minibatch_index + 1,
n_train_batches,
this_validation_loss * 100.
)
)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# main it on the main set
test_losses = test_model()
test_score = numpy.mean(test_losses)
test_confmatrices = test_confmatrix()
test_confelement = numpy.sum(test_confmatrices, axis=0)
true_pos = test_confelement[0]
true_neg = test_confelement[1]
false_pos = test_confelement[2]
false_neg = test_confelement[3]
print(('epoch %i, minibatch %i/%i, main error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(
(
'Optimization complete with best validation score of %f %%, '
'obtained at iteration %i, '
'with main performance %f %% '
) % (best_validation_loss * 100., best_iter + 1, test_score * 100.)
)
print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time)
/ 60.))
def make_theano_dataset(datasets):
data = datasets[0]
data_t = datasets[1]
data_sh = theano.shared(np.asarray(data, dtype=theano.config.floatX), borrow=True)
data_t_sh = theano.shared(np.asarray(data_t, dtype=theano.config.floatX), borrow=True)
return data_sh, T.cast(data_t_sh, "int32")
train_data_sh, train_data_t_sh = make_theano_dataset((train_data, train_data_t))
valid_data_sh, valid_data_t_sh = make_theano_dataset((valid_data, valid_data_t))
test_data_sh, test_data_t_sh = make_theano_dataset((test_data, test_data_t))
datasets = [(train_data_sh, train_data_t_sh), (valid_data_sh, valid_data_t_sh), (test_data_sh, test_data_t_sh)]
rbm_stack = RBMStack(num_dims, [500])
dbn = DBN(rbm_stack, 2)
batch_size = 100
max_epoch = 10
train_params = {
"batch_size": batch_size,
"learning_rate": 0.01,
"cd_steps": 2,
"max_epoch": max_epoch,
"persistent": True,
"finetune_learning_rate": 0.1,
}
pre_fn = dbn.pretrain_fun(train_data_sh, train_params)
num_batches = train_data_sh.get_value(borrow=True).shape[0] / batch_size
return ret_dat
traind= norm(datas[:700,:-1].astype('float32'))
trainl= one_hot(datas[:700,-1].astype('float64'))
# print(traind[0]);print(trainl[0])
testd = norm(datas[700:,:-1].astype('float32'))
testl = one_hot(datas[700:,-1].astype('float64'))
dataset=[traind,trainl,testd,testl]
classifier = DBN(
hidden_act_func='sigmoid',
output_act_func='softmax',
loss_func='cross_entropy', # gauss 激活函数会自动转换为 mse 损失函数
struct=[26, 11, 6, 11, 11,3],
lr=1e-3,
momentum=0.8,
use_for='classification',
bp_algorithm='rmsp',
epochs=100,
batch_size=20,
dropout=0.12,
units_type=['gauss','bin'],
rbm_lr=1e-3,
rbm_epochs=0,
cd_k=1,
pre_train=True)
run_sess(classifier,dataset,filename,load_saver='')
label_distribution = classifier.label_distribution
def calculate_hash(autoencoder_directory, bag_of_words):
numpy_rng = np.random.RandomState(123)
autoencoder = DBN.load_dbn_data(autoencoder_directory, numpy_rng)
bow_array = np.asarray([bag_of_words.values()], dtype="float32")
hash = autoencoder.propup_function(bow_array, is_autoencoder=True)
return hash
def test_multitask_dbn_mnist(learning_rate=0.01, training_epochs=10, batch_size=20,
finetune_lr=0.05, finetune_epoch=10, output_folder=None, dropout=0.2,
model_structure=[500, 100], isPCD=0, k=1, plot_weight=False):
"""
test_dbn_mnist(output_folder='/home/eric/Desktop/dbn_plots',
training_epochs=10,
model_structure=[500, 100],
finetune_epoch=10)
"""
assert output_folder is not None
from dbn_variants import MultiTask_DBN as DBN
#################################
# Data Constructing #
#################################
from sklearn.datasets import fetch_mldata
from xylearn.utils.data_util import get_train_test
from xylearn.utils.data_normalization import rescale
dataset = fetch_mldata('MNIST original')
data = get_train_test(rescale(dataset.data), dataset.target, useGPU=1, shuffle=True)
train_x, train_y = data['train']
test_x, test_y = data['test']
num_of_classes = max(train_y.eval()) - min(train_y.eval()) + 1
# man-made task label for testing
n_outs = [num_of_classes, num_of_classes, num_of_classes]
n_vis = train_x.get_value(borrow=True).shape[1]
n_train_batches = train_x.get_value(borrow=True).shape[0] / batch_size
print '... building the model'
dbn = DBN(n_ins=n_vis, n_outs=n_outs, dropout=dropout,
hidden_layers_sizes=model_structure, isPCD=isPCD)
print '... getting the pretraining functions'
pretrain_fns = dbn.get_pretrain_fns(train_set_x=train_x,
batch_size=batch_size,
k=k)
#################################
# Training the DBN #
#################################
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
plotting_time = 0.
start_time = time.clock()
import PIL.Image
from visualizer import tile_raster_images
# go through training epochs
for epoch in xrange(training_epochs):
# go through the training set
mean_cost = []
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for batch_index in xrange(n_train_batches):
# for each batch, we extract the gibbs chain
new_cost = pretrain_fns[i](index=batch_index, lr=learning_rate)
mean_cost += [new_cost]
if plot_weight:
# plot all weights
weight = dbn.project_back(dbn.sigmoid_layers[i].W.T,
from_layer_i=i)
# Construct image from the weight matrix
image = PIL.Image.fromarray(tile_raster_images(
X=weight,
img_shape=(dbn.input_length, dbn.input_height), tile_shape=(10, 10),
tile_spacing=(1, 1)))
image.save('pretrain_epoch_%i_layer_%i.png' % (epoch, i))
print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost)
end_time = time.clock()
pretraining_time = end_time - start_time
print ('Training took %f minutes' % (pretraining_time / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
# man-made labels, list of labels
train_ys = [train_y, train_y, train_y]
finetune_fn = dbn.get_finetune_fn(train_x, train_ys, batch_size)
print '... finetunning the model'
test_score = 0.
start_time = time.clock()
# go through the each batch
for epoch in xrange(finetune_epoch):
## Pre-train layer-wise
c = []
for batch_index in xrange(n_train_batches):
c.append(finetune_fn(index=batch_index, lr=finetune_lr))
if plot_weight:
# plot all weights
weight = dbn.project_back(dbn.sigmoid_layers[-1].W.T,
from_layer_i=dbn.n_layers - 1)
# Construct image from the weight matrix
image = PIL.Image.fromarray(tile_raster_images(
X=weight,
img_shape=(dbn.input_length, dbn.input_height), tile_shape=(10, 10),
tile_spacing=(1, 1)))
image.save('finetune_epoch_%i.png' % (epoch))
print 'Fine-tuning epoch %i, model cost ' % epoch,
print numpy.mean(c)
test_score = numpy.mean(c)
end_time = time.clock()
print(('Optimization complete with best test performance %f %%') %
(100. - test_score * 100.))
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time)
/ 60.))
########################
# Test THE MODEL #
########################
task_id = 0
print '\nTest complete with error rate ' + str(dbn.get_error_rate(test_x, test_y, task_id=task_id)) + \
' for task_' + str(task_id)
train_data_sh, train_data_t_sh = make_theano_dataset((train_data, train_data_t))
valid_data_sh, valid_data_t_sh = make_theano_dataset((valid_data, valid_data_t))
test_data_sh, test_data_t_sh = (None, None) #make_theano_dataset((test_data, test_data_t))
train_data_wp_sh, train_data_wp_t_sh = make_theano_dataset((data_wo_p, data_target_wo_p))
datasets = [(train_data_sh, train_data_t_sh),(valid_data_sh, valid_data_t_sh),(test_data_sh, test_data_t_sh)]
batch_size = 100
max_epoch = 30
train_params = {'batch_size' : batch_size, 'learning_rate' : 0.1, 'cd_steps' : 10, 'max_epoch' : max_epoch, 'persistent' : True, 'cd_steps_line' : 1, 'learning_rate_line' : 0.001, 'finetune_learning_rate' : 0.1, 'validation_frequency' : batch_size }
stack_rbm = RBMStack(num_vis = num_vis, hid_layers_size = hid_layers_size)
if not load_from_file(stack_rbm, train_params):
stack_rbm.pretrain(data_sh, train_params)
save_to_file(stack_rbm, train_params)
dbn = DBN(stack_rbm, 2)
#dbn.finetune(datasets, train_params)
#ae = AutoEncoder(stack_rbm, 100)
#ae.pretrain(train_data_sh, train_params)
#ae.finetune(train_data_sh, train_params)
get_output = theano.function([], dbn.stack[-1].output, givens=[(dbn.input,train_data_sh)])
out = get_output()
np.savetxt("/home/alexeyche/prog/sentiment/out.tsv", train_data, delimiter="\t")
np.savetxt("/home/alexeyche/prog/sentiment/answer.tsv",train_data_t,delimiter=",")
def calculate_hash(autoencoder_directory, bag_of_words):
numpy_rng = np.random.RandomState(123)
autoencoder = DBN.load_dbn_data(autoencoder_directory, numpy_rng)
bow_array = np.asarray([bag_of_words.values()], dtype="float32")
hash = autoencoder.propup_function(bow_array, is_autoencoder=True)
return hash
def main(argv):
# load preprocessed data samples
print 'loading data...\t',
data = load('../../data/vanhateren.npz')
print '[DONE]'
print
# remove DC component (first component)
data_train = data['train'][1:, :]
data_test = data['test'][1:, :]
# create 1st layer
dbn = DBN(GaussianRBM(num_visibles=data_train.shape[0], num_hiddens=100))
# hyperparameters
dbn[0].learning_rate = 5E-3
dbn[0].weight_decay = 1E-2
dbn[0].momentum = 0.9
dbn[0].sigma = 0.65
dbn[0].cd_steps = 1
dbn[0].persistent = True
# train 1st layer
print 'training...\t',
dbn.train(data_train, num_epochs=100, batch_size=100)
print '[DONE]'
# evaluate 1st layer
print 'evaluating...\t',
logptf = dbn.estimate_log_partition_function(num_ais_samples=100, beta_weights=arange(0, 1, 1E-3))
loglik = dbn.estimate_log_likelihood(data_test)
print '[DONE]'
print
print 'estimated log-partf.:\t', logptf
print 'estimated log-loss:\t', -loglik / data_test.shape[0] / log(2)
print
# create 2nd layer
dbn.add_layer(SemiRBM(num_visibles=100, num_hiddens=100))
# initialize parameters
dbn[1].L = dbn[0].W.T * dbn[0].W
dbn[1].b = dbn[0].W.T * dbn[0].b + dbn[0].c + 0.5 * asmatrix(diag(dbn[1].L)).T
dbn[1].L = dbn[1].L - asmatrix(diag(diag(dbn[1].L)))
# hyperparameters
dbn[1].learning_rate = 5E-3
dbn[1].learning_rate_lateral = 5E-4
dbn[1].weight_decay = 5E-3
dbn[1].weight_decay_lateral = 5E-3
dbn[1].momentum = 0.9
dbn[1].momentum_lateral = 0.9
dbn[1].num_lateral_updates = 20
dbn[1].damping = 0.2
dbn[1].cd_steps = 1
dbn[1].persistent = True
# train 2nd layer
print 'training...\t',
dbn.train(data_train, num_epochs=100, batch_size=100)
print '[DONE]'
# evaluate 2nd layer
print 'evaluating...\t',
logptf = dbn.estimate_log_partition_function(num_ais_samples=100, beta_weights=arange(0, 1, 1E-3))
loglik = dbn.estimate_log_likelihood(data_test, num_samples=100)
print '[DONE]'
print
print 'estimated log-partf.:\t', logptf
print 'estimated log-loss:\t', -loglik / data_test.shape[0] / log(2)
print
# fine-tune with wake-sleep
dbn[0].learning_rate /= 4.
dbn[1].learning_rate /= 4.
print 'fine-tuning...\t',
dbn.train_wake_sleep(data_train, num_epochs=10, batch_size=10)
print '[DONE]'
# reevaluate
print 'evaluating...\t',
logptf = dbn.estimate_log_partition_function(num_ais_samples=100, beta_weights=arange(0, 1, 1E-3))
loglik = dbn.estimate_log_likelihood(data_test, num_samples=100)
print '[DONE]'
print
print 'estimated log-partf.:\t', logptf
print 'estimated log-loss:\t', -loglik / data_test.shape[0] / log(2)
return 0
x_dim = X_train.shape[1]
y_dim = Y_train.shape[1]
p_dim = int(np.sqrt(x_dim))
sess = tf.Session()
# Training
select_case = 2
if select_case == 1:
classifier = DBN(output_act_func='softmax',
loss_func='cross_entropy',
use_for='classification',
bp_algorithm='adam',
dbn_lr=1e-3,
momentum=0.5,
dbn_epochs=100,
dbn_struct=[x_dim, 200, 100, y_dim],
rbm_v_type='bin',
rbm_epochs=12,
batch_size=32,
cd_k=1,
rbm_lr=1e-3,
dropout=1)
if select_case == 2:
classifier = CNN(
output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
cnn_lr=1e-3,
cnn_epochs=30,
img_shape=[p_dim, p_dim],
import sys,re,random import numpy as np from dbn import DBN from dbn.layers import * import theano.tensor as T import theano if __name__ == '__main__': data = np.hstack((np.eye(8),np.arange(8).reshape((8,1)))) data = np.vstack(100*(data,)) np.random.shuffle(data) net = DBN([ OneHotSoftmax(8), Sigmoid(3) ],8,max_epochs=1000) net.fit(data[:,:-1],data[:,-1]) print net.predict(np.eye(8,dtype=np.float32))
y_dim = datasets[1].shape[1]
p_dim = int(np.sqrt(x_dim))
tf.reset_default_graph()
# Training
select_case = 1
if select_case == 1:
classifier = DBN(
hidden_act_func='sigmoid',
output_act_func='softmax',
loss_func='cross_entropy', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, 400, 200, 100, y_dim],
lr=1e-3,
momentum=0.5,
use_for='classification',
bp_algorithm='rmsp',
epochs=10,
batch_size=32,
dropout=0.12,
units_type=['gauss', 'bin'],
rbm_lr=1e-3,
rbm_epochs=5,
cd_k=1)
if select_case == 2:
classifier = CNN(
output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
lr=1e-3,
epochs=20,
def test_DBN(finetune_lr=0.1,
pretraining_epochs=100,
pretrain_lr=0.1,
k=1,
training_epochs=5000,
batch_size=5):
datasets = load10sec_ECGII_data()
pretrain_set_x = datasets[0]
trdatasets = [datasets[1], datasets[2]]
train_set_x, train_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
dbn = DBN(numpy_rng=numpy_rng,
n_ins=2500,
hidden_layers_sizes=[1000, 1000, 1000],
n_outs=2)
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=pretrain_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
# Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
# end-snippet-2
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, train_cost_model, test_score_model, test_cost_model, test_confmatrix = dbn.build_finetune_functions(
datasets=trdatasets,
batch_size=batch_size,
)
print '... finetuning the model'
tr_cost = []
te_cost = []
test_score = 0.
start_time = time.clock()
epoch = 0
while (epoch < training_epochs):
epoch = epoch + 1
learning_rate = 0.01 / (1 + 0.001 * epoch)
# go through the training set
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index, learning_rate)
print(('In epoch %d,') % (epoch))
epoch_trcost = numpy.mean(train_cost_model())
epoch_tecost = numpy.mean(test_cost_model())
print 'Training cost = ', epoch_trcost
tr_cost.append(epoch_trcost)
print 'Testing cost = ', epoch_tecost
te_cost.append(epoch_tecost)
test_losses = test_score_model()
test_score = numpy.mean(test_losses)
test_confmatrices = test_confmatrix()
test_confelement = numpy.sum(test_confmatrices, axis=0)
true_pos = test_confelement[0]
true_neg = test_confelement[1]
false_pos = test_confelement[2]
false_neg = test_confelement[3]
f_score = (true_pos + true_neg) / float(true_pos + true_neg + false_pos +
5 * false_neg)
print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
print(('TP %i, TN %i, FP %i, FN %i') %
(true_pos, true_neg, false_pos, false_neg))
end_time = time.clock()
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
x_axis = numpy.arange(training_epochs)
plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost),
'.')
plt.show()
def test_DBN(finetune_lr=0.1, pretraining_epochs=100,
pretrain_lr=0.1, k=1, training_epochs=5000,
batch_size=5):
datasets = load10sec_ECGII_data()
pretrain_set_x = datasets[0]
trdatasets = [datasets[1], datasets[2]]
train_set_x, train_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
dbn = DBN(numpy_rng=numpy_rng, n_ins=2500,
hidden_layers_sizes=[1000, 1000, 1000],
n_outs=2)
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=pretrain_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
# Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
# end-snippet-2
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, train_cost_model, test_score_model, test_cost_model, test_confmatrix = dbn.build_finetune_functions(
datasets=trdatasets,
batch_size=batch_size,
)
print '... finetuning the model'
tr_cost = []
te_cost = []
test_score = 0.
start_time = time.clock()
epoch = 0
while (epoch < training_epochs):
epoch = epoch + 1
learning_rate = 0.01/(1+0.001*epoch)
# go through the training set
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index, learning_rate)
print(('In epoch %d,') % (epoch))
epoch_trcost = numpy.mean(train_cost_model())
epoch_tecost = numpy.mean(test_cost_model())
print 'Training cost = ', epoch_trcost
tr_cost.append(epoch_trcost)
print 'Testing cost = ', epoch_tecost
te_cost.append(epoch_tecost)
test_losses = test_score_model()
test_score = numpy.mean(test_losses)
test_confmatrices = test_confmatrix()
test_confelement = numpy.sum(test_confmatrices, axis=0)
true_pos = test_confelement[0]
true_neg = test_confelement[1]
false_pos = test_confelement[2]
false_neg = test_confelement[3]
f_score = (true_pos + true_neg)/float(true_pos + true_neg + false_pos + 5*false_neg)
print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
print(('TP %i, TN %i, FP %i, FN %i') % (true_pos, true_neg, false_pos, false_neg))
end_time = time.clock()
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time)
/ 60.))
x_axis = numpy.arange(training_epochs)
plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost), '.')
plt.show()
from data import MNIST
from dbn import DBN
import numpy as np
# An example of how to use the library
# Creates a DBN
d = DBN([28 * 28, 500, 500, 2000], 10, 0.1)
# Loads the Images and Labels from the MNIST database
images = MNIST.load_images('images')
labels = MNIST.load_labels('labels')
# Gets the training set of images
img = images[0:60000]
lbl = labels[0:60000]
# Permutes the data to ensure random batches
train_tuples = zip(img, lbl)
train_tuples = np.random.permutation(train_tuples)
(img, lbl) = zip(*train_tuples)
# Loads the test images from the MNIST database
tst_img = MNIST.load_images('test_images')
tst_lbl = MNIST.load_labels('test_labels')
# Pre trains the DBN
d.pre_train(img, 5, 50)
# Executes a 100 iterations of label training
# Attempts to classify and prints the error rate
# Splitting data
X_train, X_test, Y_train, Y_test = mnist.train.images,mnist.test.images ,mnist.train.labels, mnist.test.labels
# X_train, X_test = X_train[::100], X_test[::100]
# Y_train, Y_test = Y_train[::100], Y_test[::100]
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Training
classifier = DBN(output_act_func='softmax',
hidden_act_func='relu',
loss_fuc='cross_entropy',
use_for='classification',
dbn_lr=1e-3,
dbn_epochs=100,
dbn_struct=[784, 100, 100,10],
rbm_h_type='bin',
rbm_epochs=10,
batch_size=32,
cd_k=1,
rbm_lr=1e-3)
classifier.build_dbn()
classifier.train_dbn(X_train, Y_train,sess)
# Test
Y_pred = classifier.test_dbn(X_test, Y_test,sess)
#import matplotlib.pyplot as plt
#PX=range(0,len(Y_pred))
#plt.figure(1) # 选择图表1
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Training
select_case = 1
# gauss 激活函数会自动转换为 mse 损失函数
if select_case == 1:
classifier = DBN(output_act_func='softmax',
hidden_act_func='relu',
loss_fuc='cross_entropy',
use_for='classification',
dbn_lr=1e-3,
dbn_epochs=100,
dbn_struct=[dim, 100, 100, fault],
rbm_v_type='bin',
rbm_epochs=10,
batch_size=32,
cd_k=10,
rbm_lr=1e-3,
dropout=0.95)
if select_case == 2:
classifier = CNN(output_act_func='softmax',
hidden_act_func='relu',
loss_fuc='cross_entropy',
use_for='classification',
cnn_lr=1e-3,
cnn_epochs=100,
img_shape=[dynamic, 52],
channels=[1, 6, 6, 64, fault],
from data import MNIST
from dbn import DBN
import numpy as np
d = DBN([28 * 28, 500, 500, 2000], 10, 0.1)
images = MNIST.load_images("images")
labels = MNIST.load_labels("labels")
img = images[0:60000]
lbl = labels[0:60000]
tst_img = MNIST.load_images("test_images")
tst_lbl = MNIST.load_labels("test_labels")
d.pre_train(img, 5, 50)
for i in xrange(0, 100):
d.train_labels(img, lbl, 50, 50)
tst_class = d.classify(tst_img, 10)
print "Error over test data: {0}".format(1 - (tst_class * tst_lbl).mean() * 10)
# print d.sample(img[0:1],0)
# print d.sample(img[1:2],0)
# print 'layer 2'
# print d.sample(img[0:1],1)
# print d.sample(img[1:2],2)
# print 'layer 2'
# print d.sample(img[0:1],2)
# print d.sample(img[1:2],2)
# print 'layer 3'
# print d.sample(img[0:1],3)[0]
# print d.sample(img[1:2],3)[0]
#X_train, X_test = X_train[::100], X_test[::100]
#Y_train, Y_test = Y_train[::100], Y_test[::100]
x_dim=X_train.shape[1]
y_dim=Y_train.shape[1]
p_dim=int(np.sqrt(x_dim))
sess = tf.Session()
# Training
select_case = 1
if select_case==1:
<<<<<<< HEAD
classifier = DBN(
hidden_act_func='sigmoid',
output_act_func='softmax',
loss_func='cross_entropy', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, 100, 50, y_dim],
lr=1e-3,
=======
classifier = DBN(output_act_func='softmax',
loss_func='cross_entropy',
use_for='classification',
bp_algorithm='adam',
dbn_lr=1e-3,
>>>>>>> 4264eee5bcc3304abc64f7a1b22cf3c5b3cd37f4
momentum=0.5,
use_for='classification',
bp_algorithm='adam',
epochs=100,
batch_size=32,
dropout=0.3,
from data import MNIST
from dbn import DBN
import numpy as np
# Creates a DBN
d = DBN([28*28, 500,500, 2000], 10, 0.1)
# Loads the Images and Labels from the MNIST database
images = MNIST.load_images('images')
labels = MNIST.load_labels('labels')
# Gets the training set of images
img = images[0:60000]
lbl = labels[0:60000]
# Loads the test images from the MNIST database
tst_img = MNIST.load_images('test_images')
tst_lbl = MNIST.load_labels('test_labels')
# Pre trains the DBN
d.pre_train(img,5,50)
# Executes a 100 iterations of label training
# Attempts to classify and prints the error rate
for i in xrange(0, 100):
d.train_labels(img, lbl, 50, 50)
tst_class = d.classify(tst_img,10)
print 'Error over test data: {0}'.format(1 - (tst_class*tst_lbl).mean() * dbn.number_labels)
# Tests the DBN on test images
x_dim = X_train.shape[1]
y_dim = Y_train.shape[1]
p_dim = int(np.sqrt(x_dim))
sess = tf.Session()
# Training
select_case = 2
if select_case == 1:
classifier = DBN(output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
dbn_lr=1e-3,
dbn_epochs=100,
dbn_struct=[x_dim, 200, 100, y_dim],
rbm_v_type='bin',
rbm_epochs=10,
batch_size=32,
cd_k=1,
rbm_lr=1e-3)
if select_case == 2:
classifier = CNN(
output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
cnn_lr=1e-3,
cnn_epochs=30,
img_shape=[p_dim, p_dim],
channels=[1, 6, 6, 64, y_dim], # 前几维给 ‘Conv’ ，后几维给 ‘Full connect’
return ' '.join(sentence)
if __name__ == '__main__':
hidden_units = 200
names = [preprocess(line.strip()) for line in open(sys.argv[1], 'r')]
random.shuffle(names)
word_counter = CountVectorizer(tokenizer=wordpunct_tokenize,
stop_words=stopwords,
binary=True,
dtype=np.byte)
data = word_counter.fit_transform(names)
words = word_counter.get_feature_names()
data = data.toarray()
print data.shape
_, vocab = data.shape
n = DBN([
Sigmoid(data.shape[1]),
Sigmoid(hidden_units),
Sigmoid(hidden_units / 2)
])
n.fit(data, None)
"""
visible = r.run_hidden(np.eye(hidden_units))
out = open('assoc_words','w')
for f in range(hidden_units):
out.write(' '.join( words[i] for i in range(len(words)) if visible[f,i] ) )
out.write('\n')
"""
# Training and CV sets
trX = tr1X[:50000]
trY = tr0Y[:50000]
cvX = tr1X[50000:]
cvY = tr0Y[50000:]
# RBMs
rbmobject1 = RBM(FLAGS.out_dir, 784, 900, ['rbmw1', 'rbvb1', 'rbmhb1'], 0.3,
tf.nn.sigmoid)
rbmobject2 = RBM(FLAGS.out_dir, 900, 500, ['rbmw2', 'rbvb2', 'rbmhb2'], 0.3,
tf.nn.sigmoid)
rbmobject1.restore_weights('./out/rbmw1.chp')
rbmobject2.restore_weights('./out/rbmw2.chp')
# DBN
y_colNb = 1
dbn = DBN(FLAGS.out_dir, [rbmobject1, rbmobject2], 784, y_colNb,
FLAGS.class_nb, FLAGS.batchsize, FLAGS.epochs, 0.3)
#dbn.restore_trained_net('./out/dbn.chp')
#i=8
#dbn.predict(teX[i], teY[i])
#exit()
iterations = len(trX) / FLAGS.batchsize
## Train DBN
print 'Training DBN: ', FLAGS.epochs, ' epochs of ', iterations, ' iterations'
dbn.train(trX, trY, cvX, cvY, teX, teY)
dbn.save_trained_net('./out/dbn.chp')
