def run_dbn(geral, teste):
#epochs = np.array(([5,10,15,20,200]))
hidden = np.array(([150]))
epochs = np.array(([15, 4]))
ann = DBN(geral.shape[1], geral.shape[1], hidden, epochs)
ann.training_set(geral, geral)
ann.train_dbn()
#ann = DBN.load('dbn_teste.pk1')
dbn_output1 = result(ann.predict(geral), size)
dbn_output2 = result(ann.predict(teste), size)
print ann.predict(geral)
#ann.save(ann, 'dbn_teste')
cv2.imwrite('dbn_result1.png', dbn_output1)
cv2.imwrite('dbn_result2.png', dbn_output2)
def train_bottom_layer(train_set,
validation_set,
batch_size=20,
k=1,
layers_sizes=[40],
pretraining_epochs=[800],
pretrain_lr=[0.005],
lambda_1=0.0,
lambda_2=0.1,
rng=None,
graph_output=False):
print('Visible nodes: %i' % train_set.get_value().shape[1])
print('Output nodes: %i' % layers_sizes[-1])
dbn = DBN(numpy_rng=rng,
n_ins=train_set.get_value().shape[1],
hidden_layers_sizes=layers_sizes[:-1],
n_outs=layers_sizes[-1])
dbn.training(train_set,
batch_size,
k=k,
pretraining_epochs=pretraining_epochs,
pretrain_lr=pretrain_lr,
lambda_1=lambda_1,
lambda_2=lambda_2,
validation_set_x=validation_set,
graph_output=graph_output)
output_train_set = dbn.get_output(train_set)
if validation_set is not None:
output_val_set = dbn.get_output(validation_set)
else:
output_val_set = None
return dbn, output_train_set, output_val_set
def train_classifier(X, y):
"""
Trains a classifier using best known
parameters on given data / labels.
:param X: Samples, a numpy array of
(N, n_vis) shape where N is number of
samples and n_vis number of visible
varliables (sample dimensionality).
:param y: Labels, a numpy array of
(N, 1) shape. Each lable should be
a label index.
"""
# split data into minibatches
X_mnb, y_mnb = util.create_minibatches(X, y, __CLASS_COUNT * 20)
# create a DBN and pretrain
dbn = DBN([32 * 24, 600, 600], __CLASS_COUNT)
pretrain_params = [[80, 0.05, True, 1, 0.085, 0.1],
[80, 0.05, True, 1, 0.000, 0.0]]
dbn.pretrain(X_mnb, y_mnb, pretrain_params)
# fine-tuning
mlp = dbn.to_mlp()
mlp.train(X_mnb, y_mnb, 1000, 0.1)
return mlp
def _perform(self):
p = self.params
class_count = p[0]
layer_sizes = p[1]
# make a copy of rbm-param list because we might
# modify it for training, and we don't want to
# affect the original
rbm_params = list(p[2])
dbn = DBN(layer_sizes, class_count)
# train the first RBM as an RbmJob (to be able to reuse)
# assuming the RBN is more then one layer deep
if len(layer_sizes) >= 2:
first_rbm_params = [class_count, layer_sizes[0], layer_sizes[1]]
first_rbm_params.extend(rbm_params[0])
first_rbm_job = RbmJob(first_rbm_params, self.X_train)
if not first_rbm_job.is_done():
first_rbm_job.perform()
# create the DBN, then replace the first
# RBM with the one already trained
dbn.rbms[0] = first_rbm_job.results()[0]
# make sure it's skipped in DBN training
rbm_params[0] = None
train_res = dbn.pretrain(self.X_train, self.y_train, rbm_params)
return (dbn, train_res)
def test_all(self, n):
_dbn=DBN([784,1000,500,250,30],learning_rate=0.01,cd_k=1)
_dbn.pretrain(mnist.train.images,128,50)
_nnet = NN([784, 1000, 500, 250, 30, 250, 500, 1000, 784], 0.01, 128, 50)
_nnet.load_from_dbn_to_reconstructNN(_dbn)
_nnet.train(mnist.train.images, mnist.train.images)
_nnet.test_linear(mnist.test.images, mnist.test.images)
x_in = mnist.test.images[:30]
_predict = _nnet.predict(x_in)
_predict_img = np.concatenate(np.reshape(_predict, [-1, 28, 28]), axis=1)
x_in = np.concatenate(np.reshape(x_in, [-1, 28, 28]), axis=1)
img = Image.fromarray(
(1.0-np.concatenate((_predict_img, x_in), axis=0))*255.0)
img = img.convert('L')
img.save(str(n)+'_.jpg')
img2 = Image.fromarray(
(np.concatenate((_predict_img, x_in), axis=0))*255.0)
img2 = img2.convert('L')
img2.save(str(n)+'.jpg')
nnet_encoder=NN()
nnet_encoder.load_layers_from_NN(_nnet,0,4)
# featrue=nnet_encoder.predict(mnist.test.images)
nnet_decoder=NN()
nnet_decoder.load_layers_from_NN(_nnet,5,8)
def test_another_rbmtrain(self, n):
_dbn = DBN([784, 1000, 500, 250, 30], learning_rate=0.01, cd_k=1)
print(len(mnist.train.images))
for j in range(5):
for i in range(10):
_dbn.pretrain(mnist.train.images[i * 5500:i * 5500 + 5500],
128, 5)
_nnet = NN([784, 1000, 500, 250, 30, 250, 500, 1000, 784], 0.01, 128,
50)
_nnet.load_from_dbn_to_reconstructNN(_dbn)
_nnet.train(mnist.train.images, mnist.train.images)
_nnet.test_linear(mnist.test.images, mnist.test.images)
x_in = mnist.test.images[:30]
_predict = _nnet.predict(x_in)
_predict_img = np.concatenate(np.reshape(_predict, [-1, 28, 28]),
axis=1)
x_in = np.concatenate(np.reshape(x_in, [-1, 28, 28]), axis=1)
img = Image.fromarray((1.0 - np.concatenate(
(_predict_img, x_in), axis=0)) * 255.0)
img = img.convert('L')
img.save(str(n) + '_.jpg')
img2 = Image.fromarray((np.concatenate(
(_predict_img, x_in), axis=0)) * 255.0)
img2 = img2.convert('L')
img2.save(str(n) + '.jpg')
def completion_by_prediction(t):
train_com_x, test_com_x, train_com_y, test_com_y, train_uncom_x, test_uncom_x = datasets[
t]
train_X = np.concatenate((train_com_x, test_com_x), axis=0)
train_Y = np.concatenate((train_com_y, test_com_y), axis=0)
if t == 0: train_Y, _, prep = preprocess(train_Y, None, 'mm')
x_dim = train_X.shape[1]
y_dim = train_Y.shape[1]
if t == 0:
use_for = 'prediction'
hidden_act_func = ['tanh', 'gauss']
output_act_func = 'affine'
lr = 1e-3
else:
use_for = 'classification'
hidden_act_func = ['tanh', 'gauss']
output_act_func = 'softmax'
lr = 1e-3
tf.reset_default_graph()
classifier = DBN(
hidden_act_func=hidden_act_func,
output_act_func=output_act_func,
loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, x_dim * 40, x_dim * 20, x_dim * 10, x_dim * 2, y_dim],
lr=lr,
use_for=use_for,
bp_algorithm='rmsp',
epochs=240,
batch_size=32,
dropout=0.08,
units_type=['gauss', 'gauss'],
rbm_lr=1e-4,
rbm_epochs=60,
cd_k=1,
pre_train=True)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
classifier.train_model(train_X=train_X, train_Y=train_Y, sess=sess)
train_uncom_y = sess.run(classifier.pred,
feed_dict={
classifier.input_data: train_uncom_x,
classifier.keep_prob: 1.0
})
test_uncom_y = sess.run(classifier.pred,
feed_dict={
classifier.input_data: test_uncom_x,
classifier.keep_prob: 1.0
})
if t == 0:
train_uncom_y = prep.inverse_transform(train_uncom_y.reshape(-1, 1))
test_uncom_y = prep.inverse_transform(test_uncom_y.reshape(-1, 1))
elif t == 1:
train_uncom_y = np.argmax(train_uncom_y, axis=1).reshape(-1, 1)
test_uncom_y = np.argmax(test_uncom_y, axis=1).reshape(-1, 1)
return [train_uncom_y, test_uncom_y]
def train_top(batch_size, graph_output, joint_train_set, joint_val_set, rng):
top_DBN = DBN(numpy_rng=rng,
n_ins=joint_train_set.get_value().shape[1],
gauss=False,
hidden_layers_sizes=[24],
n_outs=3)
top_DBN.training(joint_train_set,
batch_size,
k=1,
pretraining_epochs=[800, 800],
pretrain_lr=[0.1, 0.1],
validation_set_x=joint_val_set,
graph_output=graph_output)
return top_DBN
def track(self, num_bar_states=NUM_BEAT_STATES,
min_bpm=MIN_BPM, max_bpm=MAX_BPM, gmm_model=GMM_MODEL,
tempo_change_probability=TEMPO_CHANGE_PROBABILITY,
norm_observations=NORM_OBSERVATIONS):
"""
Track the conduction with a dynamic Bayesian network.
Parameters for the transition model:
:param num_bar_states: number of cells for one beat period
:param tempo_change_probability: probability of a tempo change between
two adjacent observations
:param min_bpm: minimum tempo used for beat tracking
:param max_bpm: maximum tempo used for beat tracking
Parameters for the observation model:
:param gmm_model: load the fitted GMM model from the
given file
:param norm_observations: normalise the observations
:return: detected beat positions
"""
# convert timing information to tempo spaces
max_tempo = int(np.ceil(max_bpm * num_bar_states / (60. * self.fps)))
min_tempo = int(np.floor(min_bpm * num_bar_states / (60. * self.fps)))
tempo_states = np.arange(min_tempo, max_tempo)
# transition model
tm = TransitionModel(num_bar_states=num_bar_states,
tempo_states=tempo_states,
tempo_change_probability=tempo_change_probability)
# observation model
om = ObservationModel(gmm_model, self.features,
num_states=tm.num_states,
num_bar_states=tm.num_bar_states,
norm_observations=norm_observations,
log_probability=True, norm_probability=True,
min_probability=0.3, max_probability=0.7)
# init the DBN
dbn = DBN(transition_model=tm, observation_model=om)
# save some information (mainly for visualisation)
self.densities = om.densities.astype(np.float)
self.path = dbn.bar_states_path.astype(np.int)
self.bar_start_positions = argrelmin(self.path, mode='wrap')[0] / \
float(self.fps)
# also return the bar start positions
return self.bar_start_positions
def build_(method=1, beta=None):
tf.reset_default_graph()
# Training
if method == 1:
classifier = DBN(
hidden_act_func=['tanh', 'gauss'],
output_act_func='affine',
loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数
struct=[
x_dim, x_dim * 40, x_dim * 20, x_dim * 10, x_dim * 2, y_dim
],
lr=1e-4,
use_for='prediction',
bp_algorithm='rmsp',
epochs=240,
batch_size=32,
dropout=0.08,
units_type=['gauss', 'gauss'],
rbm_lr=1e-4,
rbm_epochs=60,
cd_k=1,
pre_train=True)
elif method == 2:
classifier = supervised_sAE(
output_func='gauss',
hidden_func='affine',
loss_func='mse',
struct=[
x_dim, x_dim * 40, x_dim * 20, x_dim * 10, x_dim * 2, y_dim
],
lr=1e-4,
use_for='prediction',
epochs=180,
batch_size=32,
dropout=0.15,
ae_type='sae', # ae | dae | sae
act_type=[
'gauss', 'affine'
], # decoder：[sigmoid] with ‘cross_entropy’ | [affine] with ‘mse’
noise_type='mn', # Gaussian noise (gs) | Masking noise (mn)
beta=beta, # DAE：噪声损失系数 | SAE：稀疏损失系数 | YAE：Y系数比重
p=0.1, # DAE：样本该维作为噪声的概率 | SAE稀疏性参数：期望的隐层平均活跃度（在训练批次上取平均）
ae_lr=1e-4,
ae_epochs=60,
pre_train=True)
return classifier
def run_(method=1, beta=None):
tf.reset_default_graph()
# Training
if method == 1:
classifier = DBN(
hidden_act_func='gauss',
output_act_func='gauss',
loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, y_dim * 20, y_dim * 10, y_dim],
# struct=[x_dim, int(x_dim/2), int(x_dim/4), int(x_dim/8), y_dim],
lr=1e-4,
use_for='classification',
bp_algorithm='rmsp',
epochs=240,
batch_size=16,
dropout=0.05,
units_type=['gauss', 'gauss'],
rbm_lr=1e-4,
rbm_epochs=45,
cd_k=1,
pre_train=True)
elif method == 2:
classifier = supervised_sAE(
output_func='gauss',
hidden_func='gauss',
loss_func='mse',
struct=[x_dim, y_dim * 60, y_dim * 30, y_dim * 10, y_dim],
lr=1e-4,
use_for='classification',
epochs=240,
batch_size=32,
dropout=0.34,
ae_type='yae', # ae | dae | sae
act_type=[
'gauss', 'affine'
], # decoder：[sigmoid] with ‘cross_entropy’ | [affine] with ‘mse’
noise_type='mn', # Gaussian noise (gs) | Masking noise (mn)
beta=beta, # DAE：噪声损失系数 | SAE：稀疏损失系数 | YAE：Y系数比重
p=0.3, # DAE：样本该维作为噪声的概率 | SAE稀疏性参数：期望的隐层平均活跃度（在训练批次上取平均）
ae_lr=1e-4,
ae_epochs=30,
pre_train=True)
run_sess(classifier, datasets, filename, load_saver='')
return classifier
def __init__(self, dbn):
"""
Prepare the sampler.
"""
if not isinstance(dbn, DBN):
if isinstance(dbn, AbstractBM):
dbn = DBN(dbn)
else:
raise TypeError('DBN or RBM expected.')
self.dbn = dbn
for l in range(len(self.dbn)):
if not hasattr(self.dbn[l], '_ais_logz'):
self.dbn[l]._ais_logz = None
self.dbn[l]._ais_samples = None
self.dbn[l]._ais_logweights = None
def test_dbn():
log.info('Testing DBN')
# trainset loading
cls_count = 9
X, y, classes = get_data(cls_count=None)
X_mnb, y_mnb = util.create_minibatches(X, y, 20 * cls_count)
# lin_eps = util.lin_reducer(0.05, 0.002, 20)
dbn = DBN([32 * 24, 588, 588], cls_count)
dbn.train(X_mnb, y_mnb, [{
'epochs': 50,
'eps': 0.05,
'spars': 0.05,
'spars_cost': 0.3
}, {
'epochs': 1,
'eps': 0.05
}])
def ruleEncodingAlgorithm(knowledgeBase):
network = []
for l in len(knowledgeBase):
current = RBM(len(knowledgeBase[l][0][0].x), len(knowledgeBase[l][0]))
for i in range(len(knowledgeBase[l])):
for j in range(len(knowledgeBase[l][i].x)):
if knowledgeBase[l][i].x[j] == True:
current.w[i][j] = knowledgeBase[l][i].c
elif knowledgeBase[l][i].x[j] == False:
current.w[i][j] = -knowledgeBase[l][i].c
network.append(current)
dbn = DBN([network[0]._input_size] + [x._output_size for x in network], [],
[])
return dbn.load_from_rbms(len(network) - 1, network)
DATA_FOLDER = './mnist'
train_dataset = torchvision.datasets.MNIST(root=DATA_FOLDER, train=True, download=False, transform=data_transform)
train_loader = data.DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=2)
test_dataset = torchvision.datasets.MNIST(root=DATA_FOLDER, train=False, download=False, transform=data_transform)
test_loader = data.DataLoader(test_dataset, batch_size=64, shuffle=True, num_workers=2)
#
# data_iter = iter(train_loader)
# images, labels = data_iter.next()
# imshow(torchvision.utils.make_grid(images))
# print(' '.join('%5s' % classes[labels[j]] for j in range(64)))
net = DBN()
print(net)
print(len(net.rbm_layers))
for i in range(len(net.rbm_layers)):
print("-------------No. {} layer's weights-------------".format(i+1))
print(net.rbm_layers[i].weights)
print(len(net.rbm_layers[i].weights))
print("-------------No. {} layer's visible_bias-------------".format(i+1))
print(net.rbm_layers[i].visible_bias)
print(len(net.rbm_layers[i].visible_bias))
print("-------------No. {} layer's hidden_bias-------------".format(i+1))
print(net.rbm_layers[i].hidden_bias)
print(len(net.rbm_layers[i].hidden_bias))
# Splitting data
X_train, X_test, Y_train, Y_test = mnist.train.images,mnist.test.images ,mnist.train.labels, mnist.test.labels
# X_train, X_test = X_train[::100], X_test[::100]
# Y_train, Y_test = Y_train[::100], Y_test[::100]
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Training
classifier = DBN(output_act_func='softmax',
hidden_act_func='relu',
loss_fuc='cross_entropy',
use_for='classification',
dbn_lr=1e-3,
dbn_epochs=100,
dbn_struct=[784, 100, 100,10],
rbm_h_type='bin',
rbm_epochs=10,
batch_size=32,
cd_k=1,
rbm_lr=1e-3)
classifier.build_dbn()
classifier.train_dbn(X_train, Y_train,sess)
# Test
Y_pred = classifier.test_dbn(X_test, Y_test,sess)
#import matplotlib.pyplot as plt
#PX=range(0,len(Y_pred))
#plt.figure(1) # 选择图表1
y_dim = Y_train.shape[1]
p_dim = int(np.sqrt(x_dim))
sess = tf.Session()
# Training
select_case = 1
if select_case == 1:
classifier = DBN(
hidden_act_func='relu',
output_act_func='softmax',
loss_func='cross_entropy', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, 200, 100, y_dim],
lr=1e-3,
momentum=0.5,
use_for='classification',
bp_algorithm='rmsp',
epochs=100,
batch_size=32,
dropout=0.1,
units_type=['gauss', 'bin'],
rbm_lr=1e-3,
rbm_epochs=16,
cd_k=1)
if select_case == 2:
classifier = CNN(
output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
lr=1e-3,
epochs=30,
#X_train, X_test = X_train[::100], X_test[::100]
#Y_train, Y_test = Y_train[::100], Y_test[::100]
x_dim=X_train.shape[1]
y_dim=Y_train.shape[1]
p_dim=int(np.sqrt(x_dim))
sess = tf.Session()
# Training
select_case = 1
if select_case==1:
<<<<<<< HEAD
classifier = DBN(
hidden_act_func='sigmoid',
output_act_func='softmax',
loss_func='cross_entropy', # gauss 激活函数会自动转换为 mse 损失函数
struct=[x_dim, 100, 50, y_dim],
lr=1e-3,
=======
classifier = DBN(output_act_func='softmax',
loss_func='cross_entropy',
use_for='classification',
bp_algorithm='adam',
dbn_lr=1e-3,
>>>>>>> 4264eee5bcc3304abc64f7a1b22cf3c5b3cd37f4
momentum=0.5,
use_for='classification',
bp_algorithm='adam',
epochs=100,
batch_size=32,
dropout=0.3,
x_dim = X_train.shape[1]
y_dim = Y_train.shape[1]
p_dim = int(np.sqrt(x_dim))
sess = tf.Session()
# Training
select_case = 2
if select_case == 1:
classifier = DBN(output_act_func='softmax',
loss_func='cross_entropy',
use_for='classification',
bp_algorithm='adam',
dbn_lr=1e-3,
momentum=0.5,
dbn_epochs=100,
dbn_struct=[x_dim, 200, 100, y_dim],
rbm_v_type='bin',
rbm_epochs=12,
batch_size=32,
cd_k=1,
rbm_lr=1e-3,
dropout=1)
if select_case == 2:
classifier = CNN(
output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
cnn_lr=1e-3,
cnn_epochs=30,
img_shape=[p_dim, p_dim],
def test_DBN(finetune_lr=0.1,
pretraining_epochs=100,
pretrain_lr=0.1,
k=1,
training_epochs=5000,
batch_size=5):
datasets = load10sec_ECGII_data()
pretrain_set_x = datasets[0]
trdatasets = [datasets[1], datasets[2]]
train_set_x, train_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
dbn = DBN(numpy_rng=numpy_rng,
n_ins=2500,
hidden_layers_sizes=[1000, 1000, 1000],
n_outs=2)
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=pretrain_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
# Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
# end-snippet-2
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, train_cost_model, test_score_model, test_cost_model, test_confmatrix = dbn.build_finetune_functions(
datasets=trdatasets,
batch_size=batch_size,
)
print '... finetuning the model'
tr_cost = []
te_cost = []
test_score = 0.
start_time = time.clock()
epoch = 0
while (epoch < training_epochs):
epoch = epoch + 1
learning_rate = 0.01 / (1 + 0.001 * epoch)
# go through the training set
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index, learning_rate)
print(('In epoch %d,') % (epoch))
epoch_trcost = numpy.mean(train_cost_model())
epoch_tecost = numpy.mean(test_cost_model())
print 'Training cost = ', epoch_trcost
tr_cost.append(epoch_trcost)
print 'Testing cost = ', epoch_tecost
te_cost.append(epoch_tecost)
test_losses = test_score_model()
test_score = numpy.mean(test_losses)
test_confmatrices = test_confmatrix()
test_confelement = numpy.sum(test_confmatrices, axis=0)
true_pos = test_confelement[0]
true_neg = test_confelement[1]
false_pos = test_confelement[2]
false_neg = test_confelement[3]
f_score = (true_pos + true_neg) / float(true_pos + true_neg + false_pos +
5 * false_neg)
print(('Test error: %f %%, F-score: %f') % (test_score * 100., f_score))
print(('TP %i, TN %i, FP %i, FN %i') %
(true_pos, true_neg, false_pos, false_neg))
end_time = time.clock()
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
x_axis = numpy.arange(training_epochs)
plt.plot(x_axis, numpy.array(tr_cost), '+', x_axis, numpy.array(te_cost),
'.')
plt.show()
from data import MNIST
from dbn import DBN
import numpy as np
# An example of how to use the library
# Creates a DBN
d = DBN([28 * 28, 500, 500, 2000], 10, 0.1)
# Loads the Images and Labels from the MNIST database
images = MNIST.load_images('images')
labels = MNIST.load_labels('labels')
# Gets the training set of images
img = images[0:60000]
lbl = labels[0:60000]
# Permutes the data to ensure random batches
train_tuples = zip(img, lbl)
train_tuples = np.random.permutation(train_tuples)
(img, lbl) = zip(*train_tuples)
# Loads the test images from the MNIST database
tst_img = MNIST.load_images('test_images')
tst_lbl = MNIST.load_labels('test_labels')
# Pre trains the DBN
d.pre_train(img, 5, 50)
# Executes a 100 iterations of label training
# Attempts to classify and prints the error rate
import sys, re, random
import numpy as np
from dbn import DBN
from dbn.layers import *
import theano.tensor as T
import theano
if __name__ == '__main__':
data = np.hstack((np.eye(8), np.arange(8).reshape((8, 1))))
data = np.vstack(100 * (data, ))
np.random.shuffle(data)
net = DBN([OneHotSoftmax(8), Sigmoid(3)], 8, max_epochs=1000)
net.fit(data[:, :-1], data[:, -1])
print net.predict(np.eye(8, dtype=np.float32))
return ret_dat
traind= norm(datas[:700,:-1].astype('float32'))
trainl= one_hot(datas[:700,-1].astype('float64'))
# print(traind[0]);print(trainl[0])
testd = norm(datas[700:,:-1].astype('float32'))
testl = one_hot(datas[700:,-1].astype('float64'))
dataset=[traind,trainl,testd,testl]
classifier = DBN(
hidden_act_func='sigmoid',
output_act_func='softmax',
loss_func='cross_entropy', # gauss 激活函数会自动转换为 mse 损失函数
struct=[26, 11, 6, 11, 11,3],
lr=1e-3,
momentum=0.8,
use_for='classification',
bp_algorithm='rmsp',
epochs=100,
batch_size=20,
dropout=0.12,
units_type=['gauss','bin'],
rbm_lr=1e-3,
rbm_epochs=0,
cd_k=1,
pre_train=True)
run_sess(classifier,dataset,filename,load_saver='')
label_distribution = classifier.label_distribution
from dbn import DBN
from skimage.color import rgb2gray
from skimage.io import imread_collection, imshow
from skimage.transform import resize
import numpy as np
import pickle
#build model
model = DBN(n_nodes=5005,rbm_epoch=100,max_epoch=2000, alpha=0.001)
#import training data
imgs_train = imread_collection('images/train/*.jpg')
print("Imported", len(imgs_train), "images")
print("The first one is",len(imgs_train[0]), "pixels tall, and",
len(imgs_train[0][0]), "pixels wide")
imgs_train = [resize(x,(77,65),mode='constant', anti_aliasing=False) for x in imgs_train]
imgs_train = [rgb2gray(x) for x in imgs_train]
imgsarr_train = [x.flatten('C') for x in imgs_train]
print(np.array(imgsarr_train).shape)
'''
X = np.array([[0.2157, 0.1255, 0.4039, 1.0, 0.0941, 0.2550],
[0.1686, 0.9529, 0.0824, 0.0980, 1.0, 0.3529],
[0.3529, 0.0824, 0.4275, 1.0, 0.1255, 0.2941],
[0.1255, 1.0, 0.1216, 0.0471, 1.0, 0.2431]])
'''
y = []
for i in range(250):
y.append(1)
for i in range(250):
y.append(0)
y = np.array([y])
def test_DBN(finetune_lr=0.01,
pretraining_epochs=100,
pretrain_lr=0.01,
k=1,
training_epochs=100,
batch_size=5):
"""
Demonstrates how to train and main a Deep Belief Network.
This is demonstrated on MNIST.
:type finetune_lr: float
:param finetune_lr: learning rate used in the finetune stage
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type k: int
:param k: number of Gibbs steps in CD/PCD
:type training_epochs: int
:param training_epochs: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
:type batch_size: int
:param batch_size: the size of a minibatch
"""
datasets = loadFeaturedData()
# datasets = load10secData()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
# construct the Deep Belief Network
# Test for debugging
# dbn = DBN(numpy_rng=numpy_rng, n_ins=10,
# hidden_layers_sizes=[100, 100, 100],
# n_outs=2)
dbn = DBN(numpy_rng=numpy_rng,
n_ins=10,
hidden_layers_sizes=[50, 50],
n_outs=2)
# start-snippet-2
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size,
k=k)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
for i in xrange(dbn.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
# end-snippet-2
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model, test_confmatrix = dbn.build_finetune_functions(
datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr)
print '... finetuning the model'
# early-stopping parameters
patience = 4 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatches before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (this_validation_loss <
best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# main it on the main set
test_losses = test_model()
test_score = numpy.mean(test_losses)
test_confmatrices = test_confmatrix()
test_confelement = numpy.sum(test_confmatrices, axis=0)
true_pos = test_confelement[0]
true_neg = test_confelement[1]
false_pos = test_confelement[2]
false_neg = test_confelement[3]
print(('epoch %i, minibatch %i/%i, main error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
print(('TP %i, TN %i, FP %i, FN %i') %
(true_pos, true_neg, false_pos, false_neg))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(('Optimization complete with best validation score of %f %%, '
'obtained at iteration %i, '
'with main performance %f %% ') %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print(('TP %i, TN %i, FP %i, FN %i') %
(true_pos, true_neg, false_pos, false_neg))
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
borrow=True)
data_t_sh = theano.shared(np.asarray(data_t, dtype=theano.config.floatX),
borrow=True)
return data_sh, T.cast(data_t_sh, 'int32')
train_data_sh, train_data_t_sh = make_theano_dataset(
(train_data, train_data_t))
valid_data_sh, valid_data_t_sh = make_theano_dataset(
(valid_data, valid_data_t))
test_data_sh, test_data_t_sh = make_theano_dataset((test_data, test_data_t))
datasets = [(train_data_sh, train_data_t_sh), (valid_data_sh, valid_data_t_sh),
(test_data_sh, test_data_t_sh)]
rbm_stack = RBMStack(num_dims, [500])
dbn = DBN(rbm_stack, 2)
batch_size = 100
max_epoch = 10
train_params = {
'batch_size': batch_size,
'learning_rate': 0.01,
'cd_steps': 2,
'max_epoch': max_epoch,
'persistent': True,
'finetune_learning_rate': 0.1
}
pre_fn = dbn.pretrain_fun(train_data_sh, train_params)
num_batches = train_data_sh.get_value(borrow=True).shape[0] / batch_size
train_data_sh, train_data_t_sh = make_theano_dataset((train_data, train_data_t))
valid_data_sh, valid_data_t_sh = make_theano_dataset((valid_data, valid_data_t))
test_data_sh, test_data_t_sh = (None, None) #make_theano_dataset((test_data, test_data_t))
train_data_wp_sh, train_data_wp_t_sh = make_theano_dataset((data_wo_p, data_target_wo_p))
datasets = [(train_data_sh, train_data_t_sh),(valid_data_sh, valid_data_t_sh),(test_data_sh, test_data_t_sh)]
batch_size = 100
max_epoch = 30
train_params = {'batch_size' : batch_size, 'learning_rate' : 0.1, 'cd_steps' : 10, 'max_epoch' : max_epoch, 'persistent' : True, 'cd_steps_line' : 1, 'learning_rate_line' : 0.001, 'finetune_learning_rate' : 0.1, 'validation_frequency' : batch_size }
stack_rbm = RBMStack(num_vis = num_vis, hid_layers_size = hid_layers_size)
if not load_from_file(stack_rbm, train_params):
stack_rbm.pretrain(data_sh, train_params)
save_to_file(stack_rbm, train_params)
dbn = DBN(stack_rbm, 2)
#dbn.finetune(datasets, train_params)
#ae = AutoEncoder(stack_rbm, 100)
#ae.pretrain(train_data_sh, train_params)
#ae.finetune(train_data_sh, train_params)
get_output = theano.function([], dbn.stack[-1].output, givens=[(dbn.input,train_data_sh)])
out = get_output()
np.savetxt("/home/alexeyche/prog/sentiment/out.tsv", train_data, delimiter="\t")
np.savetxt("/home/alexeyche/prog/sentiment/answer.tsv",train_data_t,delimiter=",")
document_placeholder = tf.placeholder(dtype=tf.float32,
shape=(None, n_input),
name='document_vector')
target_placeholder = tf.placeholder(dtype=tf.float32,
shape=(None, n_tags),
name='tag_vector')
tag_mask = tf.placeholder(dtype=tf.float32, shape=(1, n_tags), name='tag_mask')
keep_probability_placeholder = tf.placeholder(dtype=tf.float32)
dbn = None
if use_dbn and len(layer_sizes) > dbn_layer_count:
dbn = DBN(layer_sizes[:dbn_layer_count],
document_placeholder,
n_fantasy_states=dbn_batch_size,
learning_rate=1e-2,
persistent_contrastive_divergence=False,
constrastive_divergence_level=1)
if use_dbn:
activation_functions = [tf.nn.sigmoid] * (dbn_layer_count - 1) + [
tf.nn.elu
] * (len(layer_sizes) - dbn_layer_count)
stop_after = dbn_layer_count - 1
batch_normalize = [False] * (dbn_layer_count - 1) + [True] * (
len(layer_sizes) - dbn_layer_count)
else:
activation_functions = tf.nn.elu
stop_after = None
batch_normalize = True
network = FullyConnectedMultilayer(
return ' '.join(sentence)
if __name__ == '__main__':
hidden_units = 200
names = [preprocess(line.strip()) for line in open(sys.argv[1], 'r')]
random.shuffle(names)
word_counter = CountVectorizer(tokenizer=wordpunct_tokenize,
stop_words=stopwords,
binary=True,
dtype=np.byte)
data = word_counter.fit_transform(names)
words = word_counter.get_feature_names()
data = data.toarray()
print data.shape
_, vocab = data.shape
n = DBN([
Sigmoid(data.shape[1]),
Sigmoid(hidden_units),
Sigmoid(hidden_units / 2)
])
n.fit(data, None)
"""
visible = r.run_hidden(np.eye(hidden_units))
out = open('assoc_words','w')
for f in range(hidden_units):
out.write(' '.join( words[i] for i in range(len(words)) if visible[f,i] ) )
out.write('\n')
"""
# Training and CV sets
trX = tr1X[:50000]
trY = tr0Y[:50000]
cvX = tr1X[50000:]
cvY = tr0Y[50000:]
# RBMs
rbmobject1 = RBM(FLAGS.out_dir, 784, 900, ['rbmw1', 'rbvb1', 'rbmhb1'], 0.3,
tf.nn.sigmoid)
rbmobject2 = RBM(FLAGS.out_dir, 900, 500, ['rbmw2', 'rbvb2', 'rbmhb2'], 0.3,
tf.nn.sigmoid)
rbmobject1.restore_weights('./out/rbmw1.chp')
rbmobject2.restore_weights('./out/rbmw2.chp')
# DBN
y_colNb = 1
dbn = DBN(FLAGS.out_dir, [rbmobject1, rbmobject2], 784, y_colNb,
FLAGS.class_nb, FLAGS.batchsize, FLAGS.epochs, 0.3)
#dbn.restore_trained_net('./out/dbn.chp')
#i=8
#dbn.predict(teX[i], teY[i])
#exit()
iterations = len(trX) / FLAGS.batchsize
## Train DBN
print 'Training DBN: ', FLAGS.epochs, ' epochs of ', iterations, ' iterations'
dbn.train(trX, trY, cvX, cvY, teX, teY)
dbn.save_trained_net('./out/dbn.chp')
