class DBN(object):
"""
An implement of deep belief network
The hidden layers are firstly pretrained by RBM, then DBN is treated as a normal
MLP by adding a output layer.
"""
def __init__(self, n_in=784, n_out=10, hidden_layers_sizes=[500, 500]):
"""
:param n_in: int, the dimension of input
:param n_out: int, the dimension of output
:param hidden_layers_sizes: list or tuple, the hidden layer sizes
"""
# Number of layers
assert len(hidden_layers_sizes) > 0
self.n_layers = len(hidden_layers_sizes)
self.layers = [] # normal sigmoid layer
self.rbm_layers = [] # RBM layer
self.params = [] # keep track of params for training
# Define the input and output
self.x = tf.placeholder(tf.float32, shape=[None, n_in])
self.y = tf.placeholder(tf.float32, shape=[None, n_out])
# Contruct the layers of DBN
for i in range(self.n_layers):
if i == 0:
layer_input = self.x
input_size = n_in
else:
layer_input = self.layers[i-1].output
input_size = hidden_layers_sizes[i-1]
# Sigmoid layer
sigmoid_layer = HiddenLayer(inpt=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i],
activation=tf.nn.sigmoid)
self.layers.append(sigmoid_layer)
# Add the parameters for finetuning
self.params.extend(sigmoid_layer.params)
# Create the RBM layer
self.rbm_layers.append(RBM(inpt=layer_input, n_visiable=input_size, n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W, hbias=sigmoid_layer.b))
# We use the LogisticRegression layer as the output layer
self.output_layer = LogisticRegression(inpt=self.layers[-1].output, n_in=hidden_layers_sizes[-1],
n_out=n_out)
self.params.extend(self.output_layer.params)
# The finetuning cost
self.cost = self.output_layer.cost(self.y)
# The accuracy
self.accuracy = self.output_layer.accuarcy(self.y)
def pretrain(self, sess, X_train, batch_size=50, pretraining_epochs=10, lr=0.1, k=1,
display_step=1):
"""
Pretrain the layers (just train the RBM layers)
:param sess: tf.Session
:param X_train: the input of the train set (You might modidy this function if you do not use the desgined mnist)
:param batch_size: int
:param lr: float
:param k: int, use CD-k
:param pretraining_epoch: int
:param display_step: int
"""
print('Starting pretraining...\n')
start_time = timeit.default_timer()
batch_num = int(X_train.train.num_examples / batch_size)
# Pretrain layer by layer
for i in range(self.n_layers):
cost = self.rbm_layers[i].get_reconstruction_cost()
train_ops = self.rbm_layers[i].get_train_ops(learning_rate=lr, k=k, persistent=None)
for epoch in range(pretraining_epochs):
avg_cost = 0.0
for j in range(batch_num):
x_batch, _ = X_train.train.next_batch(batch_size)
# 训练
sess.run(train_ops, feed_dict={self.x: x_batch})
# 计算cost
avg_cost += sess.run(cost, feed_dict={self.x: x_batch,}) / batch_num
# 输出
if epoch % display_step == 0:
print("\tPretraing layer {0} Epoch {1} cost: {2}".format(i, epoch, avg_cost))
end_time = timeit.default_timer()
print("\nThe pretraining process ran for {0} minutes".format((end_time - start_time) / 60))
def finetuning(self, sess, trainSet, training_epochs=10, batch_size=100, lr=0.1,
display_step=1):
"""
Finetuing the network
"""
print("\nStart finetuning...\n")
start_time = timeit.default_timer()
train_op = tf.train.GradientDescentOptimizer(learning_rate=lr).minimize(
self.cost, var_list=self.params)
for epoch in range(training_epochs):
avg_cost = 0.0
batch_num = int(trainSet.train.num_examples / batch_size)
for i in range(batch_num):
x_batch, y_batch = trainSet.train.next_batch(batch_size)
# 训练
sess.run(train_op, feed_dict={self.x: x_batch, self.y: y_batch})
# 计算cost
avg_cost += sess.run(self.cost, feed_dict=
{self.x: x_batch, self.y: y_batch}) / batch_num
# 输出
if epoch % display_step == 0:
val_acc = sess.run(self.accuracy, feed_dict={self.x: trainSet.validation.images,
self.y: trainSet.validation.labels})
print("\tEpoch {0} cost: {1}, validation accuacy: {2}".format(epoch, avg_cost, val_acc))
end_time = timeit.default_timer()
print("\nThe finetuning process ran for {0} minutes".format((end_time - start_time) / 60))
class DBN(object):
"""
An implement of deep belief network
The hidden layers are firstly pretrained by RBM, then DBN is treated as a normal
MLP by adding a output layer.
"""
def __init__(self, n_in=784, n_out=10, hidden_layers_sizes=[500, 500]):
"""
:param n_in: int, the dimension of input
:param n_out: int, the dimension of output
:param hidden_layers_sizes: list or tuple, the hidden layer sizes
"""
# Number of layers
assert len(hidden_layers_sizes) > 0
self.n_layers = len(hidden_layers_sizes)
self.layers = [] # normal sigmoid layer
self.rbm_layers = [] # RBM layer
self.params = [] # keep track of params for training
# Define the input and output
self.x = tf.placeholder(tf.float32, shape=[None, n_in])
self.y = tf.placeholder(tf.float32, shape=[None, n_out])
# Contruct the layers of DBN
with tf.name_scope('DBN_layer'):
for i in range(self.n_layers):
if i == 0:
layer_input = self.x
input_size = n_in
else:
layer_input = self.layers[i - 1].output
input_size = hidden_layers_sizes[i - 1]
# Sigmoid layer
with tf.name_scope('internel_layer'):
sigmoid_layer = HiddenLayer(inpt=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=tf.nn.sigmoid)
self.layers.append(sigmoid_layer)
# Add the parameters for finetuning
self.params.extend(sigmoid_layer.params)
# Create the RBM layer
with tf.name_scope('rbm_layer'):
self.rbm_layers.append(
RBM(inpt=layer_input,
n_visiable=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b))
# We use the LogisticRegression layer as the output layer
with tf.name_scope('output_layer'):
self.output_layer = LogisticRegression(
inpt=self.layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_out)
self.params.extend(self.output_layer.params)
# The finetuning cost
with tf.name_scope('output_loss'):
self.cost = self.output_layer.cost(self.y)
# The accuracy
self.accuracy = self.output_layer.accuarcy(self.y)
def pretrain(self,
sess,
train_x,
batch_size=50,
pretraining_epochs=10,
lr=0.5,
k=1,
display_step=1):
"""
Pretrain the layers (just train the RBM layers)
:param sess: tf.Session
:param X_train: the input of the train set (You might modidy this function if you do not use the desgined mnist)
:param batch_size: int
:param lr: float
:param k: int, use CD-k
:param pretraining_epoch: int
:param display_step: int
"""
print('Starting pretraining...\n')
start_time = timeit.default_timer()
# Pretrain layer by layer
for i in range(self.n_layers):
cost = self.rbm_layers[i].get_reconstruction_cost()
train_ops = self.rbm_layers[i].get_train_ops(learning_rate=lr,
k=k,
persistent=None)
batch_num = int(train_x.shape[0] / batch_size)
for epoch in range(pretraining_epochs):
avg_cost = 0.0
for step in range(batch_num - 1):
# 训练
x_batch = train_x[batch_num * batch_size:(batch_num + 1) *
batch_size]
sess.run(train_ops, feed_dict={self.x: x_batch})
# 计算cost
avg_cost += sess.run(cost, feed_dict={
self.x: x_batch,
})
# 输出
if epoch % display_step == 0:
print("\tPretraing layer {0} Epoch {1} cost: {2}".format(
i, epoch, avg_cost))
end_time = timeit.default_timer()
print("\nThe pretraining process ran for {0} minutes".format(
(end_time - start_time) / 60))
def finetuning(self,
sess,
train_x,
train_y,
test_x,
test_y,
training_epochs=10,
batch_size=100,
lr=0.5,
display_step=1):
"""
Finetuing the network
"""
print("\nStart finetuning...\n")
start_time = timeit.default_timer()
train_op = tf.train.GradientDescentOptimizer(
learning_rate=lr).minimize(self.cost)
batch_num = int(train_x.shape[0] / batch_size)
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter("logs", sess.graph)
for epoch in range(training_epochs):
avg_cost = 0.0
for step in range(batch_num - 1):
x_batch = train_x[batch_num * batch_size:(batch_num + 1) *
batch_size]
y_batch = train_y[batch_num * batch_size:(batch_num + 1) *
batch_size]
# 训练
sess.run(train_op,
feed_dict={
self.x: x_batch,
self.y: y_batch
})
# 计算cost
avg_cost += sess.run(self.cost,
feed_dict={
self.x: x_batch,
self.y: y_batch
}) / batch_num
# 输出
if epoch % display_step == 0:
val_acc = sess.run(self.accuracy,
feed_dict={
self.x: test_x,
self.y: test_y
})
print("\tEpoch {0} cost: {1}, validation accuacy: {2}".format(
epoch, avg_cost, val_acc))
result = sess.run(merged,
feed_dict={
self.x: test_x,
self.y: test_y
}) # 输出
writer.add_summary(result, epoch)
end_time = timeit.default_timer()
print("\nThe finetuning process ran for {0} minutes".format(
(end_time - start_time) / 60))
class SdA(object):
"""
Stacked denoising autoencoder class
the model is constructed by stacking several dAs
the dA layers are used to initialize the network, after pre-training,
the SdA is similar to a normal MLP
"""
def __init__(self, n_in=784, n_out=10, hidden_layers_sizes=(500, 500),
corruption_levels=(0.1, 0.1)):
"""
:param n_in: int, the dimension of input
:param n_out: int, the dimension of output
:param hidden_layers_sizes: list or tuple, the hidden layer sizes
:param corruption_levels: list or tuple, the corruption lever for each layer
"""
assert len(hidden_layers_sizes) >= 1
assert len(hidden_layers_sizes) == len(corruption_levels)
self.corruption_levels = corruption_levels
self.n_layers = len(hidden_layers_sizes)
# define the layers
self.layers = [] # the normal layers
self.dA_layers = [] # the dA layers
self.params = [] # params
# define the input and output
self.x = tf.placeholder(tf.float32, shape=[None, n_in])
self.y = tf.placeholder(tf.float32, shape=[None, n_out])
# construct the layers
for i in range(self.n_layers):
if i == 0: # the input layer
input_size = n_in
layer_input = self.x
else:
input_size = hidden_layers_sizes[i-1]
layer_input = self.layers[i-1].output
# create the sigmoid layer
sigmoid_layer = HiddenLayer(inpt=layer_input, n_in=input_size,
n_out=hidden_layers_sizes[i], activation=tf.nn.sigmoid)
self.layers.append(sigmoid_layer)
# create the da layer
dA_layer = DA(inpt=layer_input, n_hidden=hidden_layers_sizes[i], n_visiable=input_size,
W=sigmoid_layer.W, bhid=sigmoid_layer.b)
self.dA_layers.append(dA_layer)
# collect the params
self.params.extend(sigmoid_layer.params)
# add the output layer
self.output_layer = LogisticRegression(inpt=self.layers[-1].output, n_in=hidden_layers_sizes[-1],
n_out=n_out)
self.params.extend(self.output_layer.params)
# the finetuning cost
self.finetune_cost = self.output_layer.cost(self.y)
# the accuracy
self.accuracy = self.output_layer.accuarcy(self.y)
def pretrain(self, sess, X_train, pretraining_epochs=10, batch_size=100, learning_rate=0.001,
display_step=1):
"""
Pretrain the layers
:param sess: tf.Session
:param X_train: the input of the train set
:param batch_size: int
:param learning_rate: float
"""
print('Starting pretraining...')
start_time = timeit.default_timer()
batch_num = int(X_train.train.num_examples / batch_size)
for i in range(self.n_layers):
# pretraining layer by layer
cost = self.dA_layers[i].get_cost(corruption_level=self.corruption_levels[i])
params = self.dA_layers[i].params
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost, var_list=params)
for epoch in range(pretraining_epochs):
avg_cost = 0.0
for j in range(batch_num):
x_batch, _ = X_train.train.next_batch(batch_size)
# 训练
sess.run(train_op, feed_dict={self.x: x_batch})
# 计算cost
avg_cost += sess.run(cost, feed_dict={self.x: x_batch,}) / batch_num
# 输出
if epoch % display_step == 0:
print("Pretraing layer {0} Epoch {1} cost: {2}".format(i, epoch, avg_cost))
end_time = timeit.default_timer()
print("The pretraining process ran for {0}m".format((end_time - start_time) / 60))
def finetuning(self, sess, trainSet, training_epochs=10, batch_size=100, learning_rate=0.1,
display_step=1):
"""Finetuing the network"""
print("Start finetuning...")
start_time = timeit.default_timer()
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(
self.finetune_cost, var_list=self.params)
for epoch in range(training_epochs):
avg_cost = 0.0
batch_num = int(trainSet.train.num_examples / batch_size)
for i in range(batch_num):
x_batch, y_batch = trainSet.train.next_batch(batch_size)
# 训练
sess.run(train_op, feed_dict={self.x: x_batch, self.y: y_batch})
# 计算cost
avg_cost += sess.run(self.finetune_cost, feed_dict=
{self.x: x_batch, self.y: y_batch}) / batch_num
# 输出
if epoch % display_step == 0:
val_acc = sess.run(self.accuracy, feed_dict={self.x: trainSet.validation.images,
self.y: trainSet.validation.labels})
print(" Epoch {0} cost: {1}, validation accuacy: {2}".format(epoch, avg_cost, val_acc))
end_time = timeit.default_timer()
print("The finetuning process ran for {0}m".format((end_time - start_time) / 60))
# dropout layer
layer3_dropout = DropoutLayer(layer3_fullyconn.output, keep_prob=0.5)
# the output layer
layer4_output = LogisticRegression(layer3_dropout.output,
n_in=256,
n_out=10)
# params for training
params = layer0_conv.params + layer1_conv.params + layer3_fullyconn.params + layer4_output.params
# train dicts for dropout
train_dicts = layer3_dropout.train_dicts
# prediction dicts for dropout
pred_dicts = layer3_dropout.pred_dicts
# get cost
cost = layer4_output.cost(y_)
# accuracy
accuracy = layer4_output.accuarcy(y_)
predictor = layer4_output.y_pred
# 定义训练器
train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(
cost, var_list=params)
# 初始化所有变量
init = tf.global_variables_initializer()
# 定义训练参数
training_epochs = 10
batch_size = 100
display_step = 1
# fully-connected layer
layer3_fullyconn = HiddenLayer(layer2_flatten.output, n_in=7*7*64, n_out=256, activation=tf.nn.relu)
# dropout layer
layer3_dropout = DropoutLayer(layer3_fullyconn.output, keep_prob=0.5)
# the output layer
layer4_output = LogisticRegression(layer3_dropout.output, n_in=256, n_out=10)
# params for training
params = layer0_conv.params + layer1_conv.params + layer3_fullyconn.params + layer4_output.params
# train dicts for dropout
train_dicts = layer3_dropout.train_dicts
# prediction dicts for dropout
pred_dicts = layer3_dropout.pred_dicts
# get cost
cost = layer4_output.cost(y_)
# accuracy
accuracy = layer4_output.accuarcy(y_)
predictor = layer4_output.y_pred
# 定义训练器
train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(
cost, var_list=params)
# 初始化所有变量
init = tf.global_variables_initializer()
# 定义训练参数
training_epochs = 10
batch_size = 100
display_step = 1
class DBN(object):
def __init__(self, n_in=784, n_out=10, hidden_layers_sizes=[500, 500]):
self.n_layers = len(hidden_layers_sizes)
self.layers = [] # normal sigmoid layer
self.rbm_layers = [] # RBM layer
self.params = []
self.n_in = n_in
self.n_out = n_out
# Define the input and output
self.x = tf.placeholder(tf.float32, shape=[None, self.n_in * 2])
self.y = tf.placeholder(tf.float32, shape=[None, self.n_out * 2])
total_data = np.loadtxt(
"/mnt/disk2/liuying/T-ITS/dataset/geolife/geolife_total")
total_data = total_data[:, 2:4]
self.scaler = MinMaxScaler().fit(total_data)
self.checkpoint_times = 1
# Contruct the layers of DBN
for i in range(self.n_layers):
if i == 0:
layer_input = self.x
input_size = self.n_in * 2
else:
layer_input = self.layers[i - 1].output
input_size = hidden_layers_sizes[i - 1]
# Sigmoid layer
print("n_in:{0} n_out:{1}".format(input_size,
hidden_layers_sizes[i]))
sigmoid_layer = tf.layers.dense(inputs=layer_input,
units=hidden_layers_sizes[i],
activation=tf.nn.sigmoid,
name="layer_1")
self.layers.append(sigmoid_layer)
# Add the parameters for finetuning
self.params.extend(sigmoid_layer.params)
# Create the RBM layer
self.rbm_layers.append(
RBM(inpt=layer_input,
n_visiable=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b))
# We use the LogisticRegression layer as the output layer
self.output_layer = LogisticRegression(inpt=self.layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_out * 2)
self.params.extend(self.output_layer.params)
# The finetuning cost
self.cost = self.output_layer.cost(self.y)
# The logistic regression output
self.predictor = self.output_layer.output
self.train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(
self.cost, var_list=self.params)
self.saver = tf.train.Saver(tf.global_variables())
def mean_error(self, predict_output, real_output):
"""
compute mean error
"""
error = 0.0
predict_output = predict_output.reshape(-1, 2)
real_output = real_output.reshape(-1, 2)
for i in range(predict_output.shape[0]):
distance = np.sqrt(
np.square(predict_output[i][0] - real_output[i][0]) +
np.square(predict_output[i][1] - real_output[i][1]))
error += distance
error /= predict_output.shape[0]
return error
def process_data_for_batch(self, data):
processed_data = []
info_data = []
for i in range(len(data)):
processed_data.append([elem[2] for elem in data[i]])
info_data.append(
[data[i][len(data[i]) - 1][0], data[i][len(data[i]) - 1][1]])
return processed_data, info_data
def next_batch(self, X, y, batch_size, step, forward_only=False):
if step == len(X) / batch_size - 1:
x_batch, X_info_data = self.process_data_for_batch(X[len(X) -
batch_size:])
y_batch, y_info_data = self.process_data_for_batch(y[len(y) -
batch_size:])
else:
x_batch, X_info_data = self.process_data_for_batch(
X[step * batch_size:(step + 1) * batch_size])
y_batch, y_info_data = self.process_data_for_batch(
y[step * batch_size:(step + 1) * batch_size])
x_batch = np.array(x_batch, dtype=np.float32)
y_batch = np.array(y_batch, dtype=np.float32)
x_batch = x_batch.reshape(-1, 2)
y_batch = y_batch.reshape(-1, 2)
x_batch = self.scaler.transform(x_batch)
y_batch = self.scaler.transform(y_batch)
x_batch = x_batch.reshape(-1, self.n_in * 2)
y_batch = y_batch.reshape(-1, self.n_out * 2)
return x_batch, y_batch
def pretrain(self,
sess,
dataset,
batch_size=50,
pretraining_epochs=10,
lr=0.01,
k=1,
display_step=1):
"""
Pretrain the layers (just train the RBM layers)
"""
train_X = dataset[0]
train_y = dataset[1]
print('Starting pretraining...\n')
batch_num = len(train_X) / batch_size
print("batch_num {0}".format(batch_num))
# Pretrain layer by layer
for i in range(self.n_layers):
cost = self.rbm_layers[i].get_reconstruction_cost()
train_ops = self.rbm_layers[i].get_train_ops(learning_rate=lr,
k=k,
persistent=None)
for epoch in range(pretraining_epochs):
avg_cost = 0.0
for step in range(batch_num):
x_batch, _ = self.next_batch(train_X, train_y, batch_size,
step)
# train
sess.run(train_ops, feed_dict={self.x: x_batch})
# compute cost
tmp_cost = sess.run(cost, feed_dict={
self.x: x_batch,
})
avg_cost += tmp_cost / batch_num
if (step + 1) % 500 == 0:
print(
"\t\t\tPretraing layer {0} Epoch {1} Step {2} cost: {3}"
.format((i + 1), (epoch + 1), (step + 1),
tmp_cost))
# output
if epoch % display_step == 0:
print("\tPretraing layer {0} Epoch {1} cost: {2}".format(
(i + 1), (epoch + 1), avg_cost))
def finetuning(self,
sess,
dataset,
training_epochs=10,
start=0,
batch_size=100,
display_step=1,
model_path="",
model_name="model",
load_model=0):
"""
Finetuing the network
"""
train_X = dataset[0]
train_y = dataset[1]
test_X = dataset[2]
test_y = dataset[3]
print("\nStart finetuning...\n")
best_sess = sess
global_test_error = 100000000
tolerance_count = 0
for epoch in range(start, training_epochs):
avg_cost = 0.0
batch_num = len(train_X) / batch_size
for step in range(batch_num):
x_batch, y_batch = self.next_batch(train_X, train_y,
batch_size, step)
# train
sess.run(self.train_op,
feed_dict={
self.x: x_batch,
self.y: y_batch
})
# compute cost
avg_cost += sess.run(self.cost,
feed_dict={
self.x: x_batch,
self.y: y_batch
}) / batch_num
print "epoch:", epoch + 1, "loss: ", avg_cost
if (epoch + 1) % self.checkpoint_times == 0:
count = 0
final_error = 0.0
batch_num = len(test_X) / batch_size
for step in range(batch_num):
x_batch, y_batch = self.next_batch(test_X, test_y,
batch_size, step)
count += 1
predict = sess.run(self.predictor,
feed_dict={
self.x: x_batch,
self.y: y_batch
})
error = self.mean_error(predict, y_batch)
final_error += error
test_error = (final_error / count) * 10000
print "final mean error(x10000):", test_error
if test_error < global_test_error:
tolerance_count = 0
global_test_error = test_error
self.saver.save(
best_sess,
os.path.join(model_path,
model_name + "best_model.ckpt"))
else:
tolerance_count += 1
print "The global min test error:", global_test_error
if tolerance_count >= 50:
break
print 'The final final final global min test error:', global_test_error
def test(self, sess, dataset, batch_size=100):
test_X = dataset[2]
test_y = dataset[3]
count = 0
final_error = 0.0
batch_num = len(test_X) / batch_size
for step in range(batch_num):
x_batch, y_batch = self.next_batch(test_X, test_y, batch_size,
step)
count += 1
predict = sess.run(self.predictor,
feed_dict={
self.x: x_batch,
self.y: y_batch
})
error = self.mean_error(predict, y_batch)
final_error += error
print("\nTest step :{0}, mean_error:{1}".format(step, error))
final_error /= count
print "final mean error:", final_error