def __init__(self, input, n_inp, n_hidden, n_out):
""" Initialize the parameters for the multilayer perceptron
:param input: symbolic variable that describes the input of the architecture
(one minibatch)
:param n_inp: int, number of input units, the dimension of the space in
which the datapoints lie
:param n_hidden: int, number of hidden units
:param n_out: int, number of output units, the dimension of the space in
which the labels lie
"""
"""Since we are dealing with a one hidden layer MLP, this will traslate
into a HiddenLayer with a sigmoid function connected to the LogisticRegression
layer; the activation function can be replaced by tanh or any other
nonlinear function
"""
self.hiddenLayer = HiddenLayer(input,
n_inp=n_inp,
n_out=n_hidden,
activation=tf.nn.sigmoid)
#The logistic regression layer gets as input the hidden units
#of the hidden layer
self.logRegressionLayer = LogisticRegression(self.hiddenLayer.output,
n_inp=n_hidden,
n_out=n_out)
#L1 norm; one regularization option is to enforce L1 norm to be small
self.L1 = tf.reduce_sum(tf.abs(self.hiddenLayer.W)) + tf.reduce_sum(
tf.abs(self.logRegressionLayer.W))
#square of L2 norm; one regularization option is to enforce square of L2 norm to be small
self.L2_sqr = tf.reduce_sum(self.hiddenLayer.W**2) + tf.reduce_sum(
self.logRegressionLayer.W**2)
#cross entropy cost
self.cost = self.logRegressionLayer.cost
self.accuracy = self.logRegressionLayer.accuracy
#the parameters of the model are the parameters of the two layer it is made out of
self.params = self.hiddenLayer.params + self.logRegressionLayer.params
self.input = input
def __init__(self, n_inp=784, n_out=10, hidden_layer_sizes=[500, 500]):
""" This class is made to support a variable number of layers.
:param n_inps: int, dimension of the input to the DBN
:param n_outs: int, demension of the output of the network
:param hidden_layer_sizes: list of ints, intermediate layers size, must contain
at least one value
"""
self.sigmoid_layers = []
self.layers = []
self.params = []
self.n_layers = len(hidden_layer_sizes)
assert self.n_layers > 0
#define the grape
height, weight, channel = n_inp
self.x = tf.placeholder(tf.float32, [None, height, weight, channel])
self.y = tf.placeholder(tf.float32, [None, n_out])
for i in range(self.n_layers):
# Construct the sigmoidal layer
# the size of the input is either the number of hidden units of the layer
# below or the input size if we are on the first layer
if i == 0:
input_size = height * weight * channel
else:
input_size = hidden_layer_sizes[i - 1]
# the input to this layer is either the activation of the hidden layer below
# or the input of the DBN if you are on the first layer
if i == 0:
layer_input = tf.reshape(self.x,
[-1, height * weight * channel])
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(input=layer_input,
n_inp=input_size,
n_out=hidden_layer_sizes[i],
activation=tf.nn.sigmoid)
#add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# Its arguably a philosophical question... but we are going to only
# declare that the parameters of the sigmoid_layers are parameters of the DBN.
# The visible biases in the RBM are parameters of those RBMs, but not of the DBN
self.params.extend(sigmoid_layer.params)
if i == 0:
rbm_layer = GRBM(inp=layer_input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
else:
rbm_layer = RBM(inp=layer_input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.layers.append(rbm_layer)
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_inp=hidden_layer_sizes[-1],
n_out=n_out)
self.params.extend(self.logLayer.params)
#print(self.sigmoid_layers[-1].output)
#print(hidden_layer_sizes[-1], n_out)
#compute the cost for second phase of training, defined as the cost of the
# logistic regression output layer
self.finetune_cost = self.logLayer.cost(self.y)
#compute the gradients with respect to the model parameters symbolic variable that
# points to the number of errors made on the minibatch given by self.x and self.y
self.pred = self.logLayer.pred
self.accuracy = self.logLayer.accuracy(self.y)
"""
class DBN(object):
"""Deep belief network
A deep belief network is obtained by stacking several RBMs on top of the each other.
The hidden layer of the RBM at layer 'i' becomes the input of the RBM at layer 'i+1'.
The first layer RBM gets as input the input of the network, and the hidden layer of
the last RBM represents the output. When used for classification, the DBN is treated
as a MLP, by adding a logistic regression layer on top.
"""
def __init__(self, n_inp=784, n_out=10, hidden_layer_sizes=[500, 500]):
""" This class is made to support a variable number of layers.
:param n_inps: int, dimension of the input to the DBN
:param n_outs: int, demension of the output of the network
:param hidden_layer_sizes: list of ints, intermediate layers size, must contain
at least one value
"""
self.sigmoid_layers = []
self.layers = []
self.params = []
self.n_layers = len(hidden_layer_sizes)
assert self.n_layers > 0
#define the grape
height, weight, channel = n_inp
self.x = tf.placeholder(tf.float32, [None, height, weight, channel])
self.y = tf.placeholder(tf.float32, [None, n_out])
for i in range(self.n_layers):
# Construct the sigmoidal layer
# the size of the input is either the number of hidden units of the layer
# below or the input size if we are on the first layer
if i == 0:
input_size = height * weight * channel
else:
input_size = hidden_layer_sizes[i - 1]
# the input to this layer is either the activation of the hidden layer below
# or the input of the DBN if you are on the first layer
if i == 0:
layer_input = tf.reshape(self.x,
[-1, height * weight * channel])
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(input=layer_input,
n_inp=input_size,
n_out=hidden_layer_sizes[i],
activation=tf.nn.sigmoid)
#add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# Its arguably a philosophical question... but we are going to only
# declare that the parameters of the sigmoid_layers are parameters of the DBN.
# The visible biases in the RBM are parameters of those RBMs, but not of the DBN
self.params.extend(sigmoid_layer.params)
if i == 0:
rbm_layer = GRBM(inp=layer_input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
else:
rbm_layer = RBM(inp=layer_input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.layers.append(rbm_layer)
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_inp=hidden_layer_sizes[-1],
n_out=n_out)
self.params.extend(self.logLayer.params)
#print(self.sigmoid_layers[-1].output)
#print(hidden_layer_sizes[-1], n_out)
#compute the cost for second phase of training, defined as the cost of the
# logistic regression output layer
self.finetune_cost = self.logLayer.cost(self.y)
#compute the gradients with respect to the model parameters symbolic variable that
# points to the number of errors made on the minibatch given by self.x and self.y
self.pred = self.logLayer.pred
self.accuracy = self.logLayer.accuracy(self.y)
"""
# Initialize with 0 the weights W as a matrix of shape (n_inp, n_out)
out_weights = tf.Variable(tf.zeros([hidden_layer_sizes[-1], n_out]))
# Initialize the biases b as a vector of n_out 0s
out_biases = tf.Variable(tf.zeros([n_out]))
out_ = tf.nn.softmax(tf.matmul(self.sigmoid_layers[-1].output, out_weights) + out_biases)
self.finetune_cost = -tf.reduce_mean(tf.reduce_sum(self.y * tf.log(out_)))
self.optimizer = tf.train.GradientDescentOptimizer(learning_rate = learning_rate).minimize(self.finetune_cost)
correct_prediction = tf.equal(tf.argmax(out_,1), tf.argmax(self.y,1))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
"""
def pretraining(self,
sess,
train_set_x,
batch_size=100,
pretraining_epochs=10,
learning_rate=0.0001,
k=1,
display_step=1):
""" Generates a list of functions, for performing one step of gradient descent at
a given layer. The function will require as input the minibatch index, and to train
an RBM you just need to iterate, calling the corresponding function in all minibatch
indexes.
:param train_set_x: tensor, contains all datapoints used for traing the RBM
:param batch_size: int, size of a minibatch
:param k: number of Gibbs steps to do in CD-k/ PCD-k
:param learning_rate: learning rate
:param pretraining_epochs: int, maximal number of epochs to run the optimizer
"""
#begining of a batch, given index
start_time = timeit.default_timer()
batch_num = int(train_set_x.train.num_examples / batch_size)
#Pretraining layer by layer
for i in range(self.n_layers):
# Get the cost and the updates list
#Using CD-k here for training each RBM
#TODO: change cost function to reconstruction error
#cost = self.layers[i].get_reconstruction_cost()
#train_ops = self.layers[i].get_train_ops(lr = learning_rate, persistent = None, k = k)
#print(self.rbm_layers[i].n_visible, self.rbm_layers[i].n_hidden)
if i == 0:
learning_rate = 0.001
else:
learning_rate = 0.001
cost, train_ops = self.layers[i].get_train_ops(lr=learning_rate,
persistent=None,
k=k)
#cost = self.layers[i].get_reconstruction_cost()
for epoch in range(pretraining_epochs):
avg_cost = 0.0
for j in range(batch_num):
batch_xs, batch_ys = train_set_x.train.next_batch(
batch_size)
_ = sess.run(train_ops, feed_dict={
self.x: batch_xs,
})
c = sess.run(cost, feed_dict={
self.x: batch_xs,
})
avg_cost += c / batch_num
if epoch % display_step == 0:
print("Pretraining layer {0} Epoch {1}".format(
i + 1, epoch + 1) + " cost {:.9f}".format(avg_cost))
end_time = timeit.default_timer()
print("time {0} minutes".format(
(end_time - start_time) / 60.))
#plt.imsave("new_filters_at_{0}.png".format(epoch),tile_raster_images(X = sess.run(tf.transpose(self.rbm_layers[i].W)), img_shape = (28, 28), tile_shape = (10,10), tile_spacing = (1,1)), cmap = 'gray')
#plt.show()
end_time = timeit.default_timer()
print("time {0} minutes".format((end_time - start_time) / 60.))
def fine_tuning(self,
sess,
train_set_x,
batch_size=10,
training_epochs=1000,
learning_rate=0.1,
display_step=1):
""" Genarates a function train that implements one step of finetuning, a function validate
that computes the error on a batch from the validation set, and a function test that computes
the error on a batch from the testing set
:param datasets: tensor, a list contain all the dataset
:param batch_size: int, size of a minibatch
:param learning_rate: int, learning rate
:param training_epochs: int, maximal number of epochs to run the optimizer
"""
start_time = timeit.default_timer()
train_ops = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate).minimize(self.finetune_cost,
var_list=self.params)
#Accuracy
accuracy = self.accuracy
batch_num = int(train_set_x.train.num_examples / batch_size)
ACC_max = 0
pre_max = 0
rec_max = 0
for epoch in range(training_epochs):
avg_cost = 0.0
for i in range(batch_num):
b = []
d = []
batch_xs, batch_ys = train_set_x.train.next_batch(batch_size)
_ = sess.run(train_ops,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
c = sess.run(self.finetune_cost,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
avg_cost += c / batch_num
if epoch % display_step == 0:
print("Epoch:", '%04d' % (epoch + 1), "cost:",
"{:.9f}".format(avg_cost))
#Khoa stop: acc = sess.run(accuracy, feed_dict = {self.x: train_set_x.test.segments, self.y: train_set_x.test.labels})
pr = sess.run(self.pred,
feed_dict={
self.x: train_set_x.test.segments,
self.y: train_set_x.test.labels
})
b = np.append(b, sess.run(tf.argmax(pr, axis=1)))
#np.savetxt('b.txt', b ,delimiter=',')
d = np.append(
d,
sess.run(tf.argmax(self.y, axis=1),
feed_dict={
self.x: train_set_x.test.segments,
self.y: train_set_x.test.labels
}))
#np.savetxt('d.txt', d ,delimiter=',')
a = confusion_matrix(d, b)
FP = a.sum(axis=0) - np.diag(a)
FN = a.sum(axis=1) - np.diag(a)
TP = np.diag(a)
TN = a.sum() - (FP + FN + TP)
ac = (TP + TN) / (TP + FP + FN + TN)
ACC = ac.sum() / 10
precision = precision_score(d, b,
average='weighted') #TP/ (TP+FP)
recall = recall_score(d, b, average='weighted') #TP/ (TP+FN)
if ACC > ACC_max:
ACC_max = ACC
if precision > pre_max:
pre_max = precision
if recall > rec_max:
rec_max = recall
print(ac.sum() / 10)
print(a)
print("ACCURACY: {0}, PRECISION: {1}, RECALL: {2}:".format(
ACC_max, pre_max, rec_max))
end_time = timeit.default_timer()
print("Time {0} minutes".format((end_time - start_time) / 60.))
end_time = timeit.default_timer()
print("Time {0} minutes".format((end_time - start_time) / 60.))