def fit(self, X, y):
"""
Fitting model for training data. If validation split is defined, model will use early stoping!!
"""
self.close_session()
# defining inputs and outputs of model
n_inputs = X.shape[1]
#self.classes_ = np.unique(y)
n_outputs = y.shape[1]
#building graph
self._graph = tf.Graph()
with self._graph.as_default():
self.__build_graph(n_inputs, n_outputs)
##splitting dataset
if self.validation_split:
self.X_train, self.X_valid, self.y_train, self.y_valid = self._get_validation_set(
X,
y,
validation_split=self.validation_split,
random_state=self.random_state)
else:
self.X_train, self.y_train = X, y
#early stoping params
self.checks_without_progress = 0
self.best_loss = np.infty
self.best_params = None
# Training model
self._session = tf.Session(graph=self._graph)
with self._session.as_default() as sess:
#directories
self.tensorboard_logdir, self.saver_directory = self._get_directories(
)
#summary writter
self.summary_writer = tf.summary.FileWriter(
self.tensorboard_logdir, sess.graph)
#start session
self._init.run()
for epoch in range(self.n_epochs):
#progress bar lengh
self.total_batch = int(self.X_train.shape[0] / self.batch_size)
self.bar = tqdm(range(self.total_batch), ncols=120, ascii=True)
##tensorboard variables
self.loss_per_epoch = []
self.accuracy_per_epoch = []
self.accuracy_per_epoch_val = []
for batch in self.bar:
X_batch, y_batch = self._get_random_batch(
self.X_train, self.y_train, batch_size=self.batch_size)
feed_dict = {
self._X: X_batch,
self._y: y_batch,
self._learning_rate: self.learning_rate
}
if self.batch_normalization or self.dropout is not None:
feed_dict[self._training] = True
if self._extra_ops:
sess.run(self._extra_ops, feed_dict=feed_dict)
if self.kernel_regulizer == max_norm_regularizer():
sess.run(self._clip_weights, feed_dict=feed_dict)
sess.run(self._training_op, feed_dict=feed_dict)
"""self.train_loss, self.train_accurancy = sess.run([self._loss, self._accuracy],
feed_dict=feed_dict)"""
self.train_loss, self._predictions = sess.run(
[self._loss, self._y_proba], feed_dict=feed_dict)
self.train_accurancy = self._accuracy_eval(
self._predictions, y_batch)
self.loss_per_epoch.append(self.train_loss)
self.accuracy_per_epoch.append(self.train_accurancy)
self.bar.set_description(
"Epoch: {}, Training cost: {:.6f} Traning acc: {:.3f}".
format(epoch, np.mean(self.loss_per_epoch),
np.mean(self.accuracy_per_epoch)))
self.avg_train_loss = np.mean(self.loss_per_epoch)
self.avg_acc_train = np.mean(self.accuracy_per_epoch)
if self.X_valid is not None and self.y_valid is not None:
"""self.loss_val, self.acc_val = sess.run([self._loss, self._accuracy],
feed_dict={self._X: self.X_valid, self._y: self.y_valid})"""
self.loss_val, self.prediction_val = sess.run(
[self._loss, self._y_proba],
feed_dict={
self._X: self.X_valid,
self._y: self.y_valid
})
self.val_acc = self._accuracy_eval(self.prediction_val,
self.y_valid)
self.acc_val = np.mean(self.val_acc)
self.val_loss, self.rec, self.prec, self.acc_form = sess.run(
[self._loss, self._rec, self._prec, self._acc_formula],
feed_dict={
self._X: self.X_valid,
self._y: self.y_valid
})
print(f"Valid Acc : {self.acc_form}")
if self.loss_val < self.best_loss:
self.best_params = self._get_model_params()
self.best_loss = self.loss_val
self.checks_without_progress = 0
else:
self.checks_without_progress += 1
#set description of training/validation
self.bar.set_description(
"Epoch: {}, Training cost: {:.6f}, Validation cost: {:.6f}, Traning acc: {:.3f}, Validation acc:{:.3f}"
.format(epoch, self.avg_train_loss, self.loss_val,
self.avg_acc_train * 100, self.acc_val * 100))
if self.checks_without_progress > self.early_stoping_rounds:
print("Early Stopping!")
break
##summary writting
self.summary = sess.run(
self._merged_summaries,
feed_dict={
self._loss_summary_ph: self.avg_train_loss,
self._accuracy_summary_ph: self.avg_acc_train,
self._val_loss_summary_ph: self.val_loss,
self._val_accuracy_summary_ph: self.acc_form,
self._recall_summary_ph: self.rec,
self._precision_summary_ph: self.prec
})
self.summary_writer.add_summary(self.summary, epoch)
# if early stopping works, return best params of model
if self.best_params:
self._restore_model_params(self.best_params)
return self
os.system('tensorboard --logdir=' + self.root_logdir)
def reset_graph(seed=42):
tf.compat.v1.reset_default_graph()
tf.compat.v1.set_random_seed(seed)
np.random.seed(seed)
reset_graph()
tf.reset_default_graph()
if __name__ == '__main__':
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
X_train = mnist.train.images
y_train = mnist.train.labels
dnn_clf = DNN_classifier(n_neurons=100,
n_hidden_layers=10,
validation_split=0.2,
kernel_regulizer=max_norm_regularizer(),
batch_normalization=True,
learning_rate=0.015,
use_bias=True,
batch_size=500,
dropout=True,
dropout_rate=0.4)
dnn_clf.fit(X_train, y_train)
dnn_clf.run_tensorboard()
def __build_graph(self, n_inputs, n_outputs):
"""
Build tensorflow computional graph
"""
if self.random_state:
tf.set_random_seed(self.random_state)
np.random.seed(self.random_state)
X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int32, shape=(None, n_outputs), name="y")
training = tf.placeholder_with_default(False,
shape=(),
name="is_training")
logits = self.__build_model(X, y, training=training)
y_proba = tf.nn.softmax(logits, name="Y_proba")
with tf.name_scope("loss"):
xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=y, logits=logits, name="cross_entropy")
loss = tf.reduce_mean(xentropy, name="loss")
with tf.name_scope("training"):
learning_rate = tf.placeholder(dtype=tf.float32,
shape=None,
name="learning_rate")
if self.optimizer != tf.train.MomentumOptimizer:
optimizer = self.optimizer(learning_rate=learning_rate)
else:
optimizer = self.optimizer(learning_rate=learning_rate,
momentum=0.9)
training_op = optimizer.minimize(loss)
with tf.name_scope("performance"):
#loss summary
loss_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="loss_summary")
loss_summary = tf.summary.scalar("Loss", loss_summary_ph)
#accurancy summary
accuracy_summary_ph = tf.placeholder(tf.float32,
shape=None,
name='accuracy_summary')
accuracy_summary = tf.summary.scalar('accuracy',
accuracy_summary_ph)
#val loss
val_loss_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="val_loss_summary")
val_loss_summary = tf.summary.scalar("val_Loss",
val_loss_summary_ph)
#val accurancy summary
val_accuracy_summary_ph = tf.placeholder(
tf.float32, shape=None, name='val_accuracy_summary')
val_accuracy_summary = tf.summary.scalar('val_accuracy',
val_accuracy_summary_ph)
#recall summary
recall_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="recall_summary")
recall_summary = tf.summary.scalar('recall', recall_summary_ph)
#precision symmary
precision_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="recall_summary")
precision_summary = tf.summary.scalar('precision',
recall_summary_ph)
#merged_summaries = tf.summary.merge([loss_summary, accuracy_summary, recall_summary, precision_summary])
merged_summaries = tf.summary.merge_all()
with tf.name_scope("accurancy_metrics"):
argmax_prediction = tf.argmax(y_proba, 1)
argmax_y = tf.argmax(y, 1)
#needed values for recall and precison calc
TP = tf.count_nonzero(argmax_prediction * argmax_y,
dtype=tf.float32)
TN = tf.count_nonzero((argmax_prediction - 1) * (argmax_y - 1),
dtype=tf.float32)
FP = tf.count_nonzero(argmax_prediction * (argmax_y - 1),
dtype=tf.float32)
FN = tf.count_nonzero((argmax_prediction - 1) * argmax_y,
dtype=tf.float32)
"""TP = tf.count_nonzero(y_proba * y, dtype=tf.float32)
TN = tf.count_nonzero((y_proba - 1) * (y - 1), dtype=tf.float32)
FP = tf.count_nonzero(y_proba * (y - 1), dtype=tf.float32)
FN = tf.count_nonzero((y_proba - 1) * y, dtype=tf.float32)"""
##calcs:
precision = TP / (TP + FP)
recall = TP / (TP + FN)
acc = tf.reduce_mean(
tf.cast(tf.equal(argmax_prediction, argmax_y), tf.float32))
with tf.name_scope("initialization"):
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.name_scope("extra_operations"):
extra_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if self.kernel_regulizer == max_norm_regularizer():
self._clip_weights = tf.get_collection("max_norm")
## variable sharing for graph computation periods
self._X, self._y, self._training = X, y, training
self._learning_rate = learning_rate
self._y_proba, self._loss = y_proba, loss
#self._training_op, self._accuracy = training_op, accuracy
self._training_op = training_op
self._init, self._saver = init, saver
self._extra_ops = extra_ops
self._loss_summary_ph, self._loss_summary = loss_summary_ph, loss_summary
self._accuracy_summary_ph, self._accuracy_summary = accuracy_summary_ph, accuracy_summary
self._recall_summary_ph, self._recall_summary = recall_summary_ph, recall_summary
self._precision_summary_ph, self._precision_summary = precision_summary_ph, precision_summary
self._val_loss_summary_ph, self._val_loss_summary = val_loss_summary_ph, val_loss_summary
self._val_accuracy_summary_ph, self._val_accuracy_summary = val_accuracy_summary_ph, val_accuracy_summary
self._merged_summaries = merged_summaries
##eval metrics
self._rec, self._prec, self._acc_formula = precision, recall, acc
"n_neurons": [10, 30, 50, 70, 90, 100, 120, 140, 160],
"batch_size": [50, 100, 500],
"learning_rate": [0.01, 0.02, 0.05, 0.1],
"activation":
[tf.nn.relu, tf.nn.elu,
leaky_relu(alpha=0.01),
leaky_relu(alpha=0.1)],
"n_hidden_layers": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
"optimizer": [
tf.train.AdamOptimizer,
partial(tf.train.MomentumOptimizer, momentum=0.95)
],
"batch_norm_momentum": [0.9, 0.95, 0.99, 0.999],
"dropout_rate": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6],
"use_bias": [True, False],
"kernel_regulizer": [None, max_norm_regularizer()]
}
dnn_clc = DNN_classifier(early_stoping_rounds=7,
batch_normalization=True,
validation_split=0.2,
n_epochs=30)
rnd_search = RandomizedSearchCV(dnn_clc,
param_distribs,
n_iter=10,
random_state=42,
verbose=2,
n_jobs=3)
rnd_search.fit(X_train, y_train)
best_params = rnd_search.best_params_