def generator_of_batches(batch_size, dataset_type):
if dataset_type == MyUtils_flags.FLAG_TRAIN:
dataset_length = MyUtils_dbs.get_nn_dataset_length(
MyUtils_flags.FLAG_TRAIN)
db_conn = sqlite3.connect(F.NN_TRAIN_INSTANCES_DB)
if dataset_type == MyUtils_flags.FLAG_VALID:
dataset_length = MyUtils_dbs.get_nn_dataset_length(
MyUtils_flags.FLAG_VALID)
db_conn = sqlite3.connect(F.NN_VALID_INSTANCES_DB)
if dataset_type == MyUtils_flags.FLAG_TEST:
dataset_length = MyUtils_dbs.get_nn_dataset_length(
MyUtils_flags.FLAG_TEST)
db_conn = sqlite3.connect(F.NN_TEST_INSTANCES_DB)
c = db_conn.cursor()
num_of_batches = dataset_length // batch_size + 1
half_mark_offset = dataset_length // 2
for i in range(0, num_of_batches):
start_index_pos = i * (batch_size // 2)
end_index_pos = min((i + 1) * (batch_size // 2), half_mark_offset)
start_index_neg = half_mark_offset + i * (batch_size // 2)
end_index_neg = half_mark_offset + min(
(i + 1) * (batch_size // 2), dataset_length)
c.execute("SELECT p_id, q_id, x,y FROM instances WHERE rowid IN " +
str(tuple(range(start_index_pos, end_index_pos))))
rows = c.fetchall()
c.execute("SELECT p_id, q_id, x,y FROM instances WHERE rowid IN " +
str(tuple(range(start_index_neg, end_index_neg))))
rows_neg = c.fetchall()
rows.extend(rows_neg)
batch = list(
map(
lambda elem: (str(elem[0]), str(elem[1]), json.loads(elem[2]),
int(elem[3])), rows))
yield batch
def train_NN(learning_rate = 0.01, max_epochs=1000, batch_size=32, dropout_rate=0, hiddenlayers_ls=None):
MyUtils.init_logging("train_NN.log")
hiddenlayers_ls_str = [str(num_elems) for num_elems in hiddenlayers_ls]
tensorboard_dir_path = os.path.join(F.TENSORBOARD_ANN_DIR, 'trainingset_' + str(
MyUtils_dbs.get_nn_dataset_length("train")),
"bs_" + str(batch_size),
"hls_" + "-".join(hiddenlayers_ls_str),
"lr_" + str(learning_rate),
"drop_" + str(dropout_rate) + "eps_" + str(max_epochs))
if not os.path.exists(tensorboard_dir_path):
os.makedirs(tensorboard_dir_path)
MyUtils_filesystem.clean_directory(tensorboard_dir_path)
tf.reset_default_graph()
session = tf.Session()
logging.info("Creating the placeholders for input and labels...")
(input_placeholder, labels_placeholder) = NN.get_model_placeholders(batch_size)
placeholders = (input_placeholder, labels_placeholder)
logging.info("Connecting the loss computation and forward structure...")
train_loss = NN.nn_loss_computation(logits=NN.nn_inference(input=input_placeholder, layers_hidden_units_ls=hiddenlayers_ls,
dropout_rate=dropout_rate),
labels=labels_placeholder)
lrate_tensor = tf.placeholder(shape=[], dtype=tf.float32, name="lrate_tensor")
####### Defining the optimizer
if str(learning_rate).lower() == MyUtils_flags.FLAG_ADAM:
starting_lrate = MyUtils_flags.FLAG_ADAM
optimizer = tf.train.AdamOptimizer()
else:
if str(learning_rate).lower() == MyUtils_flags.FLAG_RMSPROP:
starting_lrate = MyUtils_flags.FLAG_RMSPROP
optimizer = tf.train.RMSPropOptimizer(0.001)
else:
starting_lrate = learning_rate
optimizer = tf.train.GradientDescentOptimizer(lrate_tensor)
if str(learning_rate).lower() == MyUtils_flags.FLAG_CLR:
_best_lr, min_lr, max_lr = CLR.find_cyclical_lrate_loop(placeholders, batch_size, hiddenlayers_ls,
dropout_rate)
# Summaries, and gathering information:
train_loss_summary = tf.summary.scalar('Cross-entropy', train_loss)
predictions = tf.argmax(tf.nn.softmax(logits=NN.nn_inference(input_placeholder, hiddenlayers_ls, dropout_rate)),
axis=1, name="predictions")
tf_metric, tf_metric_update = tf.metrics.accuracy(labels=labels_placeholder, predictions=predictions,
name="accuracy")
accuracy_summary = tf.summary.scalar('Accuracy', tf_metric_update)
logging.info("Defining the optimizer's minimization task on the loss function...")
minimizer_task = optimizer.minimize(train_loss)
#Global variables are initialized after the graph structure
tf.global_variables_initializer().run(session=session)
#defining the tasks that will be run inside the training loop
training_tasks = [minimizer_task, train_loss, predictions, tf_metric_update]
validation_tasks = [tf_metric_update, predictions]
validation_writing_tasks = [accuracy_summary]
train_writing_tasks = [train_loss_summary, accuracy_summary]
tasks_dictionary = {MyUtils_flags.FLAG_TRAIN_TASKS: training_tasks,
MyUtils_flags.FLAG_WRITING_TRAIN_TASKS: train_writing_tasks,
MyUtils_flags.FLAG_VALIDATION_TASKS: validation_tasks,
MyUtils_flags.FLAG_WRITING_VALIDATION_TASKS: validation_writing_tasks}
#connection to the validation dataset
valid_db_conn = sqlite3.connect(F.NN_VALID_INSTANCES_DB)
valid_db_cursor = valid_db_conn.cursor()
if str(learning_rate).lower() == MyUtils_flags.FLAG_CLR:
CLR.training_loop_clr(tasks_dictionary, placeholders, batch_size,
max_epochs, min_lr, max_lr, valid_db_cursor, tensorboard_dir_path)
else:
training_loop(tasks_dictionary, placeholders, starting_lrate,
batch_size, max_epochs, valid_db_cursor, tensorboard_dir_path, session)
def get_num_training_iterations(batch_size):
trainset_length = MyUtils_dbs.get_nn_dataset_length(
MyUtils_flags.FLAG_TRAIN)
max_iter = trainset_length // batch_size # in 1 epoch, you can not have more iterations than batches
logging.info("Number of iterations per epoch: %s", max_iter)
return max_iter
def find_cyclical_lrate_loop(placeholders,
batch_size,
hiddenlayers_ls,
drop_rate,
lrate_start=10**(-7),
lrate_end=0.2):
trainset_length = MyUtils_dbs.get_nn_dataset_length(
MyUtils_flags.FLAG_TRAIN)
max_iter = trainset_length // batch_size # in 1 epoch, you can not have more iterations than batches
hiddenlayers_ls_str = [str(num_elems) for num_elems in hiddenlayers_ls]
tensorboard_dir_path = os.path.join(
F.TENSORBOARD_ANN_DIR,
'trainingset_' + str(MyUtils_dbs.get_nn_dataset_length("train")),
"bs_" + str(batch_size), "hls_" + "-".join(hiddenlayers_ls_str),
"lr_clr", "drop_" + str(drop_rate) + "_explore")
if not os.path.exists(tensorboard_dir_path):
os.makedirs(tensorboard_dir_path)
session = tf.Session(
) # separate session for trying to find the optimal l.r. for the Cyclical Learning Rate
logging.info("*** Session: Cyclical Learning Rate")
(input_placeholder, labels_placeholder) = placeholders
train_loss = NN.nn_loss_computation(logits=NN.nn_inference(
input=input_placeholder,
layers_hidden_units_ls=hiddenlayers_ls,
dropout_rate=drop_rate),
labels=labels_placeholder)
#train_loss_summary = tf.summary.scalar('Cross-entropy', train_loss)
predictions = tf.argmax(tf.nn.softmax(
logits=NN.nn_inference(input_placeholder, hiddenlayers_ls, drop_rate)),
axis=1)
#tf_metric, tf_metric_update = tf.metrics.accuracy(labels=labels_placeholder, predictions=predictions,
# name="CLR_train_accuracy")
running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES,
scope="CLR_train_accuracy")
#train_accuracy_summary = tf.summary.scalar('Accuracy', tf_metric_update)
lrate_tensor = tf.placeholder(shape=[],
dtype=tf.float32,
name="lrate_tensor")
optimizer = tf.train.GradientDescentOptimizer(lrate_tensor)
minimizer_task = optimizer.minimize(train_loss)
# Global variables are initialized after the graph structure
tf.global_variables_initializer().run(session=session)
running_vars_initializer = tf.variables_initializer(var_list=running_vars)
session.run(running_vars_initializer)
logging.info(
"Number of iterations per epoch (and linear steps in the search for the learning rate): %s",
max_iter)
lrate_increase = (lrate_end - lrate_start) / max_iter
logging.info(
"Step increase of the learning rate in the exploration epochs: %s",
round(lrate_increase, 7))
trial_epochs = 5
loss_matrix = np.zeros((trial_epochs, max_iter))
accuracy_matrix = np.zeros((trial_epochs, max_iter))
for i in range(1, trial_epochs + 1):
start_epoch_time = time()
logging.info(
"Search for the base learning rate; Starting training epoch n. %s",
i)
batch_generator = NN.generator_of_batches(batch_size,
MyUtils_flags.FLAG_TRAIN)
# Train, in the current epoch
for j in range(0, max_iter):
session.run(running_vars_initializer
) #new batch: re-initializing the accuracy computation
batch = \
batch_generator.__next__()
current_iteration_feed_dict = NN.fill_feed_dict(
batch, input_placeholder, labels_placeholder)
learning_rate = lrate_start + lrate_increase * j
current_iteration_feed_dict.update({lrate_tensor: learning_rate})
if j % (max_iter // 20) == 0:
logging.info("Iteration: %s on %s .", j, max_iter)
_, current_loss, b_predictions, b_labels = session.run(
[minimizer_task, train_loss, predictions, labels_placeholder],
feed_dict=current_iteration_feed_dict)
loss_matrix[i - 1][j] = current_loss
accuracy_matrix[i - 1][j] = get_batch_accuracy(
b_predictions, b_labels)
end_epoch_time = time()
logging.info(
"Searching for the base values for the cyclical learning rate. " +
"Training on epoch %s executed. Time elapsed: %s", i,
round(end_epoch_time - start_epoch_time, 3))
best_lr, min_lr, max_lr = pick_lr_boundaries(loss_matrix, lrate_start,
lrate_increase)
session.close()
#write the lr to a logfile
lrfile = open(os.path.join(tensorboard_dir_path, "found_lr.log"), "w")
lrfile.write(
"Cyclical Learning rate: applying the LR test on " +
str(trial_epochs) + "epochs ;\n " +
"the average learning rate granting the steepest descent of the loss function is: "
+ str(best_lr))
lrfile.close()
return best_lr, min_lr, max_lr