def train_predictions_with_one_environment_data(env_idx, data_gen, dataset_sizes, sess):
global logger
global tf_train_predictions, tf_train_labels
avg_accuracy, avg_soft_accuracy = [], []
for step in range(dataset_sizes['train_dataset'][env_idx] // config.BATCH_SIZE):
tr_img_id, tr_images, tr_labels = data_gen.sample_a_batch_from_data(env_idx, shuffle=False)
feed_dict = {data_gen.tf_image_ph: tr_images, data_gen.tf_label_ph: tr_labels}
train_predictions, train_actuals = sess.run([tf_train_predictions, tf_train_labels], feed_dict=feed_dict)
assert train_predictions.shape[0] == train_actuals.shape[0]
avg_soft_accuracy.append(
models_utils.soft_accuracy(
train_predictions, train_actuals,
use_argmin=False, max_thresh=max_thresh,
min_thresh=min_thresh
)
)
avg_accuracy.append(models_utils.accuracy(train_predictions, train_actuals, use_argmin=False))
if step < 2:
logger.debug('Predictions for Non-Collided data')
for pred, lbl in zip(train_predictions, train_actuals):
logger.debug('\t%s;%s', pred, lbl)
return {'accuracy': np.mean(avg_accuracy),'soft_accuracy':np.mean(avg_soft_accuracy)}
def test_predictions_with_one_environment_data(test_env_idx, test_data_gen, sess):
global tf_test_predictions, tf_test_labels
test_accuracy = [] # accuracy by argmax
soft_test_accuracy = [] # accuracy if the prediction entry for the non-zero entry of the label exceeds max thres
all_predictions, all_labels, all_img_ids, all_images = None, None, None, None
test_image_index = 0
for step in range(dataset_sizes['test_dataset'][test_env_idx] // config.BATCH_SIZE):
ts_img_id, ts_images, ts_labels = test_data_gen.sample_a_batch_from_data(test_env_idx, shuffle=False)
feed_dict = {test_data_gen.tf_image_ph: ts_images,
test_data_gen.tf_label_ph: ts_labels}
predicted_labels, test_actuals = sess.run([tf_test_predictions, tf_test_labels],
feed_dict=feed_dict
)
test_accuracy.append(models_utils.accuracy(predicted_labels, test_actuals, use_argmin=False))
soft_test_accuracy.append(
models_utils.soft_accuracy(predicted_labels, test_actuals, use_argmin=False, max_thresh=max_thresh,
min_thresh=min_thresh))
if all_predictions is None or all_labels is None:
all_predictions = predicted_labels
all_labels = test_actuals
all_img_ids = ts_img_id
all_images = ts_images
else:
all_predictions = np.append(all_predictions, predicted_labels, axis=0)
all_labels = np.append(all_labels, test_actuals, axis=0)
all_img_ids = np.append(all_img_ids, ts_img_id, axis=0)
all_images = np.append(all_images, ts_images, axis=0)
if step < 2:
logger.debug('Test Predictions (Non-Collisions)')
for pred, act in zip(predicted_labels, test_actuals):
pred_string = ''.join(['%.3f' % p + ',' for p in pred.tolist()])
act_string = ''.join(['%.1f' % a + ',' for a in act.tolist()])
predictionlogger.info('%d:%s:%s', test_image_index, act_string, pred_string)
if step < 2:
logger.debug('%d:%s:%s', test_image_index, act_string, pred_string)
test_image_index += 1
# Write predictions to log file
testPredictionLogger.info('Predictions for Non-Collisions (Epoch %d)', epoch)
test_image_index = 0
for pred, act in zip(all_predictions, all_labels):
pred_string = ''.join(['%.3f' % p + ',' for p in pred.tolist()])
act_string = ''.join(['%.3f' % a + ',' for a in act.tolist()])
testPredictionLogger.info('%d:%s:%s', test_image_index, act_string, pred_string)
test_image_index += 1
testPredictionLogger.info('\n')
# Calculating Precisio Recall
test_noncol_precision = models_utils.precision_multiclass(
all_predictions, all_labels, use_argmin=False,
max_thresh=max_thresh, min_thresh=min_thresh
)
test_noncol_recall = models_utils.recall_multiclass(
all_predictions, all_labels, use_argmin=False,
max_thresh=max_thresh, min_thresh=min_thresh
)
return {'accuracy':np.mean(test_accuracy), 'soft_accuracy':np.mean(soft_test_accuracy),
'precision': test_noncol_precision, 'recall': test_noncol_recall}
def test_the_model_5_way(sess, tf_test_labels, tf_test_predictions,
dataset_size_dict):
test_accuracy = []
soft_test_accuracy = []
all_predictions, all_labels = None, None
test_image_index = 0
for step in range(dataset_size_dict['test_dataset'] // batch_size):
predicted_labels, actual_labels = sess.run(
[tf_test_predictions, tf_test_labels])
test_accuracy.append(
models_utils.accuracy(predicted_labels,
actual_labels,
use_argmin=False))
soft_test_accuracy.append(
models_utils.soft_accuracy(predicted_labels,
actual_labels,
use_argmin=False,
max_thresh=max_thresh,
min_thresh=min_thresh))
if all_predictions is None or all_labels is None:
all_predictions = predicted_labels
all_labels = actual_labels
else:
all_predictions = np.append(all_predictions,
predicted_labels,
axis=0)
all_labels = np.append(all_labels, actual_labels, axis=0)
if step < 10:
for pred, act in zip(predicted_labels, actual_labels):
pred_string = ''.join(
['%.3f' % p + ',' for p in pred.tolist()])
act_string = ''.join([str(int(a)) + ',' for a in act.tolist()])
is_correct = np.argmax(pred) == np.argmax(act)
TestLogger.info('%s:%s:%s', act_string, pred_string,
is_correct)
if step < 5:
logger.debug('%s:%s:%s', act_string, pred_string,
is_correct)
print_start_of_new_input_pipline_to_some_logger(
TestLogger, 'Accuracy for Above: %.3f (Hard) %.3f (Soft)' %
(np.mean(test_accuracy), np.mean(soft_test_accuracy)))
test_noncol_precision = models_utils.precision_multiclass(
all_predictions,
all_labels,
use_argmin=False,
max_thresh=max_thresh,
min_thresh=min_thresh)
test_noncol_recall = models_utils.recall_multiclass(all_predictions,
all_labels,
use_argmin=False,
max_thresh=max_thresh,
min_thresh=min_thresh)
TestLogger.info('\n')
print('\t\tAverage test accuracy: %.5f ' % np.mean(test_accuracy))
print('\t\tAverage test accuracy(soft): %.5f' %
np.mean(soft_test_accuracy))
print('\t\tAverage test precision: %s', test_noncol_precision)
print('\t\tAverage test recall: %s', test_noncol_recall)
test_results = {}
test_results['noncol-accuracy-hard'] = np.mean(test_accuracy)
test_results['noncol-accuracy-soft'] = np.mean(soft_test_accuracy)
test_results['noncol-precision'] = test_noncol_precision
test_results['noncol-recall'] = test_noncol_recall
return test_results
def train_cnn_multiple_epochs(sess,
n_epochs,
test_interval,
dataset_filenames_dict,
dataset_size_dict,
train_fraction=1.0,
valid_fraction=1.0):
n_print_prediction_steps = 10
noncol_global_step = tf.Variable(0, trainable=False)
inc_noncol_gstep = inc_gstep(noncol_global_step)
tf_img_ids, tf_images, tf_labels = {}, {}, {}
tf_loss, tf_logits = {}, {}
tf_bump_loss, tf_bump_logits = {}, {}
tf_optimize, tf_mom_update_ops, tf_grads = {}, {}, {}
tf_bump_optimize, tf_bump_mom_update_ops, tf_bump_grads = {}, {}, {}
tf_mock_labels = tf.placeholder(shape=[batch_size, 1], dtype=tf.float32)
tf_grads_and_vars = {}
tf_train_predictions = {}
for direction in config.TF_DIRECTION_LABELS:
tf_img_ids[direction], tf_images[direction], tf_labels[
direction] = models_utils.build_input_pipeline(
dataset_filenames_dict['train_dataset'][direction],
batch_size,
shuffle=True,
training_data=False,
use_opposite_label=False,
inputs_for_sdae=False,
rand_valid_direction_for_bump=False)
tf_logits[direction] = logits(tf_images[direction], direction)
temp = list(config.TF_DIRECTION_LABELS)
temp.remove(direction)
tf_bump_logits[direction], tf_bump_loss[direction] = {}, {}
tf_bump_optimize[direction], tf_bump_mom_update_ops[direction] = {}, {}
# =================================================
# Defining Optimization for Opposite Direction
# =================================================
for opp_direction in temp:
bump_var_list = []
for v in tf.global_variables():
if opp_direction in v.name and config.TF_MOMENTUM_STR not in v.name:
print(v.name)
bump_var_list.append(v)
tf_bump_logits[direction][opp_direction] = logits(
tf_images[direction], opp_direction)
tf_bump_loss[direction][opp_direction] = calculate_loss(
tf_bump_logits[direction][opp_direction], tf_mock_labels)
tf_bump_optimize[direction][
opp_direction], _ = cnn_optimizer.optimize_model_naive_no_momentum(
tf_bump_loss[direction][opp_direction],
noncol_global_step,
varlist=bump_var_list)
tf_train_predictions[direction] = predictions_with_inputs(
tf_images[direction])
tf_loss[direction] = calculate_loss(tf_logits[direction],
tf_mock_labels)
var_list = []
for v in tf.global_variables():
if direction in v.name and config.TF_MOMENTUM_STR not in v.name:
print(v.name)
var_list.append(v)
tf_optimize[direction], tf_grads_and_vars[
direction] = cnn_optimizer.optimize_model_naive_no_momentum(
tf_loss[direction], noncol_global_step, varlist=var_list)
tf_valid_img_ids, tf_valid_images, tf_valid_labels = models_utils.build_input_pipeline(
dataset_filenames_dict['valid_dataset'],
batch_size,
shuffle=True,
training_data=False,
use_opposite_label=False,
inputs_for_sdae=False,
rand_valid_direction_for_bump=False)
tf_valid_predictions = predictions_with_inputs(tf_valid_images)
tf_test_img_ids, tf_test_images, tf_test_labels = \
models_utils.build_input_pipeline(dataset_filenames_dict['test_dataset'], batch_size,
shuffle=False, training_data=False, use_opposite_label=False,
inputs_for_sdae=False, rand_valid_direction_for_bump=False)
tf_test_predictions = predictions_with_inputs(tf_test_images)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
tf.global_variables_initializer().run(session=sess)
max_valid_accuracy = 0
n_valid_saturated = 0
valid_saturate_threshold = 3
for epoch in range(n_epochs):
print('=' * 80)
print('Epoch ', epoch)
print('=' * 80)
avg_loss = []
avg_train_accuracy = []
# Training with Non-Bump Data
for step in range(
int(train_fraction * dataset_size_dict['train_dataset']) //
batch_size):
rand_direction = np.random.choice(config.TF_DIRECTION_LABELS)
temp = list(config.TF_DIRECTION_LABELS)
temp.remove(rand_direction)
l1_noncol, _, pred, train_labels = \
sess.run([tf_loss[rand_direction], tf_optimize[rand_direction],
tf_train_predictions[rand_direction], tf_labels[rand_direction]],
feed_dict={tf_mock_labels: np.ones(shape=(batch_size, 1), dtype=np.float32)})
# Try doing negative suppresion for 2 times
for _ in range(2):
if 'hard-left' == rand_direction:
new_rand_direction = np.random.choice(
temp, p=[0.3, 0.3, 0.2, 0.2])
elif 'soft-left' == rand_direction:
new_rand_direction = np.random.choice(
temp, p=[0.25, 0.25, 0.25, 0.25])
elif 'soft-right' == rand_direction:
new_rand_direction = np.random.choice(
temp, p=[0.25, 0.25, 0.25, 0.25])
elif 'hard-right' in rand_direction:
new_rand_direction = np.random.choice(
temp, p=[0.2, 0.2, 0.3, 0.3])
else:
new_rand_direction = np.random.choice(temp)
l1_col, _ = sess.run(
[
tf_bump_loss[rand_direction][new_rand_direction],
tf_bump_optimize[rand_direction][new_rand_direction],
],
feed_dict={
tf_mock_labels:
np.zeros(shape=(batch_size, 1), dtype=np.float32)
})
avg_loss.append((l1_col + l1_noncol) / 2.0)
avg_train_accuracy.append(
models_utils.accuracy(pred, train_labels, use_argmin=False))
if step < n_print_prediction_steps:
for pred, lbl in zip(pred, train_labels):
is_correct = np.argmax(pred) == np.argmax(lbl)
TrainLogger.info('\t%s;%s;%s', pred, lbl, is_correct)
logger.info('\tAverage Loss for Epoch %d: %.5f' %
(epoch, np.mean(avg_loss)))
logger.info('\t\t Training accuracy: %.3f' %
np.mean(avg_train_accuracy))
valid_accuracy = []
for step in range(
int(valid_fraction * dataset_size_dict['valid_dataset']) //
batch_size):
vpred, vlabels = sess.run([tf_valid_predictions, tf_valid_labels])
valid_accuracy.append(
models_utils.accuracy(vpred, vlabels, use_argmin=False))
if step < n_print_prediction_steps:
for pred, lbl in zip(vpred, vlabels):
is_correct = np.argmax(pred) == np.argmax(lbl)
ValidLogger.info('\t%s;%s;%s', pred, lbl, is_correct)
logger.info('\tValid Accuracy: %.3f', np.mean(valid_accuracy))
if np.mean(valid_accuracy) > max_valid_accuracy:
max_valid_accuracy = np.mean(valid_accuracy)
else:
n_valid_saturated += 1
logger.info('Increase n_valid_saturated to %d', n_valid_saturated)
if n_valid_saturated >= valid_saturate_threshold:
logger.info('Stepping down collision learning rate')
sess.run(inc_noncol_gstep)
n_valid_saturated = 0
if (epoch + 1) % test_interval == 0:
test_results = test_the_model_5_way(sess, tf_test_labels,
tf_test_predictions,
dataset_size_dict)
soft_test_accuracy = test_results['noncol-accuracy-soft']
test_accuracy = test_results['noncol-accuracy-hard']
test_noncol_precision = test_results['noncol-precision']
test_noncol_recall = test_results['noncol-recall']
noncol_precision_string = ''.join([
'%.3f,' % test_noncol_precision[pi]
for pi in range(config.TF_NUM_CLASSES)
])
noncol_recall_string = ''.join([
'%.3f,' % test_noncol_recall[ri]
for ri in range(config.TF_NUM_CLASSES)
])
SummaryLogger.info('%d;%.3f;%.3f;%.5f;%.5f;%s;%s', epoch,
np.mean(test_accuracy),
np.mean(soft_test_accuracy), np.mean(avg_loss),
-1, noncol_precision_string,
noncol_recall_string)
coord.request_stop()
coord.join(threads)
l1, _, pred,train_labels = sess.run([tf_loss, tf_optimize,tf_train_predictions,tf_labels])
avg_loss.append(l1)
avg_train_accuracy.append(models_utils.soft_accuracy(pred,train_labels,use_argmin=False))
print('\tAverage Loss for Epoch %d: %.5f' %(epoch,np.mean(l1)))
print('\t\t Training accuracy: %.3f'%np.mean(avg_train_accuracy))
if (epoch+1)%3==0:
for step in range(dataset_sizes['train_bump_dataset']//batch_size):
sess.run(tf_bump_optimize)
#print('\t\t Training accuracy: %.3f' % soft_accuracy(pred, train_labels, use_argmin=False))
if (epoch+1)%5==0:
test_accuracy = []
soft_test_accuracy = []
test_image_index = 0
for step in range(dataset_sizes['test_dataset']//batch_size):
predicted_labels,actual_labels = sess.run([tf_test_predictions,tf_test_labels])
test_accuracy.append(models_utils.accuracy(predicted_labels,actual_labels,use_argmin=False))
soft_test_accuracy.append(models_utils.soft_accuracy(predicted_labels,actual_labels,use_argmin=False))
for pred,act in zip(predicted_labels,actual_labels):
pred_string = ''.join(['%.3f'%p+',' for p in pred.tolist()])
act_string = ''.join([str(int(a))+',' for a in act.tolist()])
predictionlogger.info('%d:%s:%s',test_image_index,act_string,pred_string)
test_image_index += 1
predictionlogger.info('\n')
print('\t\tAverage test accuracy: %.5f '%np.mean(test_accuracy))
print('\t\tAverage test accuracy(soft): %.5f'%np.mean(soft_test_accuracy))
bump_test_accuracy = []
bump_soft_accuracy = []
bump_test_image_index = 0
for step in range(dataset_sizes['test_bump_dataset']//batch_size):
def train_cnn():
global sess
configp = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.InteractiveSession(config=configp)
with sess.as_default():
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
tf.global_variables_initializer().run(session=sess)
max_noncol_accuracy, max_col_accuracy = 0, 0
noncol_acc_drop_count, col_acc_drop_count = 0, 0
acc_drop_threshold = 3
min_col_loss, min_noncol_loss = 10000, 10000
col_exceed_min_count, noncol_exceed_min_count = 0, 0
for epoch in range(num_epochs):
logger.info('\n')
logger.info('\tRunning epoch %d', epoch)
avg_loss, avg_bump_loss = [], []
test_accuracy, test_bump_accuracy = [], []
# ------------------------------------------------------------
# train the net
for step in range(dataset_sizes['train_dataset'] //
config.BATCH_SIZE // training_fraction):
if np.random.random() < 0.7:
l1, _, = sess.run([tf_loss, tf_optimize])
avg_loss.append(l1)
else:
bump_l1, _, = sess.run([tf_bump_loss, tf_bump_optimize])
avg_bump_loss.append(bump_l1)
# -----------------------------------------------------------
# ------------------------------------------------------------
# learning rate decay
if min_noncol_loss > np.mean(avg_loss):
min_noncol_loss = np.mean(avg_loss)
else:
noncol_exceed_min_count += 1
logger.info('Increase noncol_exceed to %d',
noncol_exceed_min_count)
# if count exceeds threshold drop the learning rate
if noncol_exceed_min_count >= acc_drop_threshold:
logger.info('Stepping down collision learning rate')
sess.run(inc_col_gstep)
noncol_exceed_min_count = 0
if min_col_loss > np.mean(avg_bump_loss):
min_col_loss = np.mean(avg_bump_loss)
else:
col_exceed_min_count += 1
logger.info('Increase col_exceed to %d', col_exceed_min_count)
# if count exceeds threshold drop the learning rate
if col_exceed_min_count >= acc_drop_threshold:
logger.info('Stepping down non-collision learning rate')
sess.run(inc_noncol_gstep)
col_exceed_min_count = 0
# ----------------------------------------------------------
# -----------------------------------------------------------
# Testing phase
if (epoch + 1) % 5 == 0:
for step in range(dataset_sizes['test_dataset'] //
config.BATCH_SIZE):
predicted_labels, actual_labels = sess.run(
[tf_test_predictions, tf_test_labels])
test_accuracy.append(
models_utils.accuracy(predicted_labels,
actual_labels,
use_argmin=False))
for step in range(dataset_sizes['test_bump_dataset'] //
config.BATCH_SIZE):
bump_predicted_labels, bump_actual_labels = sess.run(
[tf_bump_test_predictions, tf_bump_test_labels])
test_bump_accuracy.append(
models_utils.accuracy(bump_predicted_labels,
bump_actual_labels,
use_argmin=True))
avg_test_accuracy = np.mean(test_accuracy)
avg_bump_test_accuracy = np.mean(test_bump_accuracy)
logger.info(
'\tAverage accuracys: %.3f (non-collision) %.3f (collision)',
avg_test_accuracy, avg_bump_test_accuracy)
# ------------------------------------------------------------
# ------------------------------------------------------------
# Checking test accuracy if it is not increasing break the loop
if epoch > min_epochs_before_early_stop:
if avg_bump_test_accuracy > max_col_accuracy:
max_col_accuracy = avg_bump_test_accuracy
else:
col_acc_drop_count += 1
if avg_test_accuracy > max_noncol_accuracy:
max_noncol_accuracy = avg_test_accuracy
else:
noncol_acc_drop_count += 1
if col_acc_drop_count >= acc_drop_threshold and noncol_acc_drop_count >= acc_drop_threshold:
break
# ----------------------------------------------------------------
logger.debug('\t Variable count: %d', len(tf.global_variables()))
logger.debug('\t Operation count: %d',
len(sess.graph.get_operations()))
coord.request_stop()
coord.join(threads)
sess.close()
return max_noncol_accuracy, max_col_accuracy
def train_with_non_collision(sess,
tf_images_dict, tf_labels_dict, dataset_size,
tf_valid_images, tf_valid_labels, valid_dataset_size,
tf_test_images, tf_test_labels, tf_test_img_ids, test_dataset_size,
tf_bump_test_images,tf_bump_test_labels, tf_bump_test_img_ids, bump_test_dataset_size,
n_epochs, test_interval, sub_folder,include_bump_test_data, use_cross_entropy, activation):
'''
Train a CNN with a set of non-collision images and labels
Returns Train accuracy (col and non-collision) Train Loss (collision and non-collision)
:param tf_images:
:param tf_labels:
:param dataset_size:
:param tf_bump_images:
:param tf_bump_labels:
:param bump_dataset_size:
:param tf_test_images:
:param tf_test_labels:
:param test_dataset_size:
:param tf_bump_test_images:
:param tf_bump_test_labels:
:param test_bump_dataset_size:
:param n_epochs:
:param test_interval:
:param filename_suffix:
:return:
'''
global TrainLogger, ValidLogger
print(tf_images_dict)
train_results = {}
noncol_global_step = tf.Variable(0, trainable=False)
inc_noncol_gstep = inc_gstep(noncol_global_step)
tf_mock_labels = tf.placeholder(shape=[config.BATCH_SIZE,1],dtype=tf.float32)
tf_logits,tf_loss = {},{}
tf_bump_logits,tf_bump_loss = {},{}
tf_optimize,tf_bump_optimize = {},{}
tf_train_predictions = {}
for direction in ['left','straight','right']:
tf_logits[direction] = logits(tf_images_dict[direction], is_training=True, direction=direction)
tf_loss[direction] = calculate_loss_sigmoid(tf_logits[direction], tf_mock_labels)
var_list = []
for v in tf.global_variables():
if 'out' not in v.name and config.TF_MOMENTUM_STR not in v.name:
print(v.name)
var_list.append(v)
elif 'out' in v.name and direction in v.name:
var_list.append(v)
tf_optimize[direction], _ = \
cnn_optimizer.optimize_model_naive_no_momentum(
tf_loss[direction], noncol_global_step, varlist=None)
tf_train_predictions[direction] = predictions_with_inputs(tf_images_dict[direction])
temp = ['left', 'straight', 'right']
temp.remove(direction)
tf_bump_logits[direction] = {}
tf_bump_optimize[direction] = {}
tf_bump_loss[direction] = {}
for opp_direction in temp:
bump_var_list = []
for v in tf.global_variables():
if 'out' not in v.name and config.TF_MOMENTUM_STR not in v.name:
print(v.name)
bump_var_list.append(v)
if 'out' in v.name and opp_direction in v.name:
bump_var_list.append(v)
tf_bump_logits[direction][opp_direction] = \
logits(tf_images_dict[direction], is_training=True, direction=opp_direction)
tf_bump_loss[direction][opp_direction] = \
calculate_loss(tf_bump_logits[direction][opp_direction], tf_mock_labels)
tf_bump_optimize[direction][opp_direction], _ = cnn_optimizer.optimize_model_naive_no_momentum(
tf_bump_loss[direction][opp_direction], noncol_global_step, varlist=bump_var_list
)
tf_valid_predictions = predictions_with_inputs(tf_valid_images)
tf_test_predictions = predictions_with_inputs(tf_test_images)
if include_bump_test_data:
tf_bump_test_predictions = predictions_with_inputs(tf_bump_test_images)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
tf.global_variables_initializer().run()
max_valid_accuracy = 0
n_valid_accuracy_staturated = 0
valid_saturation_threshold = 3
for epoch in range(n_epochs):
avg_loss = []
avg_train_accuracy = []
print('='*80)
print('Epoch ',epoch)
print('=' * 80)
# ============================================================================
# Training Phase
# ============================================================================
for step in range( int(dataset_size) // config.BATCH_SIZE):
rand_direction = np.random.choice(['left', 'straight', 'right'])
temp = ['left', 'straight', 'right']
temp.remove(rand_direction)
if rand_direction == 'left':
new_rand_direction = np.random.choice(temp, p=[0.6, 0.4])
elif rand_direction == 'right':
new_rand_direction = np.random.choice(temp, p=[0.4, 0.6])
else:
new_rand_direction = np.random.choice(temp)
l1, _, pred, train_labels = sess.run([tf_loss[rand_direction], tf_optimize[rand_direction],
tf_train_predictions[rand_direction], tf_labels_dict[rand_direction]],
feed_dict={
tf_mock_labels: np.ones(shape=(config.BATCH_SIZE, 1), dtype=np.float32)})
l1_col, _ = sess.run([tf_bump_loss[rand_direction][new_rand_direction],
tf_bump_optimize[rand_direction][new_rand_direction]],
feed_dict={tf_mock_labels: np.zeros(shape=(config.BATCH_SIZE, 1), dtype=np.float32)})
avg_loss.append((l1+l1_col)/2.0)
avg_train_accuracy.append(
models_utils.accuracy(pred, train_labels, use_argmin=False)
)
if step < 2:
logger.debug('Predictions for Non-Collided data')
for pred, lbl in zip(pred, train_labels):
logger.debug('\t%s;%s', pred, lbl)
if step < 10:
for pred_item, label_item in zip(pred,train_labels):
is_currect = np.argmax(pred_item) == np.argmax(label_item)
TrainLogger.info('%s:%s:%s',pred_item,label_item,is_currect)
print_start_of_new_input_pipline_to_some_logger(TrainLogger,'================== END ==========================')
logger.info('\tAverage Loss for Epoch %d: %.5f' % (epoch, np.mean(avg_loss)))
logger.info('\t\t Training accuracy: %.3f' % np.mean(avg_train_accuracy))
# =========================================
# Validation Phase
# =======================================
avg_valid_accuracy = []
for step in range(valid_dataset_size // config.BATCH_SIZE):
v_pred, v_labels = sess.run(
[tf_valid_predictions, tf_valid_labels])
avg_valid_accuracy.append(
models_utils.accuracy(v_pred, v_labels, use_argmin=False)
)
for pred_item, label_item in zip(v_pred, v_labels):
is_currect = np.argmax(pred_item) == np.argmax(label_item)
ValidLogger.info('%s:%s:%s', pred_item, label_item, is_currect)
if np.mean(avg_valid_accuracy) > max_valid_accuracy:
max_valid_accuracy = np.mean(avg_valid_accuracy)
else:
n_valid_accuracy_staturated += 1
print('Increasing valid_saturated count to: %d', n_valid_accuracy_staturated)
if n_valid_accuracy_staturated>=valid_saturation_threshold:
print('Increasing global step. Validation Accuracy Saturated')
sess.run(inc_noncol_gstep)
n_valid_accuracy_staturated = 0
print_start_of_new_input_pipline_to_some_logger(ValidLogger,
'Valid Accuracy For Above: %.3f' % np.mean(avg_valid_accuracy))
# ============================================================================
# Testing Phase
# ============================================================================
if (epoch + 1) % test_interval == 0:
test_results = test_the_model(sess, tf_test_img_ids, tf_test_images,tf_test_labels,
tf_test_predictions, test_dataset_size,
tf_bump_test_img_ids, tf_bump_test_images, tf_bump_test_labels,
tf_bump_test_predictions, bump_test_dataset_size, epoch, sub_folder, include_bump_data=include_bump_test_data)
test_accuracy = test_results['noncol-accuracy-hard']
soft_test_accuracy = test_results['noncol-accuracy-soft']
bump_test_accuracy = test_results['col-accuracy-hard']
bump_soft_accuracy = test_results['col-accuracy-soft']
avg_loss = np.mean(avg_loss)
test_noncol_precision = test_results['noncol-precision']
test_noncol_recall = test_results['noncol-recall']
test_col_precision = test_results['col-precision']
test_col_recall = test_results['col-recall']
noncol_precision_string = ''.join(['%.3f;' % test_noncol_precision[pi] for pi in range(3)])
noncol_recall_string = ''.join(['%.3f;' % test_noncol_recall[ri] for ri in range(3)])
col_precision_string = ''.join(['%.3f;' % test_col_precision[pi] for pi in range(3)])
col_recall_string = ''.join(['%.3f;' % test_col_recall[ri] for ri in range(3)])
SummaryLogger.info('%d;%.3f;%.3f;%.3f;%.3f;%.5f;%.5f;%s;%s;%s;%s', epoch, np.mean(test_accuracy),
np.mean(soft_test_accuracy),
np.mean(bump_test_accuracy), np.mean(bump_soft_accuracy), np.mean(avg_loss),
-1,
noncol_precision_string, noncol_recall_string, col_precision_string,
col_recall_string)
# ============================================================================
# Persisting data
# ============================================================================
logger.info('Saving CNN Model')
cnn_model_visualizer.save_cnn_hyperparameters(sub_folder, kernel_size_dict, stride_dict, scope_list,
'hyperparams-final.pickle')
cnn_model_visualizer.save_cnn_weights_naive(sub_folder, sess, 'cnn_model-final.ckpt')
coord.request_stop()
coord.join(threads)
train_results['noncol-loss'] = np.mean(avg_loss)
train_results['noncol-accuracy'] = np.mean(avg_train_accuracy)
# average out the accuracies:
return train_results, test_results
def test_the_model(sess, tf_test_img_ids, tf_test_images,tf_test_labels,
tf_test_predictions, test_dataset_size,
tf_bump_test_img_ids, tf_bump_test_images, tf_bump_test_labels,
tf_bump_test_predictions, test_bump_dataset_size, epoch, sub_folder,include_bump_data):
'''
Test the Trained CNN by predicting all test non-collision data and collision data
Things done,
Calculate accuracies (collision and non-collision) (hard and soft)
Calculate Precision and Recall (collision and non-collision) (for each direction)
Save images categorized by the predicted navigationd direction
:param tf_test_img_ids:
:param tf_test_images:
:param tf_test_labels:
:param tf_test_predictions:
:param test_dataset_size:
:param tf_bump_test_img_ids:
:param tf_bump_test_images:
:param tf_bump_test_labels:
:param tf_bump_test_predictions:
:param test_bump_dataset_size:
:param epoch:
:return: Hard_Accuracy(Non-col), Soft Accuracy (Non-col), Hard_Accuracy(col), Soft Accuracy (col),
Precision (Non-col), Recall (Non-col), Precision (col), Recall (col),
'''
test_results = {}
test_accuracy = []
soft_test_accuracy = []
bump_test_accuracy = []
bump_soft_accuracy = []
all_predictions, all_labels, all_img_ids, all_images = None, None, None, None
all_bump_predictions, all_bump_labels, all_bump_img_ids, all_bump_images = None, None, None, None
test_image_index = 0
for step in range(test_dataset_size // config.BATCH_SIZE ):
predicted_labels, actual_labels, test_ids, test_images = sess.run(
[tf_test_predictions, tf_test_labels, tf_test_img_ids, tf_test_images])
test_accuracy.append(models_utils.accuracy(predicted_labels, actual_labels, use_argmin=False))
soft_test_accuracy.append(
models_utils.soft_accuracy(predicted_labels, actual_labels, use_argmin=False, max_thresh=max_thresh,
min_thresh=min_thresh))
if all_predictions is None or all_labels is None:
all_predictions = predicted_labels
all_labels = actual_labels
all_img_ids = test_ids
all_images = test_images
else:
all_predictions = np.append(all_predictions, predicted_labels, axis=0)
all_labels = np.append(all_labels, actual_labels, axis=0)
all_img_ids = np.append(all_img_ids, test_ids, axis=0)
all_images = np.append(all_images, test_images, axis=0)
if step < 5:
logger.debug('Test Predictions (Non-Collisions)')
for pred, act in zip(predicted_labels, actual_labels):
pred_string = ''.join(['%.3f' % p + ',' for p in pred.tolist()])
act_string = ''.join([str(int(a)) + ',' for a in act.tolist()])
is_correct = np.argmax(pred)==np.argmax(act)
TestPredictionLogger.info('%d:%s:%s:%s', test_image_index, act_string, pred_string,is_correct)
if step < 5:
logger.debug('%d:%s:%s', test_image_index, act_string, pred_string)
test_image_index += 1
TestPredictionLogger.info('\n')
print('\t\tAverage test accuracy: %.5f ' % np.mean(test_accuracy))
print('\t\tAverage test accuracy(soft): %.5f' % np.mean(soft_test_accuracy))
if include_bump_data:
for step in range(test_bump_dataset_size // config.BATCH_SIZE):
bump_predicted_labels, bump_actual_labels, bump_test_ids, bump_test_images = sess.run(
[tf_bump_test_predictions, tf_bump_test_labels, tf_bump_test_img_ids, tf_bump_test_images])
bump_test_accuracy.append(
models_utils.accuracy(bump_predicted_labels, bump_actual_labels, use_argmin=True))
bump_soft_accuracy.append(
models_utils.soft_accuracy(bump_predicted_labels, bump_actual_labels, use_argmin=True,
max_thresh=max_thresh, min_thresh=min_thresh))
if all_bump_predictions is None or all_bump_labels is None:
all_bump_predictions = bump_predicted_labels
all_bump_labels = bump_actual_labels
all_bump_img_ids = bump_test_ids
all_bump_images = bump_test_images
else:
all_bump_predictions = np.append(all_bump_predictions, bump_predicted_labels, axis=0)
all_bump_labels = np.append(all_bump_labels, bump_actual_labels, axis=0)
all_bump_img_ids = np.append(all_bump_img_ids, bump_test_ids, axis=0)
all_bump_images = np.append(all_bump_images, bump_test_images, axis=0)
print('\t\tAverage bump test accuracy: %.5f ' % np.mean(bump_test_accuracy))
print('\t\tAverage bump test (soft) accuracy: %.5f ' % np.mean(bump_soft_accuracy))
if step < 5:
logger.debug('Test Predictions (Collisions)')
test_image_index = 0
for pred, act in zip(all_bump_predictions, all_bump_labels):
bpred_string = ''.join(['%.3f' % p + ',' for p in pred.tolist()])
bact_string = ''.join([str(int(a)) + ',' for a in act.tolist()])
is_correct = np.argmin(pred)==np.argmin(act)
TestBumpPredictionLogger.info('%d:%s:%s:%s', test_image_index, bact_string, bpred_string,is_correct)
if step < 5:
logger.debug('%d:%s:%s', test_image_index, bact_string, bpred_string)
test_image_index += 1
TestBumpPredictionLogger.info('\n')
test_col_precision = models_utils.precision_multiclass(all_bump_predictions, all_bump_labels,
use_argmin=True,
max_thresh=max_thresh, min_thresh=min_thresh)
test_col_recall = models_utils.recall_multiclass(all_bump_predictions, all_bump_labels, use_argmin=True,
max_thresh=max_thresh, min_thresh=min_thresh)
print('\t\tAverage test bump precision: %s', test_col_precision)
print('\t\tAverage test bump recall: %s', test_col_recall)
test_noncol_precision = models_utils.precision_multiclass(all_predictions, all_labels, use_argmin=False,
max_thresh=max_thresh, min_thresh=min_thresh)
test_noncol_recall = models_utils.recall_multiclass(all_predictions, all_labels, use_argmin=False,
max_thresh=max_thresh, min_thresh=min_thresh)
print('\t\tAverage test precision: %s', test_noncol_precision)
print('\t\tAverage test recall: %s', test_noncol_recall)
predicted_hard_ids_sorted, predicted_bump_hard_ids_sorted = {}, {}
predicted_hard_ids_sorted_best, predicted_bump_hard_ids_sorted_best = {}, {}
for di, direct in enumerate(['left', 'straight', 'right']):
predicted_hard_ids_sorted_best[direct] = models_utils.get_id_vector_for_predicted_samples_best(
all_img_ids, all_predictions, all_labels, di, True, False, max_thresh, min_thresh
)
predicted_hard_ids_sorted[direct] = models_utils.get_id_vector_for_predicted_samples(
all_img_ids, all_predictions, all_labels, di, True, False
)
if include_bump_data:
predicted_bump_hard_ids_sorted_best[direct] = models_utils.get_id_vector_for_predicted_samples_best(
all_bump_img_ids, all_bump_predictions, all_bump_labels, di, True, True, max_thresh, min_thresh
)
predicted_bump_hard_ids_sorted[direct] = models_utils.get_id_vector_for_predicted_samples(
all_bump_img_ids, all_bump_predictions, all_bump_labels, di, True, True
)
image_list = np.split(all_images, all_images.shape[0])
id_list = all_img_ids.tolist()
dict_id_image = dict(zip(id_list, image_list))
if include_bump_data:
bump_image_list = np.split(all_bump_images, all_bump_images.shape[0])
bump_id_list = all_bump_img_ids.tolist()
bump_dict_id_image = dict(zip(bump_id_list, bump_image_list))
logger.info('correct hard img ids for: %s', predicted_hard_ids_sorted_best)
#visualizer.save_fig_with_predictions_for_direction(predicted_hard_ids_sorted_best, dict_id_image,
# sub_folder + os.sep + 'predicted_best_hard_%d.png' % (
# epoch))
#visualizer.save_fig_with_predictions_for_direction(predicted_hard_ids_sorted, dict_id_image,
# sub_folder + os.sep + 'predicted_hard_%d.png' % (
# epoch))
#if include_bump_data:
# visualizer.save_fig_with_predictions_for_direction(predicted_bump_hard_ids_sorted_best, bump_dict_id_image,
# sub_folder + os.sep + 'predicted_best_bump_%d.png' % (
# epoch))
# visualizer.save_fig_with_predictions_for_direction(predicted_bump_hard_ids_sorted, bump_dict_id_image,
# sub_folder + os.sep + 'predicted_bump_%d.png' % (
# epoch))
test_results['noncol-accuracy-hard'] = np.mean(test_accuracy)
test_results['noncol-accuracy-soft'] = np.mean(soft_test_accuracy)
test_results['noncol-precision'] = test_noncol_precision
test_results['noncol-recall'] = test_noncol_recall
if include_bump_data:
test_results['col-accuracy-hard'] = np.mean(bump_test_accuracy)
test_results['col-accuracy-soft'] = np.mean(bump_soft_accuracy)
test_results['col-precision'] = test_col_precision
test_results['col-recall'] = test_col_recall
return test_results
if (epoch + 1) % 5 == 0:
test_accuracy = []
soft_test_accuracy = []
all_predictions, all_labels, all_img_ids, all_images = None, None, None, None
test_image_index = 0
for step in range(dataset_sizes['test_dataset'] //
config.BATCH_SIZE):
predicted_labels, actual_labels, test_img_ids, test_images = sess.run(
[
tf_test_predictions, tf_test_actuals,
tf_test_img_ids, tf_test_images
])
test_accuracy.append(
models_utils.accuracy(predicted_labels,
actual_labels,
use_argmin=False))
soft_test_accuracy.append(
models_utils.soft_accuracy(predicted_labels,
actual_labels,
use_argmin=False,
max_thresh=max_thresh,
min_thresh=min_thresh))
if all_predictions is None or all_labels is None:
all_predictions = predicted_labels
all_labels = actual_labels
all_img_ids = test_img_ids
all_images = test_images
else: