def evaluate_model(model, batches, steps_per_epoch):
probs = []
labels = []
eval_start_time = time.time()
for step, batch in enumerate(batches):
if step >= steps_per_epoch:
break
labels.extend(batch['label'])
probs.extend(model.forward(batch['X']))
util.over_print("Step: %d/%d \tTime: %f\r" % (step+1, steps_per_epoch, time.time()-eval_start_time))
loss, roc_auc, accuracy = util.measure_performance(labels, probs)
return loss, roc_auc, accuracy
if best_test_roc_auc[i] < test_roc_auc and current_epoch > 0:
best_test_roc_auc[i] = test_roc_auc
model.save_model_with_name("best_test_{}".format(i), current_epoch, valid_roc_auc, test_roc_auc)
# Log training results
logger.log({
"epoch":current_epoch,
"train_loss":train_loss, "train_roc-auc":train_roc_auc,
"valid_loss":valid_loss, "valid_roc-auc":valid_roc_auc,
"test_loss":test_losses, "test_roc-auc":test_roc_aucs,
"best_test_roc-auc":list(best_test_roc_auc), "mean_best_test_roc-auc":np.mean(best_test_roc_auc),
})
print("\nBest Valid ROC-AUC: {}\t Best Test ROC-AUC: {} Mean of Best Test ROC-AUC: {}\n".format(
best_valid_roc_auc, best_test_roc_auc, np.mean(best_test_roc_auc)))
# Take Gradient Descent
loss = model.optimize_parameters(batch, lr)
util.over_print("Train Step: %d/%d \t Loss: %f \t LR: %f \t Time: %f \r"
%(step, dataloader.train_steps_per_epoch, loss, lr, time.time() - step_start_time))
step_start_time = time.time()
print('\nEnd of epoch %d / %d \t Time Taken: %d sec' % (epoch, opt.nepoch, time.time()-epoch_start_time))
# print('saving the model at the end of epoch %d'%(epoch))
# model.save(epoch)
dataloader.close_threads(coord)
print("Done Training!")
sess.close()
print("Accident Probability: %f" % probs[-1][1])
print("Label: %f" % labels[-1])
print("Frame Label: %f" % batch['frame_label'][-1])
frame = batch['X'][-1]
image = frame[:, :, 0:3].astype(np.uint8)
bbox2 = frame[:, :, 6].astype(np.uint8)
masked2 = np.ma.masked_where(bbox2 == 0, bbox2)
bbox1 = frame[:, :, 7].astype(np.uint8)
masked1 = np.ma.masked_where(bbox1 == 0, bbox1)
plt.imshow(image)
plt.imshow(masked2, 'Reds_r', alpha=0.4)
plt.imshow(masked1, 'jet', alpha=0.4)
plt.show()
util.over_print("Step: %d/%d \tTime: %f\r" %
(step + 1, dataset['test_steps_per_epoch'],
time.time() - eval_start_time))
samples_grouped_by_frame = dataset['test_samples_grouped_by_frame']
samples = list(itertools.chain.from_iterable(samples_grouped_by_frame))
result_dict['split_{}_result'.format(split)] = {
'labels': labels,
'probs': probs,
'samples': samples
}
test_loss, test_roc_auc, test_accuracy = util.measure_performance(
labels, probs)
test_roc_aucs.append(test_roc_auc)
print("\nTest - Loss: %f\t ROC-AUC: %f\t Accuracy: %f\n" %
(test_loss, test_roc_auc, test_accuracy))
def build_sample_list(scenes,
frames_per_sample,
label_method,
motion_model,
ttc_threshold,
shuffle_by_which=None):
n_samples = 0
n_acc_samples = 0
n_nonacc_samples = 0
samples_grouped_by_frame = []
for s, scene in enumerate(scenes):
for f, frame in enumerate(scene):
samples_per_frame = []
if f < frames_per_sample - 1:
continue
frame_t_minus_0_info = json.load(open(scene[f]['json_path']))
frame_t_minus_1_info = json.load(open(scene[f - 1]['json_path']))
frame_t_minus_2_info = json.load(open(scene[f - 2]['json_path']))
frame_t_minus_3_info = json.load(open(scene[f - 3]['json_path']))
frame_t_minus_4_info = json.load(open(scene[f - 4]['json_path']))
frame_t_minus_5_info = json.load(open(scene[f - 5]['json_path']))
frame_label = scene[f]['frame_label']
for bbox in frame_t_minus_0_info['vehicleInfo']:
data_driven_label, rule_based_label, rule_based_prob = get_vehicle_label(
bbox, motion_model, ttc_threshold)
if data_driven_label != None and rule_based_label != None:
sample = {
'hashcode':
bbox['hashcode'],
'frame_label':
frame_label,
'frame_t-0.0s':
frame_per_vehicle(scene[f], bbox),
'frame_t-0.1s':
frame_per_vehicle(
scene[f - 1],
get_bbox(frame_t_minus_1_info, bbox['hashcode']))
if frames_per_sample > 1 else None,
'frame_t-0.2s':
frame_per_vehicle(
scene[f - 2],
get_bbox(frame_t_minus_2_info, bbox['hashcode']))
if frames_per_sample > 2 else None,
'frame_t-0.3s':
frame_per_vehicle(
scene[f - 3],
get_bbox(frame_t_minus_3_info, bbox['hashcode']))
if frames_per_sample > 3 else None,
'frame_t-0.4s':
frame_per_vehicle(
scene[f - 4],
get_bbox(frame_t_minus_4_info, bbox['hashcode']))
if frames_per_sample > 4 else None,
'frame_t-0.5s':
frame_per_vehicle(
scene[f - 5],
get_bbox(frame_t_minus_5_info, bbox['hashcode']))
if frames_per_sample > 5 else None,
}
n_samples += 1
if label_method == "data_driven":
sample['label'] = data_driven_label
sample['label_prob'] = data_driven_label
if data_driven_label:
n_acc_samples += 1
else:
n_nonacc_samples += 1
elif label_method == "rule_based":
sample['label'] = rule_based_label
sample['label_prob'] = rule_based_label
if rule_based_label:
n_acc_samples += 1
else:
n_nonacc_samples += 1
elif label_method == "rule_based_prob":
sample['label'] = rule_based_label
sample['label_prob'] = rule_based_prob
if rule_based_label:
n_acc_samples += 1
else:
n_nonacc_samples += 1
samples_per_frame.append(sample)
samples_grouped_by_frame.append(samples_per_frame)
util.over_print("Building sample list: %d/%d \r" % (s, len(scenes)))
print()
return samples_grouped_by_frame, n_acc_samples, n_nonacc_samples, n_samples
model = CreateModel(opt, sess)
data_loader = CreateDataLoader(opt, sess)
coord = tf.train.Coordinator()
data_loader.start_enqueue_threads_for_testing(coord)
data_loader.init_test_generator()
y_true = []
y_pred = []
step_count = 0.0
for step, batch in enumerate(data_loader.test_datasets[0]['test_batch']):
probs = model.forward(batch['X'])
y_true.extend(batch['label'])
y_pred.extend(probs)
step_count += 1
util.over_print("Step count: {} \r".format(str(step_count)))
temp_y_pred = [p[1] for p in y_pred]
data_loader.close_threads(coord)
sess.close()
####################################################################################
# Filter only top n vehicles within a frame with highest collision probability #
####################################################################################
if opt.filter_top_n > 0:
samples_grouped_by_frame = data_loader.test_datasets[0][
'test_samples_grouped_by_frame']
samples = list(itertools.chain.from_iterable(samples_grouped_by_frame))
json_path = samples[0]['frame_t-0.0s']['json_path']
new_y_pred = []
preds_per_frame = []