def on_train_end(self, logs=None):
self.log['duration'] = time.time() - self.ts
misc.write_to_file(self.log, os.path.join(self.logpath, "train_log.json"))
hours, rem = divmod(self.log['duration'], 3600)
minutes, seconds = divmod(rem, 60)
print("Total Duration: {:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds))
def evaluate_regression(predictions, real_values, fname):
evas = compute_explained_variance(predictions, real_values)
rmse = compute_rmse(predictions, real_values)
highest_errors = compute_highest_regression_errors(predictions,
real_values,
n_errors=20)
dictionary = {
"evas": evas.tolist(),
"rmse": rmse.tolist(),
"highest_errors": highest_errors.tolist()
}
misc.write_to_file(dictionary, fname)
def evaluate_classification(pred_prob, pred_labels, real_labels, fname):
ave_accuracy = metrics.accuracy_score(real_labels, pred_labels)
print('Average accuracy = ', ave_accuracy)
precision = metrics.precision_score(real_labels, pred_labels)
print('Precision = ', precision)
recall = metrics.recall_score(real_labels, pred_labels)
print('Recall = ', recall)
f_score = metrics.f1_score(real_labels, pred_labels)
print('F1-score = ', f_score)
highest_errors = compute_highest_classification_errors(pred_prob,
real_labels,
n_errors=20)
dictionary = {
"ave_accuracy": ave_accuracy,
"precision": precision,
"recall": recall,
"f_score": f_score,
"highest_errors": highest_errors.tolist()
}
misc.write_to_file(dictionary, fname)
def _main():
# Load settings or use args or defaults
s = misc.load_settings_or_use_args(FLAGS)
# Misc settings
s['rescale_factor'] = s.get('rescale_factor',
1. if s['img_mode'] == "flow" else 1. / 255)
if FLAGS.test_dir is None:
s['test_dir'] = os.path.join(FLAGS.experiment_rootdir, s['test_dir'])
model_dir = os.path.join(FLAGS.experiment_rootdir, s['model_dir'])
# Set testing mode (dropout/batchnormalization)
k.set_learning_phase(0)
# Generate testing data
test_datagen = utils.DroneDataGenerator(rescale=s['rescale_factor'])
test_generator = test_datagen.flow_from_directory(
s['test_dir'],
shuffle=False,
color_mode=s['img_mode'],
target_size=(s['img_width'], s['img_height']),
crop_size=(s['crop_img_width'], s['crop_img_height']),
batch_size=1,
is_training=False)
# Load json and create model
json_model_path = os.path.join(model_dir, s['json_model_fname'])
model = utils.json_to_model(json_model_path)
# Get weights paths
weights_2_load = sorted(glob.glob(os.path.join(model_dir,
'model_weights*')),
key=os.path.getmtime)
# Prepare output directory
if s['name'] == "same":
if FLAGS.test_dir[-1] == "/":
outdir = "eval_" + os.path.split(s['test_dir'][:-1])[1]
else:
outdir = "eval_" + os.path.split(s['test_dir'])[1]
else:
outdir = "eval_" + s['name'] + "_" + os.path.split(s['test_dir'])[1]
if FLAGS.output_dir is None:
outfolder = os.path.join(FLAGS.experiment_rootdir, s['eval_dir'],
outdir)
else:
outfolder = os.path.join(FLAGS.output_dir, outdir)
if not os.path.exists(outfolder):
os.makedirs(outfolder)
# Store settings as backup
settings_out_filename = os.path.join(outfolder, s['settings_fname'])
misc.write_to_file(s, settings_out_filename, beautify=True)
# Iterate through trained models
for weights_load_path in weights_2_load:
model.load_weights(weights_load_path)
print("\nEvaluating: " + weights_load_path)
print("On: " + s['test_dir'] + "\n")
index = weights_load_path.split("_")[-1][:-3]
# Compile model
model.compile(loss='mse', optimizer='adadelta')
# Get predictions and ground truth
n_samples = test_generator.samples
nb_batches = int(np.ceil(n_samples / 1))
predictions, ground_truth, t, fnames = utils.compute_predictions_and_gt(
model, test_generator, nb_batches, verbose=1)
# Param t. t=1 steering, t=0 collision
t_mask = t == 1
# ************************* Steering evaluation ***************************
# Predicted and real steerings
pred_steerings = predictions[t_mask, 0]
real_steerings = ground_truth[t_mask, 0]
# Compute random and constant baselines for steerings
random_steerings = random_regression_baseline(real_steerings)
constant_steerings = constant_baseline(real_steerings)
# Create dictionary with filenames
dict_fname = {
'test_regression_' + str(index.zfill(4)) + '.json':
pred_steerings,
'random_regression_' + str(index.zfill(4)) + '.json':
random_steerings,
'constant_regression_' + str(index.zfill(4)) + '.json':
constant_steerings
}
# Evaluate predictions: EVA, residuals, and highest errors
for fname, pred in dict_fname.items():
abs_fname = os.path.join(outfolder, fname)
evaluate_regression(pred, real_steerings, abs_fname)
# Write predicted and real steerings
dict_test = {
'pred_steerings': pred_steerings.tolist(),
'real_steerings': real_steerings.tolist()
}
pred_fname = os.path.join(
outfolder,
'predicted_and_real_steerings_' + str(index.zfill(4)) + '.json')
misc.write_to_file(dict_test, pred_fname)
# *********************** Collision evaluation ****************************
# Predicted probabilities and real labels
pred_prob = predictions[~t_mask, 1]
pred_labels = np.zeros_like(pred_prob)
pred_labels[pred_prob >= 0.5] = 1
real_labels = ground_truth[~t_mask, 1]
# Compute random, weighted and majorirty-class baselines for collision
random_labels = random_classification_baseline(real_labels)
# Create dictionary with filenames
dict_fname = {
'test_classification_' + str(index.zfill(4)) + '.json':
pred_labels,
'random_classification_' + str(index.zfill(4)) + '.json':
random_labels
}
# Evaluate predictions: accuracy, precision, recall, F1-score, and highest errors
for fname, pred in dict_fname.items():
abs_fname = os.path.join(outfolder, fname)
evaluate_classification(pred_prob, pred, real_labels, abs_fname)
chain = itertools.chain(*fnames)
ffnames = list(chain)
# Write predicted probabilities and real labels
dict_test = {
'pred_probabilities': pred_prob.tolist(),
'real_labels': real_labels.tolist(),
'filenames': ffnames
}
pred_fname = os.path.join(
outfolder,
'predicted_and_real_labels_' + str(index.zfill(4)) + '.json')
misc.write_to_file(dict_test, pred_fname)
def _main():
global c, s
# load settings or use args or defaults
s = misc.load_settings_or_use_args(FLAGS)
# custom settings
#s['center_flow'] = True
# init misc settings
s['input_dir'] = FLAGS.input_dir
# init folders
outfolder = create_outfolder()
misc.del_and_recreate_folder(os.path.join(outfolder, s['train_dir']))
misc.del_and_recreate_folder(os.path.join(outfolder, s['val_dir']))
misc.del_and_recreate_folder(os.path.join(outfolder, s['model_dir']))
# read labels
labels_fname = os.path.join(s['input_dir'], "labels.txt")
c['all'] = file_len(labels_fname) - 2
data = read_labels(labels_fname)
# clamp and normalize steering angles
scaler = MaxAbsScaler()
data['steering'] = np.clip(data['steering'], -s['max_steer_angle'], s['max_steer_angle'])
data['steering'] = np.squeeze(scaler.fit_transform(data['steering'].reshape(-1, 1)))
# save scaler
joblib.dump(scaler, os.path.join(outfolder, s['scaler_filename']))
# split into sets and filter out unwanted elements of the data
candidates = prep_data(data)
# load cloud image for preprocessing
cloud = cv2.imread(localpath + s['cloud_file'], 0).astype(np.float)
# init flow stat collector
flowdataname = "flowstats_" + s['name'] if s['name'] != "same" else "flow_statistics"
flow_stat_collector = iu.FlowStatsCollector(outfolder, lim=200, name=flowdataname, n=s['flow_stat_bin_count'], clip=False)
# iterate through all experiments
for exp in tqdm(candidates, desc="Preparing Dataset: ", maxinterval=1, unit="Exp."):
# global counter
c['tmp'] = c['tmp'] + len(exp)
# load and validate images (check if too dark/bright)
imgs = load_set_images(exp['filename'], s['input_dir'])
# check for too bright or dark images and remove them
del_idxs = validate_set_images(imgs)
imgs = np.delete(imgs, del_idxs, 0)
exp = np.delete(exp, del_idxs, 0)
# apply quote of items that are allowed to have zero data
if len(exp) > s['min_data_per_exp']:
del_idxs = apply_zero_quote(exp)
imgs = np.delete(imgs, del_idxs, 0)
exp = np.delete(exp, del_idxs, 0)
# process and write set if still enough data available
if len(exp) > s['min_data_per_exp']:
new_imgs = apply_preprocessing(imgs, cloud)
# get stats of flow images and rescale so that the mean magnitude = 1
if s['img_mode'] in ["flow", "flow_as_rgb", "flow_as_mag"]:
flow_stat_collector.collect_exp_flow_stats(new_imgs)
# write experiment set to train or val
train_or_val = "train" if random.random() > s['train_val_qouta'] else "val"
folder = create_subfolder(exp, os.path.join(outfolder, s[train_or_val + "_dir"]))
write_set(exp, new_imgs, folder)
c[train_or_val] = c[train_or_val] + len(new_imgs)
# logging
key = get_type(exp)
c['good'] = c['good'] + len(exp)
c[key] = c[key] + len(exp)
else:
if verbose_val:
print("ID: {:5d}; \t Info: Not enough data left after image validation".format(exp['id'][0]))
c['dismissed'] = c['dismissed'] + len(exp)
# Save flow statistics
flow_stats_available = False
if s['img_mode'] in ["flow", "flow_as_rgb", "flow_as_mag"]:
flow_stat_collector.write_flow_stats()
flow_stats_available = True
# Store settings in output folders
s['img_channels'] = 3 if s['img_mode'] == "flow_as_rgb" else s['img_channels'] # set to 3 channels for rgb
s['img_channels'] = 1 if s['img_mode'] == "flow_as_mag" else s['img_channels'] # set to 1 channel for magnitude
s['img_mode'] = "rgb" if s['img_mode'] == "flow_as_rgb" else s['img_mode'] # set img mode to rgb after conversion
s['img_mode'] = "grayscale" if s['img_mode'] in ["depth", "flow_as_mag"] else s['img_mode'] # set img mode to grayscale if depth or magnitude
settings_out_filename = os.path.join(outfolder, s['settings_fname'])
misc.write_to_file(s, settings_out_filename, beautify=True)
# create eval folder
if create_subfolders:
try:
os.mkdir(os.path.join(outfolder, s['eval_dir']))
except:
print("Warning: Unable to create eval folder.")
# Output of statistics
print("\n\n## " + misc.colorize("[RESULTS]", "magenta") + " ############################################"
+ "\n\n- " + misc.colorize("INPUT", "cyan") + " ----------------------------------------------"
+ "\nFolder: " + FLAGS.input_dir
+ "\nLabels: " + labels_fname
+ "\nImages: {:d} ({:d} Experiments)".format(c['all'], c['exp_in'])
+ "\n\n- " + misc.colorize("VALIDATION", "cyan") + " -----------------------------------------"
+ "\nToo bright: {:d}".format(c['bright'])
+ "\nToo dark: {:d}".format(c['dark'])
+ "\nDismissed: {:d} ({:3.1f} %)".format(c['dismissed'], c['dismissed'] / c['all'] * 100.))
if flow_stats_available:
print("\n- " + misc.colorize("FLOW STATS", "cyan") + " -----------------------------------------")
flow_stat_collector.print_flow_stats()
print("\n- " + misc.colorize("OUTPUT", "cyan") + " ---------------------------------------------"
+ "\nFolder: " + outfolder
+ "\nGOOD: {:d} ({:3.1f} %; {:d} Experiments)".format(c['good'],
c['good'] / c['all'] * 100.,
c['exp_out'])
+ "\nClouded: {:d} ({:3.1f} %)".format(c['cloud'], c['cloud'] / c['good'] * 100.)
+ "\nSteering: {:d} (zero: {:d})".format(c['steering'], c['zero_steering'])
+ "\nCollision: {:d} (zero: {:d})".format(c['collision'], c['zero_collision'])
+ "\nTrain/Val: {:d}/{:d} "
"(Val. Quota: {:3.1f} %)".format(c['train'], c['val'], c['val'] / c['train'] * 100.)
+ "\n\n#########################################################\n")
def _main():
# Load settings or use args or defaults
s = misc.load_settings_or_use_args(FLAGS)
# Create the experiment rootdir if not already there
model_dir = os.path.join(FLAGS.experiment_rootdir, s['model_dir'])
if not os.path.exists(model_dir):
os.makedirs(model_dir)
# Image mode
if s['img_mode'] == 'rgb':
s['img_channels'] = s.get('img_channels', 3)
elif s['img_mode'] == 'flow':
s['img_channels'] = s.get('img_channels', 2)
s['rescale_factor'] = None
s['width_shift_range'] = None
s['height_shift_range'] = None
elif s['img_mode'] == 'grayscale':
s['img_channels'] = s.get('img_channels', 1)
else:
raise IOError(
"Unidentified image mode: use 'grayscale', 'flow' or 'rgb'")
# Generate training data with real-time augmentation
train_datagen = utils.DroneDataGenerator(
rotation_range=s['rotation_range'],
rescale=s['rescale_factor'],
width_shift_range=s['width_shift_range'],
height_shift_range=s['height_shift_range'],
vertical_flip=False)
train_generator = train_datagen.flow_from_directory(
os.path.join(FLAGS.experiment_rootdir, s['train_dir']),
shuffle=True,
color_mode=s['img_mode'],
target_size=(s['img_width'], s['img_height']),
crop_size=(s['crop_img_width'], s['crop_img_height']),
batch_size=s['batch_size'])
# Generate validation data with real-time augmentation
val_datagen = utils.DroneDataGenerator(rescale=s['rescale_factor'])
val_generator = val_datagen.flow_from_directory(
os.path.join(FLAGS.experiment_rootdir, s['val_dir']),
shuffle=False,
color_mode=s['img_mode'],
target_size=(s['img_width'], s['img_height']),
crop_size=(s['crop_img_width'], s['crop_img_height']),
batch_size=s['batch_size'])
# Weights to restore
if not s['restore_model']:
# In this case weights will start from random
s['initial_epoch'] = 0
s['restore_path'] = None
else:
# In this case weigths will start from the specified mode
s['restore_path'] = os.path.join(model_dir, s['weights_fname'])
# Define model
model = get_model(s['crop_img_width'], s['crop_img_height'],
s['img_channels'], 1, s['restore_path'])
# Serialize model into json
json_model_path = os.path.join(model_dir, s['json_model_fname'])
utils.model_to_json(model, json_model_path)
# Store settings
settings_out_filename = os.path.join(FLAGS.experiment_rootdir,
s['settings_fname'])
misc.write_to_file(s, settings_out_filename, beautify=True)
# Train model
train_model(train_generator, val_generator, model, s['initial_epoch'], s)
def _main():
global s
# Load settings or use args or defaults
s = misc.load_settings_or_use_args(FLAGS)
# Init variables (no need to change anything here)
last_duration = 1
json_model_path = os.path.join(FLAGS.experiment_rootdir, s['model_dir'],
s['json_model_fname'])
weights_path = os.path.join(FLAGS.experiment_rootdir, s['model_dir'],
s['weights_fname'])
s['target_size'] = s.get('target_size', (s['img_width'], s['img_height']))
s['crop_size'] = s.get('crop_size',
(s['crop_img_width'], s['crop_img_height']))
s['output_size'] = s.get('output_size', (400, 300))
extensions = ['jpg', 'png']
# Special settings for flow mode
if s['img_mode'] == "flow":
extensions = ['npy', 'bin', 'flo', 'pfm', 'png']
# Load model
model = get_model(json_model_path, weights_path)
# Penultimate layer index for grad cams
penultimate_layer = utils.find_layer_idx(model, s['penultimate_layer'])
# Check if single or multiple experiments to test
folders = misc.get_experiment_folders(FLAGS.test_dir)
# Prepare output directory
if s['name'] == "same":
if FLAGS.test_dir[-1] == "/":
outdir = "eval_" + os.path.split(s['test_dir'][:-1])[1] + "_adv"
else:
outdir = "eval_" + os.path.split(s['test_dir'])[1] + "_adv"
else:
outdir = "eval_" + s['name'] + "_" + os.path.split(
s['test_dir'])[1] + "_adv"
if FLAGS.output_dir is None:
outfolder = os.path.join(FLAGS.experiment_rootdir, s['eval_dir'],
outdir)
else:
outfolder = os.path.join(FLAGS.output_dir, outdir)
if not os.path.exists(outfolder):
os.makedirs(outfolder)
# Store settings as backup
settings_out_filename = os.path.join(outfolder, s['settings_fname'])
misc.write_to_file(s, settings_out_filename, beautify=True)
# Iterate through folders
for folder in tqdm(sorted(folders),
desc="Evaluating Dataset: ",
maxinterval=1,
unit="Exp."):
# Get filenames
imgs = []
for extension in extensions:
ipth = os.path.join(folder, 'images/*.' + extension)
imgs = sorted(glob.glob(ipth))
if len(imgs) > 0:
break
# Check if labels are available, then load them
exp_type = None
lbl = np.zeros(len(imgs)) - 1337
for key in label_files:
fullname = os.path.join(folder, label_files[key])
if os.path.isfile(fullname):
lbl = []
exp_type = key
with open(fullname, newline='') as i:
for row in csv.reader(i, delimiter=';'):
lbl.append(float(row[0]))
break
# Prepare output subfolder
foldersub = os.path.join(outfolder, os.path.split(folder)[1])
misc.del_and_recreate_folder(os.path.join(foldersub, "images"))
# # add first label on flow data (otherwise it will have one label less than the real data)
# if "flow" in folder:
# lbl.insert(0, 0)
# Check if labels and images fit
if len(lbl) >= len(imgs):
col = []
ste = []
# Iterate through imgs with n stepsize
for x in range(len(imgs)):
# Load image and prep for dronet
data = iu.load_img(imgs[x],
img_mode=s['img_mode'],
crop_size=s['crop_size'],
target_size=s['target_size'])
# Load image again for displaying
if s['img_mode'] == "flow":
# Convert for output
img = iu.flow_to_img(data)
else:
# Read img and rescale data
img = cv2.imread(imgs[x], cv2.IMREAD_COLOR)
data = np.asarray(data, dtype=np.float32) * np.float32(
1.0 / 255.0)
carry = data[np.newaxis, ...]
# Make and store predictions
img = cv2.resize(img,
s['output_size'],
interpolation=cv2.INTER_AREA)
theta, p_t, img_out = make_prediction(img, carry, model,
lbl[x])
img_out = display_predictions_as_bars(img_out, theta, p_t,
lbl[x], exp_type)
col.append(float(p_t))
ste.append(float(theta))
# Grad-CAMs
if s['show_activations']:
# Cleanup for keras-vis bug (slowdown + leaky memory)
if last_duration > 0.4:
k.clear_session()
model = get_model(json_model_path, weights_path)
# Produce activation maps for collision or steering
img_h, last_duration = get_grad_cam(
img, carry, model, penultimate_layer,
s['filter_indices'])
# Put the pieces together
img_out = np.hstack(
(img_out,
np.ones(
(s['output_size'][1], 50, 3), dtype=np.uint8) *
255, img_h))
# Write output
img_name = os.path.split(imgs[x])[1]
img_name = os.path.splitext(img_name)[0]
cv2.imwrite(
os.path.join(foldersub, "images", img_name + ".png"),
img_out)
# Show output
if show_output:
cv2.imshow('frame', img_out)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cv2.destroyAllWindows()
# Write predicted probabilities and real labels
dict_data = {
'pred_collisions': col,
'pred_steerings': ste,
'real_labels': lbl
}
jsonfilen = os.path.join(
foldersub, 'predicted_and_real_labels_' +
os.path.basename(s['weights_fname']) + '.json')
with open(jsonfilen, "w") as f:
json.dump(dict_data, f)
print("Written file {}".format(jsonfilen))
f.close()
else:
print("Something's wrong ...")
exit(1)