def on_train_end(self, logs=None): self.log['duration'] = time.time() - self.ts misc.write_to_file(self.log, os.path.join(self.logpath, "train_log.json")) hours, rem = divmod(self.log['duration'], 3600) minutes, seconds = divmod(rem, 60) print("Total Duration: {:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds))
def evaluate_regression(predictions, real_values, fname): evas = compute_explained_variance(predictions, real_values) rmse = compute_rmse(predictions, real_values) highest_errors = compute_highest_regression_errors(predictions, real_values, n_errors=20) dictionary = { "evas": evas.tolist(), "rmse": rmse.tolist(), "highest_errors": highest_errors.tolist() } misc.write_to_file(dictionary, fname)
def evaluate_classification(pred_prob, pred_labels, real_labels, fname): ave_accuracy = metrics.accuracy_score(real_labels, pred_labels) print('Average accuracy = ', ave_accuracy) precision = metrics.precision_score(real_labels, pred_labels) print('Precision = ', precision) recall = metrics.recall_score(real_labels, pred_labels) print('Recall = ', recall) f_score = metrics.f1_score(real_labels, pred_labels) print('F1-score = ', f_score) highest_errors = compute_highest_classification_errors(pred_prob, real_labels, n_errors=20) dictionary = { "ave_accuracy": ave_accuracy, "precision": precision, "recall": recall, "f_score": f_score, "highest_errors": highest_errors.tolist() } misc.write_to_file(dictionary, fname)
def _main(): # Load settings or use args or defaults s = misc.load_settings_or_use_args(FLAGS) # Misc settings s['rescale_factor'] = s.get('rescale_factor', 1. if s['img_mode'] == "flow" else 1. / 255) if FLAGS.test_dir is None: s['test_dir'] = os.path.join(FLAGS.experiment_rootdir, s['test_dir']) model_dir = os.path.join(FLAGS.experiment_rootdir, s['model_dir']) # Set testing mode (dropout/batchnormalization) k.set_learning_phase(0) # Generate testing data test_datagen = utils.DroneDataGenerator(rescale=s['rescale_factor']) test_generator = test_datagen.flow_from_directory( s['test_dir'], shuffle=False, color_mode=s['img_mode'], target_size=(s['img_width'], s['img_height']), crop_size=(s['crop_img_width'], s['crop_img_height']), batch_size=1, is_training=False) # Load json and create model json_model_path = os.path.join(model_dir, s['json_model_fname']) model = utils.json_to_model(json_model_path) # Get weights paths weights_2_load = sorted(glob.glob(os.path.join(model_dir, 'model_weights*')), key=os.path.getmtime) # Prepare output directory if s['name'] == "same": if FLAGS.test_dir[-1] == "/": outdir = "eval_" + os.path.split(s['test_dir'][:-1])[1] else: outdir = "eval_" + os.path.split(s['test_dir'])[1] else: outdir = "eval_" + s['name'] + "_" + os.path.split(s['test_dir'])[1] if FLAGS.output_dir is None: outfolder = os.path.join(FLAGS.experiment_rootdir, s['eval_dir'], outdir) else: outfolder = os.path.join(FLAGS.output_dir, outdir) if not os.path.exists(outfolder): os.makedirs(outfolder) # Store settings as backup settings_out_filename = os.path.join(outfolder, s['settings_fname']) misc.write_to_file(s, settings_out_filename, beautify=True) # Iterate through trained models for weights_load_path in weights_2_load: model.load_weights(weights_load_path) print("\nEvaluating: " + weights_load_path) print("On: " + s['test_dir'] + "\n") index = weights_load_path.split("_")[-1][:-3] # Compile model model.compile(loss='mse', optimizer='adadelta') # Get predictions and ground truth n_samples = test_generator.samples nb_batches = int(np.ceil(n_samples / 1)) predictions, ground_truth, t, fnames = utils.compute_predictions_and_gt( model, test_generator, nb_batches, verbose=1) # Param t. t=1 steering, t=0 collision t_mask = t == 1 # ************************* Steering evaluation *************************** # Predicted and real steerings pred_steerings = predictions[t_mask, 0] real_steerings = ground_truth[t_mask, 0] # Compute random and constant baselines for steerings random_steerings = random_regression_baseline(real_steerings) constant_steerings = constant_baseline(real_steerings) # Create dictionary with filenames dict_fname = { 'test_regression_' + str(index.zfill(4)) + '.json': pred_steerings, 'random_regression_' + str(index.zfill(4)) + '.json': random_steerings, 'constant_regression_' + str(index.zfill(4)) + '.json': constant_steerings } # Evaluate predictions: EVA, residuals, and highest errors for fname, pred in dict_fname.items(): abs_fname = os.path.join(outfolder, fname) evaluate_regression(pred, real_steerings, abs_fname) # Write predicted and real steerings dict_test = { 'pred_steerings': pred_steerings.tolist(), 'real_steerings': real_steerings.tolist() } pred_fname = os.path.join( outfolder, 'predicted_and_real_steerings_' + str(index.zfill(4)) + '.json') misc.write_to_file(dict_test, pred_fname) # *********************** Collision evaluation **************************** # Predicted probabilities and real labels pred_prob = predictions[~t_mask, 1] pred_labels = np.zeros_like(pred_prob) pred_labels[pred_prob >= 0.5] = 1 real_labels = ground_truth[~t_mask, 1] # Compute random, weighted and majorirty-class baselines for collision random_labels = random_classification_baseline(real_labels) # Create dictionary with filenames dict_fname = { 'test_classification_' + str(index.zfill(4)) + '.json': pred_labels, 'random_classification_' + str(index.zfill(4)) + '.json': random_labels } # Evaluate predictions: accuracy, precision, recall, F1-score, and highest errors for fname, pred in dict_fname.items(): abs_fname = os.path.join(outfolder, fname) evaluate_classification(pred_prob, pred, real_labels, abs_fname) chain = itertools.chain(*fnames) ffnames = list(chain) # Write predicted probabilities and real labels dict_test = { 'pred_probabilities': pred_prob.tolist(), 'real_labels': real_labels.tolist(), 'filenames': ffnames } pred_fname = os.path.join( outfolder, 'predicted_and_real_labels_' + str(index.zfill(4)) + '.json') misc.write_to_file(dict_test, pred_fname)
def _main(): global c, s # load settings or use args or defaults s = misc.load_settings_or_use_args(FLAGS) # custom settings #s['center_flow'] = True # init misc settings s['input_dir'] = FLAGS.input_dir # init folders outfolder = create_outfolder() misc.del_and_recreate_folder(os.path.join(outfolder, s['train_dir'])) misc.del_and_recreate_folder(os.path.join(outfolder, s['val_dir'])) misc.del_and_recreate_folder(os.path.join(outfolder, s['model_dir'])) # read labels labels_fname = os.path.join(s['input_dir'], "labels.txt") c['all'] = file_len(labels_fname) - 2 data = read_labels(labels_fname) # clamp and normalize steering angles scaler = MaxAbsScaler() data['steering'] = np.clip(data['steering'], -s['max_steer_angle'], s['max_steer_angle']) data['steering'] = np.squeeze(scaler.fit_transform(data['steering'].reshape(-1, 1))) # save scaler joblib.dump(scaler, os.path.join(outfolder, s['scaler_filename'])) # split into sets and filter out unwanted elements of the data candidates = prep_data(data) # load cloud image for preprocessing cloud = cv2.imread(localpath + s['cloud_file'], 0).astype(np.float) # init flow stat collector flowdataname = "flowstats_" + s['name'] if s['name'] != "same" else "flow_statistics" flow_stat_collector = iu.FlowStatsCollector(outfolder, lim=200, name=flowdataname, n=s['flow_stat_bin_count'], clip=False) # iterate through all experiments for exp in tqdm(candidates, desc="Preparing Dataset: ", maxinterval=1, unit="Exp."): # global counter c['tmp'] = c['tmp'] + len(exp) # load and validate images (check if too dark/bright) imgs = load_set_images(exp['filename'], s['input_dir']) # check for too bright or dark images and remove them del_idxs = validate_set_images(imgs) imgs = np.delete(imgs, del_idxs, 0) exp = np.delete(exp, del_idxs, 0) # apply quote of items that are allowed to have zero data if len(exp) > s['min_data_per_exp']: del_idxs = apply_zero_quote(exp) imgs = np.delete(imgs, del_idxs, 0) exp = np.delete(exp, del_idxs, 0) # process and write set if still enough data available if len(exp) > s['min_data_per_exp']: new_imgs = apply_preprocessing(imgs, cloud) # get stats of flow images and rescale so that the mean magnitude = 1 if s['img_mode'] in ["flow", "flow_as_rgb", "flow_as_mag"]: flow_stat_collector.collect_exp_flow_stats(new_imgs) # write experiment set to train or val train_or_val = "train" if random.random() > s['train_val_qouta'] else "val" folder = create_subfolder(exp, os.path.join(outfolder, s[train_or_val + "_dir"])) write_set(exp, new_imgs, folder) c[train_or_val] = c[train_or_val] + len(new_imgs) # logging key = get_type(exp) c['good'] = c['good'] + len(exp) c[key] = c[key] + len(exp) else: if verbose_val: print("ID: {:5d}; \t Info: Not enough data left after image validation".format(exp['id'][0])) c['dismissed'] = c['dismissed'] + len(exp) # Save flow statistics flow_stats_available = False if s['img_mode'] in ["flow", "flow_as_rgb", "flow_as_mag"]: flow_stat_collector.write_flow_stats() flow_stats_available = True # Store settings in output folders s['img_channels'] = 3 if s['img_mode'] == "flow_as_rgb" else s['img_channels'] # set to 3 channels for rgb s['img_channels'] = 1 if s['img_mode'] == "flow_as_mag" else s['img_channels'] # set to 1 channel for magnitude s['img_mode'] = "rgb" if s['img_mode'] == "flow_as_rgb" else s['img_mode'] # set img mode to rgb after conversion s['img_mode'] = "grayscale" if s['img_mode'] in ["depth", "flow_as_mag"] else s['img_mode'] # set img mode to grayscale if depth or magnitude settings_out_filename = os.path.join(outfolder, s['settings_fname']) misc.write_to_file(s, settings_out_filename, beautify=True) # create eval folder if create_subfolders: try: os.mkdir(os.path.join(outfolder, s['eval_dir'])) except: print("Warning: Unable to create eval folder.") # Output of statistics print("\n\n## " + misc.colorize("[RESULTS]", "magenta") + " ############################################" + "\n\n- " + misc.colorize("INPUT", "cyan") + " ----------------------------------------------" + "\nFolder: " + FLAGS.input_dir + "\nLabels: " + labels_fname + "\nImages: {:d} ({:d} Experiments)".format(c['all'], c['exp_in']) + "\n\n- " + misc.colorize("VALIDATION", "cyan") + " -----------------------------------------" + "\nToo bright: {:d}".format(c['bright']) + "\nToo dark: {:d}".format(c['dark']) + "\nDismissed: {:d} ({:3.1f} %)".format(c['dismissed'], c['dismissed'] / c['all'] * 100.)) if flow_stats_available: print("\n- " + misc.colorize("FLOW STATS", "cyan") + " -----------------------------------------") flow_stat_collector.print_flow_stats() print("\n- " + misc.colorize("OUTPUT", "cyan") + " ---------------------------------------------" + "\nFolder: " + outfolder + "\nGOOD: {:d} ({:3.1f} %; {:d} Experiments)".format(c['good'], c['good'] / c['all'] * 100., c['exp_out']) + "\nClouded: {:d} ({:3.1f} %)".format(c['cloud'], c['cloud'] / c['good'] * 100.) + "\nSteering: {:d} (zero: {:d})".format(c['steering'], c['zero_steering']) + "\nCollision: {:d} (zero: {:d})".format(c['collision'], c['zero_collision']) + "\nTrain/Val: {:d}/{:d} " "(Val. Quota: {:3.1f} %)".format(c['train'], c['val'], c['val'] / c['train'] * 100.) + "\n\n#########################################################\n")
def _main(): # Load settings or use args or defaults s = misc.load_settings_or_use_args(FLAGS) # Create the experiment rootdir if not already there model_dir = os.path.join(FLAGS.experiment_rootdir, s['model_dir']) if not os.path.exists(model_dir): os.makedirs(model_dir) # Image mode if s['img_mode'] == 'rgb': s['img_channels'] = s.get('img_channels', 3) elif s['img_mode'] == 'flow': s['img_channels'] = s.get('img_channels', 2) s['rescale_factor'] = None s['width_shift_range'] = None s['height_shift_range'] = None elif s['img_mode'] == 'grayscale': s['img_channels'] = s.get('img_channels', 1) else: raise IOError( "Unidentified image mode: use 'grayscale', 'flow' or 'rgb'") # Generate training data with real-time augmentation train_datagen = utils.DroneDataGenerator( rotation_range=s['rotation_range'], rescale=s['rescale_factor'], width_shift_range=s['width_shift_range'], height_shift_range=s['height_shift_range'], vertical_flip=False) train_generator = train_datagen.flow_from_directory( os.path.join(FLAGS.experiment_rootdir, s['train_dir']), shuffle=True, color_mode=s['img_mode'], target_size=(s['img_width'], s['img_height']), crop_size=(s['crop_img_width'], s['crop_img_height']), batch_size=s['batch_size']) # Generate validation data with real-time augmentation val_datagen = utils.DroneDataGenerator(rescale=s['rescale_factor']) val_generator = val_datagen.flow_from_directory( os.path.join(FLAGS.experiment_rootdir, s['val_dir']), shuffle=False, color_mode=s['img_mode'], target_size=(s['img_width'], s['img_height']), crop_size=(s['crop_img_width'], s['crop_img_height']), batch_size=s['batch_size']) # Weights to restore if not s['restore_model']: # In this case weights will start from random s['initial_epoch'] = 0 s['restore_path'] = None else: # In this case weigths will start from the specified mode s['restore_path'] = os.path.join(model_dir, s['weights_fname']) # Define model model = get_model(s['crop_img_width'], s['crop_img_height'], s['img_channels'], 1, s['restore_path']) # Serialize model into json json_model_path = os.path.join(model_dir, s['json_model_fname']) utils.model_to_json(model, json_model_path) # Store settings settings_out_filename = os.path.join(FLAGS.experiment_rootdir, s['settings_fname']) misc.write_to_file(s, settings_out_filename, beautify=True) # Train model train_model(train_generator, val_generator, model, s['initial_epoch'], s)
def _main(): global s # Load settings or use args or defaults s = misc.load_settings_or_use_args(FLAGS) # Init variables (no need to change anything here) last_duration = 1 json_model_path = os.path.join(FLAGS.experiment_rootdir, s['model_dir'], s['json_model_fname']) weights_path = os.path.join(FLAGS.experiment_rootdir, s['model_dir'], s['weights_fname']) s['target_size'] = s.get('target_size', (s['img_width'], s['img_height'])) s['crop_size'] = s.get('crop_size', (s['crop_img_width'], s['crop_img_height'])) s['output_size'] = s.get('output_size', (400, 300)) extensions = ['jpg', 'png'] # Special settings for flow mode if s['img_mode'] == "flow": extensions = ['npy', 'bin', 'flo', 'pfm', 'png'] # Load model model = get_model(json_model_path, weights_path) # Penultimate layer index for grad cams penultimate_layer = utils.find_layer_idx(model, s['penultimate_layer']) # Check if single or multiple experiments to test folders = misc.get_experiment_folders(FLAGS.test_dir) # Prepare output directory if s['name'] == "same": if FLAGS.test_dir[-1] == "/": outdir = "eval_" + os.path.split(s['test_dir'][:-1])[1] + "_adv" else: outdir = "eval_" + os.path.split(s['test_dir'])[1] + "_adv" else: outdir = "eval_" + s['name'] + "_" + os.path.split( s['test_dir'])[1] + "_adv" if FLAGS.output_dir is None: outfolder = os.path.join(FLAGS.experiment_rootdir, s['eval_dir'], outdir) else: outfolder = os.path.join(FLAGS.output_dir, outdir) if not os.path.exists(outfolder): os.makedirs(outfolder) # Store settings as backup settings_out_filename = os.path.join(outfolder, s['settings_fname']) misc.write_to_file(s, settings_out_filename, beautify=True) # Iterate through folders for folder in tqdm(sorted(folders), desc="Evaluating Dataset: ", maxinterval=1, unit="Exp."): # Get filenames imgs = [] for extension in extensions: ipth = os.path.join(folder, 'images/*.' + extension) imgs = sorted(glob.glob(ipth)) if len(imgs) > 0: break # Check if labels are available, then load them exp_type = None lbl = np.zeros(len(imgs)) - 1337 for key in label_files: fullname = os.path.join(folder, label_files[key]) if os.path.isfile(fullname): lbl = [] exp_type = key with open(fullname, newline='') as i: for row in csv.reader(i, delimiter=';'): lbl.append(float(row[0])) break # Prepare output subfolder foldersub = os.path.join(outfolder, os.path.split(folder)[1]) misc.del_and_recreate_folder(os.path.join(foldersub, "images")) # # add first label on flow data (otherwise it will have one label less than the real data) # if "flow" in folder: # lbl.insert(0, 0) # Check if labels and images fit if len(lbl) >= len(imgs): col = [] ste = [] # Iterate through imgs with n stepsize for x in range(len(imgs)): # Load image and prep for dronet data = iu.load_img(imgs[x], img_mode=s['img_mode'], crop_size=s['crop_size'], target_size=s['target_size']) # Load image again for displaying if s['img_mode'] == "flow": # Convert for output img = iu.flow_to_img(data) else: # Read img and rescale data img = cv2.imread(imgs[x], cv2.IMREAD_COLOR) data = np.asarray(data, dtype=np.float32) * np.float32( 1.0 / 255.0) carry = data[np.newaxis, ...] # Make and store predictions img = cv2.resize(img, s['output_size'], interpolation=cv2.INTER_AREA) theta, p_t, img_out = make_prediction(img, carry, model, lbl[x]) img_out = display_predictions_as_bars(img_out, theta, p_t, lbl[x], exp_type) col.append(float(p_t)) ste.append(float(theta)) # Grad-CAMs if s['show_activations']: # Cleanup for keras-vis bug (slowdown + leaky memory) if last_duration > 0.4: k.clear_session() model = get_model(json_model_path, weights_path) # Produce activation maps for collision or steering img_h, last_duration = get_grad_cam( img, carry, model, penultimate_layer, s['filter_indices']) # Put the pieces together img_out = np.hstack( (img_out, np.ones( (s['output_size'][1], 50, 3), dtype=np.uint8) * 255, img_h)) # Write output img_name = os.path.split(imgs[x])[1] img_name = os.path.splitext(img_name)[0] cv2.imwrite( os.path.join(foldersub, "images", img_name + ".png"), img_out) # Show output if show_output: cv2.imshow('frame', img_out) if cv2.waitKey(1) & 0xFF == ord('q'): break cv2.destroyAllWindows() # Write predicted probabilities and real labels dict_data = { 'pred_collisions': col, 'pred_steerings': ste, 'real_labels': lbl } jsonfilen = os.path.join( foldersub, 'predicted_and_real_labels_' + os.path.basename(s['weights_fname']) + '.json') with open(jsonfilen, "w") as f: json.dump(dict_data, f) print("Written file {}".format(jsonfilen)) f.close() else: print("Something's wrong ...") exit(1)