def evaluate_generator(self, annotations, comet_experiment=None, iou_threshold=0.5, max_detections=200): """ Evaluate prediction model using a csv fit_generator Args: annotations (str): Path to csv label file, labels are in the format -> path/to/image.png,x1,y1,x2,y2,class_name iou_threshold(float): IoU Threshold to count for a positive detection (defaults to 0.5) max_detections (int): Maximum number of bounding box predictions comet_experiment(object): A comet experiment class objects to track Return: mAP: Mean average precision of the evaluated data """ #Format args for CSV generator classes_file = utilities.create_classes(annotations) arg_list = utilities.format_args(annotations, classes_file, self.config) args = parse_args(arg_list) #create generator validation_generator = CSVGenerator( args.annotations, args.classes, image_min_side=args.image_min_side, image_max_side=args.image_max_side, config=args.config, shuffle_groups=False, ) average_precisions = evaluate(validation_generator, self.prediction_model, iou_threshold=iou_threshold, score_threshold=args.score_threshold, max_detections=max_detections, save_path=args.save_path, comet_experiment=comet_experiment) # print evaluation total_instances = [] precisions = [] for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), validation_generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) total_instances.append(num_annotations) precisions.append(average_precision) if sum(total_instances) == 0: print('No test instances found.') return print('mAP using the weighted average of precisions among classes: {:.4f}'.format( sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances))) mAP = sum(precisions) / sum(x > 0 for x in total_instances) print('mAP: {:.4f}'.format(mAP)) return mAP
def on_epoch_end(self, epoch, logs={}): # run evaluation average_precisions = evaluate(self.generator, self.model, iou_threshold=self.iou_threshold, score_threshold=self.score_threshold, max_detections=self.max_detections, save_path=self.save_path) self.mean_ap = sum( average_precisions.values()) / len(average_precisions) if self.tensorboard is not None and self.tensorboard.writer is not None: import tensorflow as tf summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = self.mean_ap summary_value.tag = "mAP_train" if self.train else "mAP_test" self.tensorboard.writer.add_summary(summary, epoch) if self.verbose == 1: for label, average_precision in average_precisions.items(): print(self.generator.label_to_name(label), '{:.4f}'.format(average_precision)) print('mAP: {:.4f}'.format(self.mean_ap))
def main(args=None): from keras import backend as K # parse arguments if args is None: args = sys.argv[1:] args = parse_args(args) print('Arguments: {}'.format(args)) # create object that stores backbone information backbone = models.backbone(args.backbone) # make sure keras is the minimum required version check_keras_version() # optionally choose specific GPU if args.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu keras.backend.tensorflow_backend.set_session(get_session()) # create the generators validation_generator = create_generators(args, backbone.preprocess_image) # create the model weights = args.weights # default to imagenet if nothing else is specified if weights is None and args.imagenet_weights: weights = backbone.download_imagenet() print('Creating model, this may take a second...') model, prediction_model = create_models( backbone_retinanet=backbone.retinanet, num_classes=validation_generator.num_classes(), weights=weights, multi_gpu=args.multi_gpu, freeze_backbone=args.freeze_backbone) # evaluate model print('Evaluating model from {}, this may take a while...'.format( args.weights)) average_precisions = evaluate( validation_generator, prediction_model, iou_threshold=0.5, score_threshold=0.05, max_detections=300, ) # print evaluation present_classes = 0 precision = 0 for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), validation_generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) if num_annotations > 0: present_classes += 1 precision += average_precision print('mAP: {:.4f}'.format(precision / present_classes))
def main1(): # for file in glob.glob("./snapshots/*_05.h5"): file = './snapshots/resnet50_csv_05.h5' # file = 'C:/Projects/OLD-keras-retinanet-master/snapshots/resnet50_csv_01.h5' map_total = 0 for i in range(50, 100, 5): i = i / 100 keras.backend.tensorflow_backend.set_session(get_session()) model = keras.models.load_model(file, custom_objects=custom_objects) val_generator = CSVGenerator( csv_data_file='c:/MTSD/Updated/test - copy.csv', csv_class_file='c:/MTSD/Updated/classes.csv', base_dir='c:/MTSD/Updated/detection/', image_min_side=1440, image_max_side=2560, min_size=25) # analyse_images(val_generator) my_eval = eval.evaluate(val_generator, model, score_threshold=0.5, iou_threshold=0.5, save_path='C:/video-out/', ground_truth=False) print(my_eval) print(sum(my_eval.values()) / 39) keras.backend.clear_session() break
def test_model(model, generator, score_threshold): result = evaluate(generator, models.convert_model(model, None), score_threshold=score_threshold, iou_threshold=0.5, max_detections=100, save_path=None) return result
def main(config, args=None): # parse arguments if args is None: args = sys.argv[1:] args = parse_args(args) # make sure keras is the minimum required version check_keras_version() # optionally choose specific GPU if args.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu keras.backend.tensorflow_backend.set_session(get_session()) # make save path if it doesn't exist if args.save_path is not None and not os.path.exists(args.save_path): os.makedirs(args.save_path) # create the generator generator = create_generator(args, config) # load the model print('Loading model, this may take a second...') model = models.load_model(args.model, backbone_name=args.backbone, convert=args.convert_model) # print model summary # print(model.summary()) # start evaluation if args.dataset_type == 'coco': from keras_retinanet.utils.coco_eval import evaluate_coco evaluate_coco(generator, model, args.score_threshold) else: average_precisions = evaluate(generator, model, iou_threshold=args.iou_threshold, score_threshold=args.score_threshold, max_detections=args.max_detections, save_path=args.save_path) # print evaluation present_classes = 0 precision = 0 for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) if num_annotations > 0: present_classes += 1 precision += average_precision print('mAP: {:.4f}'.format(precision / present_classes))
def on_epoch_end(self, epoch, logs=None): logs = logs or {} # run evaluation average_precisions = evaluate( self.generator, self.model, iou_threshold=self.iou_threshold, score_threshold=self.score_threshold, max_detections=self.max_detections, save_path=self.save_path ) # compute per class average precision present_classes = 0 precision = 0 for label, (average_precision, num_annotations ) in average_precisions.items(): if self.verbose == 1: print('{:.0f} instances of class'.format(num_annotations), self.generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) if self.save_map_path is not None: out = open(self.save_map_path, 'a') out.write('{:.0f} instances of class {} with average precision: {:.4f}\n'.format(num_annotations, self.generator.label_to_name(label), average_precision)) out.close() if num_annotations > 0: present_classes += 1 precision += average_precision self.mean_ap = precision / present_classes if self.tensorboard is not None and self.tensorboard.writer is not None: import tensorflow as tf summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = self.mean_ap summary_value.tag = "mAP" self.tensorboard.writer.add_summary(summary, epoch) logs['mAP'] = self.mean_ap if self.save_map_path is not None: out = open(self.save_map_path, 'a') out.write('Ep {}: mAP: {:.4f}\n'.format(epoch + 1, self.mean_ap)) out.close() if self.verbose == 1: print('mAP: {:.4f}'.format(self.mean_ap))
def on_epoch_end(self, epoch, logs=None): logs = logs or {} # run evaluation average_precisions = evaluate(self.generator, self.model, iou_threshold=self.iou_threshold, score_threshold=self.score_threshold, max_detections=self.max_detections, save_path=self.save_path, experiment=self.experiment) # compute per class average precision total_instances = [] precisions = [] for label, (average_precision, num_annotations) in average_precisions.items(): if self.verbose == 1: print( '{:.0f} instances of class'.format(num_annotations), self.generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) total_instances.append(num_annotations) precisions.append(average_precision) if self.weighted_average: self.mean_neon_ap = sum([ a * b for a, b in zip(total_instances, precisions) ]) / sum(total_instances) else: self.mean_neon_ap = sum(precisions) / sum(x > 0 for x in total_instances) if self.verbose == 1: print('NEON mAP: {:.4f}'.format(self.mean_neon_ap)) self.experiment.log_metric("Neon mAP", self.mean_neon_ap)
test_csv_file = 'test_hpc_RMmin2_corrected.csv' class_csv_file = 'classes.csv' train_ds = pd.read_csv(test_csv_file) test_generator = csv_generator.CSVGenerator(test_csv_file, csv_class_file=class_csv_file) # In[ ]: model_path = os.path.join('training-files/retinanet_100/models_for_testing/resnet50_csv_99_1.h5') # load retinanet model model = models.load_model(model_path, backbone_name='resnet50') model = models.convert_model(model) average_precisions,recall,precision,true_positives,false_positives = evaluate( test_generator, model, save_path='./save/test/') print (average_precisions) print (true_positives) print (false_positives) myData = [precision,recall] myFile = open('pr-model_all_unfiltered.csv', 'w')
def main(data, DeepForest_config, experiment, args=None): # parse arguments if args is None: args = sys.argv[1:] args = parse_args(args) # make sure keras is the minimum required version check_keras_version() # optionally choose specific GPU if args.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu keras.backend.tensorflow_backend.set_session(get_session()) #Add seperate dir #save time for logging dirname = datetime.now().strftime("%Y%m%d_%H%M%S") experiment.log_parameter("Start Time", dirname) # make save path if it doesn't exist if args.save_path is not None and not os.path.exists(args.save_path + dirname): os.makedirs(args.save_path + dirname) # create the testing generators generator = create_generator(args, data, DeepForest_config) #create the NEON mAP generator NEON_generator = create_NEON_generator(args, site, DeepForest_config) # load the model print('Loading model, this may take a second...') model = models.load_model(args.model, backbone_name=args.backbone, convert=args.convert_model, nms_threshold=DeepForest_config["nms_threshold"]) #print(model.summary()) average_precisions = evaluate(generator, model, iou_threshold=args.iou_threshold, score_threshold=args.score_threshold, max_detections=args.max_detections, save_path=args.save_path + dirname) # print evaluation present_classes = 0 precision = 0 for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) if num_annotations > 0: present_classes += 1 precision += average_precision print('mAP: {:.4f}'.format(precision / present_classes)) experiment.log_metric("mAP", precision / present_classes) #Use field collected polygons only for Florida site #if site == "OSBS": ##Ground truth scores #jaccard=Jaccard( #generator=generator, #model=model, #score_threshold=args.score_threshold, #save_path=args.save_path, #experiment=experiment, #DeepForest_config=DeepForest_config #) #print(f" Mean IoU: {jaccard:.2f}") #experiment.log_metric("Mean IoU", jaccard) #Neon plot recall rate recall = neonRecall(site, generator, model, score_threshold=args.score_threshold, save_path=args.save_path, experiment=experiment, DeepForest_config=DeepForest_config) experiment.log_metric("Recall", recall) print(f" Recall: {recall:.2f}") #Logs the number of train and eval "trees" ntrees = sum([len(x) for x in generator.annotation_dict.values()]) experiment.log_parameter("Number of Trees", ntrees) #NEON plot mAP average_precisions = evaluate( NEON_generator, model, iou_threshold=args.iou_threshold, score_threshold=args.score_threshold, max_detections=args.max_detections, save_path=args.save_path + dirname, experiment=experiment, ) # print evaluation present_classes = 0 precision = 0 for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) if num_annotations > 0: present_classes += 1 precision += average_precision print('NEON mAP: {:.4f}'.format(precision / present_classes)) experiment.log_metric("NEON_mAP", precision / present_classes)
def main(args=None, model_filename=None): # parse arguments if args is None: args = sys.argv[1:] args = parse_args(args) # make sure keras and tensorflow are the minimum required version check_keras_version() check_tf_version() # optionally choose specific GPU if args.gpu: setup_gpu(args.gpu) # make save path if it doesn't exist if args.save_path is not None and not os.path.exists(args.save_path): os.makedirs(args.save_path) # optionally load config parameters if args.config: args.config = read_config_file(args.config) # create the generator backbone = models.backbone(args.backbone) generator = create_generator(args, backbone.preprocess_image) # optionally load anchor parameters anchor_params = None if args.config and 'anchor_parameters' in args.config: anchor_params = parse_anchor_parameters(args.config) # load the model print('Loading model, this may take a second...') if args.continual_learning_model == 'dual_memory': # Continual learning dual-memory modelling treatment base_models = LoadModels(args.historical_snapshots_folder, args.backbone, args.day_number) all_models = [] for model in base_models: generator.compute_shapes = make_shapes_callback(model) if args.convert_model: model = models.convert_model(model, anchor_params=anchor_params) all_models.append(model) (average_precisions, inference_time, detections_per_model, final_detections) = evaluate_dual_memory_model( generator, all_models, iou_threshold=args.iou_threshold, score_threshold=args.score_threshold, max_detections=args.max_detections, save_path=args.save_path) # bbox_savepath given, save bounding box coordinates from dual-memory model predictions: if args.bbox_savepath: detections_per_model = [[ [class_predictions.tolist() for class_predictions in image] for image in model_predictions ] for model_predictions in detections_per_model] detections_with_filenames = { 'final_detections': final_detections, 'annotations': args.annotations, 'detections_per_model': detections_per_model } with open(args.bbox_savepath, 'wt') as outf: json.dump(detections_with_filenames, outf) print("Finished dual memory model") print(average_precisions, inference_time) else: if model_filename is None: model_filename = args.model model = models.load_model(model_filename, backbone_name=args.backbone) generator.compute_shapes = make_shapes_callback(model) # optionally convert the model if args.convert_model: model = models.convert_model(model, anchor_params=anchor_params) # print model summary # print(model.summary()) # start evaluation if args.dataset_type == 'coco': from ..utils.coco_eval import evaluate_coco evaluate_coco(generator, model, args.score_threshold) else: average_precisions, inference_time = evaluate( generator, model, iou_threshold=args.iou_threshold, score_threshold=args.score_threshold, max_detections=args.max_detections, save_path=args.save_path) # print evaluation total_instances = [] precisions = [] #labels = [] for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) #labels.append(label) total_instances.append(num_annotations) precisions.append(average_precision) if sum(total_instances) == 0: print('No test instances found.') return print('Inference time for {:.0f} images: {:.4f}'.format( generator.size(), inference_time)) print('mAP using the weighted average of precisions among classes: {:.4f}'. format( sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances))) print('mAP: {:.4f}'.format( sum(precisions) / sum(x > 0 for x in total_instances))) #print(labels) print(precisions) print(total_instances) # Save mAP and other accuracy statistics to mAP_savepath: mAP = sum(precisions) / sum(x > 0 for x in total_instances) date = datetime.now().strftime("%Y%m%d%H%M") with open(args.mAP_savepath, 'a') as outf: outf.write( f"{date}, {mAP}, {precisions}, {total_instances}, {model_filename}, {args.continual_learning_model}" + "\n") return mAP
def main(): ax = plt.gca() #ax.set_aspect(20) score_threshold = [0.3] #, 0.4, 0.5, 0.6, 0.7, 0.8] setup_gpu(0) classes = Path( '/data/students_home/fschipani/thesis/MSc-Thesis-PJ/Dataset/KAIST_MPD/class_name_to_ID_CARS.csv' ) annotations = Path( '/data/students_home/fschipani/thesis/MSc-Thesis-PJ/Dataset/KAIST_MPD/fine_tuning_kaist_cars/ds/test_w_people.csv' ) save_path = Path( '/data/students_home/fschipani/thesis/MSc-Thesis-PJ/Dataset/tests_manual_annotations_08_cars/' ) generator = create_generator(annotations, classes) #weight = '/data/students_home/fschipani/thesis/MSc-Thesis-PJ/Dataset/weights/manual_annotation_08.h5' weight_path_ra = '/data/students_home/fschipani/rand_augment/snapshots_ra_scratch' weight_path = '/data/students_home/fschipani/thesis/MSc-Thesis-PJ/Dataset/snapshots/' # optionally load anchor parameters anchor_params = None dataframe = pd.DataFrame(columns=[ 'epoch', 'score_threshold', 'person_instances', 'cyclist_instances', 'cars_instances', 'map_person', 'map_cyclist', 'map_cars', 'weighted_map', 'map', 'false_positives', 'true_positives', 'recall', 'precision' ]) for weight in glob.iglob(weight_path + '/*.h5'): print(weight) for threshold in [0.3]: os.makedirs(save_path.joinpath(str(threshold * 100)), exist_ok=True) name_folder = weight.replace('/weights', '').replace('.h5', '') print(weight) model = models.load_model(weight, backbone_name='resnet50') model = models.convert_model(model, anchor_params=anchor_params) os.makedirs(save_path.joinpath(str(threshold * 100)).joinpath(name_folder), exist_ok=True) average_precisions, other_metrics = evaluate( generator, model, iou_threshold=0.5, score_threshold=threshold, max_detections=100, save_path=save_path.joinpath(str(threshold * 100)).joinpath(name_folder)) total_instances = [] precisions = [] for label, (average_precision, num_annotations) in average_precisions.items(): print( '{:.0f} instances of class'.format(num_annotations), generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) total_instances.append(num_annotations) precisions.append(average_precision) if sum(total_instances) == 0: print('No test instances found.') return values = { 'epoch': int( weight.replace('resnet50_csv_', '').replace('.h5', ''). replace( '/data/students_home/fschipani/thesis/MSc-Thesis-PJ/Dataset/snapshots/', '')), 'score_threshold': threshold, 'person_instances': int(average_precisions[0][1]), 'cyclist_instances': int(average_precisions[1][1]), 'cars_instances': int(average_precisions[2][1]), 'map_person': average_precisions[0][0], 'map_cyclist': average_precisions[1][0], 'map_cars': average_precisions[2][0], 'weighted_map': (sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances)), 'map': (sum(precisions) / sum(x > 0 for x in total_instances)), 'false_positives': pd.Series(other_metrics[0]), 'true_positives': pd.Series(other_metrics[1]), 'recall': pd.Series(other_metrics[2]), 'precision': pd.Series(other_metrics[3]) } dataframe = dataframe.append(values, ignore_index=True) K.clear_session() #plt.plot(other_metrics[2][2], other_metrics[3][2], label = str(threshold)) #[recall, precision][class] usually class: 0->person 1->cyclist 2->cars dataframe.to_csv('./spero_sia_l_ultimo.csv')