def __init__(self, ground_truth, test_set, weights_file, verbose=True): """ Initializes localizer class used for running localization + classification through a dataset """ # keep track of the current scoring metrics metrics = { 'tp': 0, # number of true positives 'fp': 0, # number of false positives 'fn': 0, # number of false negatives 'yolo': 0, # number of correct classifications by yolo 'svm': 0, # number of correct classifications by svm 'total': 0, # total number of images 'non-detect': 0, # total number of images where no objects were detected 'predictions': [ ] # predictions for each detected object [svm prediction, yolo prediction, true label] } # load the test sets test_images = pd.read_csv(test_set) # load the ground truths gt_data = pd.read_csv(ground_truth, header=None, dtype={0: str}) gt_data.columns = [ 'image', 'label', 'gt_x1', 'gt_y1', 'gt_x2', 'gt_y2' ] self.metrics = metrics self.dim = 416 self.test_images = test_images self.gt_data = gt_data self.yolo = Yolo('yolo/yolo-custom.cfg', weights_file, 'yolo/yolo-custom.txt') self.svm = SVM('classifier/Pretrained/svm/svm_k_4.joblib') self.iterator = 0
def evaluate_pascal_voc(root_dir, download, batch_size, conf_thres, log_every, limit, plot, save, images_results_dir): cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg') weight_file = os.path.join(HERE, '../cfg/yolov3.weights') namesfile = os.path.join(HERE, '../cfg/coco.names') class_names = load_class_names(namesfile) model = Yolo(cfg_file=cfg_file, class_names=class_names, batch_size=batch_size) model.load_weights(weight_file) image_and_target_transform = Compose([SquashResize(416), ToTensor()]) dataset = YaYoloVocDataset(root_dir=root_dir, batch_size=batch_size, transforms=image_and_target_transform, image_set='val', download=download, class_names=class_names) summary_writer = SummaryWriter(comment=f' evaluate={batch_size}') evaluate(model, dataset, summary_writer, images_results_dir, iou_thres=0.5, conf_thres=conf_thres, nms_thres=0.5, log_every=log_every, limit=limit, plot=plot, save=save) summary_writer.close()
def detect_bird(img): yolo = Yolo() bbox = yolo.detect_bbox(img, imshow=False) birds = [] for b in bbox: print(b) if b[0] == "bird": birds.append(b[1:5]) if len(bbox) == 0: print("no detection") return birds
def run_detect_cars(in_dir, out_dir, batch_size, limit, conf_thres, nms_thres, debug): assert os.path.isdir(out_dir), "directory {} does not exist".format(out_dir) now = datetime.datetime.now() now_str = now.strftime("%Y-%m-%dT%H-%M-%S") detected_dataset_dir = os.path.join(out_dir, now_str) os.mkdir(detected_dataset_dir) feed_file = os.path.join(detected_dataset_dir, "feed.json") detected_dataset_images_dir = os.path.join(detected_dataset_dir, "images") os.mkdir(detected_dataset_images_dir) cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg') weight_file = os.path.join(HERE, '../cfg/yolov3.weights') namesfile = os.path.join(HERE, '../cfg/coco.names') class_names = load_class_names(namesfile) with torch.no_grad(): model = Yolo(cfg_file=cfg_file, class_names=class_names, batch_size=batch_size) model.load_weights(weight_file) model.to(DEVICE) model.eval() image_and_target_transform = Compose([ SquashResize(416), ToTensor() ]) dataset = SimpleCarDataset( root_dir=in_dir, transforms=image_and_target_transform, batch_size=batch_size) with FileWriter(file_path=feed_file) as file_writer: car_dataset_writer = DetectedCarDatasetWriter(file_writer) detected_dataset_helper = DetectedCarDatasetHelper(car_dataset_writer=car_dataset_writer, class_names=model.class_names, conf_thres=conf_thres, nms_thres=nms_thres, batch_size=batch_size, debug=debug) cnt = detect_and_process(model=model, dataset=dataset, processor=detected_dataset_helper.process_detections, limit=limit) logger.info("Ran detection of {} images".format(cnt))
def train_coco(coco_images_dir, coco_annotations_file, batch_size, lr, conf_thres, gradient_accumulations, clip_gradients, epochs, limit, log_every, save_every, model_dir, parameters): cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg') weight_file = os.path.join(HERE, '../cfg/yolov3.weights') namesfile = os.path.join(HERE, '../cfg/coco.names') class_names = load_class_names(namesfile) model = Yolo(cfg_file=cfg_file, class_names=class_names, batch_size=batch_size) model.load_weights(weight_file) model.freeze_parameters_of_all_old_layers() if parameters is not None: logger.info(f"loading model parameters from {parameters}") model.load_state_dict(torch.load(parameters, map_location=DEVICE)) image_and_target_transform = Compose([SquashResize(416), ToTensor()]) dataset = YaYoloCocoDataset(images_dir=coco_images_dir, annotations_file=coco_annotations_file, transforms=image_and_target_transform, batch_size=batch_size) summary_writer = SummaryWriter(comment=f' evaluate={batch_size}') train(model=model, dataset=dataset, model_dir=model_dir, summary_writer=summary_writer, epochs=epochs, lr=lr, conf_thres=conf_thres, nms_thres=0.5, iou_thres=0.5, lambda_coord=5, lambda_no_obj=0.5, gradient_accumulations=gradient_accumulations, clip_gradients=clip_gradients, limit=limit, debug=False, print_every=log_every, save_every=save_every) summary_writer.close()
def train_car_make(car_make_json_file, batch_size, lr, conf_thres, gradient_accumulations, clip_gradients, epochs, limit, log_every, save_every, model_dir, parameters, lambda_no_obj): image_and_target_transform = Compose([SquashResize(416), ToTensor()]) dataset = DetectedCareMakeDataset(json_file=car_make_json_file, transforms=image_and_target_transform, batch_size=batch_size) cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg') weight_file = os.path.join(HERE, '../cfg/yolov3.weights') model = Yolo(cfg_file=cfg_file, class_names=dataset.class_names, batch_size=batch_size, coreml_mode=False) model.load_weights(weight_file) if parameters is not None: logger.info(f"loading model parameters from {parameters}") model.load_state_dict(torch.load(parameters, map_location=DEVICE)) summary_writer = SummaryWriter(comment=f' evaluate={batch_size}') # with neptune.create_experiment(name='start-with-neptune', # params=PARAMS): # neptune.append_tag('first-example') train(model=model, dataset=dataset, model_dir=model_dir, summary_writer=summary_writer, epochs=epochs, lr=lr, conf_thres=conf_thres, nms_thres=0.5, iou_thres=0.5, lambda_coord=5, lambda_no_obj=lambda_no_obj, gradient_accumulations=gradient_accumulations, clip_gradients=clip_gradients, limit=limit, debug=False, print_every=log_every, save_every=save_every) summary_writer.close()
from yolo.yolo import Yolo import cv2 if __name__ == "__main__": img = cv2.imread("yolo/imgs/messi.jpg") yolo = Yolo() bbox = yolo.detect_bbox(img, imshow=True) # label, x, y, width, height for raw in bbox: print("[*] class:{} x:{} y:{} width:{} height:{} ".format( raw[0], int(raw[1]), raw[2], raw[3], raw[4]))
def convert_pytorch_to_coreml(cfg_file, class_names, state_dict_file, dummy_input, coreml_pipeline_filename): _path = os.path.dirname(coreml_pipeline_filename) assert os.path.isdir(_path), f"path {_path} does not exist" if os.path.exists(coreml_pipeline_filename): logger.warning(f'file {coreml_pipeline_filename} exists.') HERE = os.path.dirname(os.path.realpath(__file__)) _onnx_filename = os.path.join( HERE, '../../tests/output/yolo-cars.onnx' ) #tempfile.NamedTemporaryFile(prefix='onny-yolo') #_onnx_filename = tempfile.NamedTemporaryFile(prefix='onny-yolo', suffix='.onnx', delete=False) _coreml_filename = os.path.join( HERE, '../../tests/output/yolo-cars.mlmodel' ) #tempfile.NamedTemporaryFile(prefix='coreml-yolo') #_coreml_filename = tempfile.NamedTemporaryFile(prefix='coreml-yolo', suffix='.mlmodel', delete=False) _coreml_nms_filename = os.path.join( HERE, '../../tests/output/yolo-cars-nms.mlmodel' ) # tempfile.NamedTemporaryFile(prefix='coreml-nms') yolo = Yolo(cfg_file=cfg_file, class_names=class_names, batch_size=1, coreml_mode=True) yolo.load_state_dict(torch.load(state_dict_file, map_location=DEVICE)) pytorch_to_onnx(yolo, dummy_input, _onnx_filename) onnx_model = onnx.load(_onnx_filename) coreml_model = convert(onnx_model, minimum_ios_deployment_target="13", image_input_names=['image'], preprocessing_args={ "image_scale": 1 / 255.0, "red_bias": 0, "green_bias": 0, "blue_bias": 0 }) yolo_model = coremltools.models.MLModel(coreml_model.get_spec()) yolo_model.save(_coreml_filename) # nms model nms_spec = coremltools.proto.Model_pb2.Model() nms_spec.specificationVersion = 3 # boxes yolo_boxes = yolo_model._spec.description.output[0].SerializeToString() nms_spec.description.input.add() nms_spec.description.input[0].ParseFromString(yolo_boxes) nms_spec.description.output.add() nms_spec.description.output[0].ParseFromString(yolo_boxes) nms_spec.description.output[0].name = "coordinates" # scores yolo_scores = yolo_model._spec.description.output[1].SerializeToString() nms_spec.description.input.add() nms_spec.description.input[1].ParseFromString(yolo_scores) nms_spec.description.output.add() nms_spec.description.output[1].ParseFromString(yolo_scores) nms_spec.description.output[1].name = "confidence" # coordinates ma_type = nms_spec.description.output[0].type.multiArrayType ma_type.shapeRange.sizeRanges.add() ma_type.shapeRange.sizeRanges[0].lowerBound = 0 ma_type.shapeRange.sizeRanges[0].upperBound = -1 ma_type.shapeRange.sizeRanges.add() ma_type.shapeRange.sizeRanges[1].lowerBound = 4 ma_type.shapeRange.sizeRanges[1].upperBound = 4 del ma_type.shape[:] # confidence ma_type = nms_spec.description.output[1].type.multiArrayType ma_type.shapeRange.sizeRanges.add() ma_type.shapeRange.sizeRanges[0].lowerBound = 0 ma_type.shapeRange.sizeRanges[0].upperBound = -1 ma_type.shapeRange.sizeRanges.add() ma_type.shapeRange.sizeRanges[1].lowerBound = len(class_names) ma_type.shapeRange.sizeRanges[1].upperBound = len(class_names) del ma_type.shape[:] nms = nms_spec.nonMaximumSuppression nms.coordinatesInputFeatureName = "boxes" nms.confidenceInputFeatureName = "scores" nms.coordinatesOutputFeatureName = "coordinates" nms.confidenceOutputFeatureName = "confidence" nms.iouThresholdInputFeatureName = "iouThreshold" nms.confidenceThresholdInputFeatureName = "confidenceThreshold" default_iou_threshold = 0.5 nms.iouThreshold = default_iou_threshold default_confidence_threshold = 0.7 nms.confidenceThreshold = default_confidence_threshold nms.pickTop.perClass = True nms.stringClassLabels.vector.extend(class_names) nms_model = coremltools.models.MLModel(nms_spec) nms_model.save(_coreml_nms_filename) # pipeline input_features = [("image", datatypes.Array(3, 416, 416)), ("iouThreshold", datatypes.Double()), ("confidenceThreshold", datatypes.Double())] output_features = [ "coordinates", "confidence", ] pipeline = Pipeline(input_features, output_features) pipeline.add_model(yolo_model) pipeline.add_model(nms_model) # configure in and output of pipeline pipeline.spec.description.input[0].ParseFromString( yolo_model._spec.description.input[0].SerializeToString()) pipeline.spec.description.output[0].ParseFromString( nms_model._spec.description.output[0].SerializeToString()) pipeline.spec.description.output[1].ParseFromString( nms_model._spec.description.output[1].SerializeToString()) user_defined_metadata = { "classes": ",".join(class_names), "iou_threshold": str(default_iou_threshold), "confidence_threshold": str(default_confidence_threshold) } pipeline.spec.description.metadata.userDefined.update( user_defined_metadata) pipeline.spec.specificationVersion = 3 final_model = coremltools.models.MLModel(pipeline.spec) final_model.save(coreml_pipeline_filename)
def test_yolo_객체_생성(): yolo = Yolo() assert yolo != None
class Localizer: def __init__(self, ground_truth, test_set, weights_file, verbose=True): """ Initializes localizer class used for running localization + classification through a dataset """ # keep track of the current scoring metrics metrics = { 'tp': 0, # number of true positives 'fp': 0, # number of false positives 'fn': 0, # number of false negatives 'yolo': 0, # number of correct classifications by yolo 'svm': 0, # number of correct classifications by svm 'total': 0, # total number of images 'non-detect': 0, # total number of images where no objects were detected 'predictions': [ ] # predictions for each detected object [svm prediction, yolo prediction, true label] } # load the test sets test_images = pd.read_csv(test_set) # load the ground truths gt_data = pd.read_csv(ground_truth, header=None, dtype={0: str}) gt_data.columns = [ 'image', 'label', 'gt_x1', 'gt_y1', 'gt_x2', 'gt_y2' ] self.metrics = metrics self.dim = 416 self.test_images = test_images self.gt_data = gt_data self.yolo = Yolo('yolo/yolo-custom.cfg', weights_file, 'yolo/yolo-custom.txt') self.svm = SVM('classifier/Pretrained/svm/svm_k_4.joblib') self.iterator = 0 def analyze_image(self, image_directory, name): """ Uses yolo model to predict the bounding boxes and labels of an image. Args: image_directory: Path to the image directory name: Name of the image Returns: detections: A list of detections in the same format as the gt_train.csv file image: The image which the analyzation occured on """ image, indices, boxes, class_ids = self.yolo.extract_objects( '{}/{}'.format(image_directory, name)) detections = [] for i in indices: i = i[0] box = boxes[i] x = box[0] y = box[1] w = box[2] h = box[3] label = str(self.yolo.classes[class_ids[i]]) detections.append([ name.split('.')[0], label, round(x), round(y), round(x + w), round(y + h) ]) return detections, image def draw_bounding_boxes(self, detections, ground_truth, image): """ Draws the bounding boxes of the objects on an image for demo purposes Args: detections: List of detections given in same format as gt_train.csv file ground_truth: List of the ground truths to use for comparison image: Image object of the detection Return: result_pred: Image with predictions drawn result_gt: Image with ground truth boxes drawn """ result_pred = image.copy() result_gt = image.copy() for d in detections: cv2.rectangle(result_pred, (d[2], d[3]), (d[4], d[5]), (0, 255, 0), 2) cv2.putText(result_pred, d[1], (d[2] - 10, d[3] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) for i, row in ground_truth.iterrows(): cv2.rectangle(result_gt, (row['gt_x1'], row['gt_y1']), (row['gt_x2'], row['gt_y2']), (0, 255, 0), 2) cv2.putText(result_gt, row['label'], (row['gt_x1'] - 10, row['gt_y1'] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) return result_pred, result_gt def calculate_overlap(self, pred, gt): """ Calculates the pixel overal between two detections in an image Args: pred: Coordinates for the prediction [x1, y1, x2, y2] gt: Coordinates for the ground truth [x1, y1, x2, y2] Returns: percent_overlap: Overlap in percent value """ # create binary images with object pixels set to 1 image_pred = np.full((self.dim, self.dim), False) image_pred[pred[1]:pred[3], pred[0]:pred[2]] = True image_gt = np.full((self.dim, self.dim), False) image_gt[gt[1]:gt[3], gt[0]:gt[2]] = True # perform AND operation to calculate overlap result = np.bitwise_and(image_gt, image_pred) percent_overlap = np.sum(result) / np.sum(image_gt) # return the sum of array (number of 1s) return percent_overlap def classify_svm(self, detection, image): cropped = image[detection[3]:detection[5], detection[2]:detection[4]] pred = self.svm.predict(cropped) return pred def calculate_score(self, detections, ground_truth, image): """ Calculates the localization score of the prediction and updates metrics values Args: detections: List of detections given in same format as gt_train.csv file ground_truth: List of the ground truths to use for comparison image: The original image which that detection was run on """ tp, fp, fn = 0, 0, 0 detections_copy = detections.copy() match_threshold = 0.7 correct_classifications_yolo = 0 correct_classifications_svm = 0 for i, row in ground_truth.iterrows(): gt = [row['gt_x1'], row['gt_y1'], row['gt_x2'], row['gt_y2']] # find maximum overlap max_overlap = [0.0, []] for d in detections_copy: pred = d[2:6] overlap = self.calculate_overlap(pred, gt) if overlap > max_overlap[0]: max_overlap = [overlap, d] # if max overlap exceed threshold, then mark as true positive if max_overlap[0] > match_threshold: tp += 1 # score yolo classification if max_overlap[1][1] == row['label']: self.metrics['yolo'] += 1 correct_classifications_yolo += 1 # score svm classification svm_classification = self.classify_svm(max_overlap[1], image) self.metrics['predictions'].append( [svm_classification, max_overlap[1][1], row['label']]) if svm_classification == row['label']: self.metrics['svm'] += 1 correct_classifications_svm += 1 detections_copy.remove(max_overlap[1]) else: fn += 1 # mark any other detections as false positive fp = len(detections) - tp # update global metrics self.metrics['tp'] += tp self.metrics['fp'] += fp self.metrics['fn'] += fn print('tp: {}, fp: {}, fn: {}'.format(tp, fp, fn)) print('svm: {}/{}'.format(correct_classifications_svm, tp)) print('yolo: {}/{} \n'.format(correct_classifications_yolo, tp)) # print latest total metrics print('\nTotal Metrics - tp: {}, fp: {}, fn: {} '.format( self.metrics['tp'], self.metrics['fp'], self.metrics['fn'])) print('Correct Classifications (YOLO): {}/{}'.format( self.metrics['yolo'], self.metrics['tp'])) print('Correct Classifications (SVM): {}/{} \n'.format( self.metrics['svm'], self.metrics['tp'])) def evaluate_prediction(self, detections, image, image_path, display=False): """ Evaluates the result of a prediction by validating the prediction against the ground truth and computing DICE scores for localization. Args: detections: The list of detections computed by YOLO image: The image object which the detection was performed on image_path: The path to the actual image display: Whether to show the display for visualization (False by default) """ # load ground truth for image and scale to 416x416 ground_truth = self.gt_data.loc[self.gt_data['image'] == detections[0] [0]] height, width, _ = (cv2.imread(image_path)).shape # scale the ground truth coordinates to 416x416 ground_truth['gt_x1'] = ground_truth['gt_x1'].apply( lambda x: int(x * self.dim / width)) ground_truth['gt_x2'] = ground_truth['gt_x2'].apply( lambda x: int(x * self.dim / width)) ground_truth['gt_y1'] = ground_truth['gt_y1'].apply( lambda x: int(x * self.dim / height)) ground_truth['gt_y2'] = ground_truth['gt_y2'].apply( lambda x: int(x * self.dim / height)) self.calculate_score(detections, ground_truth, image) # show bounding boxes if display is set to true if display: image_pred, image_gt = self.draw_bounding_boxes( detections, ground_truth, image) cv2.imshow('Predictions: {}'.format(detections[0][0]), image_pred) cv2.imshow('Ground Truth: {}'.format(detections[0][0]), image_gt) cv2.waitKey(0) cv2.destroyAllWindows() def run(self, image_directory, output_directory): """ Runs the localizer across the dataset and outputs all necessary results needed for the report. Args: image_directory: Path to images directory output_directory: Output file name """ # iterate through each test image for i, row in self.test_images.iterrows(): print('Analyzing image {}/{}'.format(i, self.test_images.shape[0])) self.metrics['total'] += 1 if i == 10: break # run yolo and retrieve results from image detections, image = self.analyze_image(image_directory, row[0].split('/')[-1]) # evaluate the prediction result if len(detections) > 0: self.evaluate_prediction(detections, image, '{}/{}'.format( image_directory, row[0].split('/')[-1]), display=False) else: self.metrics['non-detect'] += 1 # write predictions to a csv file predictions = pd.DataFrame(self.metrics['predictions'], columns=['svm', 'yolo', 'gt']) if not os.path.exists(output_directory): os.makedirs(output_directory) predictions.to_csv('{}/predictions.csv'.format(output_directory), index=False, header=True) # write metrics to a sav file metrics = [] metrics.append( ['Localization True Positives: {}'.format(self.metrics['tp'])]) metrics.append( ['Localization False Positives : {}'.format(self.metrics['fp'])]) metrics.append( ['Localization False Negatives: {}'.format(self.metrics['fn'])]) metrics.append([ 'Correct Classifications (SVM): {}/{}'.format( self.metrics['svm'], self.metrics['tp']) ]) metrics.append([ 'Correct Classifications (YOLO): {}/{}'.format( self.metrics['yolo'], self.metrics['tp']) ]) metrics.append([ 'Dice Score: {}'.format(2 * self.metrics['tp'] / (2 * self.metrics['tp'] + self.metrics['fp'] + self.metrics['fn'])) ]) metrics = pd.DataFrame(metrics) metrics.to_csv('{}/metrics.csv'.format(output_directory), index=False, header=False)