def __init__(self, ground_truth, test_set, weights_file, verbose=True):
"""
Initializes localizer class used for running localization + classification through a dataset
"""
# keep track of the current scoring metrics
metrics = {
'tp': 0, # number of true positives
'fp': 0, # number of false positives
'fn': 0, # number of false negatives
'yolo': 0, # number of correct classifications by yolo
'svm': 0, # number of correct classifications by svm
'total': 0, # total number of images
'non-detect':
0, # total number of images where no objects were detected
'predictions': [
] # predictions for each detected object [svm prediction, yolo prediction, true label]
}
# load the test sets
test_images = pd.read_csv(test_set)
# load the ground truths
gt_data = pd.read_csv(ground_truth, header=None, dtype={0: str})
gt_data.columns = [
'image', 'label', 'gt_x1', 'gt_y1', 'gt_x2', 'gt_y2'
]
self.metrics = metrics
self.dim = 416
self.test_images = test_images
self.gt_data = gt_data
self.yolo = Yolo('yolo/yolo-custom.cfg', weights_file,
'yolo/yolo-custom.txt')
self.svm = SVM('classifier/Pretrained/svm/svm_k_4.joblib')
self.iterator = 0
def evaluate_pascal_voc(root_dir, download, batch_size, conf_thres, log_every,
limit, plot, save, images_results_dir):
cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg')
weight_file = os.path.join(HERE, '../cfg/yolov3.weights')
namesfile = os.path.join(HERE, '../cfg/coco.names')
class_names = load_class_names(namesfile)
model = Yolo(cfg_file=cfg_file,
class_names=class_names,
batch_size=batch_size)
model.load_weights(weight_file)
image_and_target_transform = Compose([SquashResize(416), ToTensor()])
dataset = YaYoloVocDataset(root_dir=root_dir,
batch_size=batch_size,
transforms=image_and_target_transform,
image_set='val',
download=download,
class_names=class_names)
summary_writer = SummaryWriter(comment=f' evaluate={batch_size}')
evaluate(model,
dataset,
summary_writer,
images_results_dir,
iou_thres=0.5,
conf_thres=conf_thres,
nms_thres=0.5,
log_every=log_every,
limit=limit,
plot=plot,
save=save)
summary_writer.close()
def detect_bird(img):
yolo = Yolo()
bbox = yolo.detect_bbox(img, imshow=False)
birds = []
for b in bbox:
print(b)
if b[0] == "bird":
birds.append(b[1:5])
if len(bbox) == 0:
print("no detection")
return birds
def run_detect_cars(in_dir, out_dir, batch_size, limit, conf_thres, nms_thres, debug):
assert os.path.isdir(out_dir), "directory {} does not exist".format(out_dir)
now = datetime.datetime.now()
now_str = now.strftime("%Y-%m-%dT%H-%M-%S")
detected_dataset_dir = os.path.join(out_dir, now_str)
os.mkdir(detected_dataset_dir)
feed_file = os.path.join(detected_dataset_dir, "feed.json")
detected_dataset_images_dir = os.path.join(detected_dataset_dir, "images")
os.mkdir(detected_dataset_images_dir)
cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg')
weight_file = os.path.join(HERE, '../cfg/yolov3.weights')
namesfile = os.path.join(HERE, '../cfg/coco.names')
class_names = load_class_names(namesfile)
with torch.no_grad():
model = Yolo(cfg_file=cfg_file, class_names=class_names, batch_size=batch_size)
model.load_weights(weight_file)
model.to(DEVICE)
model.eval()
image_and_target_transform = Compose([
SquashResize(416),
ToTensor()
])
dataset = SimpleCarDataset(
root_dir=in_dir,
transforms=image_and_target_transform,
batch_size=batch_size)
with FileWriter(file_path=feed_file) as file_writer:
car_dataset_writer = DetectedCarDatasetWriter(file_writer)
detected_dataset_helper = DetectedCarDatasetHelper(car_dataset_writer=car_dataset_writer,
class_names=model.class_names,
conf_thres=conf_thres,
nms_thres=nms_thres,
batch_size=batch_size,
debug=debug)
cnt = detect_and_process(model=model,
dataset=dataset,
processor=detected_dataset_helper.process_detections,
limit=limit)
logger.info("Ran detection of {} images".format(cnt))
def train_coco(coco_images_dir, coco_annotations_file, batch_size, lr,
conf_thres, gradient_accumulations, clip_gradients, epochs,
limit, log_every, save_every, model_dir, parameters):
cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg')
weight_file = os.path.join(HERE, '../cfg/yolov3.weights')
namesfile = os.path.join(HERE, '../cfg/coco.names')
class_names = load_class_names(namesfile)
model = Yolo(cfg_file=cfg_file,
class_names=class_names,
batch_size=batch_size)
model.load_weights(weight_file)
model.freeze_parameters_of_all_old_layers()
if parameters is not None:
logger.info(f"loading model parameters from {parameters}")
model.load_state_dict(torch.load(parameters, map_location=DEVICE))
image_and_target_transform = Compose([SquashResize(416), ToTensor()])
dataset = YaYoloCocoDataset(images_dir=coco_images_dir,
annotations_file=coco_annotations_file,
transforms=image_and_target_transform,
batch_size=batch_size)
summary_writer = SummaryWriter(comment=f' evaluate={batch_size}')
train(model=model,
dataset=dataset,
model_dir=model_dir,
summary_writer=summary_writer,
epochs=epochs,
lr=lr,
conf_thres=conf_thres,
nms_thres=0.5,
iou_thres=0.5,
lambda_coord=5,
lambda_no_obj=0.5,
gradient_accumulations=gradient_accumulations,
clip_gradients=clip_gradients,
limit=limit,
debug=False,
print_every=log_every,
save_every=save_every)
summary_writer.close()
def train_car_make(car_make_json_file, batch_size, lr, conf_thres,
gradient_accumulations, clip_gradients, epochs, limit,
log_every, save_every, model_dir, parameters,
lambda_no_obj):
image_and_target_transform = Compose([SquashResize(416), ToTensor()])
dataset = DetectedCareMakeDataset(json_file=car_make_json_file,
transforms=image_and_target_transform,
batch_size=batch_size)
cfg_file = os.path.join(HERE, '../cfg/yolov3.cfg')
weight_file = os.path.join(HERE, '../cfg/yolov3.weights')
model = Yolo(cfg_file=cfg_file,
class_names=dataset.class_names,
batch_size=batch_size,
coreml_mode=False)
model.load_weights(weight_file)
if parameters is not None:
logger.info(f"loading model parameters from {parameters}")
model.load_state_dict(torch.load(parameters, map_location=DEVICE))
summary_writer = SummaryWriter(comment=f' evaluate={batch_size}')
# with neptune.create_experiment(name='start-with-neptune',
# params=PARAMS):
# neptune.append_tag('first-example')
train(model=model,
dataset=dataset,
model_dir=model_dir,
summary_writer=summary_writer,
epochs=epochs,
lr=lr,
conf_thres=conf_thres,
nms_thres=0.5,
iou_thres=0.5,
lambda_coord=5,
lambda_no_obj=lambda_no_obj,
gradient_accumulations=gradient_accumulations,
clip_gradients=clip_gradients,
limit=limit,
debug=False,
print_every=log_every,
save_every=save_every)
summary_writer.close()
from yolo.yolo import Yolo
import cv2
if __name__ == "__main__":
img = cv2.imread("yolo/imgs/messi.jpg")
yolo = Yolo()
bbox = yolo.detect_bbox(img, imshow=True) # label, x, y, width, height
for raw in bbox:
print("[*] class:{} x:{} y:{} width:{} height:{} ".format(
raw[0], int(raw[1]), raw[2], raw[3], raw[4]))
def convert_pytorch_to_coreml(cfg_file, class_names, state_dict_file,
dummy_input, coreml_pipeline_filename):
_path = os.path.dirname(coreml_pipeline_filename)
assert os.path.isdir(_path), f"path {_path} does not exist"
if os.path.exists(coreml_pipeline_filename):
logger.warning(f'file {coreml_pipeline_filename} exists.')
HERE = os.path.dirname(os.path.realpath(__file__))
_onnx_filename = os.path.join(
HERE, '../../tests/output/yolo-cars.onnx'
) #tempfile.NamedTemporaryFile(prefix='onny-yolo')
#_onnx_filename = tempfile.NamedTemporaryFile(prefix='onny-yolo', suffix='.onnx', delete=False)
_coreml_filename = os.path.join(
HERE, '../../tests/output/yolo-cars.mlmodel'
) #tempfile.NamedTemporaryFile(prefix='coreml-yolo')
#_coreml_filename = tempfile.NamedTemporaryFile(prefix='coreml-yolo', suffix='.mlmodel', delete=False)
_coreml_nms_filename = os.path.join(
HERE, '../../tests/output/yolo-cars-nms.mlmodel'
) # tempfile.NamedTemporaryFile(prefix='coreml-nms')
yolo = Yolo(cfg_file=cfg_file,
class_names=class_names,
batch_size=1,
coreml_mode=True)
yolo.load_state_dict(torch.load(state_dict_file, map_location=DEVICE))
pytorch_to_onnx(yolo, dummy_input, _onnx_filename)
onnx_model = onnx.load(_onnx_filename)
coreml_model = convert(onnx_model,
minimum_ios_deployment_target="13",
image_input_names=['image'],
preprocessing_args={
"image_scale": 1 / 255.0,
"red_bias": 0,
"green_bias": 0,
"blue_bias": 0
})
yolo_model = coremltools.models.MLModel(coreml_model.get_spec())
yolo_model.save(_coreml_filename)
# nms model
nms_spec = coremltools.proto.Model_pb2.Model()
nms_spec.specificationVersion = 3
# boxes
yolo_boxes = yolo_model._spec.description.output[0].SerializeToString()
nms_spec.description.input.add()
nms_spec.description.input[0].ParseFromString(yolo_boxes)
nms_spec.description.output.add()
nms_spec.description.output[0].ParseFromString(yolo_boxes)
nms_spec.description.output[0].name = "coordinates"
# scores
yolo_scores = yolo_model._spec.description.output[1].SerializeToString()
nms_spec.description.input.add()
nms_spec.description.input[1].ParseFromString(yolo_scores)
nms_spec.description.output.add()
nms_spec.description.output[1].ParseFromString(yolo_scores)
nms_spec.description.output[1].name = "confidence"
# coordinates
ma_type = nms_spec.description.output[0].type.multiArrayType
ma_type.shapeRange.sizeRanges.add()
ma_type.shapeRange.sizeRanges[0].lowerBound = 0
ma_type.shapeRange.sizeRanges[0].upperBound = -1
ma_type.shapeRange.sizeRanges.add()
ma_type.shapeRange.sizeRanges[1].lowerBound = 4
ma_type.shapeRange.sizeRanges[1].upperBound = 4
del ma_type.shape[:]
# confidence
ma_type = nms_spec.description.output[1].type.multiArrayType
ma_type.shapeRange.sizeRanges.add()
ma_type.shapeRange.sizeRanges[0].lowerBound = 0
ma_type.shapeRange.sizeRanges[0].upperBound = -1
ma_type.shapeRange.sizeRanges.add()
ma_type.shapeRange.sizeRanges[1].lowerBound = len(class_names)
ma_type.shapeRange.sizeRanges[1].upperBound = len(class_names)
del ma_type.shape[:]
nms = nms_spec.nonMaximumSuppression
nms.coordinatesInputFeatureName = "boxes"
nms.confidenceInputFeatureName = "scores"
nms.coordinatesOutputFeatureName = "coordinates"
nms.confidenceOutputFeatureName = "confidence"
nms.iouThresholdInputFeatureName = "iouThreshold"
nms.confidenceThresholdInputFeatureName = "confidenceThreshold"
default_iou_threshold = 0.5
nms.iouThreshold = default_iou_threshold
default_confidence_threshold = 0.7
nms.confidenceThreshold = default_confidence_threshold
nms.pickTop.perClass = True
nms.stringClassLabels.vector.extend(class_names)
nms_model = coremltools.models.MLModel(nms_spec)
nms_model.save(_coreml_nms_filename)
# pipeline
input_features = [("image", datatypes.Array(3, 416, 416)),
("iouThreshold", datatypes.Double()),
("confidenceThreshold", datatypes.Double())]
output_features = [
"coordinates",
"confidence",
]
pipeline = Pipeline(input_features, output_features)
pipeline.add_model(yolo_model)
pipeline.add_model(nms_model)
# configure in and output of pipeline
pipeline.spec.description.input[0].ParseFromString(
yolo_model._spec.description.input[0].SerializeToString())
pipeline.spec.description.output[0].ParseFromString(
nms_model._spec.description.output[0].SerializeToString())
pipeline.spec.description.output[1].ParseFromString(
nms_model._spec.description.output[1].SerializeToString())
user_defined_metadata = {
"classes": ",".join(class_names),
"iou_threshold": str(default_iou_threshold),
"confidence_threshold": str(default_confidence_threshold)
}
pipeline.spec.description.metadata.userDefined.update(
user_defined_metadata)
pipeline.spec.specificationVersion = 3
final_model = coremltools.models.MLModel(pipeline.spec)
final_model.save(coreml_pipeline_filename)
def test_yolo_객체_생성():
yolo = Yolo()
assert yolo != None
class Localizer:
def __init__(self, ground_truth, test_set, weights_file, verbose=True):
"""
Initializes localizer class used for running localization + classification through a dataset
"""
# keep track of the current scoring metrics
metrics = {
'tp': 0, # number of true positives
'fp': 0, # number of false positives
'fn': 0, # number of false negatives
'yolo': 0, # number of correct classifications by yolo
'svm': 0, # number of correct classifications by svm
'total': 0, # total number of images
'non-detect':
0, # total number of images where no objects were detected
'predictions': [
] # predictions for each detected object [svm prediction, yolo prediction, true label]
}
# load the test sets
test_images = pd.read_csv(test_set)
# load the ground truths
gt_data = pd.read_csv(ground_truth, header=None, dtype={0: str})
gt_data.columns = [
'image', 'label', 'gt_x1', 'gt_y1', 'gt_x2', 'gt_y2'
]
self.metrics = metrics
self.dim = 416
self.test_images = test_images
self.gt_data = gt_data
self.yolo = Yolo('yolo/yolo-custom.cfg', weights_file,
'yolo/yolo-custom.txt')
self.svm = SVM('classifier/Pretrained/svm/svm_k_4.joblib')
self.iterator = 0
def analyze_image(self, image_directory, name):
"""
Uses yolo model to predict the bounding boxes and labels of an image.
Args:
image_directory: Path to the image directory
name: Name of the image
Returns:
detections: A list of detections in the same format as the gt_train.csv file
image: The image which the analyzation occured on
"""
image, indices, boxes, class_ids = self.yolo.extract_objects(
'{}/{}'.format(image_directory, name))
detections = []
for i in indices:
i = i[0]
box = boxes[i]
x = box[0]
y = box[1]
w = box[2]
h = box[3]
label = str(self.yolo.classes[class_ids[i]])
detections.append([
name.split('.')[0], label,
round(x),
round(y),
round(x + w),
round(y + h)
])
return detections, image
def draw_bounding_boxes(self, detections, ground_truth, image):
"""
Draws the bounding boxes of the objects on an image for demo purposes
Args:
detections: List of detections given in same format as gt_train.csv file
ground_truth: List of the ground truths to use for comparison
image: Image object of the detection
Return:
result_pred: Image with predictions drawn
result_gt: Image with ground truth boxes drawn
"""
result_pred = image.copy()
result_gt = image.copy()
for d in detections:
cv2.rectangle(result_pred, (d[2], d[3]), (d[4], d[5]), (0, 255, 0),
2)
cv2.putText(result_pred, d[1], (d[2] - 10, d[3] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
for i, row in ground_truth.iterrows():
cv2.rectangle(result_gt, (row['gt_x1'], row['gt_y1']),
(row['gt_x2'], row['gt_y2']), (0, 255, 0), 2)
cv2.putText(result_gt, row['label'],
(row['gt_x1'] - 10, row['gt_y1'] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
return result_pred, result_gt
def calculate_overlap(self, pred, gt):
"""
Calculates the pixel overal between two detections in an image
Args:
pred: Coordinates for the prediction [x1, y1, x2, y2]
gt: Coordinates for the ground truth [x1, y1, x2, y2]
Returns:
percent_overlap: Overlap in percent value
"""
# create binary images with object pixels set to 1
image_pred = np.full((self.dim, self.dim), False)
image_pred[pred[1]:pred[3], pred[0]:pred[2]] = True
image_gt = np.full((self.dim, self.dim), False)
image_gt[gt[1]:gt[3], gt[0]:gt[2]] = True
# perform AND operation to calculate overlap
result = np.bitwise_and(image_gt, image_pred)
percent_overlap = np.sum(result) / np.sum(image_gt)
# return the sum of array (number of 1s)
return percent_overlap
def classify_svm(self, detection, image):
cropped = image[detection[3]:detection[5], detection[2]:detection[4]]
pred = self.svm.predict(cropped)
return pred
def calculate_score(self, detections, ground_truth, image):
"""
Calculates the localization score of the prediction and updates metrics values
Args:
detections: List of detections given in same format as gt_train.csv file
ground_truth: List of the ground truths to use for comparison
image: The original image which that detection was run on
"""
tp, fp, fn = 0, 0, 0
detections_copy = detections.copy()
match_threshold = 0.7
correct_classifications_yolo = 0
correct_classifications_svm = 0
for i, row in ground_truth.iterrows():
gt = [row['gt_x1'], row['gt_y1'], row['gt_x2'], row['gt_y2']]
# find maximum overlap
max_overlap = [0.0, []]
for d in detections_copy:
pred = d[2:6]
overlap = self.calculate_overlap(pred, gt)
if overlap > max_overlap[0]:
max_overlap = [overlap, d]
# if max overlap exceed threshold, then mark as true positive
if max_overlap[0] > match_threshold:
tp += 1
# score yolo classification
if max_overlap[1][1] == row['label']:
self.metrics['yolo'] += 1
correct_classifications_yolo += 1
# score svm classification
svm_classification = self.classify_svm(max_overlap[1], image)
self.metrics['predictions'].append(
[svm_classification, max_overlap[1][1], row['label']])
if svm_classification == row['label']:
self.metrics['svm'] += 1
correct_classifications_svm += 1
detections_copy.remove(max_overlap[1])
else:
fn += 1
# mark any other detections as false positive
fp = len(detections) - tp
# update global metrics
self.metrics['tp'] += tp
self.metrics['fp'] += fp
self.metrics['fn'] += fn
print('tp: {}, fp: {}, fn: {}'.format(tp, fp, fn))
print('svm: {}/{}'.format(correct_classifications_svm, tp))
print('yolo: {}/{} \n'.format(correct_classifications_yolo, tp))
# print latest total metrics
print('\nTotal Metrics - tp: {}, fp: {}, fn: {} '.format(
self.metrics['tp'], self.metrics['fp'], self.metrics['fn']))
print('Correct Classifications (YOLO): {}/{}'.format(
self.metrics['yolo'], self.metrics['tp']))
print('Correct Classifications (SVM): {}/{} \n'.format(
self.metrics['svm'], self.metrics['tp']))
def evaluate_prediction(self,
detections,
image,
image_path,
display=False):
"""
Evaluates the result of a prediction by validating the prediction against the
ground truth and computing DICE scores for localization.
Args:
detections: The list of detections computed by YOLO
image: The image object which the detection was performed on
image_path: The path to the actual image
display: Whether to show the display for visualization (False by default)
"""
# load ground truth for image and scale to 416x416
ground_truth = self.gt_data.loc[self.gt_data['image'] == detections[0]
[0]]
height, width, _ = (cv2.imread(image_path)).shape
# scale the ground truth coordinates to 416x416
ground_truth['gt_x1'] = ground_truth['gt_x1'].apply(
lambda x: int(x * self.dim / width))
ground_truth['gt_x2'] = ground_truth['gt_x2'].apply(
lambda x: int(x * self.dim / width))
ground_truth['gt_y1'] = ground_truth['gt_y1'].apply(
lambda x: int(x * self.dim / height))
ground_truth['gt_y2'] = ground_truth['gt_y2'].apply(
lambda x: int(x * self.dim / height))
self.calculate_score(detections, ground_truth, image)
# show bounding boxes if display is set to true
if display:
image_pred, image_gt = self.draw_bounding_boxes(
detections, ground_truth, image)
cv2.imshow('Predictions: {}'.format(detections[0][0]), image_pred)
cv2.imshow('Ground Truth: {}'.format(detections[0][0]), image_gt)
cv2.waitKey(0)
cv2.destroyAllWindows()
def run(self, image_directory, output_directory):
"""
Runs the localizer across the dataset and outputs
all necessary results needed for the report.
Args:
image_directory: Path to images directory
output_directory: Output file name
"""
# iterate through each test image
for i, row in self.test_images.iterrows():
print('Analyzing image {}/{}'.format(i, self.test_images.shape[0]))
self.metrics['total'] += 1
if i == 10:
break
# run yolo and retrieve results from image
detections, image = self.analyze_image(image_directory,
row[0].split('/')[-1])
# evaluate the prediction result
if len(detections) > 0:
self.evaluate_prediction(detections,
image,
'{}/{}'.format(
image_directory,
row[0].split('/')[-1]),
display=False)
else:
self.metrics['non-detect'] += 1
# write predictions to a csv file
predictions = pd.DataFrame(self.metrics['predictions'],
columns=['svm', 'yolo', 'gt'])
if not os.path.exists(output_directory):
os.makedirs(output_directory)
predictions.to_csv('{}/predictions.csv'.format(output_directory),
index=False,
header=True)
# write metrics to a sav file
metrics = []
metrics.append(
['Localization True Positives: {}'.format(self.metrics['tp'])])
metrics.append(
['Localization False Positives : {}'.format(self.metrics['fp'])])
metrics.append(
['Localization False Negatives: {}'.format(self.metrics['fn'])])
metrics.append([
'Correct Classifications (SVM): {}/{}'.format(
self.metrics['svm'], self.metrics['tp'])
])
metrics.append([
'Correct Classifications (YOLO): {}/{}'.format(
self.metrics['yolo'], self.metrics['tp'])
])
metrics.append([
'Dice Score: {}'.format(2 * self.metrics['tp'] /
(2 * self.metrics['tp'] +
self.metrics['fp'] + self.metrics['fn']))
])
metrics = pd.DataFrame(metrics)
metrics.to_csv('{}/metrics.csv'.format(output_directory),
index=False,
header=False)