def _get_detections(self, annotations_for_curr_img):
detections = []
for annotation in annotations_for_curr_img:
try:
curr_detection = get_blank_detection_schema()
curr_detection["id"] = annotation["id"]
curr_detection["image_id"] = annotation["image_id"]
curr_detection["label"] = self.categories[
annotation["category_id"]]
curr_detection["segmentation"] = annotation["segmentation"] \
if (annotation["segmentation"] != [] and annotation["segmentation"] != [[]]) else None
curr_detection["left"] = annotation["bbox"][0]
curr_detection["top"] = annotation["bbox"][1]
curr_detection[
"right"] = annotation["bbox"][2] + annotation["bbox"][0]
curr_detection[
"bottom"] = annotation["bbox"][3] + annotation["bbox"][1]
curr_detection["iscrowd"] = annotation["iscrowd"]
curr_detection["isbbox"] = annotation["isbbox"]
curr_detection["keypoints"] = annotation["keypoints"]
detections.append(curr_detection)
except ValueError as ve:
message(f"Cannot find selected key: {annotation} - {ve}")
return detections
def _image_dimensions(path):
try:
with Image.open(path) as image:
return True, image.width, image.height
except FileNotFoundError as e:
message(e)
return False, -1, -1
def _get_image_detection(self,
root,
image_id,
*,
image_ext="png",
folder_names):
try:
detections_fpath = os.path.join(root, "labels", f"{image_id}.txt")
detections = self._get_detections(detections_fpath, image_id)
detections = [
det for det in detections
if det["left"] < det["right"] and det["top"] < det["bottom"]
]
image_path = os.path.join(root, "images",
f"{image_id}.{image_ext}")
image_width, image_height = _image_dimensions(image_path)
return {
"image": {
"id": image_id,
"dataset_id": None,
"path": image_path,
"segmented_path": None,
"width": image_width,
"height": image_height,
"file_name": f"{image_id}.{image_ext}"
},
"detections": detections
}
except Exception as e:
message(e)
def _get_detections(self, detections_fpath, image_id):
detections = []
with open(detections_fpath) as f:
f_csv = csv.reader(f, delimiter=" ")
for row in f_csv:
if len(row) == 0:
continue
try:
x1, y1, x2, y2 = map(float, row[4:8])
label = row[0]
detections.append({
"id": image_id,
"image_id": image_id,
"label": label,
"left": x1,
"right": x2,
"top": y1,
"bottom": y2,
"area": None,
"segmentation": None,
"isbbox": True,
"iscrowd": False,
"keypoints": []
})
except ValueError as ve:
message(f"{ve} - {row}")
return detections
def validate_image_detections(image_detections):
message("Validating...")
deleted_img_detections = 0
for i, image_detection in enumerate(image_detections):
if i % 100 == 0:
message(f"Validated {i} image detections")
try:
validate_schema(image_detection, IMAGE_DETECTION_SCHEMA)
except SchemaError as se:
message(se)
image_detections.remove(image_detection)
continue
image = image_detection["image"]
for detection in image_detection["detections"]:
if detection["isbbox"] is True:
try:
if detection["right"] > image["width"]:
raise ValueError(f"Image {image} has out of bounds bounding box {detection}")
if detection["bottom"] > image["height"]:
raise ValueError(f"Image {image} has out of bounds bounding box {detection}")
if detection["right"] <= detection["left"] or detection["bottom"] <= detection["top"]:
raise ValueError(f"Image {image} has zero dimension bbox {detection}")
except Exception as ve:
message(ve)
image_detections.remove(image_detection)
deleted_img_detections += 1
break
message(f"Deleted labels for {deleted_img_detections} images")
def _get_image_detection(self, root, folder_names):
image_detections = []
path = os.path.join(root, self.default_label_file)
with open(path) as f:
data = json.load(f)
self.categories = {
category["id"]: category["name"]
for category in data["categories"]
}
annotations_base = self._create_annotations_base(data["annotations"])
for i, image_dict in enumerate(data["images"]):
if i % 1000 == 0:
message(f"Ingested {i} images")
single_img_detection = get_blank_image_detection_schema()
single_img_detection["image"]["id"] = image_dict["id"]
single_img_detection["image"]["dataset_id"] = None
single_img_detection["image"]["path"] = image_dict["path"]
single_img_detection["image"]["segmented_path"] = None
single_img_detection["image"]["width"] = image_dict["width"]
single_img_detection["image"]["height"] = image_dict["height"]
single_img_detection["image"]["file_name"] = image_dict[
"file_name"]
single_img_detection["detections"] = self._get_detections(
annotations_base[single_img_detection["image"]["id"]])
image_detections.append(single_img_detection)
return image_detections
def _get_detections(self, frame, img_id, image_width, image_height):
detections = []
for i, annotation in enumerate(frame):
curr_detection = get_blank_detection_schema()
try:
curr_detection["id"] = i
curr_detection["image_id"] = img_id
curr_detection["label"] = annotation["lbl"]
curr_detection["segmentation"] = None
curr_detection["left"] = annotation["pos"][
0] if annotation["pos"][0] >= 0 else 0
curr_detection["top"] = annotation["pos"][
1] if annotation["pos"][1] >= 0 else 0
curr_detection["right"] = annotation["pos"][0] + annotation["pos"][2] \
if annotation["pos"][0] + annotation["pos"][2] <= image_width else image_width
curr_detection["bottom"] = annotation["pos"][1] + annotation["pos"][3] \
if annotation["pos"][1] + annotation["pos"][3] <= image_height else image_height
curr_detection["iscrowd"] = False
curr_detection["isbbox"] = True
curr_detection["keypoints"] = []
detections.append(curr_detection)
except Exception as e:
message(e)
return detections
def _get_image_detection(self, root, image_id, folder_names):
if self.iii % 100 == 0:
message(f"Processed {self.iii} xmls")
self.iii += 1
image_path = os.path.join(
os.path.join(
root, os.path.join(folder_names["images"], f"{image_id}.jpg")))
if not os.path.isfile(image_path):
raise Exception(f"Expected {image_path} to exist.")
annotation_path = os.path.join(
os.path.join(
root,
os.path.join(folder_names["annotations"], f"{image_id}.xml")))
if not os.path.isfile(annotation_path):
raise Exception(
f"Expected annotation file {annotation_path} to exist.")
tree = ET.parse(annotation_path)
xml_root = tree.getroot()
segmented = False
segmented_path = None
if segmented:
segmented_path = os.path.join(root, "SegmentationObject",
f"{image_id}.png")
if not os.path.isfile(segmented_path):
raise Exception(
f"Expected segmentation file {segmented_path} to exist.")
image_width = int(xml_root.find("width").text)
image_height = int(xml_root.find("height").text)
single_img_detection = get_blank_image_detection_schema()
single_img_detection["image"]["id"] = image_id
single_img_detection["image"]["dataset_id"] = None
single_img_detection["image"]["path"] = image_path
single_img_detection["image"]["segmented_path"] = segmented_path
single_img_detection["image"]["width"] = image_width
single_img_detection["image"]["height"] = image_height
single_img_detection["image"]["file_name"] = f"{image_id}.jpg"
detections = [
self._get_detection(node, image_id, False, image_width,
image_height)
for node in xml_root.findall("vehicle")
]
detections2 = [
self._get_detection(node, image_id, True, image_width,
image_height)
for node in xml_root.findall("passengers")
]
for detection in detections2:
detections.append(detection)
single_img_detection["detections"] = detections
return single_img_detection
def _get_class_label(self, object_mask, class_segmentation_mask):
object_pixels = np.where((object_mask != 0) & (object_mask != 255))
labels = []
for x, y in zip(*object_pixels):
labels.append(class_segmentation_mask[x][y])
if len(set(labels)) != 1:
message(
f"Error with finding class from class segmentation mask, not all pixels has the same label, found "
f"labels: {set(labels)}")
return None
return self.segmentation_labels[str(labels[0])]
def _get_image_detection(self, root, folder_names):
image_detections = []
detections = {}
images = {}
det_id = 0
path_ann = [
os.path.join(root, "train", "annotations")
]
path_img = [
os.path.join(root, "train", "images")
]
for i in range(len(path_ann)):
for ann in os.scandir(path_ann[i]):
img_name_base = ann.name[:9]
with open(ann.path) as f:
f_csv = csv.reader(f, delimiter=",")
for row in f_csv:
frame_id = row[0]
img_name = f"{img_name_base}{'0000'[:4 - len(frame_id)]}{frame_id}.jpg"
img_path = os.path.join(path_img[i], img_name)
success, width, height = self._image_dimensions(img_path)
if not success:
continue
if frame_id not in images.keys():
img = {
"id": frame_id,
"dataset_id": None,
"path": img_path,
"segmented_path": None,
"width": width,
"height": height,
"file_name": img_name
}
images[frame_id] = img
success, det = self._get_detections(row, img_name, det_id)
if success:
if frame_id in detections:
detections[frame_id].append(det)
else:
detections[frame_id] = []
detections[frame_id].append(det)
det_id += 1
else:
message(f"Parsing failed, row: {row}, detection: {det}")
continue
for k, v in images.items():
image_detections.append({
"image": v,
"detections": detections[k]
})
return image_detections
def ingest(self, path, folder_names):
image_ids = self._get_image_ids(path)
image_ext = "png"
if len(image_ids):
first_image_id = image_ids[0]
image_ext = self.find_image_ext(path, first_image_id)
tmp = [
self._get_image_detection(path,
image_name,
image_ext=image_ext,
folder_names=folder_names)
for image_name in image_ids
]
message(f"size: {len(tmp)}")
return tmp
def _get_image_detection(self, root, folder_names):
annotations = []
failed_loads = 0
path = os.path.join(root, "annotations.json")
with open(path) as f:
data = json.load(f)
total_sets = len(data)
sets_processed = 0
for data_set_key, data_set_dict in data.items():
video_processed = 0
for video_key, video_dict in data_set_dict.items():
for frame_key, frame_dict in video_dict["frames"].items():
try:
single_img_detection = get_blank_image_detection_schema(
)
image_id = f"{data_set_key}_{video_key}_{frame_key}"
image_path = os.path.join(root, "images",
f"{image_id}.png")
image_width, image_height = self._image_dimensions(
image_path)
detections = self._get_detections(
frame_dict, image_id, image_width,
image_height)
single_img_detection["image"]["id"] = image_id
single_img_detection["image"]["dataset_id"] = None
single_img_detection["image"]["path"] = image_path
single_img_detection["image"][
"segmented_path"] = None
single_img_detection["image"][
"width"] = image_width
single_img_detection["image"][
"height"] = image_height
single_img_detection["image"][
"file_name"] = os.path.basename(image_path)
single_img_detection["detections"] = detections
annotations.append(single_img_detection)
except Exception as e:
message(e)
failed_loads += 1
video_processed += 1
message(
f"Processed {video_processed} videos in current set")
sets_processed += 1
message(f"Loaded {sets_processed} sets on {total_sets}...")
message(f"Unable to load {failed_loads} images")
return annotations
def _get_detections(children):
detections = []
for element in children:
if len(element) == 0:
continue
try:
x1 = element["mincol"]
x2 = element["maxcol"]
y1 = element["minrow"]
y2 = element["maxrow"]
label = daimler_dict[element["identity"]]
detections.append({
"label": label,
"left": x1,
"right": x2,
"top": y1,
"bottom": y2
})
except ValueError as ve:
message(f"{ve} - {element}")
return detections
def convert_labels(*, image_detections, expected_labels,
select_only_known_labels, filter_images_without_labels):
message("Converting labels...")
convert_dict = {}
for label, aliases in expected_labels.items():
convert_dict[label.lower()] = label
for alias in aliases:
convert_dict[alias.lower()] = label
final_image_detections = []
for i, image_detection in enumerate(image_detections):
if i % 100 == 0:
message(f"Converted {i} labels")
detections = []
try:
for detection in image_detection["detections"]:
label = detection["label"]
fallback_label = label if not select_only_known_labels else None
final_label = convert_dict.get(label.lower(), fallback_label)
if final_label:
detection["label"] = final_label
detections.append(detection)
image_detection["detections"] = detections
if detections:
final_image_detections.append(image_detection)
elif not filter_images_without_labels:
final_image_detections.append(image_detection)
except Exception as e:
message(e)
return final_image_detections
def egest(self, *, image_detections, root, folder_names):
images_dir = os.path.join(root, "images")
os.makedirs(images_dir, exist_ok=True)
labels_dir = os.path.join(root, "labels")
os.makedirs(labels_dir, exist_ok=True)
id_file = os.path.join(root, "train.txt")
for image_detection in image_detections:
image = image_detection["image"]
image_id = image["id"]
src_extension = os.path.splitext(image["path"])[-1]
try:
shutil.copyfile(
image["path"],
os.path.join(images_dir, f"{image_id}{src_extension}"))
except FileNotFoundError as e:
message(e)
continue
with open(id_file, "a") as out_image_index_file:
out_image_index_file.write(f"{image_id}\n")
out_labels_path = os.path.join(labels_dir, f"{image_id}.txt")
with open(out_labels_path, "w") as csvfile:
csvwriter = csv.writer(csvfile,
delimiter=" ",
quoting=csv.QUOTE_MINIMAL)
for detection in image_detection["detections"]:
kitti_row = [-1] * 15
kitti_row[0] = detection["label"]
kitti_row[1] = DEFAULT_TRUNCATED
kitti_row[2] = DEFAULT_OCCLUDED
x1 = detection["left"]
x2 = detection["right"]
y1 = detection["top"]
y2 = detection["bottom"]
kitti_row[4:8] = x1, y1, x2, y2
csvwriter.writerow(kitti_row)
def _get_image_detection(self, file, folder_names):
image_detection_schema = [{
"image": {
"id": "00000000",
"file_name": "temp"
},
"detections": {}
}]
df = pd.read_csv(file[0], dtype=str)
for index, row in df.iterrows():
if len(image_detection_schema) % 50 == 1000:
message(f"Processed {len(image_detection_schema)} images")
if int(image_detection_schema[-1]["image"]["id"]) < int(row["id"]) \
and image_detection_schema[-1]["image"]["file_name"] != file_name:
image_detection_schema.append({
"image": {
"id": image_id,
"path": image_path,
"dataset_id": 10,
"segmented_path": None,
"file_name": file_name,
"width": image_width,
"height": image_height
},
"detections": detections
})
detections = self._get_detections(row["id"], df)
detections = [
det for det in detections
if det["left"] < det["right"] and det["top"] < det["bottom"]
]
image_id = row["id"]
image_path = os.path.join(file, "train", f"{image_id}.jpg")
file_name = f"{image_id}.jpg"
image_width, image_height = 1920, 1080
image_detection_schema.pop(0)
return image_detection_schema
def ingest(self, path, folder_names):
try:
ret = self._get_image_detection(path, folder_names=folder_names)
message(type(ret))
return ret
except Exception as e:
message(traceback.format_exc())
message(f"Ingesting failed - {e}")
exit()
def _get_detections(self, local_id, df):
detections = []
sub_df = df.loc[df["id"] == local_id]
for index, row in sub_df.iterrows():
try:
x1 = row["x1"]
y1 = row["y1"]
x2 = row["x2"]
y2 = row["y2"]
label = labels[row["label"]]
detections.append({
"label": label,
"left": int(x1),
"right": int(x2),
"top": int(y1),
"bottom": int(y2),
"iscrowd": False,
"isbbox": True,
"keypoints": [],
"segmentation": None
})
except ValueError as ve:
message(f"{ve} - {row}")
return detections
def _get_image_detection(
self,
root,
folder_names,
image_ext="png",
):
image_detection_schema = []
labels = os.listdir(os.path.join(root, "labels"))
for filename in labels:
try:
path = os.path.join(root, "labels", filename)
with open(path) as f:
data = json.load(f)
image_id = path_base_name(data["imagename"])
image_path = os.path.join(root, "images",
f"{image_id}.{image_ext}")
detections = self._get_detections(data["children"])
detections = [
det for det in detections if det["left"] < det["right"]
and det["top"] < det["bottom"]
]
try:
image_width, image_height = _image_dimensions(
image_path)
except FileNotFoundError as e:
message(e)
continue
message(f"id: {image_id}")
image_detection_schema.append({
"image": {
"id": image_id,
"path": image_path,
"segmented_path": None,
"width": image_width,
"height": image_height
},
"detections": detections
})
except FileNotFoundError as e:
message(e)
continue
return image_detection_schema
def _get_detections(self, xml_root, img_id, segmented_classes,
segmented_objects, image_width, image_height):
if segmented_classes is None:
return [
self._get_detection(node, img_id)
for node in xml_root.findall("object")
]
else:
detections = []
sub_masks = self.create_sub_masks(Image.open(segmented_objects))
class_segmentation_array = np.asarray(
Image.open(segmented_classes), dtype="uint8")
for label, single_mask in sub_masks.items():
array_mask = np.asarray(single_mask, dtype="uint8")
contours = measure.find_contours(array_mask, 0.5)
curr_detection = get_blank_detection_schema()
curr_detection["segmentation"] = []
for contour in contours:
contour = np.flip(contour, axis=1)
segmentation = contour.ravel().tolist()
try:
rs = mask.frPyObjects([segmentation], image_height,
image_width)
if mask.area(rs)[0] > 25:
curr_detection["segmentation"].append(segmentation)
else:
continue
except Exception as e:
message(f"Cannot calculate area of polygon: {e}")
continue
if curr_detection["segmentation"]:
curr_detection["area"] = int(
sum(
mask.area(
mask.frPyObjects(
curr_detection["segmentation"],
image_height, image_width))))
else:
continue
encoded_ground_truth = mask.encode(
np.asfortranarray(array_mask))
ground_truth_bounding_box = mask.toBbox(encoded_ground_truth)
curr_detection["id"] = self.detection_counter
self.detection_counter += 1
curr_detection["image_id"] = img_id
curr_detection["label"] = self._get_class_label(
array_mask, class_segmentation_array)
bbox = ground_truth_bounding_box.tolist()
curr_detection["left"] = bbox[0]
curr_detection["top"] = bbox[1]
curr_detection["right"] = bbox[2] + bbox[0]
curr_detection["bottom"] = bbox[3] + bbox[1]
curr_detection["iscrowd"] = False
curr_detection["isbbox"] = False
curr_detection["keypoints"] = []
detections.append(curr_detection)
return detections
def egest(self, *, image_detections, root, folder_names):
message("Processing data by COCO Egestor...")
labels = {
"images": [],
"categories": self.generate_categories(),
"annotations": []
}
detection_counter = 0
for i, image_detection in enumerate(image_detections):
if i % 100 == 0:
message(f"Processed {i} image detections")
image = image_detection["image"]
new_image = {
"id": i,
"dataset_id": None,
"path": image["path"],
"width": image["width"],
"height": image["height"],
"file_name": os.path.basename(image["path"])
}
labels["images"].append(new_image)
for detection in image_detection["detections"]:
category_id = next(
(category["id"] for category in labels["categories"]
if category["name"] == detection["label"]), None)
if category_id is None:
message(
f"No available category found: detection label = {detection['label']}"
)
continue
new_detection = {"id": detection_counter}
detection_counter += 1
new_detection["image_id"] = new_image["id"]
new_detection["category_id"] = category_id
new_detection["iscrowd"] = detection["iscrowd"]
new_detection["isbbox"] = detection["isbbox"]
# Bbox = [x left upper corner, y left upper corner, width, height]
new_detection["bbox"] = [
detection["left"], detection["top"],
detection["right"] - detection["left"],
detection["bottom"] - detection["top"]
]
if new_detection["isbbox"] == True and detection[
"segmentation"] is None:
# segmentation from bbox = [x right upper corner, y right upper corner, x right lower corner,
# y right lower corner, x left lower corner, y left lower corner, x left upper corner,
# y left upper corner]
new_detection["segmentation"] = [[
detection["right"], detection["top"],
detection["right"], detection["bottom"],
detection["left"], detection["bottom"],
detection["left"], detection["top"]
]]
else:
new_detection["segmentation"] = detection["segmentation"]
detection["area"] = None
if detection["area"] is None:
try:
new_detection["area"] = int(
sum(
mask.area(
mask.frPyObjects(
new_detection["segmentation"],
new_image["height"],
new_image["width"]))))
except Exception as e:
message(
f"Unable to automatically calculate area from segmentation: {e}"
)
new_detection["area"] = 0
new_detection["keypoints"] = detection["keypoints"]
labels["annotations"].append(new_detection)
message("Saving json file...")
encoded_labels = json.dumps(labels)
message("Finished egesting COCO")
return encoded_labels
def _get_image_detection(self, root, image_id, folder_names):
if self.iii % 100 == 0:
message(f"Processed {self.iii} xmls")
self.iii += 1
image_path = os.path.join(
os.path.join(
root, os.path.join(folder_names["images"], f"{image_id}.jpg")))
if not os.path.isfile(image_path):
number = int(image_id) + 1
image_path = os.path.join(
os.path.join(
root,
os.path.join(folder_names['images'],
'image' + '{:06d}'.format(number) + ".jpg")))
if not os.path.isfile(image_path):
raise Exception(f"Expected {image_path} to exist.")
annotation_path = os.path.join(
os.path.join(
root,
os.path.join(folder_names["annotations"], f"{image_id}.xml")))
if not os.path.isfile(annotation_path):
raise Exception(
f"Expected annotation file {annotation_path} to exist.")
tree = ElementTree.parse(annotation_path)
xml_root = tree.getroot()
size = xml_root.find("size")
if xml_root.find('segmented') is None:
segmented = False
else:
segmented = (xml_root.find('segmented').text == "1")
segmented_path = None
segmented_objects = None
if segmented:
segmented_path = os.path.join(root,
folder_names["segmentation_classes"],
f"{image_id}.png")
segmented_objects = os.path.join(
root, folder_names["segmentation_object"], f"{image_id}.png")
if not os.path.isfile(segmented_path):
raise Exception(
f"Expected segmentation file {segmented_path} to exist.")
if not os.path.isfile(segmented_objects):
raise Exception(
f"Expected segmentation file {segmented_path} to exist.")
image_width = int(size.find("width").text)
image_height = int(size.find("height").text)
single_img_detection = get_blank_image_detection_schema()
single_img_detection["image"]["id"] = image_id
single_img_detection["image"]["dataset_id"] = None
single_img_detection["image"]["path"] = image_path
single_img_detection["image"]["segmented_path"] = segmented_path
single_img_detection["image"]["width"] = image_width
single_img_detection["image"]["height"] = image_height
single_img_detection["image"]["file_name"] = f"{image_id}.jpg"
single_img_detection["detections"] = self._get_detections(
xml_root, image_id, segmented_path, segmented_objects, image_width,
image_height)
return single_img_detection