def resize_coords(bbox_data: BboxData):
bbox_data.xmin = max(
0, min(int(bbox_data.xmin * (new_width / old_width)),
new_width - 1))
bbox_data.ymin = max(
0,
min(int(bbox_data.ymin * (new_height / old_height)),
new_height - 1))
bbox_data.xmax = max(
0, min(int(bbox_data.xmax * (new_width / old_width)),
new_width - 1))
bbox_data.ymax = max(
0,
min(int(bbox_data.ymax * (new_height / old_height)),
new_height - 1))
bbox_data.keypoints[:, 0] = (bbox_data.keypoints[:, 0] *
(new_width / old_width)).astype(int)
bbox_data.keypoints[:, 1] = (bbox_data.keypoints[:, 1] *
(new_height / old_height)).astype(int)
bbox_data.keypoints = bbox_data.keypoints.astype(int)
bbox_data.cropped_image = None
keypoints = []
for (x, y) in bbox_data.keypoints:
x = max(0, min(x, new_width - 1))
y = max(0, min(y, new_height - 1))
keypoints.append([x, y])
bbox_data.keypoints = np.array(keypoints).reshape(-1, 2)
for additional_bbox_data in bbox_data.additional_bboxes_data:
resize_coords(additional_bbox_data)
def _append_cropped_bbox_data_to_image_data(bbox_data: BboxData):
bbox_data.keypoints[:, 0] += src_xmin
bbox_data.keypoints[:, 1] += src_ymin
bbox_data.xmin += src_xmin
bbox_data.ymin += src_ymin
bbox_data.xmax += src_xmin
bbox_data.ymax += src_ymin
bbox_data.image = None
bbox_data.image_path = None
bbox_data.cropped_image = None
for additional_bbox_data in bbox_data.additional_bboxes_data:
_append_cropped_bbox_data_to_image_data(additional_bbox_data)
def _get_new_coords_for_bbox_data(bbox_data: BboxData, xmin: int,
ymin: int):
bbox_data.keypoints[:, 0] += xmin
bbox_data.keypoints[:, 1] += ymin
bbox_data.xmin += xmin
bbox_data.ymin += ymin
bbox_data.xmax += xmin
bbox_data.ymax += ymin
bbox_data.image = None
bbox_data.image_path = None
bbox_data.cropped_image = None
for additional_bbox_data in bbox_data.additional_bboxes_data:
_get_new_coords_for_bbox_data(additional_bbox_data, xmin, ymin)
def if_bbox_data_inside_crop(bbox_data: BboxData):
bbox_data.keypoints = bbox_data.keypoints[(
(bbox_data.keypoints[:, 0] >= 0) & (bbox_data.keypoints[:, 1] >= 0)
& (bbox_data.keypoints[:, 0] < new_height) &
(bbox_data.keypoints[:, 1] < new_width))]
bbox_data.additional_bboxes_data = [
additional_bbox_data
for additional_bbox_data in bbox_data.additional_bboxes_data
if if_bbox_data_inside_crop(additional_bbox_data)
]
return (bbox_data.xmin >= 0 and bbox_data.ymin >= 0
and bbox_data.xmax < new_width and bbox_data.ymax < new_height
and bbox_data.xmin < bbox_data.xmax
and bbox_data.ymin < bbox_data.ymax)
def parse_rectangle_labels_to_bbox_data(rectangle_label: Dict) -> BboxData:
original_height = rectangle_label['original_height']
original_width = rectangle_label['original_width']
height = rectangle_label['value']['height']
width = rectangle_label['value']['width']
xmin = rectangle_label['value']['x']
ymin = rectangle_label['value']['y']
label = rectangle_label['value']['rectanglelabels'][0]
xmax = xmin + width
ymax = ymin + height
xmin = max(0, min(original_width - 1, xmin / 100 * original_width))
ymin = max(0, min(original_height - 1, ymin / 100 * original_height))
xmax = max(0, min(original_width - 1, xmax / 100 * original_width))
ymax = max(0, min(original_height - 1, ymax / 100 * original_height))
angle = rectangle_label['value']['rotation']
points = [(xmin, ymin), (xmin, ymax), (xmax, ymin), (xmax, ymax)]
rotated_points = [
rotate_point(x=x, y=y, cx=xmin, cy=ymin, angle=angle)
for (x, y) in points
]
xmin = max(0, min([x for (x, y) in rotated_points]))
ymin = max(0, min([y for (x, y) in rotated_points]))
xmax = max([x for (x, y) in rotated_points])
ymax = max([y for (x, y) in rotated_points])
xmin = max(0, min(original_width, xmin))
ymin = max(0, min(original_height, ymin))
xmax = max(0, min(original_width, xmax))
ymax = max(0, min(original_height, ymax))
bbox_data = BboxData(xmin=xmin,
ymin=ymin,
xmax=xmax,
ymax=ymax,
label=label)
return bbox_data
def get_image_data_from_annot(
self, image_path: Union[str, Path],
annot: Union[Path, str, Dict, fsspec.core.OpenFile, List[str]]
) -> ImageData:
if isinstance(annot, str) or isinstance(annot, Path):
with fsspec.open(annot, 'r', encoding='utf8') as f:
annots = f.read()
elif isinstance(annot, fsspec.core.OpenFile):
with annot as f:
annots = f.read()
elif isinstance(annot, List):
annots = '\n'.join(annot)
width, height = get_image_size(image_path)
bboxes_data = []
for line in annots.strip().split('\n'):
idx, xcenter, ycenter, w, h = line.split(' ')
label = self.idx_to_class_name[int(idx)]
xcenter, ycenter, w, h = float(xcenter), float(ycenter), float(
w), float(h)
xcenter, w = xcenter * width, w * width
ycenter, h = ycenter * height, h * height
bboxes_data.append(
BboxData(xmin=xcenter - w / 2,
ymin=ycenter - h / 2,
xmax=xcenter + w / 2,
ymax=ycenter + h / 2,
label=label))
return ImageData(image_path=image_path, bboxes_data=bboxes_data)
def _append_bbox_data(bbox_data: BboxData, depth: int):
if additional_bboxes_data_depth is not None and depth > additional_bboxes_data_depth:
return
bboxes_data.append(bbox_data)
for additional_bbox_data in bbox_data.additional_bboxes_data:
_append_bbox_data(additional_bbox_data)
bbox_data.additional_bboxes_data = []
def _postprocess_predictions_bboxes_data(
self, bboxes_data: List[BboxData],
pred_labels_top_n: List[List[str]],
pred_scores_top_n: List[List[float]],
open_cropped_images_in_bboxes_data: bool) -> List[List[BboxData]]:
bboxes_data_res = []
for (bbox_data, pred_label_top_n,
pred_classification_score_top_n) in zip(bboxes_data,
pred_labels_top_n,
pred_scores_top_n):
cropped_image = bbox_data.cropped_image if open_cropped_images_in_bboxes_data else None
bboxes_data_res.append(
BboxData(
image_path=bbox_data.image_path,
cropped_image=cropped_image,
xmin=bbox_data.xmin,
ymin=bbox_data.ymin,
xmax=bbox_data.xmax,
ymax=bbox_data.ymax,
detection_score=bbox_data.detection_score,
label=pred_label_top_n[0],
keypoints=bbox_data.keypoints,
classification_score=pred_classification_score_top_n[0],
top_n=len(pred_label_top_n),
labels_top_n=pred_label_top_n,
classification_scores_top_n=pred_classification_score_top_n,
additional_info=bbox_data.additional_info))
return bboxes_data_res
def get_image_data_from_annot(
self, image_path: Union[str, Path, fsspec.core.OpenFile],
annot: Union[Path, str, Dict, fsspec.core.OpenFile]) -> ImageData:
if isinstance(annot, str) or isinstance(annot, Path):
with fsspec.open(annot, 'r', encoding='utf8') as f:
annot = json.load(f)
if isinstance(annot, fsspec.core.OpenFile):
with annot as f:
annot = json.load(f)
bboxes_data = []
for obj in annot['objects']:
(xmin, ymin), (xmax, ymax) = obj['points']['exterior']
label = obj['tags'][0]['name'] if obj['tags'] else None
bboxes_data.append(
BboxData(image_path=image_path,
xmin=xmin,
ymin=ymin,
xmax=xmax,
ymax=ymax,
label=label))
image_data = ImageData(image_path=image_path, bboxes_data=bboxes_data)
return image_data
def _postprocess_predictions_bboxes_data(
self, bboxes_data: List[BboxData],
pred_n_keypoints: List[List[Tuple[float, float]]],
open_cropped_images_in_bboxes_data: bool) -> List[List[BboxData]]:
bboxes_data_res = []
pred_n_keypoints = np.array(pred_n_keypoints)
for bbox_data, pred_keypoints in zip(bboxes_data, pred_n_keypoints):
width, height = bbox_data.get_image_size()
pred_keypoints[:, 0] *= width
pred_keypoints[:, 1] *= height
cropped_image = bbox_data.cropped_image if open_cropped_images_in_bboxes_data else None
bboxes_data_res.append(
BboxData(
image_path=bbox_data.image_path,
cropped_image=cropped_image,
xmin=bbox_data.xmin,
ymin=bbox_data.ymin,
xmax=bbox_data.xmax,
ymax=bbox_data.ymax,
detection_score=bbox_data.detection_score,
label=bbox_data.label,
keypoints=pred_keypoints,
classification_score=bbox_data.classification_score,
top_n=bbox_data.top_n,
labels_top_n=bbox_data.labels_top_n,
classification_scores_top_n=bbox_data.
classification_scores_top_n,
))
return bboxes_data_res
def get_true_and_pred_images_data_with_visualized_labels(
image_data_matching: ImageDataMatching,
error_type: Literal['detection', 'pipeline'],
label: str = None) -> ImageData:
"""
Create true and pred ImageData with changed label for visualization
For detection error_type, the label will be one of ["TP", "FP", "FN"]
For pipeline error_type, the label will be in format "label {matching_error_type}"
(where matching_error_type is one of ["TP", "FP", "FN", "TP (extra bbox)", "FP (extra bbox)"])
"""
for tag, tag_image_data in [('true', image_data_matching.true_image_data),
('pred', image_data_matching.pred_image_data)]:
tag_bboxes_data_with_visualized_label = []
for tag_bbox_data_matching in image_data_matching.bboxes_data_matchings:
if tag == 'true':
tag_bbox_data = tag_bbox_data_matching.true_bbox_data
elif tag == 'pred':
tag_bbox_data = tag_bbox_data_matching.pred_bbox_data
if tag_bbox_data is None:
continue
if error_type == 'detection':
label_caption = f"[{tag_bbox_data_matching.get_detection_error_type(label=label)}]"
elif error_type == 'pipeline':
pipeline_error_type = tag_bbox_data_matching.get_pipeline_error_type(
label=label)
if label is None:
label_caption = f"{tag_bbox_data.label} [{pipeline_error_type}]"
else:
label_caption = f"{tag_bbox_data.label} [{pipeline_error_type}, cls. '{label}']"
tag_bbox_data_with_visualized_label = BboxData(
image_path=tag_bbox_data.image_path,
cropped_image=tag_bbox_data.cropped_image,
xmin=tag_bbox_data.xmin,
ymin=tag_bbox_data.ymin,
xmax=tag_bbox_data.xmax,
ymax=tag_bbox_data.ymax,
detection_score=tag_bbox_data.detection_score,
label=label_caption,
classification_score=tag_bbox_data.classification_score)
tag_bboxes_data_with_visualized_label.append(
tag_bbox_data_with_visualized_label)
tag_image_data_with_visualized_labels = ImageData(
image_path=tag_image_data.image_path,
image=tag_image_data.image,
bboxes_data=tag_bboxes_data_with_visualized_label)
if tag == 'true':
true_image_data_with_visualized_labels = tag_image_data_with_visualized_labels
elif tag == 'pred':
pred_image_data_with_visualized_labels = tag_image_data_with_visualized_labels
return true_image_data_with_visualized_labels, pred_image_data_with_visualized_labels
def get_images_data_from_annots(
self,
images_dir: Union[str, Path, fsspec.core.OpenFile],
annots: Union[Path, str, Dict, fsspec.core.OpenFile]
) -> List[ImageData]:
if isinstance(annots, str) or isinstance(annots, Path):
with fsspec.open(annots, 'r', encoding='utf8') as f:
annots = json.load(f)
elif isinstance(annots, fsspec.core.OpenFile):
with annots as f:
annots = json.load(f)
images_dir = Pathy(images_dir)
images_data = []
for annot in annots:
image_name = annot['filename']
additional_info = {}
for key in annot:
if key not in ['objects', 'filename']:
additional_info[key] = annot[key]
bboxes_data = []
for obj in annot['objects']:
if 'bbox' in obj:
xmin, ymin, xmax, ymax = obj['bbox']
else:
xmin, ymin, xmax, ymax = obj
xmin, ymin, xmax, ymax = int(xmin), int(ymin), int(xmax), int(ymax)
label = obj['label'] if 'label' in obj else None
labels_top_n = obj['labels_top_n'] if 'labels_top_n' in obj else None
top_n = len(labels_top_n) if labels_top_n is not None else None
keypoints = obj['keypoints'] if 'keypoints' in obj else []
bbox_additional_info = {}
if isinstance(obj, dict):
for key in obj:
if key not in ['bbox', 'label', 'labels_top_n', 'top_n']:
bbox_additional_info[key] = obj[key]
bboxes_data.append(BboxData(
image_path=images_dir / image_name,
xmin=xmin,
ymin=ymin,
xmax=xmax,
ymax=ymax,
keypoints=keypoints,
label=label,
labels_top_n=labels_top_n,
top_n=top_n,
additional_info=bbox_additional_info
))
images_data.append(ImageData(
image_path=images_dir / image_name,
bboxes_data=bboxes_data,
additional_info=additional_info
))
return images_data
def resize_coords(bbox_data: BboxData):
bbox_data.xmin = bbox_data.xmin - xmin
bbox_data.ymin = bbox_data.ymin - ymin
bbox_data.xmax = bbox_data.xmax - xmin
bbox_data.ymax = bbox_data.ymax - ymin
bbox_data.keypoints[:, 0] -= xmin
bbox_data.keypoints[:, 1] -= ymin
bbox_data.cropped_image = None
if not allow_negative_and_large_coords:
bbox_data.xmin = max(0, min(bbox_data.xmin, new_width - 1))
bbox_data.ymin = max(0, min(bbox_data.ymin, new_height - 1))
bbox_data.xmax = max(0, min(bbox_data.xmax, new_width - 1))
bbox_data.ymax = max(0, min(bbox_data.ymax, new_height - 1))
keypoints = []
for (x, y) in bbox_data.keypoints:
x = max(0, min(x, new_width - 1))
y = max(0, min(y, new_height - 1))
keypoints.append([x, y])
bbox_data.keypoints = np.array(keypoints).reshape(-1, 2)
for additional_bbox_data in bbox_data.additional_bboxes_data:
resize_coords(additional_bbox_data)
def _postprocess_predictions(
self, images_data: List[ImageData],
n_pred_bboxes: List[List[Tuple[int, int, int, int]]],
n_k_pred_keypoints: List[List[Tuple[int, int]]],
n_pred_detection_scores: List[List[float]],
n_pred_labels_top_n: List[List[List[str]]],
n_pred_classification_scores_top_n: List[List[List[float]]],
open_images_in_images_data: bool,
open_cropped_images_in_bboxes_data: bool) -> List[ImageData]:
pred_images_data = []
for (image_data, pred_bboxes, k_pred_keypoints, pred_detection_scores,
pred_labels_top_n, pred_classification_scores_top_n) in zip(
images_data, n_pred_bboxes, n_k_pred_keypoints,
n_pred_detection_scores, n_pred_labels_top_n,
n_pred_classification_scores_top_n):
bboxes_data = []
for (pred_bbox, pred_keypoints, pred_detection_score,
pred_label_top_n, pred_classification_score_top_n) in zip(
pred_bboxes, k_pred_keypoints, pred_detection_scores,
pred_labels_top_n, pred_classification_scores_top_n):
xmin, ymin, xmax, ymax = pred_bbox
bboxes_data.append(
BboxData(
image_path=image_data.image_path,
xmin=xmin,
ymin=ymin,
xmax=xmax,
ymax=ymax,
keypoints=pred_keypoints,
detection_score=pred_detection_score,
label=pred_label_top_n[0],
classification_score=pred_classification_score_top_n[
0],
top_n=len(pred_label_top_n),
labels_top_n=pred_label_top_n,
classification_scores_top_n=
pred_classification_score_top_n))
if open_cropped_images_in_bboxes_data:
for bbox_data in bboxes_data:
bbox_data.open_cropped_image(source_image=image_data.image,
inplace=True)
image = image_data.image if open_images_in_images_data else None
pred_images_data.append(
ImageData(
image_path=image_data.image_path,
image=image,
bboxes_data=bboxes_data,
label=image_data.label,
keypoints=image_data.keypoints,
additional_info=image_data.additional_info,
))
return pred_images_data
def non_max_suppression_image_data(image_data: ImageData,
iou: float) -> ImageData:
image_data = copy.deepcopy(image_data)
current_bboxes_data = image_data.bboxes_data.copy()
new_bboxes_data = []
while len(current_bboxes_data) != 0:
current_bbox_data = current_bboxes_data[0]
success = True
if len(current_bboxes_data) > 1:
for idx, bbox_data in enumerate(current_bboxes_data):
if idx == 0:
continue
bbox_iou = intersection_over_union(current_bbox_data,
bbox_data)
if bbox_iou >= iou:
pairs_bboxes_data = [bbox_data, current_bbox_data]
pairs_scores = [
possible_bbox_data.detection_score if
possible_bbox_data.detection_score is not None else 1.
for possible_bbox_data in pairs_bboxes_data
]
top_score_idx = np.argmax(pairs_scores)
current_bboxes_data.pop(idx)
current_bboxes_data.pop(0)
current_bboxes_data.append(
BboxData(
xmin=min(bbox_data.xmin, current_bbox_data.xmin),
ymin=min(bbox_data.ymin, current_bbox_data.ymin),
xmax=max(bbox_data.xmax, current_bbox_data.xmax),
ymax=max(bbox_data.ymax, current_bbox_data.ymax),
detection_score=pairs_bboxes_data[top_score_idx].
detection_score,
label=pairs_bboxes_data[top_score_idx].label,
keypoints=pairs_bboxes_data[top_score_idx].
keypoints,
additional_bboxes_data=pairs_bboxes_data[
top_score_idx].additional_bboxes_data,
additional_info=pairs_bboxes_data[top_score_idx].
additional_info))
success = False
break
if success:
new_bboxes_data.append(current_bboxes_data.pop(0))
image_data.bboxes_data = new_bboxes_data
image_data.image_path = image_data.image_path
image_data.image = image_data.image
return image_data
def parse_polygon_label_to_bbox_data(polygon_label: Dict) -> BboxData:
original_height = polygon_label['original_height']
original_width = polygon_label['original_width']
keypoints = []
for (x, y) in polygon_label['value']['points']:
x = x / 100 * polygon_label['original_width']
y = y / 100 * polygon_label['original_height']
keypoints.append([
max(0, min(original_width - 1, x)),
max(0, min(original_height - 1, y))
])
keypoints = np.array(keypoints)
bbox_data = BboxData(xmin=np.min(keypoints[:, 0]),
ymin=np.min(keypoints[:, 1]),
xmax=np.max(keypoints[:, 0]),
ymax=np.max(keypoints[:, 1]),
keypoints=keypoints,
label=polygon_label['value']['polygonlabels'][0])
return bbox_data
def draw_overlay(self,
frame: np.ndarray,
tracked_bboxes: List[Tuple[int, int, int, int]],
tracked_ids: List[int],
ready_frames_at_the_moment: List['FrameResult'],
filter_by_labels: List[str] = None) -> np.ndarray:
image = frame.copy()
tracked_bboxes = tracked_bboxes.astype(int)
ready_tracks_ids_at_the_moment = [
ready_frame.track_id for ready_frame in ready_frames_at_the_moment
]
current_bboxes_data = []
for bbox, track_id in zip(tracked_bboxes, tracked_ids):
if track_id not in ready_tracks_ids_at_the_moment:
continue
xmin, ymin, xmax, ymax = bbox
ready_frame = ready_frames_at_the_moment[
ready_tracks_ids_at_the_moment.index(track_id)]
label = ready_frame.label
current_bbox_data = BboxData(image=image,
xmin=xmin,
ymin=ymin,
xmax=xmax,
ymax=ymax,
label=label)
current_bboxes_data.append(current_bbox_data)
image_data = ImageData(image=frame, bboxes_data=current_bboxes_data)
image = visualize_image_data(
image_data=image_data,
use_labels=self.write_labels,
filter_by_labels=filter_by_labels,
draw_base_labels_with_given_label_to_base_label_image=(
self.draw_base_labels_with_given_label_to_base_label_image),
known_labels=self.classification_inferencer.class_names)
return image
def split_image_by_grid(image: np.ndarray,
n_rows: int,
n_cols: int,
x_window_size: int,
y_window_size: int,
x_offset: int,
y_offset: int,
minimum_size: float = 0.5) -> List[BboxData]:
height, width, _ = image.shape
bboxes_data = [
BboxData(image=image,
xmin=x_offset + i * x_window_size,
ymin=y_offset + j * y_window_size,
xmax=min(width, x_offset + (i + 1) * x_window_size),
ymax=min(height, y_offset + (j + 1) * y_window_size))
for i in range(n_rows) for j in range(n_cols)
]
bboxes_data = [
bbox_data for bbox_data in bboxes_data
if ((bbox_data.xmax - bbox_data.xmin >= minimum_size * x_window_size)
and (bbox_data.ymax - bbox_data.ymin >= minimum_size *
y_window_size))
]
return bboxes_data
def run_pipeline_on_frame(
self, frame: np.ndarray, frame_idx: int, fps: float,
detection_delay: int, classification_delay: int,
detection_score_threshold: float, batch_size: int
) -> Tuple[List[Tuple[int, int, int, int]], List[int]]:
frame = frame.copy()
image_data = ImageData(image=frame)
image_data_gen = BatchGeneratorImageData(
[image_data],
batch_size=batch_size,
use_not_caught_elements_as_last_batch=True)
pred_image_data = self.detection_inferencer.predict(
images_data_gen=image_data_gen,
score_threshold=detection_score_threshold)[0]
bboxes = np.array([(bbox_data.xmin, bbox_data.ymin, bbox_data.xmax,
bbox_data.ymax)
for bbox_data in pred_image_data.bboxes_data])
detection_scores = np.array([
bbox_data.detection_score
for bbox_data in pred_image_data.bboxes_data
])
self.opencv_tracker = OpenCVTracker(bboxes, frame)
tracked_bboxes, tracked_ids = self.update_sort_tracker(
bboxes=bboxes, scores=detection_scores)
# detection_delay_frames = int(round(detection_delay * fps / 1000))
# classification_delay_frames = int(round(classification_delay * fps / 1000))
current_tracks_ids = [
frame_result.track_id
for frame_result in self.current_ready_frames_queue
]
current_not_tracked_items_idxs = [
idx for idx, tracked_id in enumerate(tracked_ids)
if tracked_id not in current_tracks_ids
]
if current_not_tracked_items_idxs:
current_not_tracked_bboxes = tracked_bboxes[
current_not_tracked_items_idxs]
current_not_tracked_ids = tracked_ids[
current_not_tracked_items_idxs]
bboxes_data = [
BboxData(image=frame,
xmin=xmin,
ymin=ymin,
xmax=xmax,
ymax=ymax)
for (xmin, ymin, xmax, ymax) in current_not_tracked_bboxes
]
bboxes_data_gen = BatchGeneratorBboxData(
[bboxes_data],
batch_size=batch_size,
use_not_caught_elements_as_last_batch=True)
pred_bboxes_data = self.classification_inferencer.predict(
bboxes_data_gen)[0]
for bbox_data, tracked_id in zip(pred_bboxes_data,
current_not_tracked_ids):
ready_at_frame = frame_idx
frame_result = FrameResult(label=bbox_data.label,
track_id=tracked_id,
ready_at_frame=ready_at_frame)
self.current_ready_frames_queue.append(frame_result)
return tracked_bboxes, tracked_ids
from cv_pipeliner.core.data import ImageData, BboxData
from cv_pipeliner.data_converters.coco import COCODataConverter
from cv_pipeliner.data_converters.brickit import BrickitDataConverter
from cv_pipeliner.data_converters.json import JSONDataConverter
from cv_pipeliner.data_converters.supervisely import SuperviselyDataConverter
import pytest
from .conftest import assert_images_datas_equal
test_data = Path(__file__).parent / 'test_data'
coco_imgs = test_data / 'coco'
images_data = [
ImageData(image_path=coco_imgs / '000000000009.jpg',
bboxes_data=[
BboxData(xmin=1, ymin=187, xmax=612, ymax=473, label='bowl'),
BboxData(xmin=311, ymin=4, xmax=631, ymax=232, label='bowl'),
BboxData(xmin=249,
ymin=229,
xmax=565,
ymax=474,
label='broccoli'),
BboxData(xmin=0, ymin=13, xmax=434, ymax=388, label='bowl'),
BboxData(xmin=376,
ymin=40,
xmax=451,
ymax=86,
label='orange'),
BboxData(xmin=465,
ymin=38,
xmax=523,
def concat_images_data(image_data_a: ImageData,
image_data_b: ImageData,
background_color_a: Tuple[int, int, int, int] = None,
background_color_b: Tuple[int, int, int, int] = None,
thumbnail_size_a: Tuple[int, int] = None,
thumbnail_size_b: Tuple[int, int] = None,
how: Literal['horizontally',
'vertically'] = 'horizontally',
mode: Literal['L', 'RGB', 'RGBA'] = 'RGBA',
background_edge_width: int = 3,
between_edge_width: int = 0) -> ImageData:
image_data_a = copy.deepcopy(image_data_a)
image_data_b = copy.deepcopy(image_data_b)
if image_data_a is None and image_data_b is not None:
return image_data_b
if image_data_a is not None and image_data_b is None:
return image_data_a
image_a = image_data_a.open_image()
image_b = image_data_b.open_image()
ha, wa = image_a.shape[:2]
hb, wb = image_b.shape[:2]
image = concat_images(image_a=image_a,
image_b=image_b,
background_color_a=background_color_a,
background_color_b=background_color_b,
thumbnail_size_a=thumbnail_size_a,
thumbnail_size_b=thumbnail_size_b,
how=how,
mode=mode,
background_edge_width=background_edge_width,
between_edge_width=between_edge_width)
image_data_a_new_xmin, image_data_a_new_ymin = None, None
image_data_b_new_xmin, image_data_b_new_ymin = None, None
if how == 'horizontally':
max_height = np.max([ha, hb])
min_ha = max_height // 2 - ha // 2
max_ha = max_height // 2 + ha // 2
min_hb = max_height // 2 - hb // 2
max_hb = max_height // 2 + hb // 2
image_data_a_new_xmin = 0
image_data_a_new_ymin = min_ha
image_data_a_new_xmax = wa
image_data_a_new_ymax = max_ha
image_data_b_new_xmin = wa + between_edge_width
image_data_b_new_ymin = min_hb
image_data_b_new_xmax = wa + between_edge_width + wb
image_data_b_new_ymax = max_hb
elif how == 'vertically':
max_width = np.max([wa, wb])
min_wa = max_width // 2 - wa // 2
max_wa = max_width // 2 + wa // 2
min_wb = max_width // 2 - wb // 2
max_wb = max_width // 2 + wb // 2
image_data_a_new_xmin = min_wa
image_data_a_new_ymin = 0
image_data_a_new_xmax = max_wa
image_data_a_new_ymax = ha
image_data_b_new_xmin = min_wb
image_data_b_new_ymin = ha + between_edge_width
image_data_b_new_xmax = max_wb
image_data_b_new_ymax = ha + between_edge_width + hb
keypoints_a = image_data_a.keypoints
keypoints_b = image_data_a.keypoints
keypoints_a[:, 0] += image_data_a_new_xmin
keypoints_a[:, 1] += image_data_a_new_ymin
keypoints_b[:, 0] += image_data_b_new_xmin
keypoints_b[:, 1] += image_data_b_new_ymin
def _get_new_coords_for_bbox_data(bbox_data: BboxData, xmin: int,
ymin: int):
bbox_data.keypoints[:, 0] += xmin
bbox_data.keypoints[:, 1] += ymin
bbox_data.xmin += xmin
bbox_data.ymin += ymin
bbox_data.xmax += xmin
bbox_data.ymax += ymin
bbox_data.image = None
bbox_data.image_path = None
bbox_data.cropped_image = None
for additional_bbox_data in bbox_data.additional_bboxes_data:
_get_new_coords_for_bbox_data(additional_bbox_data, xmin, ymin)
for bbox_data in image_data_a.bboxes_data:
_get_new_coords_for_bbox_data(bbox_data, image_data_a_new_xmin,
image_data_a_new_ymin)
if 'concat_images_data__image_data' not in [
bbox_data.label for bbox_data in image_data_a.bboxes_data
]:
bbox_data_a_into = [
BboxData(xmin=image_data_a_new_xmin,
ymin=image_data_a_new_ymin,
xmax=image_data_a_new_xmax,
ymax=image_data_a_new_ymax,
label='concat_images_data__image_data',
additional_bboxes_data=[
bbox_data for bbox_data in image_data_a.bboxes_data
if 'concat_images_data__image_data' != bbox_data.label
])
]
else:
bbox_data_a_into = []
image_data_a.bboxes_data = [
bbox_data for bbox_data in image_data_a.bboxes_data
if 'concat_images_data__image_data' == bbox_data.label
] + bbox_data_a_into
for bbox_data in image_data_b.bboxes_data:
_get_new_coords_for_bbox_data(bbox_data, image_data_b_new_xmin,
image_data_b_new_ymin)
if 'concat_images_data__image_data' not in [
bbox_data.label for bbox_data in image_data_b.bboxes_data
]:
bbox_data_b_into = [
BboxData(xmin=image_data_b_new_xmin,
ymin=image_data_b_new_ymin,
xmax=image_data_b_new_xmax,
ymax=image_data_b_new_ymax,
label='concat_images_data__image_data',
additional_bboxes_data=[
bbox_data for bbox_data in image_data_b.bboxes_data
if 'concat_images_data__image_data' != bbox_data.label
])
]
else:
bbox_data_b_into = []
image_data_b.bboxes_data = [
bbox_data for bbox_data in image_data_b.bboxes_data
if 'concat_images_data__image_data' == bbox_data.label
] + bbox_data_b_into
image_data = ImageData(image_path=None,
image=image,
bboxes_data=image_data_a.bboxes_data +
image_data_b.bboxes_data,
label=None,
keypoints=np.concatenate([keypoints_a, keypoints_b],
axis=0),
additional_info={
**image_data_a.additional_info,
**image_data_b.additional_info
})
return image_data
from cv_pipeliner.core.data import BboxData, ImageData
from cv_pipeliner.metrics.image_data_matching import ImageDataMatching
from cv_pipeliner.visualizers.core.image_data import visualize_images_data_side_by_side
from cv_pipeliner.visualizers.core.image_data_matching import visualize_image_data_matching_side_by_side
from cv_pipeliner.metrics.pipeline import get_df_pipeline_metrics
test_dir = Path(__file__).parent / 'images'
test_dir.mkdir(exist_ok=True, parents=True)
image_path = Path(__file__).parent / 'original.jpg'
# Banana
true_bbox_data_banana = BboxData( # detection: FN, pipeline: FN for banana
image_path=image_path,
xmin=296,
ymin=104,
xmax=1190,
ymax=650,
label='banana')
pred_bbox_data_banana_part1 = BboxData( # detection: FP, pipeline: FP for banana
image_path=image_path,
xmin=314,
ymin=122,
xmax=501,
ymax=314,
label='banana')
pred_bbox_data_banana_part2 = BboxData( # detection: FP
# pipeline: TP when extra_bbox_label == 'trash'
# pipeline: FP when extra_bbox_label is not 'trash'
image_path=image_path,
xmin=536,
import dataframe_image as dfi
from PIL import Image
from cv_pipeliner.core.data import BboxData, ImageData
from cv_pipeliner.visualizers.core.image_data import visualize_images_data_side_by_side
from cv_pipeliner.metrics.pipeline import get_df_pipeline_metrics
from cv_pipeliner.utils.images import concat_images
test_dir = Path(__file__).parent / 'images'
test_dir.mkdir(exist_ok=True, parents=True)
image_path = Path(__file__).parent / 'A_B_Z.jpg'
true_bbox_data_A1 = BboxData(image_path=image_path,
xmin=131,
ymin=116,
xmax=396,
ymax=373,
label='A')
pred_bbox_data_A1 = BboxData(image_path=image_path,
xmin=169,
ymin=149,
xmax=355,
ymax=341,
label='A')
true_bbox_data_A2 = BboxData(image_path=image_path,
xmin=64,
ymin=656,
xmax=321,
ymax=900,
label='A')
import dataframe_image as dfi
from PIL import Image
from cv_pipeliner.core.data import BboxData, ImageData
from cv_pipeliner.visualizers.core.image_data import visualize_images_data_side_by_side
from cv_pipeliner.metrics.classification import get_df_classification_metrics
from cv_pipeliner.utils.images import concat_images
test_dir = Path(__file__).parent / 'images'
test_dir.mkdir(exist_ok=True, parents=True)
image_path = Path(__file__).parent / 'A_B_Z.jpg'
true_bbox_data_A1 = BboxData(image_path=image_path,
xmin=131,
ymin=116,
xmax=396,
ymax=373,
label='A')
pred_bbox_data_A1 = BboxData(image_path=image_path,
xmin=169,
ymin=149,
xmax=355,
ymax=341,
label='A',
top_n=4,
labels_top_n=['A', 'B', 'Z', 'other'],
classification_scores_top_n=[0.3, 0.5, 0.1, 0.1])
true_bbox_data_A2 = BboxData(image_path=image_path,
xmin=64,
ymin=656,