def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file image = self.load_image(dataset_dict) prob = rand_range() if self.is_train: # mosaic and mixup should only apply one try: if prob <= self.mosaic_prob: image, dataset_dict = self.load_mosaic(image, dataset_dict) elif prob <= self.mixup_prob: image, dataset_dict = self.load_mixup(image, dataset_dict) except: print("mosaic or mixup augmentation error!! ") # apply cutout # if random.random() < self.cutout_prob: # image, dataset_dict = cutout(image, dataset_dict) # apply albumentations transform if self.img_format == "BGR": image = image[..., ::-1] # albumentations use rgb image as input # one of [hsv, brightness_contrast] # if self.is_train: # if random.random() < 0.5: # image = augment_hsv(image, 0.014, 0.68, 0.36) # yolov5 hyp # else: # image = augment_brightness_contrast(image) augment_anno = {"image": image, "bboxes": [], "category_id": []} if "annotations" in dataset_dict: augment_anno["bboxes"] = [ x['bbox'] for x in dataset_dict["annotations"] ] augment_anno["category_id"] = [ x['category_id'] for x in dataset_dict["annotations"] ] # do augmentation augment_anno = self._albumentations_tfm(**augment_anno) image = augment_anno["image"] if self.img_format == "BGR": image = image[..., ::-1] # translate back to bgr if len(augment_anno["bboxes"]) > 0: dataset_dict["annotations"] = [{ "category_id": category_id, "bbox": bbox, "iscrowd": 0, "area": bbox[2] * bbox[3], "bbox_mode": BoxMode.XYWH_ABS, } for bbox, category_id in zip(augment_anno["bboxes"], augment_anno["category_id"])] else: dataset_dict["annotations"] = [] # apply detectron2 transform if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) # only transform image image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor( np.ascontiguousarray(image.transpose(2, 0, 1))) # USER: Remove if you don't use pre-computed proposals. if self.load_proposals: utils.transform_proposals(dataset_dict, image_shape, transforms, self.min_box_side_len, self.proposal_topk) # if not self.is_train: # # USER: Modify this if you want to keep them for some reason. # dataset_dict.pop("annotations", None) # dataset_dict.pop("sem_seg_file_name", None) # return dataset_dict if "annotations" in dataset_dict: # USER: Modify this if you want to keep them for some reason. for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) # USER: Implement additional transformations if you have other types of data # transform only annotations, because the image has transformed before annos = [ utils.transform_instance_annotations( obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = utils.annotations_to_instances( annos, image_shape, mask_format=self.mask_format) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ # get sample information and read image dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below image = utils.read_image(dataset_dict["file_name"], format=self.img_format) # check real image size match the sample description in dataset_dict utils.check_image_size(dataset_dict, image) # IMAGE AUGMENTATION if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor( image.transpose(2, 0, 1).astype("float32")) # Can use uint8 if it turns out to be slow some day # these information no needed in train mode if not self.is_train: dataset_dict.pop("annotations", None) dataset_dict.pop("multi_labels", None) dataset_dict.pop("multi_label_names", None) dataset_dict.pop("seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: # USER: Implement additional transformations if you have other types of data annos = [ utils.transform_instance_annotations(obj, transforms, image_shape) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = utils.annotations_to_instances(annos, image_shape) # 虽然用不上,但还是保留吧 # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) if "seg_file_name" in dataset_dict: with PathManager.open(dataset_dict.pop("seg_file_name"), "rb") as f: sem_seg_gt = Image.open(f) sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8") sem_seg_gt = transforms.apply_segmentation(sem_seg_gt) # TODO: one-hot encoding after transformation sem_gt = np.zeros( (self.num_seg_class, sem_seg_gt.shape[0], sem_seg_gt.shape[1]), dtype="uint8") for c in range(self.num_seg_class): sem_gt[c][sem_seg_gt == c] = 1 # sem_gt = np.transpose(sem_gt, (1, 2, 0)) # for later transform sem_gt = torch.as_tensor(sem_gt.astype("long")) dataset_dict["sem_seg"] = sem_gt if "multi_labels" in dataset_dict: dataset_dict["multi_labels"] = dataset_dict.pop("multi_labels") return dataset_dict
def __call__(self, dataset_dict: dict) -> dict: """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file image: np.ndarray = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) # TODO simplify this if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( transform_gens=([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, img=image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict['image'] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) if not self.is_train: dataset_dict.pop('annotations', None) return dataset_dict panel_annos = [] label_annos = [] for obj in dataset_dict.pop('annotations'): panel_obj = { 'bbox': obj['panel_bbox'], 'bbox_mode': obj['bbox_mode'] } panel_annos.append(utils.transform_instance_annotations(annotation=panel_obj, transforms=transforms, image_size=image_shape)) if 'label' in obj: if 'label_bbox' in obj: label_obj = { 'bbox': obj['label_bbox'], 'bbox_mode': obj['bbox_mode'], 'label': obj['label'] } label_annos.append( utils.transform_instance_annotations(annotation=label_obj, transforms=transforms, image_size=image_shape)) else: logging.error("Error with annotation: %s has a 'label' field"\ " but no corresponding 'label_bbox' field.", obj) elif 'label_bbox' in obj: logging.error("Inconsistent label annotation:"\ " obj['label']=%s and obj['label_bbox']=%s", obj['label'], obj['label_bbox']) panel_instances, label_instances = self._annotations_to_instances(panel_annos=panel_annos, label_annos=label_annos, image_size=image_shape) # dataset_dict["instances"] = utils.filter_empty_instances(instances) # TODO check if we have to adapt this for handling panel_boxes and label_boxes # (see method above) # dataset_dict["instances"] = utils.filter_empty_instances(instances) dataset_dict['panel_instances'] = panel_instances dataset_dict['label_instances'] = label_instances return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below image = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w dataset_dict["image"] = torch.as_tensor( image.transpose(2, 0, 1).astype("float32")) if not self.is_train: dataset_dict.pop("annotations", None) return dataset_dict for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) annos = [ transform_rotated_boxes_annotations(obj, transforms) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = rotated_annotations_to_instances(annos, image_shape) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) del annos, instances return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file image = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor( image.transpose(2, 0, 1).astype("float32")) # Can use uint8 if it turns out to be slow some day # USER: Remove if you don't use pre-computed proposals. if self.load_proposals: utils.transform_proposals(dataset_dict, image_shape, transforms, self.min_box_side_len, self.proposal_topk) if not self.is_train: dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: # USER: Modify this if you want to keep them for some reason. for annotation in dataset_dict["annotations"]: if not self.mask_on: annotation.pop("segmentation", None) if not self.keypoint_on: annotation.pop("keypoints", None) if not self.fiberwidth_on: annotation.pop("fiberwidth", None) if not self.fiberlength_on: annotation.pop("fiberlength", None) annotations = [ obj for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] num_keypoints = self.cfg.MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS annotations = [ transformation.interpolate_keypoints(obj, num_keypoints) for obj in annotations ] annotations = [ utils.transform_instance_annotations( obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices) for obj in annotations ] annotations = [ transformation.transform_instance_keypoint_order( obj, self.cfg) for obj in annotations ] instances = utils.annotations_to_instances( annotations, image_shape, mask_format=self.mask_format) if len(annotations ) and "fiberwidth" in annotations[0] and self.fiberwidth_on: gt_fiberwidth = torch.tensor( [obj["fiberwidth"] for obj in annotations]) instances.gt_fiberwidth = gt_fiberwidth if len(annotations) and "fiberlength" in annotations[ 0] and self.fiberlength_on: gt_fiberlength = torch.tensor( [obj["fiberlength"] for obj in annotations]) instances.gt_fiberlength = gt_fiberlength # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) return dataset_dict
def test_gen_crop_outside_boxes(self): instance = {"bbox": [10, 10, 100, 100], "bbox_mode": BoxMode.XYXY_ABS} with self.assertRaises(AssertionError): detection_utils.gen_crop_transform_with_instance((10, 10), (15, 15), instance)
def _original_call(self, dataset_dict): """ Modified from detectron2's original __call__ in DatasetMapper """ dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below image = self._read_image(dataset_dict, format=self.img_format) if not self.backfill_size: utils.check_image_size(dataset_dict, image) image, dataset_dict = self._custom_transform(image, dataset_dict) inputs = AugInput(image=image) if "annotations" not in dataset_dict: transforms = AugmentationList( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens)(inputs) image = inputs.image else: # pass additional arguments, will only be used when the Augmentation # takes `annotations` as input inputs.annotations = dataset_dict["annotations"] # Crop around an instance if there are instances in the image. if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) transforms = AugmentationList(self.tfm_gens)(inputs) image = inputs.image if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w if image.ndim == 2: image = np.expand_dims(image, 2) dataset_dict["image"] = torch.as_tensor( image.transpose(2, 0, 1).astype("float32")) # Can use uint8 if it turns out to be slow some day if self.load_proposals: utils.transform_proposals( dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk, min_box_size=self.proposal_min_box_size, ) if not self.is_train: dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) annos = [ utils.transform_instance_annotations( obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices, ) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = utils.annotations_to_instances( annos, image_shape, mask_format=self.mask_format) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) if "sem_seg_file_name" in dataset_dict: sem_seg_gt = read_image_with_prefetch( dataset_dict.pop("sem_seg_file_name"), "L", prefetched=dataset_dict.get(PREFETCHED_SEM_SEG_FILE_NAME, None), ).squeeze(2) sem_seg_gt = transforms.apply_segmentation(sem_seg_gt) sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long")) dataset_dict["sem_seg"] = sem_seg_gt # extend standard D2 semantic segmentation to support multiple segmentation # files, each file can represent a class if "multi_sem_seg_file_names" in dataset_dict: raise NotImplementedError() if "_post_process_" in dataset_dict: proc_func = dataset_dict.pop("_post_process_") dataset_dict = proc_func(dataset_dict) return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file import os if not os.path.exists(dataset_dict["file_name"]): name = dataset_dict["file_name"].replace("hico/train2015", "coco2014/train2014") else: name = dataset_dict["file_name"] image = utils.read_image(name, format=self.img_format) # import ipdb;ipdb.set_trace() if dataset_dict['height'] != image.shape[0]: image = image.transpose([1, 0, 2]) # print(dataset_dict) # np.asarray().transpose() # print(image.shape, type(image)) utils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor( np.ascontiguousarray(image.transpose(2, 0, 1))) if not self.is_train: # USER: Modify this if you want to keep them for some reason. dataset_dict.pop("annotations", None) return dataset_dict if "annotations" in dataset_dict: # USER: Implement additional transformations if you have other types of data annos = [ utils.transform_instance_annotations( obj, transforms, image_shape, ) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = annotations_to_instances(annos, image_shape) dataset_dict["instances"] = utils.filter_empty_instances(instances) return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file image = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image ) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor( image.transpose(2, 0, 1).astype("float32") ).contiguous() # Can use uint8 if it turns out to be slow some day if not self.is_train: dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) dataset_dict.pop("pan_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: annos = [ utils.transform_instance_annotations( obj, transforms, image_shape ) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = annotations_to_instances( annos, image_shape, mask_format=self.mask_format ) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() instances = utils.filter_empty_instances(instances) # TODO: design a data structure for storing relation mat if self.relation_on: instances, gt_relation = get_relation_gt(instances) dataset_dict["relation"] = gt_relation if instances.has("bit_masks"): instances.remove("bit_masks") dataset_dict["instances"] = instances if "sem_seg_file_name" in dataset_dict: with PathManager.open(dataset_dict.pop("sem_seg_file_name"), "rb") as f: sem_seg_gt = Image.open(f) sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8") sem_seg_gt = transforms.apply_segmentation(sem_seg_gt) sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long")) if self.fcn_roi_on: fcn_roi_gt = get_fcn_roi_gt( sem_seg_gt, dataset_dict["instances"], self.fcn_roi_size) dataset_dict["fcn_roi"] = fcn_roi_gt dataset_dict["sem_seg"] = sem_seg_gt if "pan_seg_file_name" in dataset_dict: with PathManager.open(dataset_dict.pop("pan_seg_file_name"), "rb") as f: pan_seg_gt = Image.open(f) pan_seg_gt = np.asarray(pan_seg_gt, dtype="uint32") pan_seg_gt = pan_id2channel_id( pan_seg_gt, dataset_dict["instances"].pan_id, self.stuff_cls_num) dataset_dict["instances"].remove("pan_id") pan_seg_gt = transforms.apply_segmentation(pan_seg_gt) pan_seg_gt = torch.as_tensor(pan_seg_gt.astype("long")) dataset_dict["pan_seg"] = pan_seg_gt return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file image = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor( np.ascontiguousarray(image.transpose(2, 0, 1))) # USER: Remove if you don't use pre-computed proposals. # Most users would not need this feature. if self.load_proposals: utils.transform_proposals( dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk, min_box_size=self.proposal_min_box_size, ) # HACK Keep annotations for test # if not self.is_train: # # USER: Modify this if you want to keep them for some reason. # dataset_dict.pop("annotations", None) # dataset_dict.pop("sem_seg_file_name", None) # return dataset_dict if "annotations" in dataset_dict: # USER: Modify this if you want to keep them for some reason. for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) # USER: Implement additional transformations if you have other types of data annos = [ utils.transform_instance_annotations( obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = utils.annotations_to_instances( annos, image_shape, mask_format=self.mask_format) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) return dataset_dict
def _original_call(self, dataset_dict): """ Modified from detectron2's original __call__ in DatasetMapper """ dataset_dict = copy.deepcopy( dataset_dict) # it will be modified by code below image = self._read_image(dataset_dict, format=self.img_format) if not self.backfill_size: utils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w dataset_dict["image"] = torch.as_tensor( image.transpose(2, 0, 1).astype("float32")) # Can use uint8 if it turns out to be slow some day assert not self.load_proposals, "Not supported!" if not self.is_train: dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) # Convert dataset_dict["annotations"] to dataset_dict["instances"] annotations = [ obj for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] # Convert either rotated box or horizontal box to XYWHA_ABS format original_boxes = [ BoxMode.convert( box=obj["bbox"], from_mode=obj["bbox_mode"], to_mode=BoxMode.XYWHA_ABS, ) for obj in annotations ] transformed_boxes = transforms.apply_rotated_box( np.array(original_boxes, dtype=np.float64)) instances = Instances(image_shape) instances.gt_classes = torch.tensor( [obj["category_id"] for obj in annotations], dtype=torch.int64) instances.gt_boxes = RotatedBoxes(transformed_boxes) instances.gt_boxes.clip(image_shape) dataset_dict["instances"] = instances[ instances.gt_boxes.nonempty()] return dataset_dict
def __call__(self, dataset_dict): dataset_dict = copy.deepcopy(dataset_dict) img = utils.read_image(dataset_dict['file_name'], format=self.img_format) boxes = [ann['bbox'] for ann in dataset_dict['annotations']] labels = [ann['category_id'] for ann in dataset_dict['annotations']] augm_annotation = self.aug(image=img, bboxes=boxes, category_id=labels) img = augm_annotation['image'] h, w, _ = img.shape augm_boxes = np.array(augm_annotation['bboxes'], dtype=np.float32) # sometimes bbox annotations go beyond image augm_boxes[:, :] = augm_boxes[:, :].clip(min=[0, 0, 0, 0], max=[w, h, w, h]) augm_labels = np.array(augm_annotation['category_id']) dataset_dict['annotations'] = [ { 'iscrowd': 0, 'bbox': augm_boxes[i].tolist(), 'category_id': augm_labels[i], 'bbox_mode': BoxMode.XYWH_ABS, } for i in range(len(augm_boxes)) ] if self.crop_gen: # image crop using detectron tools crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(img.shape[:2]), img.shape[:2], np.random.choice(dataset_dict["annotations"]) ) img = crop_tfm.apply_image(img) # image resize using detectron tools img, transforms = T.apply_transform_gens(self.resize_gen, img) if self.crop_gen: transforms = crop_tfm + transforms annos = [ utils.transform_instance_annotations( obj, transforms, img.shape[:2] ) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] dataset_dict['annotations'] = annos instances = utils.annotations_to_instances( annos, img.shape[:2] ) dataset_dict["instances"] = utils.filter_empty_instances(instances) dataset_dict['height'] = img.shape[0] dataset_dict['width'] = img.shape[1] dataset_dict["image"] = torch.as_tensor( img.transpose(2, 0, 1).astype("float32") ).contiguous() return dataset_dict
def __call__(self, dataset_dict): dataset_dict = copy.deepcopy(dataset_dict) # Robust video sampling video_path = dataset_dict['video_path'] frame_number = dataset_dict['video_frame_number'] frame_time = dataset_dict['video_frame_time'] OVERALL_ATTEMPTS = 5 image = get_frame_without_crashing( video_path, frame_number, frame_time, OVERALL_ATTEMPTS) d2_dutils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = d2_transforms.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image ) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = d2_dutils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = \ d2_transforms.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to # shared-memory, but not efficient on large generic data structures due # to the use of pickle & mp.Queue. Therefore it's important to use # torch.Tensor. dataset_dict["image"] = torch.as_tensor( image.transpose(2, 0, 1).astype("float32") ).contiguous() # Can use uint8 if it turns out to be slow some day if not self.is_train: dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: # USER: Implement additional transformations if you have other # types of data annos = [ d2_dutils.transform_instance_annotations( obj, transforms, image_shape, ) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = d2_dutils.annotations_to_instances( annos, image_shape, ) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = \ d2_dutils.filter_empty_instances(instances) return dataset_dict
def __call__(self, dataset_dict): dataset_dict = copy.deepcopy(dataset_dict) image = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) # this place you can add your own code to change the input image if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image) else: if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] dataset_dict["image"] = torch.as_tensor( np.ascontiguousarray(image.transpose(2, 0, 1))) if self.load_proposals: utils.transform_proposals(dataset_dict, image_shape, transforms, self.min_box_side_len, self.proposal_topk) if not self.is_train: dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) if not self.attribute_on: anno.pop("attribute_ids") annos = [ utils.transform_instance_annotations( obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = annotations_to_instances_with_attributes( annos, image_shape, mask_format=self.mask_format, load_attributes=self.attribute_on, max_attr_per_ins=self.max_attr_per_ins) if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) if "sem_seg_file_name" in dataset_dict: with PathManager.open(dataset_dict.pop("sem_seg_file_name"), "rb") as f: sem_seg_gt = Image.open(f) sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8") sem_seg_gt = transforms.apply_segmentation(sem_seg_gt) sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long")) dataset_dict["sem_seg"] = sem_seg_gt return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in centernet Dataset format. Returns: dict: a format that builtin models in centernet accept """ dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file image = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image ) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor(image.transpose(2, 0, 1).astype("float32")) # Can use uint8 if it turns out to be slow some day # USER: Remove if you don't use pre-computed proposals. if self.load_proposals: utils.transform_proposals( dataset_dict, image_shape, transforms, self.min_box_side_len, self.proposal_topk ) if not self.is_train and not self.eval_with_gt: dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: # USER: Modify this if you want to keep them for some reason. for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) # USER: Implement additional transformations if you have other types of data annos = [ utils.transform_instance_annotations( obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices ) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = utils.annotations_to_instances( annos, image_shape, mask_format=self.mask_format ) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) # USER: Remove if you don't do semantic/panoptic segmentation. if "sem_seg_file_name" in dataset_dict: with PathManager.open(dataset_dict.pop("sem_seg_file_name"), "rb") as f: sem_seg_gt = Image.open(f) sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8") sem_seg_gt = transforms.apply_segmentation(sem_seg_gt) sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long")) dataset_dict["sem_seg"] = sem_seg_gt return dataset_dict
def __call__(self, dataset_dict): """ Args: dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. Returns: dict: a format that builtin models in detectron2 accept """ dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below # USER: Write your own image loading if it's not from a file image = utils.read_image(dataset_dict["file_name"], format=self.img_format) utils.check_image_size(dataset_dict, image) for i in range(len(dataset_dict["annotations"])): dataset_dict["annotations"][i]["segmentation"] = [] ### my code ## image, dataset_dict = self.aug_handler(image=image, dataset_dict_detectron=dataset_dict) ### my code ## if "annotations" not in dataset_dict: image, transforms = T.apply_transform_gens( ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image ) else: # Crop around an instance if there are instances in the image. # USER: Remove if you don't use cropping if self.crop_gen: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict["annotations"]), ) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen: transforms = crop_tfm + transforms image_shape = image.shape[:2] # h, w # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, # but not efficient on large generic data structures due to the use of pickle & mp.Queue. # Therefore it's important to use torch.Tensor. dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) # USER: Remove if you don't use pre-computed proposals. if self.load_proposals: utils.transform_proposals( dataset_dict, image_shape, transforms, self.min_box_side_len, self.proposal_topk ) if not self.is_train: # USER: Modify this if you want to keep them for some reason. dataset_dict.pop("annotations", None) dataset_dict.pop("sem_seg_file_name", None) return dataset_dict if "annotations" in dataset_dict: # USER: Modify this if you want to keep them for some reason. for anno in dataset_dict["annotations"]: if not self.mask_on: anno.pop("segmentation", None) if not self.keypoint_on: anno.pop("keypoints", None) # USER: Implement additional transformations if you have other types of data annos = [ utils.transform_instance_annotations( obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices ) for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0 ] instances = utils.annotations_to_instances( annos, image_shape, mask_format=self.mask_format ) # Create a tight bounding box from masks, useful when image is cropped if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) # USER: Remove if you don't do semantic/panoptic segmentation. if "sem_seg_file_name" in dataset_dict: with PathManager.open(dataset_dict.pop("sem_seg_file_name"), "rb") as f: sem_seg_gt = Image.open(f) sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8") sem_seg_gt = transforms.apply_segmentation(sem_seg_gt) sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long")) dataset_dict["sem_seg"] = sem_seg_gt #### this uses local variable, use with caution ###### if True: vis_img = image.copy() bbox_list = [BBox.from_list(vals) for vals in dataset_dict["instances"].gt_boxes.tensor.numpy().tolist()] # seg_list = [Segmentation([Polygon.from_list(poly.tolist(), demarcation=False) for poly in seg_polys]) for seg_polys in dataset_dict["instances"].gt_masks.polygons] for bbox in (bbox_list): # if len(seg) > 0 and False: # vis_img = draw_segmentation(img=vis_img, segmentation=seg, transparent=True) vis_img = draw_bbox(img=vis_img, bbox=bbox) aug_vis.step(vis_img) return dataset_dict
def test_gen_crop(self): instance = {"bbox": [10, 10, 100, 100], "bbox_mode": BoxMode.XYXY_ABS} t = detection_utils.gen_crop_transform_with_instance((10, 10), (150, 150), instance) # the box center must fall into the cropped region self.assertTrue(t.x0 <= 55 <= t.x0 + t.w)
def __call__(self, dataset_dict): assert 'annotations' in dataset_dict, '今回はセグメンテーションのみを対象にする' assert not 'sem_seg_file_name' in dataset_dict, 'パノプティックセグメンテーションは行わない' dataset_dict = copy.deepcopy(dataset_dict) image = utils.read_image(dataset_dict['file_name'], format=self.img_format) utils.check_image_size(dataset_dict, image) # 明るさ・コントラスト・彩度・カットアウト if self.cont_gen is not None: tfm = self.cont_gen.get_transform(image) image = tfm.apply_image(image) if self.bright_gen is not None: tfm = self.bright_gen.get_transform(image) image = tfm.apply_image(image) if self.sat_gen is not None: tfm = self.sat_gen.get_transform(image) image = tfm.apply_image(image) if self.cutout_gen is not None: tfm = self.cutout_gen.get_transform(image) image = tfm.apply_image(image) # アフィン if self.rotate_gen is not None: rotate_tfm = self.rotate_gen.get_transform(image) image = rotate_tfm.apply_image(image) if self.shear_gen is not None: shear_tfm = self.shear_gen.get_transform(image) image = shear_tfm.apply_image(image) if self.extent_gen is not None: extent_tfm = self.extent_gen.get_transform(image) image = extent_tfm.apply_image(image) if self.crop_gen is not None: crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(image.shape[:2]), image.shape[:2], np.random.choice(dataset_dict['annotations'])) image = crop_tfm.apply_image(image) image, transforms = T.apply_transform_gens(self.tfm_gens, image) if self.crop_gen is not None: transforms = crop_tfm + transforms if self.extent_gen is not None: transforms = extent_tfm + transforms if self.shear_gen is not None: transforms = shear_tfm + transforms if self.rotate_gen is not None: transforms = rotate_tfm + transforms # テストの場合はアノテーションがいらないので削除して終了 if not self.is_train: dataset_dict.pop('annotations', None) dataset_dict.pop('sem_seg_file_name', None) return dataset_dict image_shape = image.shape[:2] # h, w dataset_dict['image'] = torch.as_tensor( np.ascontiguousarray(image.transpose(2, 0, 1))) annos = [ utils.transform_instance_annotations(obj, transforms, image_shape, keypoint_hflip_indices=None) for obj in dataset_dict.pop('annotations') if obj.get("iscrowd", 0) == 0 ] instances = utils.annotations_to_instances( annos, image_shape, mask_format=self.mask_format) # マスクからバウンディングボックスを作成 if self.crop_gen and instances.has("gt_masks"): instances.gt_boxes = instances.gt_masks.get_bounding_boxes() dataset_dict["instances"] = utils.filter_empty_instances(instances) return dataset_dict
dataset_dict['annotations'] = [ { 'iscrowd': 0, 'bbox': augm_boxes[i].tolist(), 'masks': augm_masks 'category_id': augm_labels[i], 'bbox_mode': BoxMode.XYWH_ABS, } for i in range(len(augm_boxes)) ] if self.crop_gen: # image crop using detectron tools crop_tfm = utils.gen_crop_transform_with_instance( self.crop_gen.get_crop_size(img.shape[:2]), img.shape[:2], np.random.choice(dataset_dict["annotations"]) ) img = crop_tfm.apply_image(img) # image resize using detectron tools img, transforms = T.apply_transform_gens(self.resize_gen, img) if self.crop_gen: transforms = crop_tfm + transforms annos = [ utils.transform_instance_annotations( obj, transforms, img.shape[:2] ) for obj in dataset_dict.pop("annotations")