def create_augimg(self, img, img_info): """ Create an augmented image and key points. Parameters img: as numpy array img_info: tuple with label and keypoints Return augmented image and keypoints """ label, points = img_info # print(points) kps_merge = [point for key_points in points for point in key_points] kps = KeypointsOnImage(kps_merge, shape=img.shape) # print(kps) if img.shape != self.dim: img = ia.imresize_single_image(img, self.dim[0:2]) kps = kps.on(img) # Augment keypoints and images. img_aug, kps_aug = seq(image=img, keypoints=kps) aug_points = [[kp.x, kp.y] for kp in kps_aug.keypoints] aug_points_dic = {'label': label, 'points': aug_points} return img_aug, aug_points_dic
def _read_labelme(_path_list, _pos): for _path_list_i, name in enumerate(_path_list): pos = (_pos + _path_list_i) with open(os.path.join(self.label_path, name[:name.rfind(".")] + ".json"), encoding=self.encoding) as f: jdata = f.read() data = json.loads(jdata) if self.img_path is None: img64 = data['imageData'] img = Image.open(BytesIO(base64.b64decode(img64))) else: img = Image.open(os.path.join(self.img_path, name)) zoom_r = (np.array(img.size) / np.array( (self.label_size[1], self.label_size[0]))) img = _process_img(img, self.img_size) kps = [] indexes = [] for data_i in range(len(data['shapes'])): label = data["shapes"][data_i]["label"] if label in self.class_names: index = self.class_names.index(label) indexes.append(index) point = np.array(data['shapes'][data_i]['points']) point = point.squeeze() / zoom_r kps.append(Keypoint(x=point[0], y=point[1])) kps = KeypointsOnImage(kps, shape=self.label_size) _imgaug_to_array(img, kps, pos, indexes)
def __call__(self, image, keypoints): """ transform the image and label together :param img: PIL grayscale image :param label: 21 landmarks :return: PIL grayscale image, 21 landmarks """ kps = KeypointsOnImage([Keypoint(*kp) for kp in keypoints], shape=image.shape) _max = keypoints.max(0).astype(int) _min = keypoints.min(0).astype(int) max_x = np.min([_max[0] + self.margin, image.shape[1]]) max_y = np.min([_max[1] + self.margin, image.shape[0]]) min_x = np.max([_min[0] - self.margin, 0]) min_y = np.max([_min[1] - self.margin, 0]) seq = iaa.Sequential([ iaa.Crop(px=(min_y, image.shape[1] - max_x, image.shape[0] - max_y, min_x)) ]) # Augment BBs and images. image_aug, kps_aug = seq(image=image, keypoints=kps) kps_fixed = np.asarray([[kp.x, kp.y] for kp in kps_aug.keypoints], dtype=np.float32) return image_aug, kps_fixed
def __call__(self, sample): #image, keypoints = sample['image'], sample['points'] image, keypoints, valids = sample im = np.array(image) kpt = KeypointsOnImage([Keypoint(x, y) for x, y in keypoints], shape=im.shape) seq = iaa.Sequential([ iaa.Multiply((1.2, 1.5)), iaa.Affine(scale=(0.8, 1.2), translate_px={ "x": (-20, 20), "y": (-20, 20) }, rotate=(5, 10), shear=(1, 3)), iaa.LinearContrast((0.8, 1.2)) ]) img_aug, kpt_aug = seq(image=im, keypoints=kpt) keypoints_aug = None for i, point in enumerate(kpt_aug): if i == 0: keypoints_aug = [[point.x, point.y]] else: keypoints_aug = np.append(keypoints_aug, [[point.x, point.y]], axis=0) img_aug = Image.fromarray(img_aug) keypoints_aug = np.array(keypoints_aug) return img_aug, keypoints_aug, valids
def get_kps_bbs(part, box, image, index): # Key points kps = KeypointsOnImage([ Keypoint(x=int(part[0].attributes['x'].value), y=int(part[0].attributes['y'].value)), Keypoint(x=int(part[1].attributes['x'].value), y=int(part[1].attributes['y'].value)), Keypoint(x=int(part[2].attributes['x'].value), y=int(part[2].attributes['y'].value)), Keypoint(x=int(part[3].attributes['x'].value), y=int(part[3].attributes['y'].value)), ], shape=image.shape) # Bounding box top = int(box[index].attributes['top'].value) left = int(box[index].attributes['left'].value) width = int(box[index].attributes['width'].value) height = int(box[index].attributes['height'].value) bbs = BoundingBoxesOnImage( [BoundingBox(x1=left, y1=top, x2=width + left, y2=height + top)], shape=image.shape) return kps, bbs
def augment_3D_train_data(self): sometimes = lambda aug: iaa.Sometimes(0.5, aug) for idx in range (len(self.train_imgs)): aug_img = np.zeros(shape=(512,512,3)) aug_label = np.zeros(shape=(1)) X = self.train_imgs[idx] Y = KeypointsOnImage([Keypoint(x=self.train_labels[idx][0]*512, y=self.train_labels[idx][1]*512), Keypoint(x=self.train_labels[idx][2]*512, y=self.train_labels[idx][3]*512), Keypoint(x=self.train_labels[idx][4]*512, y=self.train_labels[idx][5]*512), Keypoint(x=self.train_labels[idx][6]*512, y=self.train_labels[idx][7]*512), Keypoint(x=self.train_labels[idx][8]*512, y=self.train_labels[idx][9]*512), Keypoint(x=self.train_labels[idx][10]*512, y=self.train_labels[idx][11]*512), Keypoint(x=self.train_labels[idx][12]*512, y=self.train_labels[idx][13]*512), Keypoint(x=self.train_labels[idx][14]*512, y=self.train_labels[idx][15]*512)], shape=X.shape) seq2D = iaa.Sequential([ # Small gaussian blur with random sigma between 0 and 0.5. # But we only blur about 50% of all images. iaa.Sometimes(0.5, iaa.GaussianBlur(sigma=(0, 0.5)) ), # Strengthen or weaken the contrast in each image. iaa.ContrastNormalization((0.75, 1.5)), # Add gaussian noise. # For 50% of all images, we sample the noise once per pixel. # For the other 50% of all images, we sample the noise per pixel AND # channel. This can change the color (not only brightness) of the # pixels. iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # Make some images brighter and some darker. # In 20% of all cases, we sample the multiplier once per channel, # which can end up changing the color of the images. iaa.Multiply((0.8, 1.2), per_channel=0.2), # Apply affine transformations to each image. # Scale/zoom them, translate/move them, rotate them and shear them. iaa.Affine( scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, ) ], random_order=True) # apply augmenters in random order seq2D.deterministic = True aug_img, aug_label = seq2D(image=X, keypoints=Y) aug_label = [(aug_label.keypoints[0].x/512), (aug_label.keypoints[0].y/512), (aug_label.keypoints[1].x/512), (aug_label.keypoints[1].y/512), (aug_label.keypoints[2].x/512), (aug_label.keypoints[2].y/512), (aug_label.keypoints[3].x/512), (aug_label.keypoints[3].y/512), (aug_label.keypoints[4].x/512), (aug_label.keypoints[4].y/512), (aug_label.keypoints[5].x/512), (aug_label.keypoints[5].y/512), (aug_label.keypoints[6].x/512), (aug_label.keypoints[6].y/512), (aug_label.keypoints[7].x/512), (aug_label.keypoints[7].y/512)] self.aug_train_imgs.append(aug_img) self.aug_train_labels.append(aug_label)
def __call__(self, image, keypoints): seq = iaa.Sequential([iaa.Fliplr(0.5)]) kps = KeypointsOnImage([Keypoint(*kp) for kp in keypoints], shape=image.shape) image_aug, kps_aug = seq(image=image, keypoints=kps) kps_fixed = np.asarray([[kp.x, kp.y] for kp in kps_aug.keypoints], dtype=np.float32) return image_aug, kps_fixed
def augment2(x, y, x_new, y_new, deg): seq = iaa.Sequential([ iaa.Affine(rotate=deg) ]) for dpt_x, dpt_y, dpt_x_new, dpt_y_new in zip(x, y, x_new, y_new): keypts = KeypointsOnImage([Keypoint(x=dpt_y[i] * img_size_cropped, y=dpt_y[i+1] * img_size_cropped) for i in range(0, 50, 2)]) img_aug, keypts_aug = seq(image=dpt_x, keypoints=keypts) dpt_x_new[:,:,:] = img_aug dpt_y_new[:] = [t / img_size_cropped for x in [(keypt.x, keypt.y) for keypt in keypts_aug.keypoints] for t in x]
def get_image_and_keypoints(image_dict): (x1,y1), (x2,y2) = image_dict["A"], image_dict["B"] image = image_dict["image"] shape = image.shape[:2] kps = KeypointsOnImage([Keypoint(x=x1,y=y1), Keypoint(x=x2,y=y2)], shape=shape) heatmaps = None if "masks" in image_dict: # Optionally return heatmeps heatmaps = HeatmapsOnImage(image_dict["masks"], shape) return image, kps, heatmaps
def generate_keypoints(points, img_shape): """ From point stored in [[x coordinate],[y coordinate]] (e.g [[20],[70]]) create Keypoint objects. Keypoint object is needed for imgaug library. """ key_points = [] for point in points: key_points.append(Keypoint(x=point[0], y=point[1])) return KeypointsOnImage(key_points, img_shape)
def augmentation(images, annotations, distributed=False): height, width, _ = images[0].shape keypoints = [KeypointsOnImage( [ Keypoint(x=0, y=annotation[0]*height), Keypoint(x=annotation[1]*width, y=annotation[2]*height), Keypoint(x=width, y=annotation[3]*height) ], shape=(height, width)) for annotation in annotations] seq = iaa.Sequential( [ iaa.Fliplr(0.5), iaa.Sometimes(0.5, iaa.Crop(percent=(0, 0.125))), iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.01*255), per_channel=0.5), iaa.Sometimes(0.5, drop_light_shadow_generator), iaa.SomeOf((0, 3), [ iaa.Multiply((0.75, 1.5), per_channel=False), iaa.BlendAlphaFrequencyNoise( exponent=(-1, 1), foreground=iaa.Multiply((0.7, 1.2)), background=iaa.LinearContrast((0.75, 1.5)) ), iaa.MotionBlur(k=[3, 9]), iaa.Add((-20, 20), per_channel=0.5), iaa.LinearContrast((0.75, 1.5), per_channel=0.5) ], random_order=True) ], random_order=False).to_deterministic() if distributed: data = np.asarray(Parallel(n_jobs=multiprocessing.cpu_count())(delayed(seq)(image=img, keypoints=kps) for img, kps in zip(images, keypoints)), dtype=object) augmented_images, augmented_keypoints = data[:, 0], data[:, 1] else: augmented_images, augmented_keypoints = seq(images=images, keypoints=keypoints) augmented_annotations = [] for i, k in enumerate(augmented_keypoints): if k[0].x > k[2].x: k = k[::-1] peak = (-1, -1) if annotations[i][1] == -1 and annotations[i][2] == -1: x, y = [k[0].x, k[2].x], [k[0].y, k[2].y] elif k[1].x < 0 or (k[0].y < 0 and k[1].y < 0) or (k[0].y > height and k[1].y > height): x, y = [k[1].x, k[2].x], [k[1].y, k[2].y] elif k[1].x > width or (k[1].y < 0 and k[2].y < 0) or (k[1].y > height and k[2].y > height): x, y = [k[0].x, k[1].x], [k[0].y, k[1].y] else: x, y = [k[0].x, k[1].x, k[2].x], [k[0].y, k[1].y, k[2].y] peak = (x[1]/width, np.interp(x[1], x, y)/height) augmented_annotation = [np.interp(0, x, y)/height, peak[0], peak[1], np.interp(width, x, y)/height] if augmented_annotation[0] < 0 and augmented_annotation[3] < 0: augmented_annotation = [0, -1, -1, 0] elif augmented_annotation[0] > 1 and augmented_annotation[2] > 1 and augmented_annotation[3] > 1: augmented_annotation = [1, -1, -1, 1] augmented_annotations.append(augmented_annotation) return augmented_images, np.asarray(augmented_annotations)
def __call__(self, image, keypoints): if not isinstance(image, np.ndarray): image = np.array(image) seq = iaa.Sequential([iaa.Resize(self.size)]) kps = KeypointsOnImage([Keypoint(*kp) for kp in keypoints], shape=image.shape) image_aug, kps_aug = seq(image=image, keypoints=kps) kps_fixed = np.asarray([[kp.x, kp.y] for kp in kps_aug.keypoints], dtype=np.float32) return image_aug, kps_fixed
def __getitem__(self, index): img = io.imread(self.img_paths[index]) mask = io.imread(self.mask_paths[index]) mask = mask / mask.max() mask = mask.astype(np.uint8) corners = self.csv.iloc[index] poly = np.zeros([5, 2]) nodes = np.zeros([self.L, 2]) for c in range(4): poly[c, 0] = np.float(corners[1 + 2 * c]) poly[c, 1] = np.float(corners[2 + 2 * c]) poly[4, :] = poly[0, :] [tck, u] = interpolate.splprep([poly[:, 0], poly[:, 1]], s=2, k=1, per=1) [nodes[:, 0], nodes[:, 1]] = interpolate.splev(np.linspace(0, 1, self.L), tck) sample = { 'image': img, 'label/segmentation': mask, 'label/nodes': nodes } # do image augmentations if (self.augmentations is not None): orig_shape = sample['image'].shape aug_det = self.augmentations.to_deterministic() sample['image'] = aug_det.augment_image(sample['image']) truth = sample['label/segmentation'] truth = ia.SegmentationMapOnImage(truth, shape=truth.shape, nb_classes=2) truth = aug_det.augment_segmentation_maps( [truth])[0].get_arr_int()[..., np.newaxis] sample['label/segmentation'] = truth if ('label/nodes' in sample.keys()): kp = sample['label/nodes'] kp = KeypointsOnImage( [Keypoint(x=r[1], y=r[0] - orig_shape[0]) for r in kp], shape=orig_shape) sample['label/nodes'] = aug_det.augment_keypoints( kp).to_xy_array() # do image normalization sample['image_unnormalized'] = sample['image'] sample['image'] = self.normalization.augment_image(sample['image']) return sample
def get_template_info(self, image, label, template_type=None): # get fore ground points from label points, label = self.get_points(label) points = [Keypoint(x=point[0], y=point[1]) for point in points] points = KeypointsOnImage(keypoints=points, shape=image.shape) # get fore ground mask from getting region from label mask = self.get_template_mask( label, template_type) if template_type else np.ones_like(image) mask = SegmentationMapsOnImage(arr=mask, shape=image.shape) return image, label, mask, points
def main(argv=None): with open(FLAGS.processed_data_dir, 'r') as f: data_dict = json.load(f) Path(FLAGS.save_to_path).mkdir(parents=True, exist_ok=True) brightness_range = tuple([float(n) for n in FLAGS.brightness_range]) rotation_range = tuple([float(n) for n in FLAGS.rotation_range]) scale_range = tuple([float(n) for n in FLAGS.scale_range]) seq = iaa.Sequential([ iaa.Multiply(brightness_range), iaa.Affine( rotate=rotation_range, scale=scale_range)]) for n in tqdm(range(FLAGS.num_augmentations+1), desc='Epochs', total=FLAGS.num_augmentations+1): fnm = os.path.join(FLAGS.save_to_path, 'augmentation_{}.tfrecord'.format(n)) with tf.io.TFRecordWriter(fnm) as writer: for info in tqdm(data_dict.values(), desc='Pics', total=len(data_dict), leave=False): image = np.array(info['image']).astype(np.float32) key_pts = KeypointsOnImage([ Keypoint(info['left_eye']['center'][0], info['left_eye']['center'][1]), Keypoint(info['left_eye']['inner_corner'][0], info['left_eye']['inner_corner'][1]), Keypoint(info['left_eye']['outer_corner'][0], info['left_eye']['outer_corner'][1]), Keypoint(info['right_eye']['center'][0], info['right_eye']['center'][1]), Keypoint(info['right_eye']['inner_corner'][0], info['right_eye']['inner_corner'][1]), Keypoint(info['right_eye']['outer_corner'][0], info['right_eye']['outer_corner'][1]), Keypoint(info['left_eyebrow']['inner_end'][0], info['left_eyebrow']['inner_end'][1]), Keypoint(info['left_eyebrow']['outer_end'][0], info['left_eyebrow']['outer_end'][1]), Keypoint(info['right_eyebrow']['inner_end'][0], info['right_eyebrow']['inner_end'][1]), Keypoint(info['right_eyebrow']['outer_end'][0], info['right_eyebrow']['outer_end'][1]), Keypoint(info['mouth']['left_corner'][0], info['mouth']['left_corner'][1]), Keypoint(info['mouth']['right_corner'][0], info['mouth']['right_corner'][1]), Keypoint(info['mouth']['center_top_lip'][0], info['mouth']['center_top_lip'][1]), Keypoint(info['mouth']['center_bottom_lip'][0], info['mouth']['center_bottom_lip'][1]), Keypoint(info['nose']['tip'][0], info['nose']['tip'][1]) ], shape=image.shape) if n == 0: image_aug = image key_pts_aug_arr = key_pts.to_xy_array() else: image_aug, key_pts_aug = seq(image=image, keypoints=key_pts) key_pts_aug_arr = key_pts_aug.to_xy_array() image_aug /= 255 key_pts_aug_arr[:, 0] /= image_aug.shape[1] key_pts_aug_arr[:, 1] /= image_aug.shape[0] example = serialize_example(image_aug.tostring(), key_pts_aug_arr.tostring()) writer.write(example) return
def next_batch(self, plotting=False): while True: ( batch_images, joint_ids, batch_joints, data_items, sm_size, target_size, ) = self.get_batch() pipeline = self.build_augmentation_pipeline(height=target_size[0], width=target_size[1], apply_prob=0.5) batch_images, batch_joints = pipeline(images=batch_images, keypoints=batch_joints) # If you would like to check the augmented images, script for saving # the images with joints on: if plotting: for i in range(self.batch_size): joints = batch_joints[i] kps = KeypointsOnImage( [Keypoint(x=joint[0], y=joint[1]) for joint in joints], shape=batch_images[i].shape, ) im = kps.draw_on_image(batch_images[i]) # imageio.imwrite(data_items[i].im_path.split('/')[-1],im) imageio.imwrite( os.path.join(self.cfg["project_path"], str(i) + ".png"), im) image_shape = arr(batch_images).shape[1:3] batch = {Batch.inputs: arr(batch_images).astype(np.float64)} if self.has_gt: targetmaps = self.get_targetmaps_update( joint_ids, batch_joints, data_items, sm_size, image_shape) batch.update(targetmaps) # if returndata: # return batch_images,batch_joints,targetmaps batch = { key: data_to_input_batch(data) for (key, data) in batch.items() } batch[Batch.data_item] = data_items return batch
def add_level(self, crate_images, height): n = 0 num_crates_on_level = np.random.randint(2, 6) for i in range(num_crates_on_level): crate = np.copy(crate_images[n]) if n == len(crate_images) - 1: n = 0 else: n = n + 1 crate_height, crate_width = crate.shape[:2] self.startY = self.startY - height if self.startY < 40: break kps = KeypointsOnImage([ Keypoint(x=self.startX, y=self.startY), Keypoint(x=self.startX + crate_width, y=self.startY), Keypoint(x=self.startX + crate_width, y=self.startY + crate_height), Keypoint(x=self.startX, y=self.startY + crate_height) ], shape=(imgH, imgW, 3)) self.list_kps.append(kps) brightness_delta = np.random.randint(0, 100) contrast_delta = np.random.randint(-30, 30) crate[:, :, :3] = apply_brightness_contrast( crate[:, :, :3], 255 - brightness_delta, 127 - contrast_delta) self.img = np.zeros((imgH, imgW, 4), dtype=np.uint8) self.img[self.startY:self.startY + crate_height, self.startX:self.startX + crate_width, :] = crate # self.class1=class1 # self.class2=class2 # self.class3=class3 # Construct final image of the scene by superimposing: bg, img1, img2 and img3 mask = self.img[:, :, 3] self.mask = np.stack([mask] * 3, -1) self.final = np.where(self.mask, self.img[:, :, 0:3], self.final)
def create_multi_augimg(self, img, img_info): """ This method create image augmentation of multiple classes per image. Parameters img: as numpy array img_info: tuple with label and keypoints Return augmented image and keypoints """ labels, points_list = img_info kps_merge = list() shapes_dict = list() index = 0 for label, key_points in zip(labels, points_list): kps_merge += key_points dict_aux = {} dict_aux['label'] = label dict_aux['index'] = (index, index + len(key_points)) shapes_dict.append(dict_aux) index += len(key_points) kps_merge = [ point for key_points in points_list for point in key_points ] kps_oi = KeypointsOnImage(kps_merge, shape=img.shape) # Resize image if shapes are not equals if img.shape != self.dim: img = ia.imresize_single_image(img, self.dim[0:2]) kps_oi = kps_oi.on(img) # Augment keypoints and images. if self.augmentation: seq_det = seq.to_deterministic() img_aug = seq_det.augment_images([img])[0] kps_aug = seq_det.augment_keypoints([kps_oi])[0] else: img_aug = img kps_aug = kps_oi # Add points to each dictionary for shape in shapes_dict: first, last = shape['index'] shape['points'] = [[kp.x, kp.y] for kp in kps_aug.keypoints[first:last]] return img_aug, shapes_dict
def __getitem__(self, idx): try: image = cv2.imread(self.filename_list[idx]) if self.mode != 'test': img_fname = self.filename_list[idx] json_fname = '.'.join(img_fname.split('.')[:-1]) + '.json' j = json.load(open(json_fname)) pts = j['shapes'][0]['points'] if self.mode == 'train': seq = self.train_seq else: seq = self.val_seq if self.mode != 'test': kps = KeypointsOnImage( [Keypoint(x=pt[0], y=pt[1]) for pt in pts], shape=image.shape) else: kps = [] image_aug, kps_aug = seq(image=image, keypoints=kps) if self.mode != 'test': minx = min([pt.x for pt in kps_aug]) maxx = max([pt.x for pt in kps_aug]) target = [minx / image_aug.shape[1], maxx / image_aug.shape[1]] sample = { 'image': torch.tensor(image_aug).permute(2, 0, 1).type(torch.FloatTensor), 'bbox': torch.tensor(target).type(torch.FloatTensor) } else: sample = { 'image': torch.tensor(image_aug).permute(2, 0, 1).type(torch.FloatTensor) } return sample except Exception as e: print(e) raise
def chapter_examples_keypoints_simple(): import imgaug as ia import imgaug.augmenters as iaa from imgaug.augmentables import Keypoint, KeypointsOnImage ia.seed(1) image = ia.quokka(size=(256, 256)) kps = KeypointsOnImage([ Keypoint(x=65, y=100), Keypoint(x=75, y=200), Keypoint(x=100, y=100), Keypoint(x=200, y=80) ], shape=image.shape) seq = iaa.Sequential([ iaa.Multiply( (1.2, 1.5)), # change brightness, doesn't affect keypoints iaa.Affine(rotate=10, scale=( 0.5, 0.7 )) # rotate by exactly 10deg and scale to 50-70%, affects keypoints ]) # Augment keypoints and images. image_aug, kps_aug = seq(image=image, keypoints=kps) # print coordinates before/after augmentation (see below) # use after.x_int and after.y_int to get rounded integer coordinates for i in range(len(kps.keypoints)): before = kps.keypoints[i] after = kps_aug.keypoints[i] print("Keypoint %d: (%.8f, %.8f) -> (%.8f, %.8f)" % (i, before.x, before.y, after.x, after.y)) # image with keypoints before/after augmentation (shown below) image_before = kps.draw_on_image(image, size=7) image_after = kps_aug.draw_on_image(image_aug, size=7) # ------------ save("examples_keypoints", "simple.jpg", grid([image_before, image_after], cols=2, rows=1), quality=90)
def __call__(self, image, keypoints): """ transform the image and label together :param img: PIL grayscale image :param label: 21 landmarks :return: PIL grayscale image, 21 landmarks """ kps = KeypointsOnImage([Keypoint(*kp) for kp in keypoints], shape=image.shape) cval = int(np.random.uniform(20, 230)) seq = iaa.Sequential([ iaa.Affine(rotate=self.min_max_angle, cval=cval), ]) # Augment BBs and images. image_aug, kps_aug = seq(image=image, keypoints=kps) kps_fixed = np.asarray([[kp.x, kp.y] for kp in kps_aug.keypoints], dtype=np.float32) return image_aug, kps_fixed
def augment(img, keypoints, output_dir_img, output_dir_kpt, new_idx, show=False, mode='kpt', depth_img=None, depth_output_dir_img=None): seq = seq_kpts if mode == 'kpt' else seq_bbox kps = [Keypoint(x, y) for x, y in keypoints] kps = KeypointsOnImage(kps, shape=img.shape) img_aug, kps_aug = seq(image=img, keypoints=kps) #seq = seq.to_deterministic() #depth_img_aug = seq(image=depth_img) vis_img_aug = img_aug.copy() kps_aug = kps_aug.to_xy_array().astype(int) for i, (u, v) in enumerate(kps_aug): (r, g, b) = colorsys.hsv_to_rgb(float(i) / keypoints.shape[0], 1.0, 1.0) R, G, B = int(255 * r), int(255 * g), int(255 * b) # cv2.circle(vis_img_aug,(u,v),4,(R,G,B), -1) # if show: # cv2.imshow("img", img_aug) # cv2.waitKey(0) cv2.imwrite(os.path.join(output_dir_img, "%05d.png" % new_idx), img_aug) if depth_img is not None: cv2.imwrite(os.path.join(depth_output_dir_img, "%05d.png" % new_idx), depth_img_aug) if mode == 'kpt': kps = np.array([kps_aug[0], keypoints[1]]) # print(kps) np.save(os.path.join(output_dir_kpt, "%05d.npy" % new_idx), kps) else: # BBOX process_bbox_annots(os.path.join(output_dir_kpt, "%05d.xml" % new_idx), kps_aug)
def __getitem__(self, index): img_filename = self.df.iloc[index][0] assert os.path.exists(img_filename), 'image file does not exists' image = cv2.imread(img_filename) center_x = self.df.iloc[index][1] center_y = self.df.iloc[index][2] #shape: height, width, resize: width, height # if (self.image_shape is not None) and (image.shape[:2] != self.image_shape[:2]): # center_x /= (image.shape[1] / self.image_shape[1]) # center_y /= (image.shape[0] / self.image_shape[0]) # image = cv2.resize(image, (self.image_shape[1], self.image_shape[0])) kps = KeypointsOnImage([ Keypoint(x=center_x, y=center_y), ], shape=image.shape) image, kps_aug = self.imgaug_iaa(image=image, keypoints=kps) # before = kps.keypoints[0] after = kps_aug.keypoints[0] center_x, center_y = after.x, after.y # print(img_filename) # print(center_x, center_y) # (H,W,C)->(C,H,W) , normalization /255, -> Pytorch Tensor x = transforms.ToTensor()(image) if not self.test_mode: y = np.array((center_x, center_y), dtype=np.float32) y = torch.from_numpy(y) return x, y else: return x
import imgaug.augmenters as iaa from imgaug.augmentables import Keypoint, KeypointsOnImage ia.seed(1) image = imageio.imread("input.jpg") x1 = 125.5 y1 = 485.5 x2 = 400.5 y2 = 680.5 kps = KeypointsOnImage([ Keypoint(x=x1, y=y1), Keypoint(x=x2, y=y1), Keypoint(x=x2, y=y2), Keypoint(x=x1, y=y2) ], shape=image.shape) seq = iaa.Sequential([ iaa.Multiply((1.2, 1.5)), # change brightness, doesn't affect keypoints iaa.Affine(rotate=45) ]) # Augment keypoints and images. image_aug, kps_aug = seq(image=image, keypoints=kps) # print coordinates before/after augmentation (see below) # use after.x_int and after.y_int to get rounded integer coordinates for i in range(len(kps.keypoints)):
def next_batch(self, plotting=False): while True: ( batch_images, joint_ids, batch_joints, data_items, ) = self.get_batch() # Scale is sampled only once (per batch) to transform all of the images into same size. target_size, sm_size = self.calc_target_and_scoremap_sizes() scale = np.mean(target_size / self.default_size) augmentation.update_crop_size(self.pipeline, *target_size) batch_images, batch_joints = self.pipeline(images=batch_images, keypoints=batch_joints) batch_images = np.asarray(batch_images) image_shape = batch_images.shape[1:3] # Discard keypoints whose coordinates lie outside the cropped image batch_joints_valid = [] joint_ids_valid = [] for joints, ids in zip(batch_joints, joint_ids): inside = np.logical_and.reduce(( joints[:, 0] < image_shape[1], joints[:, 0] > 0, joints[:, 1] < image_shape[0], joints[:, 1] > 0, )) batch_joints_valid.append(joints[inside]) temp = [] start = 0 for array in ids: end = start + array.size temp.append(array[inside[start:end]]) start = end joint_ids_valid.append(temp) # If you would like to check the augmented images, script for saving # the images with joints on: if plotting: for i in range(self.batch_size): joints = batch_joints_valid[i] kps = KeypointsOnImage( [Keypoint(x=joint[0], y=joint[1]) for joint in joints], shape=batch_images[i].shape, ) im = kps.draw_on_image(batch_images[i]) # imageio.imwrite(data_items[i].im_path.split('/')[-1],im) imageio.imwrite( os.path.join(self.cfg["project_path"], str(i) + ".png"), im) batch = {Batch.inputs: batch_images.astype(np.float64)} if self.has_gt: targetmaps = self.get_targetmaps_update( joint_ids_valid, batch_joints_valid, data_items, (sm_size[1], sm_size[0]), scale, ) batch.update(targetmaps) batch = {key: np.asarray(data) for (key, data) in batch.items()} batch[Batch.data_item] = data_items return batch
def landmarks_to_kps(self, image, landmarks): kp_list = [] for i in range(landmarks.shape[0]): kp_list.append(Keypoint(x=landmarks[i][0], y=landmarks[i][1])) kps = KeypointsOnImage(kp_list, shape=image.shape) return kps
Augmentations.append([augtype, seq]) for ind, imname in enumerate(Dataframe.index): image = imresize(imread(os.path.join('montblanc_images', imname)), size=scale) ny, nx, nc = np.shape(image) kpts = [] for i in individuals: for b in bodyparts: x, y = Dataframe.iloc[ind][scorer][i][b]['x'], Dataframe.iloc[ind][ scorer][i][b]['y'] if np.isfinite(x) and np.isfinite(y): kpts.append(Keypoint(x=x * scale, y=y * scale)) kps = KeypointsOnImage(kpts, shape=image.shape) cells = [] # image with keypoints before augmentation image_before = kps.draw_on_image(image, color=color, size=size, alpha=alpha) cells.append(image_before) for name, seq in Augmentations: image_aug, kps_aug = seq(image=image, keypoints=kps) image_after = kps_aug.draw_on_image(image_aug, color=color, size=size,
def __getitem__(self, idx): file_name = self.files[idx] image = cv2.imread(file_name, cv2.IMREAD_GRAYSCALE) # Resize w, h = image.shape[1], image.shape[0] w_new, h_new = process_resize(w, h, self.resize) if self.resize_float: image = cv2.resize(image.astype('float32'), (w_new, h_new)) else: image = cv2.resize(image, (w_new, h_new)).astype('float32') # extract keypoints of the original image kps_orig_np, descs_orig, scores_orig = self.detector.detect(image) kps_orig = KeypointsOnImage([Keypoint(x=kp[0], y=kp[1]) for kp in kps_orig_np], shape=image.shape) all_matches = [] i = 0 while len(all_matches) < self.min_keypoints and i < 10: i += 1 image_aug1, kps_proj_aug1 = self.aug_seq(image=image, keypoints=kps_orig) image_aug2, kps_proj_aug2 = self.aug_seq(image=image, keypoints=kps_orig) kps_proj_aug_np1 = np.array([(kp.x, kp.y) for kp in kps_proj_aug1.keypoints]) kps_proj_aug_np2 = np.array([(kp.x, kp.y) for kp in kps_proj_aug2.keypoints]) # extract keypoints from augmented images kp_det_aug_np1, descs_aug1, scores_aug1 = self.detector.detect(image_aug1) kp_det_aug_np2, descs_aug2, scores_aug2 = self.detector.detect(image_aug2) # image = torch.from_numpy(image / 255.)[None].float() image_aug1 = torch.from_numpy(image_aug1 / 255.)[None].float() image_aug2 = torch.from_numpy(image_aug2 / 255.)[None].float() # finding GT matches if len(kp_det_aug_np1) < self.min_keypoints or len(kp_det_aug_np2) < self.min_keypoints: continue if len(kp_det_aug_np1.shape) != 2 or len(kp_det_aug_np2.shape) != 2 or len(kps_proj_aug_np1.shape) != 2 or len(kps_proj_aug_np2.shape) != 2: continue matches1 = self.find_matches(kps_proj_aug_np1, kp_det_aug_np1) matches2 = self.find_matches(kps_proj_aug_np2, kp_det_aug_np2) matches1 = {m[0]: m[1] for m in matches1} matches2 = {m[0]: m[1] for m in matches2} all_matches = np.array([(v, matches2[k]) for k, v in matches1.items() if k in matches2]) # width, height = image.shape[:2] # corners = np.array([[0, 0], [0, height], [width, 0], [width, height]], dtype=np.float32) # # # kp1, kp2, all_matches = [], [], [] # i = 0 # while i < 100: # warp = np.random.randint(-224, 224, size=(4, 2)).astype(np.float32) # # # get the corresponding warped image # M = cv2.getPerspectiveTransform(corners, corners + warp) # warped = cv2.warpPerspective(src=image, M=M, dsize=(image.shape[1], image.shape[0])) # return an image type # # # extract keypoints of the image pair using SIFT # kp1_np, descs1, scores1_np = self.detector.detect(image) # kp2_np, descs2, scores2_np = self.detector.detect(warped) # # # skip this image pair if no keypoints detected in image # if len(kp1_np) < self.min_keypoints: # break # # if len(kp2_np) < self.min_keypoints: # i += 1 # continue # # image = torch.from_numpy(image / 255.)[None].float() # warped = torch.from_numpy(warped / 255.)[None].float() # # # obtain the matching matrix of the image pair # # matched = self.matcher.match(descs1, descs2) # kp1_projected = cv2.perspectiveTransform(kp1_np.reshape((1, -1, 2)), M)[0, :, :] # dists = cdist(kp1_projected, kp2_np) # # min1 = np.argmin(dists, axis=0) # min2 = np.argmin(dists, axis=1) # # min1v = np.min(dists, axis=1) # min1f = min2[min1v < 3] # # xx = np.where(min2[min1] == np.arange(min1.shape[0]))[0] # matches = np.intersect1d(min1f, xx) # # missing1 = np.setdiff1d(np.arange(kp1_np.shape[0]), min1[matches]) # missing2 = np.setdiff1d(np.arange(kp2_np.shape[0]), matches) # # MN = np.concatenate([min1[matches][np.newaxis, :], matches[np.newaxis, :]]) # MN2 = np.concatenate([missing1[np.newaxis, :], (len(kp2)) * np.ones((1, len(missing1)), dtype=np.int64)]) # MN3 = np.concatenate([(len(kp1)) * np.ones((1, len(missing2)), dtype=np.int64), missing2[np.newaxis, :]]) # all_matches = MN.T # np.concatenate([MN, MN2, MN3], axis=1).T # all_matches = np.unique(all_matches, axis=0) # if len(all_matches) < 2: # continue # kp1_np = kp1_np.reshape((2, -1)) # kp2_np = kp2_np.reshape((2, -1)) # scores1_np = scores1_np.reshape((1, -1)) # scores2_np = scores2_np.reshape((1, -1)) # descs1 = np.pad((descs1 / 256.).T, pad_width=((0, 128), (0, 0)), constant_values=0.0) # descs2 = np.pad((descs2 / 256.).T, pad_width=((0, 128), (0, 0)), constant_values=0.0) if len(all_matches) < self.min_keypoints: continue # Padding pad_value = - 10 ** 6 kp1_np, mask1 = self._pad_and_mask(kp_det_aug_np1, axis=0, val=pad_value) kp2_np, mask2 = self._pad_and_mask(kp_det_aug_np2, axis=0, val=pad_value) descs1, _ = self._pad_and_mask(descs_aug1, axis=1, val=pad_value) descs2, _ = self._pad_and_mask(descs_aug2, axis=1, val=pad_value) scores1_np = np.pad(scores_aug1, pad_width=((0, self.nfeatures - scores_aug1.shape[0]),), mode='constant', constant_values=pad_value) scores2_np = np.pad(scores_aug2, pad_width=((0, self.nfeatures - scores_aug2.shape[0]),), mode='constant', constant_values=pad_value) all_matches, all_matches_mask = self._pad_and_mask(all_matches, axis=0, val=0) # if np.random.rand(1) > 0.5: # kp1_np, kp2_np = kp2_np, kp1_np # descs1, descs2 = descs2, descs1 # scores1_np, scores2_np = scores2_np, scores1_np # mask1, mask2 = mask2, mask1 # image, warped = warped, image # all_matches[:, 0], all_matches[:, 1] = np.copy(all_matches[:, 1]), np.copy(all_matches[:, 0]) return { 'keypoints0': kp1_np, 'keypoints1': kp2_np, 'descriptors0': descs1, 'descriptors1': descs2, 'scores0': scores1_np, 'scores1': scores2_np, 'mask0': mask1, 'mask1': mask2, 'image0': image_aug1, 'image1': image_aug2, 'all_matches': all_matches, 'all_matches_mask': all_matches_mask, 'file_name': file_name } # image_aug1 = torch.from_numpy(image_aug1 / 255.)[None].float() # image_aug2 = torch.from_numpy(image_aug2 / 255.)[None].float() return { 'keypoints0': np.zeros([self.nfeatures, 2]), 'keypoints1': np.zeros([self.nfeatures, 2]), 'descriptors0': np.zeros([self.desc_dim, self.nfeatures]), 'descriptors1': np.zeros([self.desc_dim, self.nfeatures]), 'scores0': np.zeros([self.nfeatures, ]), 'scores1': np.zeros([self.nfeatures, ]), 'mask0': np.zeros([self.nfeatures, ]), 'mask1': np.zeros([self.nfeatures, ]), 'all_matches': np.zeros([self.nfeatures, 2], dtype=int), 'all_matches_mask': np.zeros([self.nfeatures, ]), 'image0': image_aug1, 'image1': image_aug2, 'file_name': file_name }
import numpy as np from scipy import misc import imageio import cv2 import imgaug as ia import imgaug.augmenters as iaa from imgaug.augmentables import Keypoint, KeypointsOnImage ia.seed(1) image = ia.quokka(size=(256, 256)) kps = KeypointsOnImage([ Keypoint(x=65, y=100), Keypoint(x=75, y=200), Keypoint(x=100, y=100), Keypoint(x=200, y=80) ], shape=image.shape) seq = iaa.Sequential([ iaa.Multiply((1.2, 1.5)), # change brightness, doesn't affect keypoints iaa.Affine(rotate=10, scale=( 0.5, 0.7)) # rotate by exactly 10deg and scale to 50-70%, affects keypoints ]) # Augment keypoints and images. image_aug, kps_aug = seq(image=image, keypoints=kps) # print coordinates before/after augmentation (see below)
def __get_rotated_coco(self, img, anns, num_objs): kpts = [] kpts_tmp = [] for k in range(num_objs): ann = anns[k] ann_rotated = get_annotation_with_angle(ann) ann_rotated[4] = ann_rotated[4] rot = rotate_bbox(*ann_rotated) kpts.extend([Keypoint(*x) for x in rot]) if self.num_keypoints > 0: if 'keypoints' not in ann: ann['keypoints'] = np.zeros((3 * self.num_keypoints, )) kpt = [ Keypoint(*x) for x in np.array(ann['keypoints']).reshape(-1, 3)[:, :2] ] kpts_tmp.extend(kpt) idx_boxes = len(kpts) if self.num_keypoints > 0: kpts.extend(kpts_tmp) kpts = KeypointsOnImage(kpts, shape=img.shape) if self.augmentation is not None: img_aug, kpts_aug = self.augmentation(image=img, keypoints=kpts) else: img_aug, kpts_aug = np.copy(img), kpts.copy() img_aug, kpts_aug = self.resize(image=img_aug, keypoints=kpts_aug) img = (img_aug.astype(np.float32) / 255.) inp = (img - self.mean) / self.std inp = inp.transpose(2, 0, 1) output_h = self.input_size[1] // self.down_ratio output_w = self.input_size[0] // self.down_ratio num_classes = self.num_classes hm = np.zeros((num_classes, output_h, output_w), dtype=np.float32) wh = np.zeros((self.max_detections, 3), dtype=np.float32) reg = np.zeros((self.max_detections, 2), dtype=np.float32) ind = np.zeros((self.max_detections), dtype=np.int64) reg_mask = np.zeros((self.max_detections), dtype=np.uint8) gt_det = np.zeros( (self.max_detections, 7 if self.use_rotated_boxes else 6), dtype=np.float32) gt_areas = np.zeros((self.max_detections), dtype=np.float32) if self.num_keypoints > 0: kp = np.zeros((self.max_detections, self.num_keypoints * 2), dtype=np.float32) gt_kp = np.zeros((self.max_detections, self.num_keypoints, 2), dtype=np.float32) kp_reg_mask = np.zeros( (self.max_detections, self.num_keypoints * 2), dtype=np.uint8) kpts_aug = self.resize_out(keypoints=kpts_aug) box_kpts_aug, kpts_aug = kpts_aug[:idx_boxes], kpts_aug[idx_boxes:] assert num_objs == len(box_kpts_aug) // 4 for k in range(num_objs): ann = anns[k] points = [] for p in box_kpts_aug[k * 4:k * 4 + 4]: box_kp = list( (np.clip(p.x, 0, output_w - 1), np.clip(p.y, 0, output_h - 1))) points.append(box_kp) points = np.array(points).astype(np.float32) cv_ct, cv_wh, cv_angle = cv2.minAreaRect(points) if cv_wh[0] == 0 or cv_wh[1] == 0: continue cx, cy, w, h, angle = get_annotation_with_angle({ 'rbbox': np.array([cv_ct[0], cv_ct[1], cv_wh[0], cv_wh[1], cv_angle]) }) ct = np.array((cx, cy)) cls_id = int(self.cat_mapping[ann['category_id']]) if h > 0 and w > 0: radius = gaussian_radius((np.ceil(h), np.ceil(w))) radius = max(0, int(radius)) ct_int = ct.astype(np.int32) draw_gaussian(hm[cls_id], ct_int, radius) wh[k] = w, h, angle ind[k] = ct_int[1] * output_w + ct_int[0] reg[k] = ct - ct_int reg_mask[k] = 1 gt_det[k] = ([ct[0], ct[1], w, h, angle, 1, cls_id]) if self.num_keypoints > 0: valid = np.array(ann["keypoints"]).reshape(-1, 3)[:, -1] for i, p in enumerate( kpts_aug[k * self.num_keypoints:k * self.num_keypoints + self.num_keypoints]): kp[k][i * 2] = p.x - ct_int[0] kp[k][i * 2 + 1] = p.y - ct_int[1] is_valid = valid[i] == 2 and not p.is_out_of_image( (output_w, output_w)) kp_reg_mask[k, i * 2] = int(is_valid) kp_reg_mask[k, i * 2 + 1] = int(is_valid) gt_kp[k][i] = p.x, p.y if "area" not in ann: gt_areas[k] = w * h else: gt_areas[k] = ann["area"] del box_kpts_aug del img_aug gt_det = np.array(gt_det, dtype=np.float32) if len(gt_det) > 0 else np.zeros( (1, 7), dtype=np.float32) ret = { 'input': inp, 'hm': hm, 'reg_mask': reg_mask, 'ind': ind, 'wh': wh, 'reg': reg, 'gt_dets': gt_det, 'gt_areas': gt_areas, } if self.num_keypoints > 0: ret['kps'] = kp ret['gt_kps'] = gt_kp ret['kp_reg_mask'] = kp_reg_mask del kpts_aug return ret