def extract_instances_from_image_voc(name, loader, instance_root): inst_mask = np.array(Image.open(os.path.join(instance_root, name + '.png'))) gt_bboxes, _, gt_cats, _, _, _ = loader.read_annotations(name) instance_inds = np.unique(inst_mask) instance_inds = instance_inds[(instance_inds != 0) * (instance_inds != 255)] masks, cats = [], [] for iind in instance_inds: mask = np.ones(inst_mask.shape[:2], dtype=bool) mask[inst_mask == iind] = 0 coords_y, coords_x = map(np.unique, np.where(mask == 0)) x_min, y_min, x_max, y_max = 0, 0, mask.shape[0], mask.shape[1] if len(coords_x) > 0: x_min, x_max = get_ends(coords_x) if len(coords_y) > 0: y_min, y_max = get_ends(coords_y) mask_box = [x_min, y_min, x_max - x_min + 1, y_max - y_min + 1] mask_box = np.reshape(mask_box, (-1, 4)) ious = batch_iou(gt_bboxes, mask_box).ravel() best_match = np.argmax(ious) cat = gt_cats[best_match] masks.append(mask) cats.append(cat) return masks, cats
def dynamic_place_instance(self, name, image, bbox, blending_list): """Finds fitting instance given a box to place in, and copies there""" if type(image) != list: backgrounds = [image.copy() for i in range(len(blending_list))] else: backgrounds = [i.copy() for i in image] width, height = image.size xmin, ymin, xmax, ymax = wh2xy(bbox) mask, foreground, w, h, scale = self.find_instance(name, bbox) inst_bbox = [0, 0, w, h] if self.config['colorize']: foreground = self.color_augment(foreground) foreground, mask, inst_bbox = self.rescale_instance( foreground, mask, scale, wh2xy(inst_bbox)) w, h = xy2wh(inst_bbox)[2:] fits = False for i in range(50): new_xc = np.random.randint(xmin - 0.25 * w, xmax + 0.25 * w) new_yc = np.random.randint(ymin - 0.25 * h, ymax + 0.25 * h) # new_xc = np.random.randint(xmin - 0.75*w, xmax + 0.75*w) # new_yc = np.random.randint(ymin - 0.75*h, ymax + 0.75*h) xmin, ymin = new_xc - int(w / 2), new_yc - int(h / 2) new_bbox, fits = self.check_bbox([xmin, ymin, w, h], width, height) iou = batch_iou(new_bbox, bbox) if (iou > 0.2 and iou < 0.7 or self.no_spatial_constraints) and fits: backgrounds = self.blend(np.array([xmin, ymin]), mask, foreground, backgrounds, blending_list) return backgrounds, new_bbox return None, None
def delete_covered_boxes(self, new_bbox, bboxes, cats, diff, given_cat): """Handles occlusions""" ious, intersections = batch_iou(bboxes, np.reshape(new_bbox, (1, 4)), return_intersection=True) assert bboxes.shape[0] == len(cats) == len(diff), \ 'Have a problem with number of examples' areas = bboxes[:, 2] * bboxes[:, 3] inter_fraq = (intersections.ravel() / areas) not_occluded = inter_fraq < 0.7 bboxes = bboxes[not_occluded] cats = np.array(cats)[not_occluded].tolist() diff = np.array(diff)[not_occluded].tolist() bboxes = np.concatenate([bboxes, np.reshape(new_bbox, (1, 4))], axis=0) cats.append(given_cat) diff.append(0) return bboxes, cats, diff
def duplicate_instance(self, image, mask_file, fg_image, blending_list): """Performs copy-pasting of an instance""" foreground = fg_image.copy() width, height = image.size if type(image) != list: backgrounds = [image.copy() for i in range(len(blending_list))] else: backgrounds = [i.copy() for i in image] scale = np.random.uniform(1 / 2, 2) foreground, mask, old_coord = self.extract_mask(mask_file, foreground) if foreground is None: return None, None x_min, y_min, w, h = old_coord foreground, mask, new_coord = self.rescale_instance( foreground, mask, scale, wh2xy(old_coord)) if self.config['colorize']: foreground = self.color_augment(foreground) xmin, ymin, xmax, ymax = new_coord old_bbox = xy2wh(new_coord) wi, hi = old_bbox[2:] new_wi, new_hi = xmax - xmin, ymax - ymin fits = False for i in range(50): new_xc = np.random.randint(xmin - self.radius * wi, xmax + self.radius * wi) new_yc = np.random.randint(ymin - self.radius * hi, ymax + self.radius * hi) xmin, ymin = new_xc - int(new_wi / 2), new_yc - int(new_hi / 2) new_bbox, fits = self.check_bbox([xmin, ymin, new_wi, new_hi], width, height) iou = batch_iou(new_bbox, old_bbox) if (iou > 0.2 and iou < 0.7 or self.no_spatial_constraints) and fits: backgrounds = self.blend(np.array([xmin, ymin]), mask, foreground, backgrounds, blending_list) return backgrounds, new_bbox return None, None
def sample_negative(self, name=None): """Samples a background contextual image""" cat = None while cat is None: name = np.random.choice(self.loader.filenames) if name is None else name bboxes, _, cats, w, h, diff = self.loader.read_annotations(name) image = self.loader.load_image(name) if len(cats) == 0: bbox = self.draw_bbox(w, h) cat = 0 else: for i in range(self.max_sample_tries): bbox = self.draw_bbox(w, h) ious = batch_iou(bboxes, bbox) area = bbox[0, 2] * bbox[0, 3] areas = bboxes[:, 2] * bboxes[:, 3] inter_fraq = (ious * (areas + area) / (1 + ious)) / areas if np.max(inter_fraq) < 0.3: cat = 0 break self.name = name return [image, bbox, cat, w, h]
def manipulate_image(self, name, bboxes, cats, diff): """Perform enlargement data augmentation for an image""" image = Image.open(self.loader.get_image_path(name)) init_image = image.copy() new_bboxes = np.array(bboxes) try: instances = self.name2instances[name] except KeyError: print('No instance segmentation gt for %s' % name) return image, bboxes, np.array(cats), np.array(diff) N_min = np.min([self.config['min_objects'], len(instances)]) N_max = np.min([self.config['max_objects'], len(instances)]) + 1 N = np.random.choice(np.arange(N_min, N_max)) np.random.shuffle(instances) instances = instances[:N] random_bboxes = np.array(bboxes) np.random.shuffle(random_bboxes) random_bboxes = random_bboxes[:N] blendings = np.random.choice(self.blending_list, N) for k in range(N): i = instances[k] blending = blendings[k] images, new_bbox, old_bbox = self.enlarge_instance( image, i, init_image, [blending]) if images is not None: image = images[0] ind = np.argmax(batch_iou(new_bboxes, old_bbox), axis=0) new_bboxes[ind] = new_bbox # new_bboxes, cats, diff = self.delete_covered_boxes( # new_bbox, new_bboxes, cats, diff, self.instance2cat(i)) return image, new_bboxes, np.array(cats), np.array(diff)
def match_anchors_to_instances(anchor_dict, excluded, loader): """for each candidate box finds a set of instances that could be inserted inside Creates a dict structure that for each image retrieves -> a set of confident context candidate boxes and for each of them -> all possible instance (with corresponding scaling factor) fitting there """ keys = ['bboxes', 'cats', 'scores', 'inst_paths', 'scales'] man = DynamicInstanceManipulator(loader) # contains instance dataset n_scales = 16 all_scales = np.linspace(0.5, 2.0, n_scales) n_inst = len(man.ars) # extracting width and height for all instance boxes in the instance dataset all_whs = [] for i in range(n_inst): wh = np.array(man.boxes[i][2:]) whs = np.array([wh * scale for scale in all_scales]) all_whs.append(whs) all_whs = np.array(all_whs) box2instance = dict() bar = progressbar.ProgressBar() for name in bar(anchor_dict): # iterating over the candidates for one image for i in range(anchor_dict[name].shape[0]): bbox = anchor_dict[name][i, :4] cat = anchor_dict[name][i, 4] score = anchor_dict[name][i, 5] # picking [w, h] of instances belonging to the current category cat_inds = np.arange(len(man.cats))[np.array(man.cats) == cat] cat_whs = all_whs[cat_inds] n_inst = cat_whs.shape[0] # trying to fit instances into the candidate box by re-scaling them # by n_scales different factors init_bboxes = np.tile(bbox.reshape(1, 1, 4), (n_inst, n_scales, 1)) new_bboxes = np.concatenate([init_bboxes[:, :, :2], cat_whs], axis=-1) ious = batch_iou(bbox.reshape((1, 4)), new_bboxes.reshape([-1, 4])).ravel() # picking the ones with high IoI good_ones = (ious > 0.7).reshape([n_inst, n_scales]) good_instances = good_ones.sum(axis=1) > 0 instances = np.arange(good_instances.shape[0])[good_instances] inst_paths = [man.inst_paths[cat_inds[i]] for i in instances] scales = [all_scales[m] for m in good_ones[instances]] scales = { inst_paths[i]: np.array([np.min(scales[i]), np.max(scales[i])]) for i in np.arange(len(scales)) } if np.any(good_ones): storage = box2instance.get(name, {k: list() for k in keys}) vals = { 'bboxes': bbox.astype(int), 'cats': cat, 'scores': score, 'inst_paths': inst_paths, 'scales': scales } for key in keys: storage[key].append(vals[key]) box2instance[name] = storage return box2instance