def extract_instances_from_image_voc(name, loader, instance_root):
inst_mask = np.array(Image.open(os.path.join(instance_root, name + '.png')))
gt_bboxes, _, gt_cats, _, _, _ = loader.read_annotations(name)
instance_inds = np.unique(inst_mask)
instance_inds = instance_inds[(instance_inds != 0) * (instance_inds != 255)]
masks, cats = [], []
for iind in instance_inds:
mask = np.ones(inst_mask.shape[:2], dtype=bool)
mask[inst_mask == iind] = 0
coords_y, coords_x = map(np.unique, np.where(mask == 0))
x_min, y_min, x_max, y_max = 0, 0, mask.shape[0], mask.shape[1]
if len(coords_x) > 0:
x_min, x_max = get_ends(coords_x)
if len(coords_y) > 0:
y_min, y_max = get_ends(coords_y)
mask_box = [x_min, y_min, x_max - x_min + 1, y_max - y_min + 1]
mask_box = np.reshape(mask_box, (-1, 4))
ious = batch_iou(gt_bboxes, mask_box).ravel()
best_match = np.argmax(ious)
cat = gt_cats[best_match]
masks.append(mask)
cats.append(cat)
return masks, cats
def dynamic_place_instance(self, name, image, bbox, blending_list):
"""Finds fitting instance given a box to place in, and copies there"""
if type(image) != list:
backgrounds = [image.copy() for i in range(len(blending_list))]
else:
backgrounds = [i.copy() for i in image]
width, height = image.size
xmin, ymin, xmax, ymax = wh2xy(bbox)
mask, foreground, w, h, scale = self.find_instance(name, bbox)
inst_bbox = [0, 0, w, h]
if self.config['colorize']:
foreground = self.color_augment(foreground)
foreground, mask, inst_bbox = self.rescale_instance(
foreground, mask, scale, wh2xy(inst_bbox))
w, h = xy2wh(inst_bbox)[2:]
fits = False
for i in range(50):
new_xc = np.random.randint(xmin - 0.25 * w, xmax + 0.25 * w)
new_yc = np.random.randint(ymin - 0.25 * h, ymax + 0.25 * h)
# new_xc = np.random.randint(xmin - 0.75*w, xmax + 0.75*w)
# new_yc = np.random.randint(ymin - 0.75*h, ymax + 0.75*h)
xmin, ymin = new_xc - int(w / 2), new_yc - int(h / 2)
new_bbox, fits = self.check_bbox([xmin, ymin, w, h], width, height)
iou = batch_iou(new_bbox, bbox)
if (iou > 0.2 and iou < 0.7
or self.no_spatial_constraints) and fits:
backgrounds = self.blend(np.array([xmin,
ymin]), mask, foreground,
backgrounds, blending_list)
return backgrounds, new_bbox
return None, None
def delete_covered_boxes(self, new_bbox, bboxes, cats, diff, given_cat):
"""Handles occlusions"""
ious, intersections = batch_iou(bboxes,
np.reshape(new_bbox, (1, 4)),
return_intersection=True)
assert bboxes.shape[0] == len(cats) == len(diff), \
'Have a problem with number of examples'
areas = bboxes[:, 2] * bboxes[:, 3]
inter_fraq = (intersections.ravel() / areas)
not_occluded = inter_fraq < 0.7
bboxes = bboxes[not_occluded]
cats = np.array(cats)[not_occluded].tolist()
diff = np.array(diff)[not_occluded].tolist()
bboxes = np.concatenate([bboxes, np.reshape(new_bbox, (1, 4))], axis=0)
cats.append(given_cat)
diff.append(0)
return bboxes, cats, diff
def duplicate_instance(self, image, mask_file, fg_image, blending_list):
"""Performs copy-pasting of an instance"""
foreground = fg_image.copy()
width, height = image.size
if type(image) != list:
backgrounds = [image.copy() for i in range(len(blending_list))]
else:
backgrounds = [i.copy() for i in image]
scale = np.random.uniform(1 / 2, 2)
foreground, mask, old_coord = self.extract_mask(mask_file, foreground)
if foreground is None:
return None, None
x_min, y_min, w, h = old_coord
foreground, mask, new_coord = self.rescale_instance(
foreground, mask, scale, wh2xy(old_coord))
if self.config['colorize']:
foreground = self.color_augment(foreground)
xmin, ymin, xmax, ymax = new_coord
old_bbox = xy2wh(new_coord)
wi, hi = old_bbox[2:]
new_wi, new_hi = xmax - xmin, ymax - ymin
fits = False
for i in range(50):
new_xc = np.random.randint(xmin - self.radius * wi,
xmax + self.radius * wi)
new_yc = np.random.randint(ymin - self.radius * hi,
ymax + self.radius * hi)
xmin, ymin = new_xc - int(new_wi / 2), new_yc - int(new_hi / 2)
new_bbox, fits = self.check_bbox([xmin, ymin, new_wi, new_hi],
width, height)
iou = batch_iou(new_bbox, old_bbox)
if (iou > 0.2 and iou < 0.7
or self.no_spatial_constraints) and fits:
backgrounds = self.blend(np.array([xmin,
ymin]), mask, foreground,
backgrounds, blending_list)
return backgrounds, new_bbox
return None, None
def sample_negative(self, name=None):
"""Samples a background contextual image"""
cat = None
while cat is None:
name = np.random.choice(self.loader.filenames) if name is None else name
bboxes, _, cats, w, h, diff = self.loader.read_annotations(name)
image = self.loader.load_image(name)
if len(cats) == 0:
bbox = self.draw_bbox(w, h)
cat = 0
else:
for i in range(self.max_sample_tries):
bbox = self.draw_bbox(w, h)
ious = batch_iou(bboxes, bbox)
area = bbox[0, 2] * bbox[0, 3]
areas = bboxes[:, 2] * bboxes[:, 3]
inter_fraq = (ious * (areas + area) / (1 + ious)) / areas
if np.max(inter_fraq) < 0.3:
cat = 0
break
self.name = name
return [image, bbox, cat, w, h]
def manipulate_image(self, name, bboxes, cats, diff):
"""Perform enlargement data augmentation for an image"""
image = Image.open(self.loader.get_image_path(name))
init_image = image.copy()
new_bboxes = np.array(bboxes)
try:
instances = self.name2instances[name]
except KeyError:
print('No instance segmentation gt for %s' % name)
return image, bboxes, np.array(cats), np.array(diff)
N_min = np.min([self.config['min_objects'], len(instances)])
N_max = np.min([self.config['max_objects'], len(instances)]) + 1
N = np.random.choice(np.arange(N_min, N_max))
np.random.shuffle(instances)
instances = instances[:N]
random_bboxes = np.array(bboxes)
np.random.shuffle(random_bboxes)
random_bboxes = random_bboxes[:N]
blendings = np.random.choice(self.blending_list, N)
for k in range(N):
i = instances[k]
blending = blendings[k]
images, new_bbox, old_bbox = self.enlarge_instance(
image, i, init_image, [blending])
if images is not None:
image = images[0]
ind = np.argmax(batch_iou(new_bboxes, old_bbox), axis=0)
new_bboxes[ind] = new_bbox
# new_bboxes, cats, diff = self.delete_covered_boxes(
# new_bbox, new_bboxes, cats, diff, self.instance2cat(i))
return image, new_bboxes, np.array(cats), np.array(diff)
def match_anchors_to_instances(anchor_dict, excluded, loader):
"""for each candidate box finds a set of instances that could
be inserted inside
Creates a dict structure that for each image retrieves -> a set of
confident context candidate boxes and for each of them -> all possible
instance (with corresponding scaling factor) fitting there
"""
keys = ['bboxes', 'cats', 'scores', 'inst_paths', 'scales']
man = DynamicInstanceManipulator(loader) # contains instance dataset
n_scales = 16
all_scales = np.linspace(0.5, 2.0, n_scales)
n_inst = len(man.ars)
# extracting width and height for all instance boxes in the instance dataset
all_whs = []
for i in range(n_inst):
wh = np.array(man.boxes[i][2:])
whs = np.array([wh * scale for scale in all_scales])
all_whs.append(whs)
all_whs = np.array(all_whs)
box2instance = dict()
bar = progressbar.ProgressBar()
for name in bar(anchor_dict):
# iterating over the candidates for one image
for i in range(anchor_dict[name].shape[0]):
bbox = anchor_dict[name][i, :4]
cat = anchor_dict[name][i, 4]
score = anchor_dict[name][i, 5]
# picking [w, h] of instances belonging to the current category
cat_inds = np.arange(len(man.cats))[np.array(man.cats) == cat]
cat_whs = all_whs[cat_inds]
n_inst = cat_whs.shape[0]
# trying to fit instances into the candidate box by re-scaling them
# by n_scales different factors
init_bboxes = np.tile(bbox.reshape(1, 1, 4), (n_inst, n_scales, 1))
new_bboxes = np.concatenate([init_bboxes[:, :, :2], cat_whs],
axis=-1)
ious = batch_iou(bbox.reshape((1, 4)),
new_bboxes.reshape([-1, 4])).ravel()
# picking the ones with high IoI
good_ones = (ious > 0.7).reshape([n_inst, n_scales])
good_instances = good_ones.sum(axis=1) > 0
instances = np.arange(good_instances.shape[0])[good_instances]
inst_paths = [man.inst_paths[cat_inds[i]] for i in instances]
scales = [all_scales[m] for m in good_ones[instances]]
scales = {
inst_paths[i]: np.array([np.min(scales[i]),
np.max(scales[i])])
for i in np.arange(len(scales))
}
if np.any(good_ones):
storage = box2instance.get(name, {k: list() for k in keys})
vals = {
'bboxes': bbox.astype(int),
'cats': cat,
'scores': score,
'inst_paths': inst_paths,
'scales': scales
}
for key in keys:
storage[key].append(vals[key])
box2instance[name] = storage
return box2instance