def prepare_instance(self, idx):
"""
Prepare a single instance, can be both multicomponent
or just a single component
"""
instance = self.instances[idx]
if self.opts['skip_multicomponent']:
# Skip_multicomponent is true even during test because we use only
# 1 bbox and no polys
assert len(
instance['components']) == 1, 'Found multicomponent instance\
with skip_multicomponent set to True!'
component = instance['components'][0]
results = self.prepare_component(instance, component)
if 'test' in self.mode:
results['instance'] = instance
else:
if 'test' in self.mode:
component = instance['components'][0]
results = self.prepare_component(instance, component)
if self.opts['ext_points']:
all_comp_gt_poly = []
for component in instance['components']:
if component['area'] < self.opts['min_area']:
continue
else:
comp = self.extract_crop(
component, instance,
results['context_expansion'])
all_comp_gt_poly.extend(comp['poly'].tolist())
all_comp_gt_poly = np.array(
all_comp_gt_poly) * self.opts['img_side']
ex_0, ex_1, ex_2, ex_3 = utils.extreme_points(
all_comp_gt_poly)
nodes = [ex_0, ex_1, ex_2, ex_3]
point_annotation = utils.make_gt(nodes,
h=self.opts['img_side'],
w=self.opts['img_side'])
results['annotation_prior'] = point_annotation
elif 'train' in self.mode:
component = random.choice(instance['components'])
results = self.prepare_component(instance, component)
results['instance'] = instance
# When we have multicomponents turned on, also send the whole instance
# In test, this is used to calculate IoU. In train(RL/Evaluator),
# this is used to calculate the reward
return results
def prepare_component(self, instance, component):
"""
Prepare a single component within an instance
"""
get_gt_poly = 'train' in self.mode or 'oracle' in self.mode
max_num = self.opts['p_num']
pnum = self.opts['p_num']
cp_num = self.opts['cp_num']
# create circle polygon data
pointsnp = np.zeros(shape=(cp_num, 2), dtype=np.float32)
for i in range(cp_num):
thera = 1.0 * i / cp_num * 2 * np.pi
x = np.cos(thera)
y = -np.sin(thera)
pointsnp[i, 0] = x
pointsnp[i, 1] = y
fwd_poly = (0.7 * pointsnp + 1) / 2
arr_fwd_poly = np.ones((cp_num, 2), np.float32) * 0.
arr_fwd_poly[:, :] = fwd_poly
lo, hi = self.opts['random_context']
context_expansion = random.uniform(lo, hi)
crop_info = self.extract_crop(component, instance, context_expansion)
img = crop_info['img']
train_dict = {}
if get_gt_poly:
poly = crop_info['poly']
orig_poly = poly.copy()
gt_orig_poly = poly.copy()
gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28)
# Get masks
vertex_mask = np.zeros((28, 28), np.float32)
edge_mask = np.zeros((28, 28), np.float32)
vertex_mask = utils.get_vertices_mask(gt_orig_poly, vertex_mask)
edge_mask = utils.get_edge_mask(gt_orig_poly, edge_mask)
if self.opts['get_point_annotation']:
gt_poly_224 = np.floor(orig_poly *
self.opts['img_side']).astype(np.int32)
if self.opts['ext_points']:
ex_0, ex_1, ex_2, ex_3 = utils.extreme_points(
gt_poly_224, pert=self.opts['ext_points_pert'])
nodes = [ex_0, ex_1, ex_2, ex_3]
point_annotation = utils.make_gt(nodes,
h=self.opts['img_side'],
w=self.opts['img_side'])
target_annotation = np.array([[0, 0]])
gt_poly = self.uniformsample(poly, pnum)
sampled_poly = self.uniformsample(poly, 70)
arr_gt_poly = np.ones((pnum, 2), np.float32) * 0.
arr_gt_poly[:, :] = gt_poly
ff = utils.poly01_to_poly0g(arr_gt_poly, 36)
#poly_mask = np.zeros((224, 224), np.float32)
#poly_mask = utils.get_vertices_mask(ff, poly_mask)
#poly_mask11 = np.ones((224, 224), np.float32) * 0.
#poly_mask11[:, :] = poly_mask
# Numpy doesn't throw an error if the last index is greater than size
if self.opts['get_point_annotation']:
train_dict = {
'target_annotation': target_annotation,
'sampled_poly': sampled_poly,
'orig_poly': orig_poly,
'gt_poly': arr_gt_poly,
'annotation_prior': point_annotation
}
else:
train_dict = {
'sampled_poly': sampled_poly,
'orig_poly': orig_poly,
'gt_poly': arr_gt_poly
}
boundry_dic = {
'vertex_mask': vertex_mask,
'gt_orig_poly': gt_orig_poly,
'poly_mask': ff,
'edge_mask': edge_mask
}
train_dict.update(boundry_dic)
if 'train' in self.mode:
train_dict['label'] = instance['label']
# for Torch, use CHW, instead of HWC
img = img.transpose(2, 0, 1)
# blank_image
return_dict = {
'img': img,
'fwd_poly': arr_fwd_poly,
'img_path': instance['img_path'],
'patch_w': crop_info['patch_w'],
'starting_point': crop_info['starting_point'],
'context_expansion': context_expansion
}
return_dict.update(train_dict)
return return_dict
def prepare_component(self, instance, component):
"""
Prepare a single component within an instance
"""
get_gt_poly = 'train' in self.mode or 'oracle' in self.mode
max_num = self.opts['p_num']
pnum = self.opts['p_num']
cp_num = self.opts['cp_num']
# create circle polygon data
pointsnp = np.zeros(shape=(cp_num, 2), dtype=np.float32)
for i in range(cp_num):
thera = 1.0 * i / cp_num * 2 * np.pi
x = np.cos(thera)
y = -np.sin(thera)
pointsnp[i, 0] = x
pointsnp[i, 1] = y
fwd_poly = (0.7 * pointsnp + 1) / 2
arr_fwd_poly = np.ones((cp_num, 2), np.float32) * 0.
arr_fwd_poly[:, :] = fwd_poly
lo, hi = self.opts['random_context']
context_expansion = random.uniform(lo, hi)
crop_info = self.extract_crop(component, instance, context_expansion)
img = crop_info['img']
## get the onehot labels and dist_maps for boundary loss
train_dict = {}
if get_gt_poly:
poly = crop_info['poly']
orig_poly = poly.copy()
gt_orig_poly = poly.copy()
gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28)
# Get masks
vertex_mask = np.zeros((28, 28), np.float32)
edge_mask = np.zeros((28, 28), np.float32)
vertex_mask = utils.get_vertices_mask(gt_orig_poly, vertex_mask)
edge_mask = utils.get_edge_mask(gt_orig_poly, edge_mask)
if 'train' in self.mode:
mask = np.asarray(crop_info['mask']) # wh
mask_tensor = torch.from_numpy(mask)
onehot_label = utils.class2one_hot(mask_tensor, 2)[0]
mask_distmap = utils.one_hot2dist(onehot_label)
if self.opts['get_point_annotation']:
gt_poly_224 = np.floor(orig_poly * self.opts['img_side']).astype(np.int32)
if self.opts['ext_points']:
ex_0, ex_1, ex_2, ex_3 = utils.extreme_points(gt_poly_224, pert=self.opts['ext_points_pert'])
nodes = [ex_0, ex_1, ex_2, ex_3]
point_annotation = utils.make_gt(nodes, h=self.opts['img_side'], w=self.opts['img_side'])
target_annotation = np.array([[0, 0]])
gt_poly = self.uniformsample(poly, pnum)
sampled_poly = self.uniformsample(poly, 70)
#sampled_interactive = self.uniformsample(poly, 40)
## uniformsample by 1280%32
interactive_temp = []
for i in range(pnum): #pnum: 1280
if i% 32 == 0:
interactive_temp.append(gt_poly[i,:])
sampled_interactive = np.array(interactive_temp)
#sampled_interactive = self.uniformsample(gt_poly, 40)
# gt_poly = self.uniformsample(sampled_interactive, pnum)
arr_gt_poly = np.ones((pnum, 2), np.float32) * 0.
arr_gt_poly[:, :] = gt_poly
# Numpy doesn't throw an error if the last index is greater than size
if self.opts['get_point_annotation']:
train_dict = {
'target_annotation': target_annotation,
'sampled_poly': sampled_poly,
'orig_poly': orig_poly,
'gt_poly': arr_gt_poly,
'annotation_prior':point_annotation,
'sampled_interactive': sampled_interactive,
}
else:
train_dict = {
'sampled_poly': sampled_poly,
'orig_poly': orig_poly,
'gt_poly': arr_gt_poly,
'sampled_interactive': sampled_interactive,
}
if 'train' in self.mode:
train_dict['onehot_label'] = np.array(onehot_label),
train_dict['mask_distmap'] = np.array(mask_distmap) # cwh
boundry_dic = {
'vertex_mask':vertex_mask,
'edge_mask':edge_mask
}
train_dict.update(boundry_dic)
if 'train' in self.mode:
train_dict['label'] = instance['label']
# for Torch, use CHW, instead of HWC
img = img.transpose(2, 0, 1)
# blank_image
return_dict = {
'img': img,
'fwd_poly': arr_fwd_poly,
'img_path': instance['img_path'],
'patch_w': crop_info['patch_w'],
'starting_point': crop_info['starting_point'],
'context_expansion': context_expansion
}
return_dict.update(train_dict)
return return_dict
def prepare_instance(self, idx):
"""
Prepare a single instance, can be both multicomponent
or just a single component
"""
instance = self.instances[idx]
n_component = len(instance['components'])
n_sample_point = int(self.opts['p_num'] / n_component)
n_additional_point = self.opts['p_num'] - n_component * n_sample_point
if self.opts['skip_multicomponent']:
# Skip_multicomponent is true even during test because we use only
# 1 bbox and no polys
assert len(
instance['components']) == 1, 'Found multicomponent instance\
with skip_multicomponent set to True!'
component = instance['components'][0]
results = self.prepare_component(instance, component)
results['gt_img'] = np.zeros(
(self.opts['diff_iou_dim'], self.opts['diff_iou_dim']),
dtype=np.int32)
results['vertex_mask'] = np.zeros((28, 28), np.float32)
results['edge_mask'] = np.zeros((28, 28), np.float32)
all_comp_gt_poly_list = []
all_comp_gt_poly = []
comp = self.extract_crop(component, instance,
results['context_expansion'])
all_comp_gt_poly_list.append(comp['poly'])
all_comp_gt_poly.extend(comp['poly'].tolist())
all_comp_gt_poly_img_side = np.array(
all_comp_gt_poly) * self.opts['img_side']
ex_0, ex_1, ex_2, ex_3 = utils.extreme_points(
all_comp_gt_poly_img_side)
nodes = [ex_0, ex_1, ex_2, ex_3]
point_annotation = utils.make_gt(nodes,
h=self.opts['img_side'],
w=self.opts['img_side'])
results['annotation_prior'] = point_annotation
for gt_poly in all_comp_gt_poly_list:
gt_orig_poly = gt_poly.copy()
gt_poly = np.array(gt_poly * self.opts['diff_iou_dim'],
dtype=np.int32)
img_mask = utils.masks_from_poly(gt_poly,
self.opts['diff_iou_dim'],
self.opts['diff_iou_dim'])
results['gt_img'] = results['gt_img'] + img_mask.astype(
np.int32)
gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28)
results['vertex_mask'] = utils.get_vertices_mask(
gt_orig_poly, results['vertex_mask'])
results['edge_mask'] = utils.get_edge_mask(
gt_orig_poly, results['edge_mask'])
results['gt_img'][results['gt_img'] > 0] = 255
if 'test' in self.mode:
results['instance'] = instance
else:
if 'test' in self.mode:
component = instance['components'][0]
results = self.prepare_component(instance, component)
if self.opts['ext_points']:
all_comp_gt_poly = []
for component in instance['components']:
if component['area'] < self.opts['min_area']:
continue
else:
comp = self.extract_crop(
component, instance,
results['context_expansion'])
all_comp_gt_poly.extend(comp['poly'].tolist())
all_comp_gt_poly = np.array(
all_comp_gt_poly) * self.opts['img_side']
ex_0, ex_1, ex_2, ex_3 = utils.extreme_points(
all_comp_gt_poly)
nodes = [ex_0, ex_1, ex_2, ex_3]
point_annotation = utils.make_gt(nodes,
h=self.opts['img_side'],
w=self.opts['img_side'])
results['annotation_prior'] = point_annotation
else:
component = random.choice(instance['components'])
results = self.prepare_component(instance, component)
results['gt_img'] = np.zeros(
(self.opts['diff_iou_dim'], self.opts['diff_iou_dim']),
dtype=np.int32)
results['gt_edge_img'] = np.zeros(
(self.opts['diff_iou_dim'], self.opts['diff_iou_dim']),
dtype=np.int32)
results['vertex_mask'] = np.zeros((28, 28), np.float32)
results['edge_mask'] = np.zeros((28, 28), np.float32)
all_comp_gt_poly_list = []
all_comp_gt_poly = []
all_comp_sample_poly_list = []
for component in instance['components']:
if component['area'] < self.opts['min_area']:
continue
else:
comp = self.extract_crop(component, instance,
results['context_expansion'])
all_comp_gt_poly_list.append(comp['poly'])
all_comp_gt_poly.extend(comp['poly'].tolist())
sampled_poly = self.uniformsample(
comp['poly'], n_sample_point)
all_comp_sample_poly_list.append(sampled_poly)
all_comp_sample_poly_list.append(
sampled_poly[:n_additional_point])
all_comp_sample_poly_array = np.vstack(
all_comp_sample_poly_list)
all_comp_gt_poly_img_side = np.array(
all_comp_gt_poly) * self.opts['img_side']
ex_0, ex_1, ex_2, ex_3 = utils.extreme_points(
all_comp_gt_poly_img_side)
nodes = [ex_0, ex_1, ex_2, ex_3]
point_annotation = utils.make_gt(nodes,
h=self.opts['img_side'],
w=self.opts['img_side'])
results['annotation_prior'] = point_annotation
for gt_poly in all_comp_gt_poly_list:
gt_orig_poly = gt_poly.copy()
gt_poly = np.array(gt_poly * self.opts['diff_iou_dim'],
dtype=np.int32)
img_mask = utils.masks_from_poly(gt_poly,
self.opts['diff_iou_dim'],
self.opts['diff_iou_dim'])
results['gt_img'] = results['gt_img'] + img_mask.astype(
np.int32)
results['gt_edge_img'] = utils.get_edge_mask(
gt_poly, results['gt_edge_img'])
gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28)
results['vertex_mask'] = utils.get_vertices_mask(
gt_orig_poly, results['vertex_mask'])
results['edge_mask'] = utils.get_edge_mask(
gt_orig_poly, results['edge_mask'])
results['gt_edge_img'][results['gt_edge_img'] > 0] = 255
results['gt_img'][results['gt_img'] > 0] = 255
results['gt_poly_for_chamfer'] = all_comp_sample_poly_array
results['instance'] = instance
return results