def prepare_instance(self, idx): """ Prepare a single instance, can be both multicomponent or just a single component """ instance = self.instances[idx] if self.opts['skip_multicomponent']: # Skip_multicomponent is true even during test because we use only # 1 bbox and no polys assert len( instance['components']) == 1, 'Found multicomponent instance\ with skip_multicomponent set to True!' component = instance['components'][0] results = self.prepare_component(instance, component) if 'test' in self.mode: results['instance'] = instance else: if 'test' in self.mode: component = instance['components'][0] results = self.prepare_component(instance, component) if self.opts['ext_points']: all_comp_gt_poly = [] for component in instance['components']: if component['area'] < self.opts['min_area']: continue else: comp = self.extract_crop( component, instance, results['context_expansion']) all_comp_gt_poly.extend(comp['poly'].tolist()) all_comp_gt_poly = np.array( all_comp_gt_poly) * self.opts['img_side'] ex_0, ex_1, ex_2, ex_3 = utils.extreme_points( all_comp_gt_poly) nodes = [ex_0, ex_1, ex_2, ex_3] point_annotation = utils.make_gt(nodes, h=self.opts['img_side'], w=self.opts['img_side']) results['annotation_prior'] = point_annotation elif 'train' in self.mode: component = random.choice(instance['components']) results = self.prepare_component(instance, component) results['instance'] = instance # When we have multicomponents turned on, also send the whole instance # In test, this is used to calculate IoU. In train(RL/Evaluator), # this is used to calculate the reward return results
def prepare_component(self, instance, component): """ Prepare a single component within an instance """ get_gt_poly = 'train' in self.mode or 'oracle' in self.mode max_num = self.opts['p_num'] pnum = self.opts['p_num'] cp_num = self.opts['cp_num'] # create circle polygon data pointsnp = np.zeros(shape=(cp_num, 2), dtype=np.float32) for i in range(cp_num): thera = 1.0 * i / cp_num * 2 * np.pi x = np.cos(thera) y = -np.sin(thera) pointsnp[i, 0] = x pointsnp[i, 1] = y fwd_poly = (0.7 * pointsnp + 1) / 2 arr_fwd_poly = np.ones((cp_num, 2), np.float32) * 0. arr_fwd_poly[:, :] = fwd_poly lo, hi = self.opts['random_context'] context_expansion = random.uniform(lo, hi) crop_info = self.extract_crop(component, instance, context_expansion) img = crop_info['img'] train_dict = {} if get_gt_poly: poly = crop_info['poly'] orig_poly = poly.copy() gt_orig_poly = poly.copy() gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28) # Get masks vertex_mask = np.zeros((28, 28), np.float32) edge_mask = np.zeros((28, 28), np.float32) vertex_mask = utils.get_vertices_mask(gt_orig_poly, vertex_mask) edge_mask = utils.get_edge_mask(gt_orig_poly, edge_mask) if self.opts['get_point_annotation']: gt_poly_224 = np.floor(orig_poly * self.opts['img_side']).astype(np.int32) if self.opts['ext_points']: ex_0, ex_1, ex_2, ex_3 = utils.extreme_points( gt_poly_224, pert=self.opts['ext_points_pert']) nodes = [ex_0, ex_1, ex_2, ex_3] point_annotation = utils.make_gt(nodes, h=self.opts['img_side'], w=self.opts['img_side']) target_annotation = np.array([[0, 0]]) gt_poly = self.uniformsample(poly, pnum) sampled_poly = self.uniformsample(poly, 70) arr_gt_poly = np.ones((pnum, 2), np.float32) * 0. arr_gt_poly[:, :] = gt_poly ff = utils.poly01_to_poly0g(arr_gt_poly, 36) #poly_mask = np.zeros((224, 224), np.float32) #poly_mask = utils.get_vertices_mask(ff, poly_mask) #poly_mask11 = np.ones((224, 224), np.float32) * 0. #poly_mask11[:, :] = poly_mask # Numpy doesn't throw an error if the last index is greater than size if self.opts['get_point_annotation']: train_dict = { 'target_annotation': target_annotation, 'sampled_poly': sampled_poly, 'orig_poly': orig_poly, 'gt_poly': arr_gt_poly, 'annotation_prior': point_annotation } else: train_dict = { 'sampled_poly': sampled_poly, 'orig_poly': orig_poly, 'gt_poly': arr_gt_poly } boundry_dic = { 'vertex_mask': vertex_mask, 'gt_orig_poly': gt_orig_poly, 'poly_mask': ff, 'edge_mask': edge_mask } train_dict.update(boundry_dic) if 'train' in self.mode: train_dict['label'] = instance['label'] # for Torch, use CHW, instead of HWC img = img.transpose(2, 0, 1) # blank_image return_dict = { 'img': img, 'fwd_poly': arr_fwd_poly, 'img_path': instance['img_path'], 'patch_w': crop_info['patch_w'], 'starting_point': crop_info['starting_point'], 'context_expansion': context_expansion } return_dict.update(train_dict) return return_dict
def prepare_component(self, instance, component): """ Prepare a single component within an instance """ get_gt_poly = 'train' in self.mode or 'oracle' in self.mode max_num = self.opts['p_num'] pnum = self.opts['p_num'] cp_num = self.opts['cp_num'] # create circle polygon data pointsnp = np.zeros(shape=(cp_num, 2), dtype=np.float32) for i in range(cp_num): thera = 1.0 * i / cp_num * 2 * np.pi x = np.cos(thera) y = -np.sin(thera) pointsnp[i, 0] = x pointsnp[i, 1] = y fwd_poly = (0.7 * pointsnp + 1) / 2 arr_fwd_poly = np.ones((cp_num, 2), np.float32) * 0. arr_fwd_poly[:, :] = fwd_poly lo, hi = self.opts['random_context'] context_expansion = random.uniform(lo, hi) crop_info = self.extract_crop(component, instance, context_expansion) img = crop_info['img'] ## get the onehot labels and dist_maps for boundary loss train_dict = {} if get_gt_poly: poly = crop_info['poly'] orig_poly = poly.copy() gt_orig_poly = poly.copy() gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28) # Get masks vertex_mask = np.zeros((28, 28), np.float32) edge_mask = np.zeros((28, 28), np.float32) vertex_mask = utils.get_vertices_mask(gt_orig_poly, vertex_mask) edge_mask = utils.get_edge_mask(gt_orig_poly, edge_mask) if 'train' in self.mode: mask = np.asarray(crop_info['mask']) # wh mask_tensor = torch.from_numpy(mask) onehot_label = utils.class2one_hot(mask_tensor, 2)[0] mask_distmap = utils.one_hot2dist(onehot_label) if self.opts['get_point_annotation']: gt_poly_224 = np.floor(orig_poly * self.opts['img_side']).astype(np.int32) if self.opts['ext_points']: ex_0, ex_1, ex_2, ex_3 = utils.extreme_points(gt_poly_224, pert=self.opts['ext_points_pert']) nodes = [ex_0, ex_1, ex_2, ex_3] point_annotation = utils.make_gt(nodes, h=self.opts['img_side'], w=self.opts['img_side']) target_annotation = np.array([[0, 0]]) gt_poly = self.uniformsample(poly, pnum) sampled_poly = self.uniformsample(poly, 70) #sampled_interactive = self.uniformsample(poly, 40) ## uniformsample by 1280%32 interactive_temp = [] for i in range(pnum): #pnum: 1280 if i% 32 == 0: interactive_temp.append(gt_poly[i,:]) sampled_interactive = np.array(interactive_temp) #sampled_interactive = self.uniformsample(gt_poly, 40) # gt_poly = self.uniformsample(sampled_interactive, pnum) arr_gt_poly = np.ones((pnum, 2), np.float32) * 0. arr_gt_poly[:, :] = gt_poly # Numpy doesn't throw an error if the last index is greater than size if self.opts['get_point_annotation']: train_dict = { 'target_annotation': target_annotation, 'sampled_poly': sampled_poly, 'orig_poly': orig_poly, 'gt_poly': arr_gt_poly, 'annotation_prior':point_annotation, 'sampled_interactive': sampled_interactive, } else: train_dict = { 'sampled_poly': sampled_poly, 'orig_poly': orig_poly, 'gt_poly': arr_gt_poly, 'sampled_interactive': sampled_interactive, } if 'train' in self.mode: train_dict['onehot_label'] = np.array(onehot_label), train_dict['mask_distmap'] = np.array(mask_distmap) # cwh boundry_dic = { 'vertex_mask':vertex_mask, 'edge_mask':edge_mask } train_dict.update(boundry_dic) if 'train' in self.mode: train_dict['label'] = instance['label'] # for Torch, use CHW, instead of HWC img = img.transpose(2, 0, 1) # blank_image return_dict = { 'img': img, 'fwd_poly': arr_fwd_poly, 'img_path': instance['img_path'], 'patch_w': crop_info['patch_w'], 'starting_point': crop_info['starting_point'], 'context_expansion': context_expansion } return_dict.update(train_dict) return return_dict
def prepare_instance(self, idx): """ Prepare a single instance, can be both multicomponent or just a single component """ instance = self.instances[idx] n_component = len(instance['components']) n_sample_point = int(self.opts['p_num'] / n_component) n_additional_point = self.opts['p_num'] - n_component * n_sample_point if self.opts['skip_multicomponent']: # Skip_multicomponent is true even during test because we use only # 1 bbox and no polys assert len( instance['components']) == 1, 'Found multicomponent instance\ with skip_multicomponent set to True!' component = instance['components'][0] results = self.prepare_component(instance, component) results['gt_img'] = np.zeros( (self.opts['diff_iou_dim'], self.opts['diff_iou_dim']), dtype=np.int32) results['vertex_mask'] = np.zeros((28, 28), np.float32) results['edge_mask'] = np.zeros((28, 28), np.float32) all_comp_gt_poly_list = [] all_comp_gt_poly = [] comp = self.extract_crop(component, instance, results['context_expansion']) all_comp_gt_poly_list.append(comp['poly']) all_comp_gt_poly.extend(comp['poly'].tolist()) all_comp_gt_poly_img_side = np.array( all_comp_gt_poly) * self.opts['img_side'] ex_0, ex_1, ex_2, ex_3 = utils.extreme_points( all_comp_gt_poly_img_side) nodes = [ex_0, ex_1, ex_2, ex_3] point_annotation = utils.make_gt(nodes, h=self.opts['img_side'], w=self.opts['img_side']) results['annotation_prior'] = point_annotation for gt_poly in all_comp_gt_poly_list: gt_orig_poly = gt_poly.copy() gt_poly = np.array(gt_poly * self.opts['diff_iou_dim'], dtype=np.int32) img_mask = utils.masks_from_poly(gt_poly, self.opts['diff_iou_dim'], self.opts['diff_iou_dim']) results['gt_img'] = results['gt_img'] + img_mask.astype( np.int32) gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28) results['vertex_mask'] = utils.get_vertices_mask( gt_orig_poly, results['vertex_mask']) results['edge_mask'] = utils.get_edge_mask( gt_orig_poly, results['edge_mask']) results['gt_img'][results['gt_img'] > 0] = 255 if 'test' in self.mode: results['instance'] = instance else: if 'test' in self.mode: component = instance['components'][0] results = self.prepare_component(instance, component) if self.opts['ext_points']: all_comp_gt_poly = [] for component in instance['components']: if component['area'] < self.opts['min_area']: continue else: comp = self.extract_crop( component, instance, results['context_expansion']) all_comp_gt_poly.extend(comp['poly'].tolist()) all_comp_gt_poly = np.array( all_comp_gt_poly) * self.opts['img_side'] ex_0, ex_1, ex_2, ex_3 = utils.extreme_points( all_comp_gt_poly) nodes = [ex_0, ex_1, ex_2, ex_3] point_annotation = utils.make_gt(nodes, h=self.opts['img_side'], w=self.opts['img_side']) results['annotation_prior'] = point_annotation else: component = random.choice(instance['components']) results = self.prepare_component(instance, component) results['gt_img'] = np.zeros( (self.opts['diff_iou_dim'], self.opts['diff_iou_dim']), dtype=np.int32) results['gt_edge_img'] = np.zeros( (self.opts['diff_iou_dim'], self.opts['diff_iou_dim']), dtype=np.int32) results['vertex_mask'] = np.zeros((28, 28), np.float32) results['edge_mask'] = np.zeros((28, 28), np.float32) all_comp_gt_poly_list = [] all_comp_gt_poly = [] all_comp_sample_poly_list = [] for component in instance['components']: if component['area'] < self.opts['min_area']: continue else: comp = self.extract_crop(component, instance, results['context_expansion']) all_comp_gt_poly_list.append(comp['poly']) all_comp_gt_poly.extend(comp['poly'].tolist()) sampled_poly = self.uniformsample( comp['poly'], n_sample_point) all_comp_sample_poly_list.append(sampled_poly) all_comp_sample_poly_list.append( sampled_poly[:n_additional_point]) all_comp_sample_poly_array = np.vstack( all_comp_sample_poly_list) all_comp_gt_poly_img_side = np.array( all_comp_gt_poly) * self.opts['img_side'] ex_0, ex_1, ex_2, ex_3 = utils.extreme_points( all_comp_gt_poly_img_side) nodes = [ex_0, ex_1, ex_2, ex_3] point_annotation = utils.make_gt(nodes, h=self.opts['img_side'], w=self.opts['img_side']) results['annotation_prior'] = point_annotation for gt_poly in all_comp_gt_poly_list: gt_orig_poly = gt_poly.copy() gt_poly = np.array(gt_poly * self.opts['diff_iou_dim'], dtype=np.int32) img_mask = utils.masks_from_poly(gt_poly, self.opts['diff_iou_dim'], self.opts['diff_iou_dim']) results['gt_img'] = results['gt_img'] + img_mask.astype( np.int32) results['gt_edge_img'] = utils.get_edge_mask( gt_poly, results['gt_edge_img']) gt_orig_poly = utils.poly01_to_poly0g(gt_orig_poly, 28) results['vertex_mask'] = utils.get_vertices_mask( gt_orig_poly, results['vertex_mask']) results['edge_mask'] = utils.get_edge_mask( gt_orig_poly, results['edge_mask']) results['gt_edge_img'][results['gt_edge_img'] > 0] = 255 results['gt_img'][results['gt_img'] > 0] = 255 results['gt_poly_for_chamfer'] = all_comp_sample_poly_array results['instance'] = instance return results