def selective_search(I, color_spaces = ['rgb'], ks = [100], feature_masks = [features.SimilarityMask(1, 1, 1, 1)], eraseMap=None, n_jobs = -1): parameters = itertools.product(color_spaces, ks, feature_masks) region_set = joblib.Parallel(n_jobs=n_jobs)(joblib.delayed(_selective_search_one)(I, color, k, mask, eraseMap) for (color, k, mask) in parameters) # region_set = region_set[0:len(region_set):4] #flatten list of list of tuple to list of tuple # regions = sum(region_set, []) return region_set # sorted(regions), region_labels
def test_similarity_user_all(self, monkeypatch): monkeypatch.setattr(features.Features, '_Features__sim_size', lambda self, i, j: 1) monkeypatch.setattr(features.Features, '_Features__sim_texture', lambda self, i, j: 1) monkeypatch.setattr(features.Features, '_Features__sim_color', lambda self, i, j: 1) monkeypatch.setattr(features.Features, '_Features__sim_fill', lambda self, i, j: 1) w = features.SimilarityMask(1, 1, 1, 1) f = features.Features(self.dummy_image, self.dummy_label, 1, w) assert f.similarity(0, 1) == 4
def __parameter_changed(self): # obtain parameters color_spaces = [color.lower() for color in self.chosen_colors] ks = [float(k) for k in self.chosen_ks] similarity_masks = [ features.SimilarityMask('S' in mask, 'C' in mask, 'T' in mask, 'F' in mask) for mask in self.chosen_similarities ] self.regions = selective_search.selective_search( self.ndimg, color_spaces, ks, similarity_masks) self.slider.setMaximum(len(self.regions)) self.slider.setValue(int(len(self.regions) / 4)) self.__draw()
def hierarchical_segmentation(I, k=100, feature_mask=features.SimilarityMask(1, 1, 1, 1)): F0, n_region = segment.segment_label(I, 0.8, k, 100) adj_mat, A0 = _calc_adjacency_matrix(F0, n_region) feature_extractor = features.Features(I, F0, n_region) # stores list of regions sorted by their similarity S = _build_initial_similarity_set(A0, feature_extractor) # stores region label and its parent (empty if initial). R = {i: () for i in range(n_region)} A = [A0] # stores adjacency relation for each step F = [F0] # stores label image for each step # greedy hierarchical grouping loop while len(S): (s, (i, j)) = S.pop() t = feature_extractor.merge(i, j) # record merged region (larger region should come first) R[t] = ( i, j) if feature_extractor.size[j] < feature_extractor.size[i] else ( j, i) Ak = _new_adjacency_dict(A[-1], i, j, t) A.append(Ak) S = _merge_similarity_set(feature_extractor, Ak, S, i, j, t) F.append(_new_label_image(F[-1], i, j, t)) # bounding boxes for each hierarchy L = feature_extractor.bbox return (R, F, L)
def gen_regions(image, dims, pad, ks): """ Generates candidate regions for object detection using selective search. """ print "Generating cropped regions..." assert (len(dims) == 3) regions = selective_search(image, ks=[ks], feature_masks=[ features.SimilarityMask( size=1, color=1, texture=1, fill=1, ) ]) crops = [] for conf, (y0, x0, y1, x1) in regions: if x0 - pad >= 0: x0 = x0 - pad if y0 - pad >= 0: y0 = y0 - pad if x1 + pad <= dims[0]: x1 = x1 + pad if y1 + pad <= dims[0]: y1 = y1 + pad # Images are rows, then columns, then channels. region = image[y0:y1, x0:x1, :] candidate = resize(region, dims) crops.append((conf, candidate, region, (x0, y0, x1, y1))) print "Generated {} crops".format(len(crops)) return crops
parser.add_argument('-k', '--k', type=int, default=100, help='threshold k for initial segmentation') parser.add_argument('-c', '--color', nargs=1, default='rgb', choices=['rgb', 'lab', 'rgi', 'hsv', 'nrgb', 'hue'], help='color space') parser.add_argument('-f', '--feature', nargs="+", default=['texture', 'fill'], choices=['size', 'color', 'texture', 'fill'], help='feature for similarity calculation') parser.add_argument('-o', '--output', type=str, default='result', help='prefix of resulting images') parser.add_argument('-a', '--alpha', type=float, default=1.0, help='alpha value for compositing result image with input image') args = parser.parse_args() img = skimage.io.imread(args.image) if len(img.shape) == 2: img = skimage.color.gray2rgb(img) print('k:', args.k) print('color:', args.color) print('feature:', ' '.join(args.feature)) mask = features.SimilarityMask('size' in args.feature, 'color' in args.feature, 'texture' in args.feature, 'fill' in args.feature) (R, F, L) = selective_search.hierarchical_segmentation(img, args.k, mask) print('result filename: %s_[0000-%04d].png' % (args.output, len(F) - 1)) # suppress warning when saving result images warnings.filterwarnings("ignore", category = UserWarning) step_title = "[INFO] Generating segments: " widgets = [step_title, progressbar.Percentage(), " ", progressbar.Bar(), " ", progressbar.ETA()] pbar = progressbar.ProgressBar(maxval=len(F), widgets=widgets).start() colors = generate_color_table(R) for depth, label in enumerate(F): result = colors[label] result = (result * args.alpha + img * (1. - args.alpha)).astype(numpy.uint8)
def hierarchical_segmentation(I, feature_mask=features.SimilarityMask(1, 1, 1, 1), F0=None, k=100, return_stacks=False): """ I: Image Returns: R: Merge dictionary (key: label of parent, value: (label of child1, label of child2)) scales: scale dictionary (key: label, value: scale) """ relabeled = False #pass #Remap if labels are not contiguous sorted_labels = np.asarray(sorted(np.unique(F0).ravel())) if (np.any((sorted_labels[1:] - sorted_labels[0:-1]) > 1)): relabeled = True map_dict = {sorted_labels[i]: i for i in range(sorted_labels.shape[0])} F0 = relabel(F0, map_dict) n_region = np.unique(F0.ravel()).shape[0] adj_mat, A0 = _calc_adjacency_matrix(F0, n_region) feature_extractor = features.Features(I, F0, n_region, feature_mask) # stores list of regions sorted by their similarity S = _build_initial_similarity_set(A0, feature_extractor) #Initialize scale dictionary unique_labels = np.unique(F0.ravel()).tolist() g = nx.DiGraph() g.add_nodes_from(unique_labels, n_elems=1, stack=0) # stores region label and its parent (empty if initial). R = {i: () for i in range(n_region)} A = [A0] # stores adjacency relation for each step F = [F0] # stores label image for each step # greedy hierarchical grouping loop stack = 0 while len(S): stack += 1 (s, (i, j)) = S.pop() t = feature_extractor.merge(i, j) n_elems_left = nx.get_node_attributes(g, 'n_elems')[i] n_elems_right = nx.get_node_attributes(g, 'n_elems')[j] new_n_elems = n_elems_left + n_elems_right g.add_node(t, n_elems=new_n_elems, stack=stack) g.add_edge(t, i, diff_n_elems=new_n_elems - n_elems_left) g.add_edge(t, j, diff_n_elems=new_n_elems - n_elems_right) # record merged region (larger region should come first) R[t] = ( i, j) if feature_extractor.size[j] < feature_extractor.size[i] else ( j, i) Ak = _new_adjacency_dict(A[-1], i, j, t) A.append(Ak) S = _merge_similarity_set(feature_extractor, Ak, S, i, j, t) F.append(_new_label_image(F[-1], i, j, t)) # bounding boxes for each hierarchy L = feature_extractor.bbox if (relabeled): inv_map = {v: k for k, v in map_dict.items()} g = nx.relabel_nodes(g, inv_map, copy=True) #g = nx.relabel_nodes(g,map_dict) if (relabeled and return_stacks): print('relabeling stacks') with progressbar.ProgressBar(maxval=len(F)) as bar: for i in range(len(F)): bar.update(i) F[i] = relabel(F[i], inv_map) if (return_stacks): return (R, F, g) else: return g
def hierarchical_segmentation(I, k = 100, feature_mask = features.SimilarityMask(1, 1, 1, 1), eraseMap=None): # F0, n_region = segment.segment_label(I, 0.8, k, 100) F0, n_region = segment.segment_label(I, 0.5, k, 500) n_region = F0.max() + 1 # ++ calculate outside region labels if eraseMap is not None: eraseLabels = set(list(F0[numpy.where(eraseMap == 1)].flatten())) else: eraseLabels = [] adj_mat, A0 = _calc_adjacency_matrix(F0, n_region) feature_extractor = features.Features(I, F0, n_region) # stores list of regions sorted by their similarity S = _build_initial_similarity_set(A0, feature_extractor) # ++ pop out regions outside circle if eraseMap is not None: ii = len(S) - 1 while ii >= 0: if (S[ii][1][0] in eraseLabels) or (S[ii][1][1] in eraseLabels): S.pop(ii) ii -= 1 # stores region label and its parent (empty if initial). R = {i : () for i in range(n_region)} A = [A0] # stores adjacency relation for each step F = [F0] # stores label image for each step # greedy hierarchical grouping loop while len(S): (s, (i, j)) = S.pop() t = feature_extractor.merge(i, j) # record merged region (larger region should come first) R[t] = (i, j) if feature_extractor.size[j] < feature_extractor.size[i] else (j, i) Ak = _new_adjacency_dict(A[-1], i, j, t) A.append(Ak) S = _merge_similarity_set(feature_extractor, Ak, S, i, j, t) # ++ pop out regions outside circle if eraseMap is not None: ii = len(S) - 1 while ii >= 0: if (S[ii][1][0] in eraseLabels) or (S[ii][1][1] in eraseLabels): S.pop(ii) ii -= 1 F.append(_new_label_image(F[-1], i, j, t)) # bounding boxes for each hierarchy L = feature_extractor.bbox L_regions = {} for r, l in R.iteritems(): if r < n_region: L_regions[r] = [r] else: ll = [] if l[0] >= n_region: ll = ll + L_regions[l[0]] else: ll.append(l[0]) if l[1] >= n_region: ll = ll + L_regions[l[1]] else: ll.append(l[1]) L_regions[r] = ll return (R, F, L, L_regions, eraseLabels)
def hierarchical_segmentation_M(paras, feature_mask = features.SimilarityMask(1, 1, 1, 1)): # F0, n_region = segment.segment_label(I, 0.8, k, 100) # F0, n_region = segment.segment_label(I, 0.5, k, 500) # ++ calculate outside region labels train = paras['train'] is_rotate = paras['is_rotate'] if train: eraseRegionLabels = paras['eraseRegionLabels'] F0, region_labels, eraseLabels = rcm.region_special_map(paras) # regions_special = [] # regions_rest = [] # regions_common = [] # for _, v in region_labels.iteritems(): # regions_special = regions_special + v[0] # regions_rest = regions_rest + v[1] # regions_common = regions_common + v[2] coordinates_special = numpy.zeros((0, 2)) print region_labels for ls in region_labels.values()[0][0]: # print 'ls: ' + str(ls) coordinates_special = numpy.vstack([coordinates_special, numpy.argwhere(F0==ls)]) pca = PCA() pca.fit(coordinates_special) components = pca.components_ main_ax = components[0] angle = math.atan(main_ax[0] / main_ax[1]) * (180.0 / math.pi) # print 'F0 before: ' # print F0.dtype # eraseLabels = list(eraseLabels) # el = eraseLabels[0] # for ell in eraseLabels: # F0[numpy.where(F0==ell)] = el # F0 = numpy.array(rotate(F0, angle).round(), dtype='i') # print 'F0 after: ' # print F0.dtype n_region = len(set(list(F0.flatten()))) I = paras['im'] # print 'I before: ' # print I.dtype # I = numpy.array(rotate(I, angle), dtype='i') # print 'I after: ' # print I.dtype eraseRegions_fb = {'fg': [], 'bg': []} for rl in range(3): if rl in eraseRegionLabels: eraseRegions_fb['fg'] = eraseRegions_fb['fg'] + region_labels.values()[0][rl] else: eraseRegions_fb['bg'] = eraseRegions_fb['bg'] + region_labels.values()[0][rl] Rd = {} Fd = {} Ld = {} L_regionsd = {} for fb_k, fb_v in eraseRegions_fb.iteritems(): print 'n_region', n_region print fb_k, fb_v eraseLabels_k = set(fb_v + list(eraseLabels)) adj_mat, A0 = _calc_adjacency_matrix(F0, n_region) feature_extractor = features.Features(I, F0, n_region) # stores list of regions sorted by their similarity S = _build_initial_similarity_set(A0, feature_extractor) # ++ pop out regions outside circle and selected common ii = len(S) - 1 while ii >= 0: if (S[ii][1][0] in eraseLabels_k) or (S[ii][1][1] in eraseLabels_k): S.pop(ii) ii -= 1 # stores region label and its parent (empty if initial). R = {i : () for i in range(n_region)} A = [A0] # stores adjacency relation for each step F = [F0] # stores label image for each step # greedy hierarchical grouping loop while len(S): (s, (i, j)) = S.pop() t = feature_extractor.merge(i, j) # record merged region (larger region should come first) R[t] = (i, j) if feature_extractor.size[j] < feature_extractor.size[i] else (j, i) Ak = _new_adjacency_dict(A[-1], i, j, t) A.append(Ak) S = _merge_similarity_set(feature_extractor, Ak, S, i, j, t) # ++ pop out regions outside circle and selected common ii = len(S) - 1 while ii >= 0: if (S[ii][1][0] in eraseLabels_k) or (S[ii][1][1] in eraseLabels_k): S.pop(ii) ii -= 1 F.append(_new_label_image(F[-1], i, j, t)) # bounding boxes for each hierarchy L = feature_extractor.bbox L_regions = {} for r, l in R.iteritems(): if r < n_region: L_regions[r] = [r] else: ll = [] if l[0] >= n_region: ll = ll + L_regions[l[0]] else: ll.append(l[0]) if l[1] >= n_region: ll = ll + L_regions[l[1]] else: ll.append(l[1]) L_regions[r] = ll Rd[fb_k] = R Fd[fb_k] = F Ld[fb_k] = L L_regionsd[fb_k] = L_regions return (Rd, Fd, Ld, L_regionsd, eraseRegions_fb) else: # if is_rotate: # F0, region_labels, eraseLabels, special_labels = rcm.region_special_map(paras) # coordinates_special = numpy.zeros((0, 2)) # for ls in special_labels: # print 'ls: ' + str(ls) # coordinates_special = numpy.vstack([coordinates_special, numpy.argwhere(F0 == ls)]) # print 'coordinates_special', coordinates_special # pca = PCA() # pca.fit(coordinates_special) # # components = pca.components_ # main_ax = components[0] # angle = math.atan(main_ax[0] / main_ax[1]) * (180.0 / math.pi) # else: # F0, region_labels, eraseLabels = rcm.region_special_map(paras) F0, region_labels, eraseLabels, _ = rcm.region_special_map(paras) eraseLabels = set(region_labels + list(eraseLabels)) n_region = len(set(list(F0.flatten()))) I = paras['im'] special_labels = [x for x in range(n_region) if x not in eraseLabels] # print special_labels if len(special_labels) == 0: angle = 0 else: coordinates_special = numpy.zeros((0, 2)) for ls in special_labels: # print 'ls: ' + str(ls) coordinates_special = numpy.vstack([coordinates_special, numpy.argwhere(F0 == ls)]) # print 'coordinates_special', coordinates_special pca = PCA() pca.fit(coordinates_special) components = pca.components_ main_ax = components[0] angle = math.atan(main_ax[0] / main_ax[1]) * (180.0 / math.pi) # if eraseMap is not None: # eraseLabels = set(list(F0[numpy.where(eraseMap == 1)].flatten())) adj_mat, A0 = _calc_adjacency_matrix(F0, n_region) feature_extractor = features.Features(I, F0, n_region) # stores list of regions sorted by their similarity S = _build_initial_similarity_set(A0, feature_extractor) # ++ pop out regions outside circle and selected common ii = len(S) - 1 while ii >= 0: if (S[ii][1][0] in eraseLabels) or (S[ii][1][1] in eraseLabels): S.pop(ii) ii -= 1 # stores region label and its parent (empty if initial). R = {i : () for i in range(n_region)} A = [A0] # stores adjacency relation for each step F = [F0] # stores label image for each step # greedy hierarchical grouping loop while len(S): (s, (i, j)) = S.pop() t = feature_extractor.merge(i, j) # record merged region (larger region should come first) R[t] = (i, j) if feature_extractor.size[j] < feature_extractor.size[i] else (j, i) Ak = _new_adjacency_dict(A[-1], i, j, t) A.append(Ak) S = _merge_similarity_set(feature_extractor, Ak, S, i, j, t) # ++ pop out regions outside circle and selected common ii = len(S) - 1 while ii >= 0: if (S[ii][1][0] in eraseLabels) or (S[ii][1][1] in eraseLabels): S.pop(ii) ii -= 1 F.append(_new_label_image(F[-1], i, j, t)) # bounding boxes for each hierarchy L = feature_extractor.bbox L_regions = {} for r, l in R.iteritems(): if r < n_region: L_regions[r] = [r] else: ll = [] if l[0] >= n_region: ll = ll + L_regions[l[0]] else: ll.append(l[0]) if l[1] >= n_region: ll = ll + L_regions[l[1]] else: ll.append(l[1]) L_regions[r] = ll if is_rotate: return (R, F, L, L_regions, eraseLabels, angle) else: return (R, F, L, L_regions, eraseLabels)
def generate_with_selective_search_3d(l_start, l_end, dataset, gt_set, im_in_DIR, im_out_DIR, mat_out_DIR, data, cfg): """ generate the proposals with the selective search method """ assert data in ['MASK','MAT'], \ '[ERROR] unknow type of data: {}'.format(data) im_names = dataset.sets[gt_set]["im_names"] timer = Timer() for im_i in xrange(l_start, l_end): im_nm = im_names[im_i] im_out_pn = osp.join(im_out_DIR, im_nm + '.png') mat_out_pn = osp.join(mat_out_DIR, im_nm + '.mat') _, im_pn = dataset.built_im_path(im_nm, im_in_DIR) im_RAW = None timer.tic() if data == 'MASK' and not osp.exists(im_out_pn): im_RAW = cv2.imread(im_pn) im_RAW = cv2.cvtColor(im_RAW, cv2.COLOR_BGR2RGB) mask = features.SimilarityMask('size' in cfg.SEGM_SS_FEATURE, 'color' in cfg.SEGM_SS_FEATURE, 'texture' in cfg.SEGM_SS_FEATURE, 'fill' in cfg.SEGM_SS_FEATURE) (R, F, _) = selective_search.hierarchical_segmentation( im_RAW, cfg.SEGM_SS_K, mask) colors = generate_color_table(R) is_mask = False for depth, label in enumerate(F): result = colors[label] result = (result * cfg.SEGM_SS_ALPHA + im_RAW * (1. - cfg.SEGM_SS_ALPHA)).astype(np.uint8) # extract on targeted depth if depth == cfg.SEGM_SS_DEPTH: mask_RAW = result is_mask = True break assert is_mask == True, "[ERROR] unable to extract segm on depth {}".format( im_nm) # save raw-img output mask_RAW = np.array(mask_RAW, dtype=np.int8) #print 'mask_RAW.shape: {}'.format(mask_RAW.shape) cv2.imwrite(im_out_pn, mask_RAW) timer.toc() print '[INFO] %d/%d use time (s) %f' % (im_i + 1, len(im_names), timer.average_time) """ parse proposals as a MAT for easy handle """ if data == 'MAT' and not osp.exists(mat_out_pn): assert osp.exists(im_out_pn), \ '[ERROR] missing MASK for {}. Please generate MASK first.'.format(im_out_pn) #print 'im_out_pn: {}'.format(im_out_pn) mask_RAW = cv2.imread(im_out_pn) mask_RAW = cv2.cvtColor(mask_RAW, cv2.COLOR_BGR2RGB) R, G, B = cv2.split(mask_RAW) R_uniq_vals = np.unique(R) G_uniq_vals = np.unique(G) B_uniq_vals = np.unique(B) #print 'len(R_uniq_vals): {}'.format(len(R_uniq_vals)) #print 'len(G_uniq_vals): {}'.format(len(G_uniq_vals)) #print 'len(B_uniq_vals): {}'.format(len(B_uniq_vals)) labels = [] bboxes = [] superpixels = np.zeros((mask_RAW.shape[0], mask_RAW.shape[1]), dtype=np.float) #num_ids = min(len(R_uniq_vals)*len(G_uniq_vals)*len(B_uniq_vals),255) num_ids = len(R_uniq_vals) * len(G_uniq_vals) * len(B_uniq_vals) labels_ids = np.random.permutation(xrange(1, num_ids + 1)) #print 'num_ids: {}'.format(num_ids) #print 'len(labels_ids): {}'.format(len(labels_ids)) #print 'labels_ids: {}'.format(labels_ids) idx = 0 for R_val in R_uniq_vals: for G_val in G_uniq_vals: for B_val in B_uniq_vals: [row_ids, col_ids] = np.array(np.where((R == R_val) & \ (G == G_val) & \ (B == B_val))) if len(row_ids) > 0 and len(row_ids) == len(col_ids): y1 = np.min(row_ids) x1 = np.min(col_ids) y2 = np.max(row_ids) x2 = np.max(col_ids) box = np.array([x1, y1, x2, y2]) assert idx < num_ids, \ '[ERROR] idx out labels range: {} vs. {}'.format(idx,num_ids) label = labels_ids[idx] superpixels[row_ids, col_ids] = label labels.append(label) bboxes.append(box) idx += 1 im_mat = { "labels": labels, "bboxes": bboxes, "superpixels": superpixels } # save mat-img output sio.savemat(mat_out_pn, im_mat, do_compression=True) timer.toc() print '[INFO] %d/%d use time (s) %f' % (im_i + 1, len(im_names), timer.average_time)
def test_similarity_user_all(self): self.setup_method() w = features.SimilarityMask(1, 1, 1, 1) f = features.Features(self.dummy_image, self.dummy_label, 1, w) assert f.similarity(0, 1) == 4