def fit(self, X, y, idx):#img is unused here but is needed for the pipeline self.unaryEnergy = np.ascontiguousarray(-self.MULTIPLIER*np.log(X)).astype('int32') #energy=-log(probability) nClass = self.unaryEnergy.shape[2] params = np.logspace(-3,3,10) #grid search values for Potts compatibility best_score = 0.0 print('Fitting Grid MRF...') for i in range(params.shape[0]): pairwiseEnergy = (self.MULTIPLIER*params[i]*(1-np.eye(nClass))).astype('int32') out = gco.cut_simple(unary_cost=self.unaryEnergy,\ pairwise_cost=pairwiseEnergy,n_iter=self.INFERENCE_NITER, algorithm='swap').ravel() score = np.sum(out[idx]==y)/float(y.size) if score > best_score: self.cost = params[i] best_score = score params = np.logspace(np.log10(self.cost)-1,np.log10(self.cost)+1,30) best_score = 0.0 print('Finetuning Grid MRF...') for i in range(params.shape[0]): pairwiseEnergy = (self.MULTIPLIER*params[i]*(1-np.eye(nClass))).astype('int32') out = gco.cut_simple(unary_cost=self.unaryEnergy,\ pairwise_cost=pairwiseEnergy,n_iter=self.INFERENCE_NITER, algorithm='swap').ravel() score = np.sum(out[idx]==y)/float(y.size) if score > best_score: self.cost = params[i] best_score = score
def example_multinomial(): # generate dataset with three stripes np.random.seed(15) x = np.zeros((10, 12, 3)) x[:, :4, 0] = -1 x[:, 4:8, 1] = -1 x[:, 8:, 2] = -1 unaries = x + 1.5 * np.random.normal(size=x.shape) x = np.argmin(x, axis=2) unaries = (unaries * 10).astype(np.int32) x_thresh = np.argmin(unaries, axis=2) # potts potential pairwise_potts = -2 * np.eye(3, dtype=np.int32) result = cut_simple(unaries, 10 * pairwise_potts) # potential that penalizes 0-1 and 1-2 less thann 0-2 pairwise_1d = -15 * np.eye(3, dtype=np.int32) - 8 pairwise_1d[-1, 0] = 0 pairwise_1d[0, -1] = 0 print(pairwise_1d) result_1d = cut_simple(unaries, pairwise_1d) plt.subplot(141, title="original") plt.imshow(x, interpolation="nearest") plt.subplot(142, title="thresholded unaries") plt.imshow(x_thresh, interpolation="nearest") plt.subplot(143, title="potts potentials") plt.imshow(result, interpolation="nearest") plt.subplot(144, title="1d topology potentials") plt.imshow(result_1d, interpolation="nearest") plt.show()
def test_label_costs_simple(self): """Test the label_costs argument with cut_simple.""" unaries, pairwise, edges, expected = self.binary_data() # Give a slight preference to class 0 unaries[:, :, 1] += 1 result = cut_simple(unaries, pairwise, label_cost=1) self.assertTrue(np.array_equal(result, expected)) # Try again with a very high label cost to collapse to a single label result = cut_simple(unaries, pairwise, label_cost=1000) self.assertTrue(np.array_equal(result, np.zeros_like(result)))
def predict(self, X): #self.cost = 2. self.unaryEnergy = np.ascontiguousarray(-self.MULTIPLIER*np.log(X)).astype('int32') nClass = self.unaryEnergy.shape[2] pairwiseEnergy = (self.MULTIPLIER*self.cost*(1-np.eye(nClass))).astype('int32') return gco.cut_simple(unary_cost=self.unaryEnergy,\ pairwise_cost=pairwiseEnergy,n_iter=self.INFERENCE_NITER,algorithm='swap')
def original_example(img1, img2, max_disp): # Code was based on this example unaries = (unaries_ssd(img1, img2, max_disp) * 100).astype(np.int32) n_disps = unaries.shape[2] newshape = unaries.shape[:2] potts_cut1 = cut_simple(unaries, -5 * np.eye(n_disps, dtype=np.int32)) potts_cut2 = cut_simple(unaries, -5 * np.eye(n_disps, dtype=np.int32), n_iter=10) x, y = np.ogrid[:n_disps, :n_disps] one_d_topology = np.abs(x - y).astype(np.int32).copy("C") one_d_cut1 = cut_simple(unaries, 5 * one_d_topology) one_d_cut2 = cut_simple(unaries, 5 * one_d_topology, n_iter=10) return one_d_cut1, one_d_cut2, potts_cut1, potts_cut2
def test_cut_simple(self): """Test the cut_simple method.""" unaries, pairwise, edges, expected = self.binary_data() result = cut_simple(unaries, pairwise) self.assertTrue(np.array_equal(result, expected))
def Post_Processing(prob_map, height, width, num_classes, y_test, test_indexes): Gamma = [10, 20, 30, 50, 100, 150, 200] # Gamma = [20] SL = np.zeros([len(Gamma), height, width]) SAE = np.zeros([len(Gamma), num_classes]) SA = np.zeros([len(Gamma)]) for j in range(len(Gamma)): gamma = Gamma[j] unaries = (-gamma * np.log(prob_map + 1e-4)).astype( np.int32) # 20 15 una = unaries_reshape(unaries, width, height, num_classes) one_d_topology = (np.ones(num_classes) - np.eye(num_classes)).astype( np.int32).copy("C") Seg_Label = cut_simple(una, 100 * one_d_topology) # 30 200 Seg_Label = Seg_Label + 1 seg_Label = Seg_Label.transpose().flatten() test_indexes = test_indexes.astype(np.int32) cmat = confusion_matrix(y_test, seg_Label[test_indexes]) cmat.astype(float) a = cmat.diagonal() a = np.array([float(a[i]) for i in range(0, len(a))]) b = cmat.sum(axis=1) seg_accuracy_each = a / b seg_accuracy = accuracy_score(y_test, seg_Label[test_indexes]) SL[j, :, :] = Seg_Label SAE[j, :] = seg_accuracy_each SA[j] = seg_accuracy SA = SA.tolist() max_ind = SA.index(max(SA)) seg_accuracy = SA[max_ind] Seg_Label = SL[max_ind, :, :] seg_accuracy_each = SAE[max_ind, :] return Seg_Label, seg_accuracy, seg_accuracy_each
def segment_simple_cut(self): # noinspection PyUnresolvedReferences from pygco import cut_simple unaries = -np.log(np.array([np.clip(self.posterior_images[o],a_min=0.01, a_max=1.0) for o in self.candidate_objects])).transpose((1, 2, 0)) num_labels = unaries.shape[2] p_same_label = 0.99 pairwise = -np.log(1 - p_same_label) * (1 - np.eye(num_labels)) - np.log(p_same_label) * np.eye(num_labels) k = 10 # scaling factor for potentials to reduce aliasing because the potentials need to be converted to integers self.segmentation = cut_simple(np.copy((k * unaries).astype('int32'), order='C'), np.copy((k * pairwise).astype('int32'), order='C'))
def Post_Processing(prob_map, height, width, num_classes, y_test, test_indexes): unaries = (-100 * np.log(prob_map + 1e-4)).astype(np.int32) una = unaries_reshape(unaries, width, height, num_classes) one_d_topology = (np.ones(num_classes) - np.eye(num_classes)).astype( np.int32).copy("C") Seg_Label = cut_simple(una, 50 * one_d_topology) Seg_Label = Seg_Label + 1 seg_Label = Seg_Label.transpose().flatten() seg_accuracy = accuracy_score(y_test, seg_Label[test_indexes]) return Seg_Label, seg_accuracy
def segmentation(depth_file, opts): depth = preprocess_depth(depth_file, opts['preprocess_depth']) # Potts pairwise potential used for graph cut algorithm pairwise = (-100 * np.eye(2)).astype(np.int32) depth = depth.astype(np.int32) depth = depth - opts['board_depth'] depth = (np.dstack([depth, -depth]).copy("C")).astype(np.int32) segmask = cut_simple(depth.astype(np.int32), pairwise.astype(np.int32)) # Invert mask segmask = (segmask+1) % 2 return segmask
def example_binary(): # generate trivial data x = np.ones((10, 10)) x[:, 5:] = -1 x_noisy = x + np.random.normal(0, 0.8, size=x.shape) x_thresh = x_noisy > 0.0 # create unaries unaries = x_noisy # as we convert to int, we need to multipy to get sensible values unaries = (10 * np.dstack([unaries, -unaries]).copy("C")).astype(np.int32) # create potts pairwise pairwise = -10 * np.eye(2, dtype=np.int32) # do simple cut result = cut_simple(unaries, pairwise) # generalized Potts potentials pix_nums = np.r_[: 10 * 10].reshape(10, 10) pairwise_cost = dict( [(tuple(sorted(pair)), 30) for pair in zip(pix_nums[:, :-1].flatten(), pix_nums[:, 1:].flatten())] + [(tuple(sorted(pair)), 0) for pair in zip(pix_nums[:-1, :].flatten(), pix_nums[1:, :].flatten())] ) result_gp = cut_simple_gen_potts(unaries, pairwise_cost) # use the gerneral graph algorithm # first, we construct the grid graph inds = np.arange(x.size).reshape(x.shape) horz = np.c_[inds[:, :-1].ravel(), inds[:, 1:].ravel()] vert = np.c_[inds[:-1, :].ravel(), inds[1:, :].ravel()] edges = np.vstack([horz, vert]).astype(np.int32) # we flatten the unaries result_graph = cut_from_graph(edges, unaries.reshape(-1, 2), pairwise) # generalized Potts potentials result_graph_gp = cut_from_graph_gen_potts(unaries.reshape(-1, 2), pairwise_cost) # plot results plt.subplot(231, title="original") plt.imshow(x, interpolation="nearest") plt.subplot(232, title="noisy version") plt.imshow(x_noisy, interpolation="nearest") plt.subplot(233, title="rounded to integers") plt.imshow(unaries[:, :, 0], interpolation="nearest") plt.subplot(234, title="thresholding result") plt.imshow(x_thresh, interpolation="nearest") plt.subplot(235, title="cut_simple") plt.imshow(result, interpolation="nearest") plt.subplot(236, title="cut_from_graph") plt.imshow(result_graph.reshape(x.shape), interpolation="nearest") plt.show()
def potts_example(): img1 = np.asarray(Image.open("scene1.row3.col1.ppm")) / 255. img2 = np.asarray(Image.open("scene1.row3.col2.ppm")) / 255. unaries = (stereo_unaries(img1, img2) * 100).astype(np.int32) n_disps = unaries.shape[2] newshape = unaries.shape[:2] potts_cut = cut_simple(unaries, -5 * np.eye(n_disps, dtype=np.int32)) x, y = np.ogrid[:n_disps, :n_disps] one_d_topology = np.abs(x - y).astype(np.int32).copy("C") one_d_cut = cut_simple(unaries, 5 * one_d_topology) plt.subplot(231, xticks=(), yticks=()) plt.imshow(img1) plt.subplot(232, xticks=(), yticks=()) plt.imshow(img2) plt.subplot(233, xticks=(), yticks=()) plt.imshow(np.argmin(unaries, axis=2), interpolation='nearest') plt.subplot(223, xticks=(), yticks=()) plt.imshow(potts_cut.reshape(newshape), interpolation='nearest') plt.subplot(224, xticks=(), yticks=()) plt.imshow(one_d_cut.reshape(newshape), interpolation='nearest') plt.show()
def segment_simple_cut(self): # noinspection PyUnresolvedReferences from pygco import cut_simple unaries = -np.log( np.array([ np.clip(self.posterior_images[o], a_min=0.01, a_max=1.0) for o in self.candidate_objects ])).transpose((1, 2, 0)) num_labels = unaries.shape[2] p_same_label = 0.99 pairwise = -np.log(1 - p_same_label) * ( 1 - np.eye(num_labels)) - np.log(p_same_label) * np.eye(num_labels) k = 10 # scaling factor for potentials to reduce aliasing because the potentials need to be converted to integers self.segmentation = cut_simple( np.copy((k * unaries).astype('int32'), order='C'), np.copy((k * pairwise).astype('int32'), order='C'))
def graphcut(self,label_costs, l=100): num_classes = len(self.colors) pairwise_costs = np.zeros((num_classes, num_classes)) for ii in range(num_classes): for jj in range(num_classes): c1 = np.array(self.colors[ii]) c2 = np.array(self.colors[jj]) pairwise_costs[ii,jj] = np.linalg.norm(c1-c2) label_costs_int32 = np.ascontiguousarray(label_costs).astype('int32') pairwise_costs_int32 = (l*pairwise_costs).astype('int32') if self.sobel: edges = self.get_edges() vv_int32 = edges.astype('int32') vh_int32 = edges.astype('int32') new_labels = pygco.cut_simple_vh(label_costs_int32, pairwise_costs_int32, vv_int32, vh_int32, n_iter=10, algorithm='swap') else: new_labels = pygco.cut_simple(label_costs_int32, pairwise_costs_int32, n_iter=10, algorithm='swap') return new_labels
def example_binary(): # generate trivial data x = np.ones((10, 10)) x[:, 5:] = -1 x_noisy = x + np.random.normal(0, 0.8, size=x.shape) x_thresh = x_noisy > .0 # create unaries unaries = x_noisy # as we convert to int, we need to multipy to get sensible values unaries = (10 * np.dstack([unaries, -unaries]).copy("C")).astype(np.int32) # create potts pairwise pairwise = -10 * np.eye(2, dtype=np.int32) # do simple cut result = cut_simple(unaries, pairwise) print unaries print result
def example_binary(): # generate trivial data x = np.ones((10, 10)) x[:, 5:] = -1 x_noisy = x + np.random.normal(0, 0.8, size=x.shape) x_thresh = x_noisy > .0 # create unaries unaries = x_noisy # as we convert to int, we need to multipy to get sensible values unaries = (10 * np.dstack([unaries, -unaries]).copy("C")).astype(np.int32) # create potts pairwise pairwise = -10 * np.eye(2, dtype=np.int32) # do simple cut result = cut_simple(unaries, pairwise) # use the gerneral graph algorithm # first, we construct the grid graph inds = np.arange(x.size).reshape(x.shape) horz = np.c_[inds[:, :-1].ravel(), inds[:, 1:].ravel()] vert = np.c_[inds[:-1, :].ravel(), inds[1:, :].ravel()] edges = np.vstack([horz, vert]).astype(np.int32) # we flatten the unaries result_graph = cut_from_graph(edges, unaries.reshape(-1, 2), pairwise) # plot results plt.subplot(231, title="original") plt.imshow(x, interpolation='nearest') plt.subplot(232, title="noisy version") plt.imshow(x_noisy, interpolation='nearest') plt.subplot(233, title="rounded to integers") plt.imshow(unaries[:, :, 0], interpolation='nearest') plt.subplot(234, title="thresholding result") plt.imshow(x_thresh, interpolation='nearest') plt.subplot(235, title="cut_simple") plt.imshow(result, interpolation='nearest') plt.subplot(236, title="cut_from_graph") plt.imshow(result_graph.reshape(x.shape), interpolation='nearest') plt.show()
def graph_cut(img_list, gaussian_size, unary_scale, pair_scale, n_iter): imGray_list = [] for img in img_list: imGray_list.append(cvt_to_grayscale(img)) n = len(imGray_list) unary_cost = [] ii, jj = np.meshgrid(range(n), range(n)) pairwise_cost = np.abs(ii - jj) * pair_scale for imGray in imGray_list: gray_img = imGray.astype(np.float32) / 255. grad = np.exp(-(cv2.Sobel(gray_img, cv2.CV_32F, 1, 1)**2)) unary_cost.append( cv2.GaussianBlur(grad, (gaussian_size, gaussian_size), 0) * unary_scale) unary_cost = normalization(np.stack(unary_cost, axis=-1)) * unary_scale graph_img = cut_simple(unary_cost.astype(np.int32), pairwise_cost.astype(np.int32), n_iter) return graph_img
def some_cut(unaries, binaries, K, n_iter): # Performs the graph cut n_disps = unaries.shape[2] newshape = unaries.shape[:2] cut = cut_simple(unaries, K * binaries, n_iter) return cut
def regularized_fine(lenses, fine_costs, disp, penalty1, penalty2, max_cost, conf_tec='mlm', conf_sigma=0.3, min_thresh=2.0, eps=0.0000001): fine_depths = dict() fine_depths_interp = dict() fine_depths_val = dict() wta_depths = dict() wta_depths_interp = dict() wta_depths_val = dict() num_lenses = len(lenses) confidence = dict() for i, l in enumerate(fine_costs): if i%1000==0: print("Regularization: Processing lens {0}/{1}".format(i, num_lenses)) lens = lenses[l] # prepare the cost shape: disparity axis is third axis (index [2] instead of [0]) F = np.flipud(np.rot90(fine_costs[l].T)) # the regularized cost volume sgm_cost = rtxsgm.sgm(lenses[l].img, F, lens.mask, penalty1, penalty2, False, max_cost) # plain minima fine_depths[l] = np.argmin(sgm_cost, axis=2) # interpolated minima and values #fine_depths_interp[l], fine_depths_val[l] = rtxdisp.cost_minima_interp(sgm_cost, disp) # cost should be C-Contiguous sgm_cost_c = sgm_cost.copy(order='C') # also in int32 sgm_cost_c_int = (sgm_cost_c * 1000).astype(np.int32) # number of disparities n_disps = F.shape[2] cut = 'unary' if cut == 'potts': # potts model depth_cut = cut_simple(sgm_cost_c_int, -5 * np.eye(sgm_cost.shape[2], dtype=np.int32)) elif cut == 'unary': # unary model x, y = np.ogrid[:n_disps, :n_disps] one_d_topology = np.abs(x - y).astype(np.int32).copy("C") #pdb.set_trace() depth_cut = cut_simple(sgm_cost_c_int, 5 * one_d_topology) else: print("No cut recognised. Do you wish to use potts or unary model?") pdb.set_trace() fine_depths_interp[l] = depth_cut #if i%1000==0: # print("max interp: {0}".format(np.amax(fine_depths_interp[l]))) # plain winner takes all minima wta_depths[l] = np.argmin(F, axis=2) # interpolated minima and values from the unregularized cost volume wta_depths_interp[l], wta_depths_val[l] = rtxdisp.cost_minima_interp(F, disp) fine_depths_val[l] = wta_depths_val[l] ### CALCULATE THE CONFIDENCE USING A METHOD minimum_costs = np.min(sgm_cost, axis=2) #pdb.set_trace() if conf_tec == 'oev': # TODO max does not work on arrays num_denom = 0 dmax = np.max(sgm_cost) dmin = np.min(sgm_cost) denom_denom = max(sgm_cost - minimum_costs[:,:,None], 1) for n in range(0, sgm_cost.shape[2]): index_map = np.ones((sgm_cost.shape[0], sgm_cost.shape[1])) * n tmp_num = np.pow(max(min(index_map - fine_depths[l], (dmax - dmin)/3), 0), 2) num_denom += tmp_num / denom_denom[:,:,n] confidence[l] = 1 / num_denom elif conf_tec == 'rtvbf': confidence[l] = np.sum(np.exp(-((sgm_cost - fine_depths_val[l][:, :, None])**2) / conf_sigma), axis=2) - 1 # confidence measure used in "Real-Time Visibility-Based Fusion of Depth Maps" # substract 1 at the end since the "real" optimium is included in the vectorized operations # confidence[l][fine_depths_val[l] > min_thresh] = 0.0 #TODO: calculate wta confidence, scale the sigma accordingly #confidence[l] = np.sum(np.exp(-((F - wta_depths_val[l][:, :, None])**2) / conf_sigma), axis=2) - 1 # avoid overflow in division ind = confidence[l] > eps confidence[l][confidence[l] <= 0] = 0.0 confidence[l][ind] = 1.0 / confidence[l][ind] else: # TODO # check overflow in exp # and division for zero #conf_tec == 'mlm': exp_cost = np.exp(-minimum_costs/(2*np.power(conf_sigma,2))) denom_cost = np.sum(np.exp(-sgm_cost/(2*np.power(conf_sigma,2))), axis=2) #zeros = denom_cost == 0 #denom_cost = denom_cost + zeros * np.max(denom_cost) confidence_map = exp_cost / denom_cost confidence_map[np.isnan(confidence_map)] = 0 confidence[l] = confidence_map return fine_depths, fine_depths_interp, fine_depths_val, wta_depths, wta_depths_interp, wta_depths_val, confidence