def castRayBetweenRegions(p1, q1, ls_list, junctions, reg_i, reg_j): # collect intersecting edges intersec_set = set() for i, ls in enumerate(ls_list): k, l = ls p2, q2 = junctions[k], junctions[l] if doIntersect(p1, q1, p2, q2): intersec_set.add((k, l)) # print(k, l) # print(intersec_set) # # DEBUG # comb_reg = np.stack([reg_i, reg_j]) # comb_reg = np.clip(np.sum(comb_reg, 0), 0, 1) # comb_reg = Image.fromarray(comb_reg*255.0).convert('RGB') # dr = ImageDraw.Draw(comb_reg) # x1, y1 = p1 # x2, y2 = q1 # x3, y3 = p2 # x4, y4 = q2 # dr.line((x1, y1, x2, y2), fill='green', width=4) # dr.line((x3, y3, x4, y4), fill='red', width=1) # print(doIntersect(p1, q1, p2, q2)) # plt.figure() # plt.imshow(comb_reg) # plt.show() return intersec_set
def castRay(pt, th, ls_list, junctions, large_region, region_small, other_regions, ray_length=1000.0, thresh=0.0): # compute ray x1, y1 = int(pt[0]), int(pt[1]) rad = np.radians(th) dy = np.sin(rad) * ray_length dx = np.cos(rad) * ray_length x2, y2 = x1 + dx, y1 + dy # collect intersecting edges intersec_set = set() for i, ls in enumerate(ls_list): k, l = ls p1, q1 = (x1, y1), (x2, y2) p2, q2 = junctions[k], junctions[l] if doIntersect(p1, q1, p2, q2): intersec_set.add((k, l)) # # DEBUG # deb = Image.fromarray(region_small*255.0).convert('RGB') # dr = ImageDraw.Draw(deb) # for ls in list(intersec_set): # k, l = ls # p2, q2 = junctions[k], junctions[l] # x3, y3 = p2 # x4, y4 = q2 # dr.line((x3, y3, x4, y4), fill='red', width=1) # dr.line((x1, y1, x2, y2), fill='green', width=4) # plt.imshow(deb) # plt.show() # check intersection with other regions intersec_region = False if len(other_regions) > 0: comb_reg = np.clip(np.sum(np.array(other_regions), 0), 0, 1) # cast ray ray_im = Image.new('L', (256, 256)) dr = ImageDraw.Draw(ray_im) dr.line((x1, y1, x2, y2), fill='white', width=8) ray = np.array(ray_im) / 255.0 intersec = np.array(np.where(np.logical_and(ray, comb_reg) > 0)) intersec_region = (intersec.shape[1] > 0) # # DEBUG # print(comb_reg.shape) # comb_reg = Image.fromarray(comb_reg*255.0).convert('RGB') # dr = ImageDraw.Draw(comb_reg) # dr.line((x1, y1, x2, y2), fill='green', width=4) # print(intersec) # print(intersec_region) # plt.figure() # plt.imshow(comb_reg) # plt.show() # check self intersection ray_im = Image.new('L', (256, 256)) dr = ImageDraw.Draw(ray_im) dr.line((x1, y1, x2, y2), fill='white', width=8) dr.ellipse((x1 - 4, y1 - 4, x1 + 4, y1 + 4), fill='black') ray = np.array(ray_im) / 255.0 self_intersect = np.logical_and(ray, region_small).sum() / (ray.sum() + 1e-8) self_inter = False if self_intersect > thresh: intersec_region = True self_inter = True # # DEBUG # print(self_intersect) # deb = Image.fromarray(region_small*255.0).convert('RGB') # dr = ImageDraw.Draw(deb) # # for k, l in list(intersec_set): # # p2, q2 = junctions[k], junctions[l] # # x3, y3 = p2 # # x4, y4 = q2 # # dr.line((x3, y3, x4, y4), fill='red', width=1) # print(intersec_region) # dr.line((x1, y1, x2, y2), fill='green', width=1) # plt.imshow(deb) # plt.show() # # DEBUG return intersec_set, intersec_region, self_inter
def castRayRegion(pt, th, ls_list, junctions, sm_reg_i, sm_other_regs, l_reg, l_other_regs, other_regs_id, ray_length=1000.0, thresh=0.0): # compute ray x1, y1 = int(pt[0]), int(pt[1]) rad = np.radians(th) dy = np.sin(rad) * ray_length dx = np.cos(rad) * ray_length x2, y2 = x1 + dx, y1 + dy p1, q1 = (x1, y1), (x2, y2) # render ray ray_im = Image.new('L', (256, 256)) draw = ImageDraw.Draw(ray_im) draw.line((x1, y1, x2, y2), fill='white') ray = np.array(ray_im) # check region intersection background = np.zeros((256, 256)) comb_reg = np.zeros((256, 256)) for _id, l_reg_j, sm_reg_j in zip(other_regs_id, l_other_regs, sm_other_regs): reg_j_tag = np.array(sm_reg_j) inds = np.where(reg_j_tag > 0) reg_j_tag[inds] = _id + 1 comb_reg += reg_j_tag background += l_reg_j background += l_reg intersect = np.logical_and(ray, comb_reg) inds = np.where(intersect > 0) intersect_tags = np.array(comb_reg[inds]) # DEBUG comb_reg = np.clip(comb_reg, 0, 1) comb_reg = Image.fromarray(comb_reg * 255.0).convert('RGB') dr = ImageDraw.Draw(comb_reg) dr.line((x1, y1, x2, y2), fill='green') dr.ellipse((x1 - 2, y1 - 2, x1 + 2, y1 + 2), fill='blue') # DEBUG # compute distances intersec_set = set() self_intersect = None region_id = None inds = np.array(inds) if inds.shape[1] > 0: px = np.array([x1, y1])[np.newaxis, :] pts = inds.transpose(1, 0)[:, ::-1] dists = np.sqrt((np.sum((pts - px)**2, -1))) closest_pt = pts[np.argmin(dists), :] region_id = intersect_tags[np.argmin(dists)] - 1 xc, yc = closest_pt # collect intersecting edges for i, ls in enumerate(ls_list): k, l = ls p2, q2 = junctions[k], junctions[l] if doIntersect(p1, closest_pt, p2, q2): intersec_set.add((k, l)) # DEBUG -- Draw intersecting edges for ls in list(intersec_set): k, l = ls p2, q2 = junctions[k], junctions[l] x3, y3 = p2 x4, y4 = q2 dr.line((x3, y3, x4, y4), fill='red', width=1) # DEBUG # DEBUG dr.ellipse((xc - 2, yc - 2, xc + 2, yc + 2), fill='magenta') #print(region_id) # DEBUG # check self intersection ray_im = Image.new('L', (256, 256)) dr = ImageDraw.Draw(ray_im) dr.line((x1, y1, xc, yc), fill='white', width=16) dr.ellipse((x1 - 4, y1 - 4, x1 + 4, y1 + 4), fill='black') ray = np.array(ray_im) / 255.0 self_intersect = np.logical_and(ray, sm_reg_i).sum() / (ray.sum() + 1e-8) self_intersect = self_intersect > thresh # render ray ray_im = Image.new('L', (256, 256)) draw = ImageDraw.Draw(ray_im) draw.line((x1, y1, xc, yc), fill='white', width=16) ray = np.array(ray_im) / 255.0 # check background intersection background_im = Image.fromarray(background * 255.0) background_im = background_im.filter(ImageFilter.MaxFilter(9)) background = 1.0 - (np.array(background_im) / 255.0) background[background > 0.5] = 1.0 background[background <= 0.5] = 0.0 background_intersection = np.logical_and(ray, background).sum() if background_intersection > 0: self_intersect = True # # DEBUG # deb = Image.fromarray(background*255.0).convert('RGB') # dr = ImageDraw.Draw(deb) # dr.line((x1, y1, xc, yc), fill='green') # print(background_intersection) # plt.imshow(deb) # plt.show() # # DEBUG # print(self_intersect) # plt.imshow(comb_reg) # plt.show() return intersec_set, region_id, self_intersect
def reconstructBuildingBaseline(junctions, edge_map, regions=None, with_weighted_junctions=True, with_corner_variables=False, with_edge_confidence=False, with_corner_edge_confidence=False, \ lw_from_cls=None, use_edge_classifier=False,\ corner_min_degree_constraint=False, ignore_invalid_corners=False, use_junctions_with_var=False, \ use_regions=False, corner_suppression=False, corner_penalty=False, \ junction_suppression=False, \ intersection_constraint=False, angle_constraint=False, use_junctions=False, use_loops=False, \ dist_thresh=None, angle_thresh=None, edge_threshold=None, corner_edge_thresh=None, thetas=None, corner_confs=None, \ theta_threshold=0.2, region_hit_threshold=None, theta_confs=None, filter_size=11, \ region_weight=1000.0, post_process=False, \ edge_map_weight=1.0, junctions_weight=1.0, inter_region_weight=1.0, wrong_dir_weight=1.0, closed_region_weight=1.0, closed_region_lowerbound=False, closed_region_upperbound=False,\ region_intersection_constraint=False, inter_region_constraint=False,\ junctions_soft=False): # create a new model m = Model("building_reconstruction_baseline") m.setParam('OutputFlag', False) obj = LinExpr(0) num_junc = len(junctions) # list primitives js_list = [k for k in range(num_junc)] if ignore_invalid_corners: js_list = [k for k in js_list if len(thetas[k]) >= 2] ls_list = [(k, l) for k in js_list for l in js_list if l > k] # create variables if with_corner_variables: js_var_dict = {} for j in js_list: js_var_dict[j] = m.addVar(vtype=GRB.BINARY, name="junc_{}".format(j)) ls_var_dict = {} for k, l in ls_list: ls_var_dict[(k, l)] = m.addVar(vtype=GRB.BINARY, name="line_{}_{}".format(k, l)) # edgeness objective if with_edge_confidence: for k, l in ls_list: if use_edge_classifier: try: lw = lw_from_cls[(k, l)] except: lw = lw_from_cls[(l, k)] else: lw = getLineWeight(edge_map, junctions[k], junctions[l]) obj += (lw - edge_threshold) * ls_var_dict[ (k, l)] # favor edges with over .5? elif with_corner_edge_confidence: for k, l in ls_list: if use_edge_classifier: try: lw = lw_from_cls[(k, l)] except: lw = lw_from_cls[(l, k)] else: lw = getLineWeight(edge_map, junctions[k], junctions[l]) #obj += (lw-0.1)*ls_var_dict[(k, l)] #obj += (corner_confs[k]-corner_threshold)*(corner_confs[l]-corner_threshold)*(lw-edge_threshold)*ls_var_dict[(k, l)] # favor edges with over .5? #print((np.prod([corner_confs[k], corner_confs[l], lw])-corner_edge_thresh)) obj += edge_map_weight * ( np.prod([corner_confs[k], corner_confs[l], lw]) - corner_edge_thresh) * ls_var_dict[ (k, l)] # favor edges with over .5? else: for k, l in ls_list: obj += ls_var_dict[(k, l)] if with_corner_variables: # corner-edge connectivity constraint for k, l in ls_list: m.addConstr((js_var_dict[k] + js_var_dict[l] - 2) * ls_var_dict[(k, l)] == 0, "c_{}_{}".format(k, l)) ########################################################################################################## ############################################### OPTIONAL ################################################# ########################################################################################################## if use_regions: reg_list = [] reg_var_ls = {} reg_sm = {} reg_contour = {} for i, reg in enumerate(regions): # apply min filter reg_small = Image.fromarray(reg * 255.0) reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size)) reg_small = np.array(reg_small) / 255.0 # ignore too small regions inds = np.argwhere(reg_small > 0) if np.array(inds).shape[0] > 0: ret, thresh = cv2.threshold( np.array(reg_small * 255.0).astype('uint8'), 127, 255, 0) _, contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) lg_contour = [None, 0] for c in contours: cont = Image.new('L', (256, 256)) dr = ImageDraw.Draw(cont) c = c.reshape(-1, 2) c = [(x, y) for x, y in c] if len(c) <= 2: continue dr.polygon(c, fill='white') # plt.imshow(cont) # plt.show() size = (np.array(cont) / 255.0).sum() if size > lg_contour[1]: lg_contour = [c, size] contours = lg_contour[0] contours = np.array(contours) if len(contours.shape) > 0: reg_contour[i] = contours.reshape(-1, 2) inds = np.linspace(0, reg_contour[i].shape[0], min(int(reg_contour[i].shape[0] / 2), reg_contour[i].shape[0]), endpoint=False).astype('int') reg_contour[i] = reg_contour[i][inds, :] reg_list.append(i) reg_var_ls[i] = m.addVar(vtype=GRB.BINARY, name="reg_{}".format(i)) reg_sm[i] = reg_small obj += region_weight * reg_var_ls[i] for i in reg_list: if region_intersection_constraint: # compute intersection constraint for k, l in ls_list: intersec = getIntersection(reg_sm[i], junctions[k], junctions[l]) if intersec >= region_hit_threshold: m.addConstr(ls_var_dict[(k, l)] * reg_var_ls[i] == 0, "r1_{}_{}".format(k, l)) # add loop constraints ths, pts = compute_normals(reg_contour[i], reg_sm[i]) # # DEBUG -- SAMPLED POINTS # deb = Image.fromarray(reg_sm[i]*255.0).convert('RGB') # dr = ImageDraw.Draw(deb) # for pt in pts: # x, y = pt # dr.ellipse((x-2, y-2, x+2, y+2), fill='green') # plt.imshow(deb) # plt.show() other_regions = [regions[j] for j in reg_list if i != j] sm_other_regions = [reg_sm[j] for j in reg_list if i != j] for pt, th in zip(pts, ths): # closed region soft constraint -- upperbound intersec_edges, intersec_region, self_intersect = castRay( pt, th, ls_list, junctions, regions[i], reg_sm[i], other_regions, ray_length=1000.0) # if self_intersect: # intersec_edges, intersec_region, self_intersect = castRay(pt, th, ls_list, junctions, regions[i], reg_sm[i], other_regions, ray_length=20.0) sum_in_set = LinExpr(0) for e in list(intersec_edges): k, l = e sum_in_set += ls_var_dict[(k, l)] if not intersec_region and closed_region_upperbound: slack_var = m.addVar(vtype=GRB.INTEGER, name="slack_var_{}_{}".format(th, i)) m.addConstr( sum_in_set * reg_var_ls[i] <= reg_var_ls[i] + slack_var, "r2_{}_{}".format(th, i)) obj -= closed_region_weight * slack_var m.addConstr(slack_var >= 0) if closed_region_lowerbound: # closed region hard constraint -- lowerbound slack_var = m.addVar(vtype=GRB.INTEGER) m.addConstr( sum_in_set * reg_var_ls[i] >= reg_var_ls[i] - slack_var, "r2_{}".format(th)) obj -= closed_region_weight * slack_var m.addConstr(slack_var >= 0) if inter_region_constraint: # inter region soft constraint other_regions_id = [j for j in reg_list if i != j] intersec_set, region_id, self_intersect = castRayRegion( pt, th, ls_list, junctions, reg_sm[i], sm_other_regions, regions[i], other_regions, other_regions_id) if (region_id is not None) and (not self_intersect): sum_in_set = LinExpr(0) # DEBUG # deb = Image.fromarray(reg_sm[i]*255.0).convert('RGB') # dr = ImageDraw.Draw(deb) # DEBUG for e in list(intersec_set): k, l = e sum_in_set += ls_var_dict[(k, l)] # # DEBUG # p2, q2 = junctions[k], junctions[l] # x3, y3 = p2 # x4, y4 = q2 # dr.line((x3, y3, x4, y4), fill='red', width=1) # plt.imshow(deb) # plt.show() # DEBUG slack_var_up = m.addVar( vtype=GRB.INTEGER, name="slack_var_inter_up_{}_{}".format(th, i)) slack_var_low = m.addVar( vtype=GRB.INTEGER, name="slack_var_inter_low_{}_{}".format(th, i)) #m.addConstr(sum_in_set*reg_var_ls[i] == reg_var_ls[i] + slack_var, "r3_{}_{}".format(th, i)) m.addConstr( sum_in_set * reg_var_ls[i] >= 1 - slack_var_low) m.addConstr( sum_in_set * reg_var_ls[i] <= 1 + slack_var_up) m.addConstr(slack_var_low >= 0) m.addConstr(slack_var_up >= 0) obj -= inter_region_weight * slack_var_low + inter_region_weight * slack_var_up # if corner_penalty: # for j in js_list: # obj -= 2*js_var_dict[j] if intersection_constraint: # intersection constraint for k, (j0, j1) in enumerate(ls_list): for l, (j2, j3) in enumerate(ls_list): if l > k: p1, q1 = junctions[j0], junctions[j1] p2, q2 = junctions[j2], junctions[j3] if doIntersect(p1, q1, p2, q2): m.addConstr( ls_var_dict[(j0, j1)] * ls_var_dict[(j2, j3)] == 0, "i_{}_{}_{}_{}".format(j0, j1, j2, j3)) if use_junctions_with_var or use_junctions: for j1 in js_list: # consider only valid degrees # if len(thetas[j1]) >= 2: # create list of lines for each junction lines_sets = [LinExpr(0) for _ in range(len(thetas[j1]) + 1)] lines_sets_deb = [list() for _ in range(len(thetas[j1]) + 1)] lines_max_in_sets = [0.0 for _ in range(len(thetas[j1]) + 1)] for j2 in js_list: if j1 != j2: # get line var if (j1, j2) in ls_var_dict: ls_var = ls_var_dict[(j1, j2)] else: ls_var = ls_var_dict[(j2, j1)] # check each line angle at junction in_sets = False for i, a in enumerate(thetas[j1]): lb = (a - angle_thresh) if ( a - angle_thresh) >= 0 else 360.0 + (a - angle_thresh) up = (a + angle_thresh) % 360.0 pt1 = junctions[j1] pt2 = junctions[j2] ajl = getAngle(pt1, pt2) if inBetween(ajl, lb, up): if use_edge_classifier: try: lw = lw_from_cls[(j1, j2)] except: lw = lw_from_cls[(j2, j1)] else: lw = getLineWeight(edge_map, pt1, pt2) lines_max_in_sets[i] = max(lines_max_in_sets[i], lw) lines_sets[i] += ls_var in_sets = True #print(i, j1, j2, ajl, a, lb, up, inBetween(ajl, lb, up)) # not in any direction set if not in_sets: lines_sets[-1] += ls_var # print(lines_sets_deb) # # Debug # x1, y1 = junctions[j1] # for angle_i, line_set in enumerate(lines_sets_deb): # im_deb = Image.new('RGB', (256, 256)) # dr = ImageDraw.Draw(im_deb) # dr.ellipse((x1-2, y1-2, x1+2, y1+2), fill='blue') # for v in line_set: # x2, y2 = junctions[v] # dr.line((x1, y1, x2, y2), fill='green', width=2) # dr.ellipse((x2-2, y2-2, x2+2, y2+2), fill='red') # if angle_i < len(thetas[j1]): # print(thetas[j1][angle_i]) # else: # print('Others') # plt.imshow(im_deb) # plt.show() # add to constraints #set_sum = QuadExpr(0) # add all sets for i in range(len(thetas[j1])): if use_junctions_with_var: junc_th_var = m.addVar(vtype=GRB.BINARY, name="angle_{}".format(j1)) obj += junctions_weight * junc_th_var * ( np.prod([corner_confs[j1], theta_confs[j1][i]]) - theta_threshold) m.addConstr(lines_sets[i] == junc_th_var, "a_{}_{}".format(i, j1)) # OLD #obj += (np.prod([lines_max_in_sets[i], theta_confs[j1][i]])-theta_threshold)*junc_th_var #set_sum += junc_th_var*lines_sets[i] #m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1)) else: m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1)) # OLD #set_sum += junc_th_var*lines_sets[i] # # add not in set -- SOFT if junctions_soft: slack_var = m.addVar(vtype=GRB.INTEGER, name="slack_var_{}".format(j1)) m.addConstr(lines_sets[-1] - slack_var == 0, "a_{}_{}".format(-1, j1)) obj -= wrong_dir_weight * slack_var m.addConstr(slack_var >= 0) else: # add not in set -- HARD m.addConstr(lines_sets[-1] == 0, "a_{}_{}".format(-1, j1)) # OLD #set_sum += junc_th_var*lines_sets[-1] # if use_junctions_with_var: # # final constraint # m.addConstr(set_sum == junc_th_var*len(thetas[j1]), "a_sum_{}".format(j1)) if corner_suppression: # junction spatial constraint junc_sets = set() for j1 in js_list: junc_intersec_set = [] for j2 in js_list: pt1 = np.array(junctions[j1]) pt2 = np.array(junctions[j2]) dist = np.linalg.norm(pt1 - pt2) if dist < dist_thresh: junc_intersec_set.append(j2) junc_intersec_set = tuple(np.sort(junc_intersec_set)) junc_sets.add(junc_intersec_set) # avoid duplicated constraints for js_tuple in junc_sets: junc_expr = LinExpr(0) for j in np.array(js_tuple): junc_expr += js_var_dict[j] m.addConstr(junc_expr <= 1, "s_{}".format(j1)) if junction_suppression: angs = [] for j1 in js_list: # compute angles and degree at each junction deg_j1 = QuadExpr(0) angs = [] for j2 in js_list: if j1 != j2: deg_j1 += ls_var_dict[(j1, j2)] if ( j1, j2) in ls_var_dict else ls_var_dict[(j2, j1)] pt1 = junctions[j1] pt2 = junctions[j2] a12 = getAngle(pt1, pt2) angs.append(a12) else: angs.append(0.0) angs = np.array(angs) ang_diffs = np.abs(180.0 - np.abs(angs[:, np.newaxis] - angs[np.newaxis, :])) inds = np.array(np.where(ang_diffs <= 10.0)).transpose(1, 0) keep_track = [] for j2, j3 in inds: if j1 != j2 and j2 != j3 and j3 != j1: js = tuple(np.sort([j1, j2, j3])) if js not in keep_track: keep_track.append(js) ls_var_12 = ls_var_dict[(j1, j2)] if ( j1, j2) in ls_var_dict else ls_var_dict[(j2, j1)] ls_var_13 = ls_var_dict[(j1, j3)] if ( j1, j3) in ls_var_dict else ls_var_dict[(j3, j1)] m.addConstr((deg_j1 - 2) * js_var_dict[j1] >= ls_var_12 * ls_var_13, "j_3_{}_{}_{}".format(j1, j2, j3)) if corner_min_degree_constraint: # degree constraint for j in js_list: # degree expression deg_j = QuadExpr(0) for k, l in ls_list: if (j == k) or (j == l): deg_j += ls_var_dict[(k, l)] # degree constraint - active junctions must have degree >= 2 m.addConstr(deg_j * js_var_dict[j] >= 2 * js_var_dict[j], "d_1_{}".format(j)) # set optimizer m.setObjective(obj, GRB.MAXIMIZE) m.optimize() # parse solution juncs_on = [] lines_on = [] regs_sm_on = [] for v in m.getVars(): if 'junc' in v.varName and v.x >= .5: juncs_on.append(int(v.varName.split('_')[-1])) elif 'line' in v.varName and v.x >= .5: lines_on.append( (int(v.varName.split('_')[-2]), int(v.varName.split('_')[-1]))) elif 'reg' in v.varName and v.x >= .5: #print('REGION ON') reg_id = int(v.varName.split('_')[-1]) reg = regions[reg_id] reg_small = Image.fromarray(reg * 255.0) reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size)) regs_sm_on.append(reg_small) elif 'slack_var_inter' in v.varName: print(v.varName, v.x) if not with_corner_variables: juncs_on = np.array(list(set(sum(lines_on, ())))) if use_regions: if post_process: juncs_on, lines_on = remove_junctions(junctions, juncs_on, lines_on) return junctions, juncs_on, lines_on, regs_sm_on if post_process: juncs_on, lines_on = remove_junctions(junctions, juncs_on, lines_on) return junctions, juncs_on, lines_on
def reconstructBuildingBaseline(junctions, edge_map, regions=None, with_weighted_junctions=True, with_corner_variables=False, with_edge_confidence=False, with_corner_edge_confidence=False, \ corner_min_degree_constraint=False, ignore_invalid_corners=False, use_junctions_with_var=False, \ use_regions=False, corner_suppression=False, corner_penalty=False, \ intersection_constraint=False, angle_constraint=False, use_junctions=False, use_loops=False, \ dist_thresh=None, angle_thresh=None, edge_threshold=None, corner_edge_thresh=None, thetas=None, corner_confs=None, \ theta_threshold=0.2, region_hit_threshold=None, theta_confs=None, filter_size=11, \ region_weight=1000.0): # create a new model m = Model("building_reconstruction_baseline") obj = LinExpr(0) num_junc = len(junctions) # list primitives js_list = [k for k in range(num_junc)] if ignore_invalid_corners: js_list = [k for k in js_list if len(thetas[k]) >= 2] ls_list = [(k, l) for k in js_list for l in js_list if l > k] # create variables if with_corner_variables: js_var_dict = {} for j in js_list: js_var_dict[j] = m.addVar(vtype=GRB.BINARY, name="junc_{}".format(j)) ls_var_dict = {} for k, l in ls_list: ls_var_dict[(k, l)] = m.addVar(vtype=GRB.BINARY, name="line_{}_{}".format(k, l)) # edgeness objective if with_edge_confidence: for k, l in ls_list: lw = getLineWeight(edge_map, junctions[k], junctions[l]) obj += (lw - edge_threshold) * ls_var_dict[ (k, l)] # favor edges with over .5? elif with_corner_edge_confidence: for k, l in ls_list: lw = getLineWeight(edge_map, junctions[k], junctions[l]) #obj += (lw-0.1)*ls_var_dict[(k, l)] #obj += (corner_confs[k]-corner_threshold)*(corner_confs[l]-corner_threshold)*(lw-edge_threshold)*ls_var_dict[(k, l)] # favor edges with over .5? #print((np.prod([corner_confs[k], corner_confs[l], lw])-corner_edge_thresh)) obj += (np.prod([corner_confs[k], corner_confs[l], lw]) - corner_edge_thresh) * ls_var_dict[ (k, l)] # favor edges with over .5? else: for k, l in ls_list: obj += ls_var_dict[(k, l)] if with_corner_variables: # corner-edge connectivity constraint for k, l in ls_list: m.addConstr((js_var_dict[k] + js_var_dict[l] - 2) * ls_var_dict[(k, l)] == 0, "c_{}_{}".format(k, l)) ########################################################################################################## ############################################### OPTIONAL ################################################# ########################################################################################################## if use_regions: reg_list = [] reg_var_ls = {} reg_sm = {} reg_contour = {} for i, reg in enumerate(regions): # apply min filter reg_small = Image.fromarray(reg * 255.0) reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size)) reg_small = np.array(reg_small) / 255.0 # ignore too small regions inds = np.argwhere(reg_small > 0) if np.array(inds).shape[0] > 0: reg_list.append(i) reg_var_ls[i] = m.addVar(vtype=GRB.BINARY, name="reg_{}".format(i)) reg_sm[i] = reg_small obj += region_weight * reg_var_ls[i] ret, thresh = cv2.threshold( np.array(reg_small * 255.0).astype('uint8'), 127, 255, 0) _, contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours = np.concatenate(contours, 0) contours = np.array(contours) reg_contour[i] = contours.reshape(-1, 2) #print(np.array(contours).shape) for i in reg_list: # compute intersection constraint for k, l in ls_list: intersec = getIntersection(reg_sm[i], junctions[k], junctions[l]) if intersec >= region_hit_threshold: m.addConstr(ls_var_dict[(k, l)] * reg_var_ls[i] == 0, "r1_{}_{}".format(k, l)) # closed region constraint inds = np.linspace(0, reg_contour[i].shape[0], min(10, reg_contour[i].shape[0]), endpoint=False).astype('int') sampled_pts_1 = reg_contour[i][inds, :] # # DEBUG -- SAMPLED POINTS # deb = Image.fromarray(reg_sm[i]*255.0).convert('RGB') # dr = ImageDraw.Draw(deb) # for pt in sampled_pts_1: # x, y = pt # dr.ellipse((x-2, y-2, x+2, y+2), fill='green') # plt.imshow(deb) # plt.show() other_regions = [regions[j] for j in reg_list if i != j] for pt in sampled_pts_1: for th in range(0, 360, 10): intersec_edges, intersec_region = castRay( pt, th, ls_list, junctions, regions[i], reg_sm[i], other_regions) sum_in_set = LinExpr(0) for e in list(intersec_edges): k, l = e sum_in_set += ls_var_dict[(k, l)] if not intersec_region: slack_var = m.addVar(vtype=GRB.INTEGER, name="slack_var_{}_{}".format( th, i)) m.addConstr(sum_in_set - slack_var <= reg_var_ls[i], "r2_{}_{}".format(th, i)) obj -= 0.05 * slack_var # m.addConstr(sum_in_set >= reg_var_ls[i], "r2_{}".format(th)) else: m.addConstr(sum_in_set >= reg_var_ls[i], "r2_{}".format(th)) # # inter region constraint # other_regions_id = [j for j in reg_list if i != j] # inds1 = np.array(np.argwhere(reg_sm[i]>0)) # sampled_pts_1 = inds1[np.random.choice(inds1.shape[0], min(10, inds1.shape[0]), replace=False), :] # for j in other_regions_id: # inds2 = np.array(np.argwhere(reg_sm[j]>0)) # sampled_pts_2 = inds2[np.random.choice(inds2.shape[0], min(1, inds2.shape[0]), replace=False), :] # if i > j: # make order # for pt1 in sampled_pts_1: # for pt2 in sampled_pts_2: # intersec_edges = castRayBetweenRegions(pt1[::-1], pt2[::-1], ls_list, junctions, reg_sm[i], reg_sm[j]) # sum_in_set = LinExpr(0) # for e in list(intersec_edges): # k, l = e # sum_in_set += ls_var_dict[(k, l)] # m.addConstr(sum_in_set >= reg_var_ls[i]*reg_var_ls[j], "r3_{}_{}".format(k, l)) #if corner_penalty: # corner penalty # for j in js_list: # obj -= 0.1*js_var_dict[j] if intersection_constraint: # intersection constraint for k, (j0, j1) in enumerate(ls_list): for l, (j2, j3) in enumerate(ls_list): if l > k: p1, q1 = junctions[j0], junctions[j1] p2, q2 = junctions[j2], junctions[j3] if doIntersect(p1, q1, p2, q2): m.addConstr( ls_var_dict[(j0, j1)] * ls_var_dict[(j2, j3)] == 0, "i_{}_{}_{}_{}".format(j0, j1, j2, j3)) if use_junctions_with_var or use_junctions: for j1 in js_list: # consider only valid degrees # if len(thetas[j1]) >= 2: # create list of lines for each junction lines_sets = [LinExpr(0) for _ in range(len(thetas[j1]) + 1)] lines_sets_deb = [list() for _ in range(len(thetas[j1]) + 1)] lines_max_in_sets = [0.0 for _ in range(len(thetas[j1]) + 1)] for j2 in js_list: if j1 != j2: # get line var if (j1, j2) in ls_var_dict: ls_var = ls_var_dict[(j1, j2)] else: ls_var = ls_var_dict[(j2, j1)] # check each line angle at junction in_sets = False for i, a in enumerate(thetas[j1]): lb = (a - angle_thresh) if ( a - angle_thresh) >= 0 else 360.0 + (a - angle_thresh) up = (a + angle_thresh) % 360.0 pt1 = junctions[j1] pt2 = junctions[j2] ajl = getAngle(pt1, pt2) if inBetween(ajl, lb, up): lw = getLineWeight(edge_map, pt1, pt2) lines_max_in_sets[i] = max(lines_max_in_sets[i], lw) lines_sets[i] += ls_var in_sets = True #print(i, j1, j2, ajl, a, lb, up, inBetween(ajl, lb, up)) # not in any direction set if not in_sets: lines_sets[-1] += ls_var # print(lines_sets_deb) # # Debug # x1, y1 = junctions[j1] # for angle_i, line_set in enumerate(lines_sets_deb): # im_deb = Image.new('RGB', (256, 256)) # dr = ImageDraw.Draw(im_deb) # dr.ellipse((x1-2, y1-2, x1+2, y1+2), fill='blue') # for v in line_set: # x2, y2 = junctions[v] # dr.line((x1, y1, x2, y2), fill='green', width=2) # dr.ellipse((x2-2, y2-2, x2+2, y2+2), fill='red') # if angle_i < len(thetas[j1]): # print(thetas[j1][angle_i]) # else: # print('Others') # plt.imshow(im_deb) # plt.show() # add to constraints set_sum = QuadExpr(0) # add all sets for i in range(len(thetas[j1])): if use_junctions_with_var: junc_th_var = m.addVar(vtype=GRB.BINARY, name="angle_{}".format(j1)) #obj += lines_sets[i] * (np.min([lines_max_in_sets[i], corner_confs[j1], theta_confs[j1][i]]) - theta_threshold) #obj += (np.prod([lines_max_in_sets[i], theta_confs[j1][i]])-theta_threshold)*junc_th_var m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1)) set_sum += junc_th_var * lines_sets[i] else: m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1)) #set_sum += junc_th_var*lines_sets[i] # add not in set slack_var = m.addVar(vtype=GRB.INTEGER, name="slack_var_{}_{}".format(i, j1)) m.addConstr(lines_sets[-1] - slack_var == 0, "a_{}_{}".format(-1, j1)) obj -= 0.1 * slack_var set_sum += junc_th_var * lines_sets[-1] # if use_junctions_with_var: # # final constraint # m.addConstr(set_sum == junc_th_var*len(thetas[j1]), "a_sum_{}".format(j1)) if corner_suppression: # junction spatial constraint junc_sets = set() for j1 in js_list: junc_intersec_set = [] for j2 in js_list: pt1 = np.array(junctions[j1]) pt2 = np.array(junctions[j2]) dist = np.linalg.norm(pt1 - pt2) if dist < dist_thresh: junc_intersec_set.append(j2) junc_intersec_set = tuple(np.sort(junc_intersec_set)) junc_sets.add(junc_intersec_set) # avoid duplicated constraints for js_tuple in junc_sets: junc_expr = LinExpr(0) for j in np.array(js_tuple): junc_expr += js_var_dict[j] m.addConstr(junc_expr <= 1, "s_{}".format(j1)) if corner_min_degree_constraint: # degree constraint for j in js_list: # degree expression deg_j = QuadExpr(0) for k, l in ls_list: if (j == k) or (j == l): deg_j += ls_var_dict[(k, l)] # degree constraint - active junctions must have degree >= 2 m.addConstr(deg_j * js_var_dict[j] >= 2 * js_var_dict[j], "d_1_{}".format(j)) # set optimizer m.setObjective(obj, GRB.MAXIMIZE) m.optimize() # parse solution juncs_on = [] lines_on = [] regs_sm_on = [] for v in m.getVars(): if 'junc' in v.varName and v.x >= .5: juncs_on.append(int(v.varName.split('_')[-1])) elif 'line' in v.varName and v.x >= .5: lines_on.append( (int(v.varName.split('_')[-2]), int(v.varName.split('_')[-1]))) elif 'reg' in v.varName and v.x >= .5: print('REGION ON') reg_id = int(v.varName.split('_')[-1]) reg = regions[reg_id] reg_small = Image.fromarray(reg * 255.0) reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size)) regs_sm_on.append(reg_small) if not with_corner_variables: juncs_on = np.array(list(set(sum(lines_on, ())))) if use_regions: return junctions, juncs_on, lines_on, regs_sm_on return junctions, juncs_on, lines_on
def extract_regions_v2(junctions, juncs_on, lines_on): # is_closed, degree_list = check_polygon(lines_on, degree_list=True) # print(is_closed, degree_list) # handle intersections edges_intersect = set() for k, (p1, q1) in enumerate(lines_on): for l, (p2, q2) in enumerate(lines_on): if k > l: if doIntersect(junctions[p1], junctions[q1], junctions[p2], junctions[q2]): edges_intersect.add((p1, q1)) edges_intersect.add((p2, q2)) # remove dangling edges junctions, juncs_on, lines_on = remove_dangling(junctions, juncs_on, lines_on) # init variables lines_on = [tuple([x, y]) for x, y in lines_on] tracker = set() # (clockwise, anticlockwise) if len(junctions) == 0: return [], [] # find contour tracker = find_contour(junctions, lines_on, tracker) # extract regions start_edge = lines_on[0] regions = [] while True: # find loop, updating tracker region, updated_tracker, is_repeated = find_loop(junctions, start_edge, start_edge[1], lines_on, tracker) if region is None: region, updated_tracker, is_repeated = find_loop(junctions, start_edge, start_edge[0], lines_on, tracker) tracker = updated_tracker if not is_repeated: regions.append(region) # find next edge that does not appear twice in tracker start_edge = None for (i, j) in lines_on: if ((i, j) not in tracker): start_edge = (i, j) tracker.add((i, j)) break if ((j, i) not in tracker): start_edge = (j, i) tracker.add((j, i)) break # else stop if start_edge is None: break # mark regions using edges with intersection false_pos = [] for l, poly in enumerate(regions): for k in range(len(poly)-1): i, j = poly[k], poly[k+1] if ((i, j) in edges_intersect) or ((j, i) in edges_intersect): false_pos.append(l) break # extract region masks region_mks = [] for poly in regions: rm = Image.new('L', (256, 256)) dr = ImageDraw.Draw(rm) poly_coords = [tuple(junctions[x]) for x in poly] dr.polygon(poly_coords, fill='white') region_mks.append(np.array(rm)/255.0) # ### DEBUG # print(false_pos) # print(poly_coords) # plt.imshow(rm) # plt.show() # print('test:', len(region_mks)) return region_mks, false_pos