def castRayBetweenRegions(p1, q1, ls_list, junctions, reg_i, reg_j):
# collect intersecting edges
intersec_set = set()
for i, ls in enumerate(ls_list):
k, l = ls
p2, q2 = junctions[k], junctions[l]
if doIntersect(p1, q1, p2, q2):
intersec_set.add((k, l))
# print(k, l)
# print(intersec_set)
# # DEBUG
# comb_reg = np.stack([reg_i, reg_j])
# comb_reg = np.clip(np.sum(comb_reg, 0), 0, 1)
# comb_reg = Image.fromarray(comb_reg*255.0).convert('RGB')
# dr = ImageDraw.Draw(comb_reg)
# x1, y1 = p1
# x2, y2 = q1
# x3, y3 = p2
# x4, y4 = q2
# dr.line((x1, y1, x2, y2), fill='green', width=4)
# dr.line((x3, y3, x4, y4), fill='red', width=1)
# print(doIntersect(p1, q1, p2, q2))
# plt.figure()
# plt.imshow(comb_reg)
# plt.show()
return intersec_set
def castRay(pt,
th,
ls_list,
junctions,
large_region,
region_small,
other_regions,
ray_length=1000.0,
thresh=0.0):
# compute ray
x1, y1 = int(pt[0]), int(pt[1])
rad = np.radians(th)
dy = np.sin(rad) * ray_length
dx = np.cos(rad) * ray_length
x2, y2 = x1 + dx, y1 + dy
# collect intersecting edges
intersec_set = set()
for i, ls in enumerate(ls_list):
k, l = ls
p1, q1 = (x1, y1), (x2, y2)
p2, q2 = junctions[k], junctions[l]
if doIntersect(p1, q1, p2, q2):
intersec_set.add((k, l))
# # DEBUG
# deb = Image.fromarray(region_small*255.0).convert('RGB')
# dr = ImageDraw.Draw(deb)
# for ls in list(intersec_set):
# k, l = ls
# p2, q2 = junctions[k], junctions[l]
# x3, y3 = p2
# x4, y4 = q2
# dr.line((x3, y3, x4, y4), fill='red', width=1)
# dr.line((x1, y1, x2, y2), fill='green', width=4)
# plt.imshow(deb)
# plt.show()
# check intersection with other regions
intersec_region = False
if len(other_regions) > 0:
comb_reg = np.clip(np.sum(np.array(other_regions), 0), 0, 1)
# cast ray
ray_im = Image.new('L', (256, 256))
dr = ImageDraw.Draw(ray_im)
dr.line((x1, y1, x2, y2), fill='white', width=8)
ray = np.array(ray_im) / 255.0
intersec = np.array(np.where(np.logical_and(ray, comb_reg) > 0))
intersec_region = (intersec.shape[1] > 0)
# # DEBUG
# print(comb_reg.shape)
# comb_reg = Image.fromarray(comb_reg*255.0).convert('RGB')
# dr = ImageDraw.Draw(comb_reg)
# dr.line((x1, y1, x2, y2), fill='green', width=4)
# print(intersec)
# print(intersec_region)
# plt.figure()
# plt.imshow(comb_reg)
# plt.show()
# check self intersection
ray_im = Image.new('L', (256, 256))
dr = ImageDraw.Draw(ray_im)
dr.line((x1, y1, x2, y2), fill='white', width=8)
dr.ellipse((x1 - 4, y1 - 4, x1 + 4, y1 + 4), fill='black')
ray = np.array(ray_im) / 255.0
self_intersect = np.logical_and(ray,
region_small).sum() / (ray.sum() + 1e-8)
self_inter = False
if self_intersect > thresh:
intersec_region = True
self_inter = True
# # DEBUG
# print(self_intersect)
# deb = Image.fromarray(region_small*255.0).convert('RGB')
# dr = ImageDraw.Draw(deb)
# # for k, l in list(intersec_set):
# # p2, q2 = junctions[k], junctions[l]
# # x3, y3 = p2
# # x4, y4 = q2
# # dr.line((x3, y3, x4, y4), fill='red', width=1)
# print(intersec_region)
# dr.line((x1, y1, x2, y2), fill='green', width=1)
# plt.imshow(deb)
# plt.show()
# # DEBUG
return intersec_set, intersec_region, self_inter
def castRayRegion(pt,
th,
ls_list,
junctions,
sm_reg_i,
sm_other_regs,
l_reg,
l_other_regs,
other_regs_id,
ray_length=1000.0,
thresh=0.0):
# compute ray
x1, y1 = int(pt[0]), int(pt[1])
rad = np.radians(th)
dy = np.sin(rad) * ray_length
dx = np.cos(rad) * ray_length
x2, y2 = x1 + dx, y1 + dy
p1, q1 = (x1, y1), (x2, y2)
# render ray
ray_im = Image.new('L', (256, 256))
draw = ImageDraw.Draw(ray_im)
draw.line((x1, y1, x2, y2), fill='white')
ray = np.array(ray_im)
# check region intersection
background = np.zeros((256, 256))
comb_reg = np.zeros((256, 256))
for _id, l_reg_j, sm_reg_j in zip(other_regs_id, l_other_regs,
sm_other_regs):
reg_j_tag = np.array(sm_reg_j)
inds = np.where(reg_j_tag > 0)
reg_j_tag[inds] = _id + 1
comb_reg += reg_j_tag
background += l_reg_j
background += l_reg
intersect = np.logical_and(ray, comb_reg)
inds = np.where(intersect > 0)
intersect_tags = np.array(comb_reg[inds])
# DEBUG
comb_reg = np.clip(comb_reg, 0, 1)
comb_reg = Image.fromarray(comb_reg * 255.0).convert('RGB')
dr = ImageDraw.Draw(comb_reg)
dr.line((x1, y1, x2, y2), fill='green')
dr.ellipse((x1 - 2, y1 - 2, x1 + 2, y1 + 2), fill='blue')
# DEBUG
# compute distances
intersec_set = set()
self_intersect = None
region_id = None
inds = np.array(inds)
if inds.shape[1] > 0:
px = np.array([x1, y1])[np.newaxis, :]
pts = inds.transpose(1, 0)[:, ::-1]
dists = np.sqrt((np.sum((pts - px)**2, -1)))
closest_pt = pts[np.argmin(dists), :]
region_id = intersect_tags[np.argmin(dists)] - 1
xc, yc = closest_pt
# collect intersecting edges
for i, ls in enumerate(ls_list):
k, l = ls
p2, q2 = junctions[k], junctions[l]
if doIntersect(p1, closest_pt, p2, q2):
intersec_set.add((k, l))
# DEBUG -- Draw intersecting edges
for ls in list(intersec_set):
k, l = ls
p2, q2 = junctions[k], junctions[l]
x3, y3 = p2
x4, y4 = q2
dr.line((x3, y3, x4, y4), fill='red', width=1)
# DEBUG
# DEBUG
dr.ellipse((xc - 2, yc - 2, xc + 2, yc + 2), fill='magenta')
#print(region_id)
# DEBUG
# check self intersection
ray_im = Image.new('L', (256, 256))
dr = ImageDraw.Draw(ray_im)
dr.line((x1, y1, xc, yc), fill='white', width=16)
dr.ellipse((x1 - 4, y1 - 4, x1 + 4, y1 + 4), fill='black')
ray = np.array(ray_im) / 255.0
self_intersect = np.logical_and(ray,
sm_reg_i).sum() / (ray.sum() + 1e-8)
self_intersect = self_intersect > thresh
# render ray
ray_im = Image.new('L', (256, 256))
draw = ImageDraw.Draw(ray_im)
draw.line((x1, y1, xc, yc), fill='white', width=16)
ray = np.array(ray_im) / 255.0
# check background intersection
background_im = Image.fromarray(background * 255.0)
background_im = background_im.filter(ImageFilter.MaxFilter(9))
background = 1.0 - (np.array(background_im) / 255.0)
background[background > 0.5] = 1.0
background[background <= 0.5] = 0.0
background_intersection = np.logical_and(ray, background).sum()
if background_intersection > 0:
self_intersect = True
# # DEBUG
# deb = Image.fromarray(background*255.0).convert('RGB')
# dr = ImageDraw.Draw(deb)
# dr.line((x1, y1, xc, yc), fill='green')
# print(background_intersection)
# plt.imshow(deb)
# plt.show()
# # DEBUG
# print(self_intersect)
# plt.imshow(comb_reg)
# plt.show()
return intersec_set, region_id, self_intersect
def reconstructBuildingBaseline(junctions, edge_map, regions=None, with_weighted_junctions=True, with_corner_variables=False, with_edge_confidence=False,
with_corner_edge_confidence=False, \
lw_from_cls=None, use_edge_classifier=False,\
corner_min_degree_constraint=False, ignore_invalid_corners=False, use_junctions_with_var=False, \
use_regions=False, corner_suppression=False, corner_penalty=False, \
junction_suppression=False, \
intersection_constraint=False, angle_constraint=False, use_junctions=False, use_loops=False, \
dist_thresh=None, angle_thresh=None, edge_threshold=None, corner_edge_thresh=None, thetas=None, corner_confs=None, \
theta_threshold=0.2, region_hit_threshold=None, theta_confs=None, filter_size=11, \
region_weight=1000.0, post_process=False, \
edge_map_weight=1.0, junctions_weight=1.0, inter_region_weight=1.0, wrong_dir_weight=1.0, closed_region_weight=1.0,
closed_region_lowerbound=False, closed_region_upperbound=False,\
region_intersection_constraint=False, inter_region_constraint=False,\
junctions_soft=False):
# create a new model
m = Model("building_reconstruction_baseline")
m.setParam('OutputFlag', False)
obj = LinExpr(0)
num_junc = len(junctions)
# list primitives
js_list = [k for k in range(num_junc)]
if ignore_invalid_corners:
js_list = [k for k in js_list if len(thetas[k]) >= 2]
ls_list = [(k, l) for k in js_list for l in js_list if l > k]
# create variables
if with_corner_variables:
js_var_dict = {}
for j in js_list:
js_var_dict[j] = m.addVar(vtype=GRB.BINARY,
name="junc_{}".format(j))
ls_var_dict = {}
for k, l in ls_list:
ls_var_dict[(k, l)] = m.addVar(vtype=GRB.BINARY,
name="line_{}_{}".format(k, l))
# edgeness objective
if with_edge_confidence:
for k, l in ls_list:
if use_edge_classifier:
try:
lw = lw_from_cls[(k, l)]
except:
lw = lw_from_cls[(l, k)]
else:
lw = getLineWeight(edge_map, junctions[k], junctions[l])
obj += (lw - edge_threshold) * ls_var_dict[
(k, l)] # favor edges with over .5?
elif with_corner_edge_confidence:
for k, l in ls_list:
if use_edge_classifier:
try:
lw = lw_from_cls[(k, l)]
except:
lw = lw_from_cls[(l, k)]
else:
lw = getLineWeight(edge_map, junctions[k], junctions[l])
#obj += (lw-0.1)*ls_var_dict[(k, l)]
#obj += (corner_confs[k]-corner_threshold)*(corner_confs[l]-corner_threshold)*(lw-edge_threshold)*ls_var_dict[(k, l)] # favor edges with over .5?
#print((np.prod([corner_confs[k], corner_confs[l], lw])-corner_edge_thresh))
obj += edge_map_weight * (
np.prod([corner_confs[k], corner_confs[l], lw]) -
corner_edge_thresh) * ls_var_dict[
(k, l)] # favor edges with over .5?
else:
for k, l in ls_list:
obj += ls_var_dict[(k, l)]
if with_corner_variables:
# corner-edge connectivity constraint
for k, l in ls_list:
m.addConstr((js_var_dict[k] + js_var_dict[l] - 2) *
ls_var_dict[(k, l)] == 0, "c_{}_{}".format(k, l))
##########################################################################################################
############################################### OPTIONAL #################################################
##########################################################################################################
if use_regions:
reg_list = []
reg_var_ls = {}
reg_sm = {}
reg_contour = {}
for i, reg in enumerate(regions):
# apply min filter
reg_small = Image.fromarray(reg * 255.0)
reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size))
reg_small = np.array(reg_small) / 255.0
# ignore too small regions
inds = np.argwhere(reg_small > 0)
if np.array(inds).shape[0] > 0:
ret, thresh = cv2.threshold(
np.array(reg_small * 255.0).astype('uint8'), 127, 255, 0)
_, contours, _ = cv2.findContours(thresh, cv2.RETR_TREE,
cv2.CHAIN_APPROX_NONE)
lg_contour = [None, 0]
for c in contours:
cont = Image.new('L', (256, 256))
dr = ImageDraw.Draw(cont)
c = c.reshape(-1, 2)
c = [(x, y) for x, y in c]
if len(c) <= 2:
continue
dr.polygon(c, fill='white')
# plt.imshow(cont)
# plt.show()
size = (np.array(cont) / 255.0).sum()
if size > lg_contour[1]:
lg_contour = [c, size]
contours = lg_contour[0]
contours = np.array(contours)
if len(contours.shape) > 0:
reg_contour[i] = contours.reshape(-1, 2)
inds = np.linspace(0,
reg_contour[i].shape[0],
min(int(reg_contour[i].shape[0] / 2),
reg_contour[i].shape[0]),
endpoint=False).astype('int')
reg_contour[i] = reg_contour[i][inds, :]
reg_list.append(i)
reg_var_ls[i] = m.addVar(vtype=GRB.BINARY,
name="reg_{}".format(i))
reg_sm[i] = reg_small
obj += region_weight * reg_var_ls[i]
for i in reg_list:
if region_intersection_constraint:
# compute intersection constraint
for k, l in ls_list:
intersec = getIntersection(reg_sm[i], junctions[k],
junctions[l])
if intersec >= region_hit_threshold:
m.addConstr(ls_var_dict[(k, l)] * reg_var_ls[i] == 0,
"r1_{}_{}".format(k, l))
# add loop constraints
ths, pts = compute_normals(reg_contour[i], reg_sm[i])
# # DEBUG -- SAMPLED POINTS
# deb = Image.fromarray(reg_sm[i]*255.0).convert('RGB')
# dr = ImageDraw.Draw(deb)
# for pt in pts:
# x, y = pt
# dr.ellipse((x-2, y-2, x+2, y+2), fill='green')
# plt.imshow(deb)
# plt.show()
other_regions = [regions[j] for j in reg_list if i != j]
sm_other_regions = [reg_sm[j] for j in reg_list if i != j]
for pt, th in zip(pts, ths):
# closed region soft constraint -- upperbound
intersec_edges, intersec_region, self_intersect = castRay(
pt,
th,
ls_list,
junctions,
regions[i],
reg_sm[i],
other_regions,
ray_length=1000.0)
# if self_intersect:
# intersec_edges, intersec_region, self_intersect = castRay(pt, th, ls_list, junctions, regions[i], reg_sm[i], other_regions, ray_length=20.0)
sum_in_set = LinExpr(0)
for e in list(intersec_edges):
k, l = e
sum_in_set += ls_var_dict[(k, l)]
if not intersec_region and closed_region_upperbound:
slack_var = m.addVar(vtype=GRB.INTEGER,
name="slack_var_{}_{}".format(th, i))
m.addConstr(
sum_in_set * reg_var_ls[i] <=
reg_var_ls[i] + slack_var, "r2_{}_{}".format(th, i))
obj -= closed_region_weight * slack_var
m.addConstr(slack_var >= 0)
if closed_region_lowerbound:
# closed region hard constraint -- lowerbound
slack_var = m.addVar(vtype=GRB.INTEGER)
m.addConstr(
sum_in_set * reg_var_ls[i] >=
reg_var_ls[i] - slack_var, "r2_{}".format(th))
obj -= closed_region_weight * slack_var
m.addConstr(slack_var >= 0)
if inter_region_constraint:
# inter region soft constraint
other_regions_id = [j for j in reg_list if i != j]
intersec_set, region_id, self_intersect = castRayRegion(
pt, th, ls_list, junctions, reg_sm[i],
sm_other_regions, regions[i], other_regions,
other_regions_id)
if (region_id is not None) and (not self_intersect):
sum_in_set = LinExpr(0)
# DEBUG
# deb = Image.fromarray(reg_sm[i]*255.0).convert('RGB')
# dr = ImageDraw.Draw(deb)
# DEBUG
for e in list(intersec_set):
k, l = e
sum_in_set += ls_var_dict[(k, l)]
# # DEBUG
# p2, q2 = junctions[k], junctions[l]
# x3, y3 = p2
# x4, y4 = q2
# dr.line((x3, y3, x4, y4), fill='red', width=1)
# plt.imshow(deb)
# plt.show()
# DEBUG
slack_var_up = m.addVar(
vtype=GRB.INTEGER,
name="slack_var_inter_up_{}_{}".format(th, i))
slack_var_low = m.addVar(
vtype=GRB.INTEGER,
name="slack_var_inter_low_{}_{}".format(th, i))
#m.addConstr(sum_in_set*reg_var_ls[i] == reg_var_ls[i] + slack_var, "r3_{}_{}".format(th, i))
m.addConstr(
sum_in_set * reg_var_ls[i] >= 1 - slack_var_low)
m.addConstr(
sum_in_set * reg_var_ls[i] <= 1 + slack_var_up)
m.addConstr(slack_var_low >= 0)
m.addConstr(slack_var_up >= 0)
obj -= inter_region_weight * slack_var_low + inter_region_weight * slack_var_up
# if corner_penalty:
# for j in js_list:
# obj -= 2*js_var_dict[j]
if intersection_constraint:
# intersection constraint
for k, (j0, j1) in enumerate(ls_list):
for l, (j2, j3) in enumerate(ls_list):
if l > k:
p1, q1 = junctions[j0], junctions[j1]
p2, q2 = junctions[j2], junctions[j3]
if doIntersect(p1, q1, p2, q2):
m.addConstr(
ls_var_dict[(j0, j1)] * ls_var_dict[(j2, j3)] == 0,
"i_{}_{}_{}_{}".format(j0, j1, j2, j3))
if use_junctions_with_var or use_junctions:
for j1 in js_list:
# consider only valid degrees
# if len(thetas[j1]) >= 2:
# create list of lines for each junction
lines_sets = [LinExpr(0) for _ in range(len(thetas[j1]) + 1)]
lines_sets_deb = [list() for _ in range(len(thetas[j1]) + 1)]
lines_max_in_sets = [0.0 for _ in range(len(thetas[j1]) + 1)]
for j2 in js_list:
if j1 != j2:
# get line var
if (j1, j2) in ls_var_dict:
ls_var = ls_var_dict[(j1, j2)]
else:
ls_var = ls_var_dict[(j2, j1)]
# check each line angle at junction
in_sets = False
for i, a in enumerate(thetas[j1]):
lb = (a - angle_thresh) if (
a -
angle_thresh) >= 0 else 360.0 + (a - angle_thresh)
up = (a + angle_thresh) % 360.0
pt1 = junctions[j1]
pt2 = junctions[j2]
ajl = getAngle(pt1, pt2)
if inBetween(ajl, lb, up):
if use_edge_classifier:
try:
lw = lw_from_cls[(j1, j2)]
except:
lw = lw_from_cls[(j2, j1)]
else:
lw = getLineWeight(edge_map, pt1, pt2)
lines_max_in_sets[i] = max(lines_max_in_sets[i],
lw)
lines_sets[i] += ls_var
in_sets = True
#print(i, j1, j2, ajl, a, lb, up, inBetween(ajl, lb, up))
# not in any direction set
if not in_sets:
lines_sets[-1] += ls_var
# print(lines_sets_deb)
# # Debug
# x1, y1 = junctions[j1]
# for angle_i, line_set in enumerate(lines_sets_deb):
# im_deb = Image.new('RGB', (256, 256))
# dr = ImageDraw.Draw(im_deb)
# dr.ellipse((x1-2, y1-2, x1+2, y1+2), fill='blue')
# for v in line_set:
# x2, y2 = junctions[v]
# dr.line((x1, y1, x2, y2), fill='green', width=2)
# dr.ellipse((x2-2, y2-2, x2+2, y2+2), fill='red')
# if angle_i < len(thetas[j1]):
# print(thetas[j1][angle_i])
# else:
# print('Others')
# plt.imshow(im_deb)
# plt.show()
# add to constraints
#set_sum = QuadExpr(0)
# add all sets
for i in range(len(thetas[j1])):
if use_junctions_with_var:
junc_th_var = m.addVar(vtype=GRB.BINARY,
name="angle_{}".format(j1))
obj += junctions_weight * junc_th_var * (
np.prod([corner_confs[j1], theta_confs[j1][i]]) -
theta_threshold)
m.addConstr(lines_sets[i] == junc_th_var,
"a_{}_{}".format(i, j1))
# OLD
#obj += (np.prod([lines_max_in_sets[i], theta_confs[j1][i]])-theta_threshold)*junc_th_var
#set_sum += junc_th_var*lines_sets[i]
#m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1))
else:
m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1))
# OLD
#set_sum += junc_th_var*lines_sets[i]
# # add not in set -- SOFT
if junctions_soft:
slack_var = m.addVar(vtype=GRB.INTEGER,
name="slack_var_{}".format(j1))
m.addConstr(lines_sets[-1] - slack_var == 0,
"a_{}_{}".format(-1, j1))
obj -= wrong_dir_weight * slack_var
m.addConstr(slack_var >= 0)
else:
# add not in set -- HARD
m.addConstr(lines_sets[-1] == 0, "a_{}_{}".format(-1, j1))
# OLD
#set_sum += junc_th_var*lines_sets[-1]
# if use_junctions_with_var:
# # final constraint
# m.addConstr(set_sum == junc_th_var*len(thetas[j1]), "a_sum_{}".format(j1))
if corner_suppression:
# junction spatial constraint
junc_sets = set()
for j1 in js_list:
junc_intersec_set = []
for j2 in js_list:
pt1 = np.array(junctions[j1])
pt2 = np.array(junctions[j2])
dist = np.linalg.norm(pt1 - pt2)
if dist < dist_thresh:
junc_intersec_set.append(j2)
junc_intersec_set = tuple(np.sort(junc_intersec_set))
junc_sets.add(junc_intersec_set)
# avoid duplicated constraints
for js_tuple in junc_sets:
junc_expr = LinExpr(0)
for j in np.array(js_tuple):
junc_expr += js_var_dict[j]
m.addConstr(junc_expr <= 1, "s_{}".format(j1))
if junction_suppression:
angs = []
for j1 in js_list:
# compute angles and degree at each junction
deg_j1 = QuadExpr(0)
angs = []
for j2 in js_list:
if j1 != j2:
deg_j1 += ls_var_dict[(j1, j2)] if (
j1, j2) in ls_var_dict else ls_var_dict[(j2, j1)]
pt1 = junctions[j1]
pt2 = junctions[j2]
a12 = getAngle(pt1, pt2)
angs.append(a12)
else:
angs.append(0.0)
angs = np.array(angs)
ang_diffs = np.abs(180.0 - np.abs(angs[:, np.newaxis] -
angs[np.newaxis, :]))
inds = np.array(np.where(ang_diffs <= 10.0)).transpose(1, 0)
keep_track = []
for j2, j3 in inds:
if j1 != j2 and j2 != j3 and j3 != j1:
js = tuple(np.sort([j1, j2, j3]))
if js not in keep_track:
keep_track.append(js)
ls_var_12 = ls_var_dict[(j1, j2)] if (
j1, j2) in ls_var_dict else ls_var_dict[(j2, j1)]
ls_var_13 = ls_var_dict[(j1, j3)] if (
j1, j3) in ls_var_dict else ls_var_dict[(j3, j1)]
m.addConstr((deg_j1 - 2) * js_var_dict[j1] >=
ls_var_12 * ls_var_13,
"j_3_{}_{}_{}".format(j1, j2, j3))
if corner_min_degree_constraint:
# degree constraint
for j in js_list:
# degree expression
deg_j = QuadExpr(0)
for k, l in ls_list:
if (j == k) or (j == l):
deg_j += ls_var_dict[(k, l)]
# degree constraint - active junctions must have degree >= 2
m.addConstr(deg_j * js_var_dict[j] >= 2 * js_var_dict[j],
"d_1_{}".format(j))
# set optimizer
m.setObjective(obj, GRB.MAXIMIZE)
m.optimize()
# parse solution
juncs_on = []
lines_on = []
regs_sm_on = []
for v in m.getVars():
if 'junc' in v.varName and v.x >= .5:
juncs_on.append(int(v.varName.split('_')[-1]))
elif 'line' in v.varName and v.x >= .5:
lines_on.append(
(int(v.varName.split('_')[-2]), int(v.varName.split('_')[-1])))
elif 'reg' in v.varName and v.x >= .5:
#print('REGION ON')
reg_id = int(v.varName.split('_')[-1])
reg = regions[reg_id]
reg_small = Image.fromarray(reg * 255.0)
reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size))
regs_sm_on.append(reg_small)
elif 'slack_var_inter' in v.varName:
print(v.varName, v.x)
if not with_corner_variables:
juncs_on = np.array(list(set(sum(lines_on, ()))))
if use_regions:
if post_process:
juncs_on, lines_on = remove_junctions(junctions, juncs_on,
lines_on)
return junctions, juncs_on, lines_on, regs_sm_on
if post_process:
juncs_on, lines_on = remove_junctions(junctions, juncs_on, lines_on)
return junctions, juncs_on, lines_on
def reconstructBuildingBaseline(junctions, edge_map, regions=None, with_weighted_junctions=True, with_corner_variables=False, with_edge_confidence=False,
with_corner_edge_confidence=False, \
corner_min_degree_constraint=False, ignore_invalid_corners=False, use_junctions_with_var=False, \
use_regions=False, corner_suppression=False, corner_penalty=False, \
intersection_constraint=False, angle_constraint=False, use_junctions=False, use_loops=False, \
dist_thresh=None, angle_thresh=None, edge_threshold=None, corner_edge_thresh=None, thetas=None, corner_confs=None, \
theta_threshold=0.2, region_hit_threshold=None, theta_confs=None, filter_size=11, \
region_weight=1000.0):
# create a new model
m = Model("building_reconstruction_baseline")
obj = LinExpr(0)
num_junc = len(junctions)
# list primitives
js_list = [k for k in range(num_junc)]
if ignore_invalid_corners:
js_list = [k for k in js_list if len(thetas[k]) >= 2]
ls_list = [(k, l) for k in js_list for l in js_list if l > k]
# create variables
if with_corner_variables:
js_var_dict = {}
for j in js_list:
js_var_dict[j] = m.addVar(vtype=GRB.BINARY,
name="junc_{}".format(j))
ls_var_dict = {}
for k, l in ls_list:
ls_var_dict[(k, l)] = m.addVar(vtype=GRB.BINARY,
name="line_{}_{}".format(k, l))
# edgeness objective
if with_edge_confidence:
for k, l in ls_list:
lw = getLineWeight(edge_map, junctions[k], junctions[l])
obj += (lw - edge_threshold) * ls_var_dict[
(k, l)] # favor edges with over .5?
elif with_corner_edge_confidence:
for k, l in ls_list:
lw = getLineWeight(edge_map, junctions[k], junctions[l])
#obj += (lw-0.1)*ls_var_dict[(k, l)]
#obj += (corner_confs[k]-corner_threshold)*(corner_confs[l]-corner_threshold)*(lw-edge_threshold)*ls_var_dict[(k, l)] # favor edges with over .5?
#print((np.prod([corner_confs[k], corner_confs[l], lw])-corner_edge_thresh))
obj += (np.prod([corner_confs[k], corner_confs[l], lw]) -
corner_edge_thresh) * ls_var_dict[
(k, l)] # favor edges with over .5?
else:
for k, l in ls_list:
obj += ls_var_dict[(k, l)]
if with_corner_variables:
# corner-edge connectivity constraint
for k, l in ls_list:
m.addConstr((js_var_dict[k] + js_var_dict[l] - 2) *
ls_var_dict[(k, l)] == 0, "c_{}_{}".format(k, l))
##########################################################################################################
############################################### OPTIONAL #################################################
##########################################################################################################
if use_regions:
reg_list = []
reg_var_ls = {}
reg_sm = {}
reg_contour = {}
for i, reg in enumerate(regions):
# apply min filter
reg_small = Image.fromarray(reg * 255.0)
reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size))
reg_small = np.array(reg_small) / 255.0
# ignore too small regions
inds = np.argwhere(reg_small > 0)
if np.array(inds).shape[0] > 0:
reg_list.append(i)
reg_var_ls[i] = m.addVar(vtype=GRB.BINARY,
name="reg_{}".format(i))
reg_sm[i] = reg_small
obj += region_weight * reg_var_ls[i]
ret, thresh = cv2.threshold(
np.array(reg_small * 255.0).astype('uint8'), 127, 255, 0)
_, contours, _ = cv2.findContours(thresh, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
contours = np.concatenate(contours, 0)
contours = np.array(contours)
reg_contour[i] = contours.reshape(-1, 2)
#print(np.array(contours).shape)
for i in reg_list:
# compute intersection constraint
for k, l in ls_list:
intersec = getIntersection(reg_sm[i], junctions[k],
junctions[l])
if intersec >= region_hit_threshold:
m.addConstr(ls_var_dict[(k, l)] * reg_var_ls[i] == 0,
"r1_{}_{}".format(k, l))
# closed region constraint
inds = np.linspace(0,
reg_contour[i].shape[0],
min(10, reg_contour[i].shape[0]),
endpoint=False).astype('int')
sampled_pts_1 = reg_contour[i][inds, :]
# # DEBUG -- SAMPLED POINTS
# deb = Image.fromarray(reg_sm[i]*255.0).convert('RGB')
# dr = ImageDraw.Draw(deb)
# for pt in sampled_pts_1:
# x, y = pt
# dr.ellipse((x-2, y-2, x+2, y+2), fill='green')
# plt.imshow(deb)
# plt.show()
other_regions = [regions[j] for j in reg_list if i != j]
for pt in sampled_pts_1:
for th in range(0, 360, 10):
intersec_edges, intersec_region = castRay(
pt, th, ls_list, junctions, regions[i], reg_sm[i],
other_regions)
sum_in_set = LinExpr(0)
for e in list(intersec_edges):
k, l = e
sum_in_set += ls_var_dict[(k, l)]
if not intersec_region:
slack_var = m.addVar(vtype=GRB.INTEGER,
name="slack_var_{}_{}".format(
th, i))
m.addConstr(sum_in_set - slack_var <= reg_var_ls[i],
"r2_{}_{}".format(th, i))
obj -= 0.05 * slack_var
# m.addConstr(sum_in_set >= reg_var_ls[i], "r2_{}".format(th))
else:
m.addConstr(sum_in_set >= reg_var_ls[i],
"r2_{}".format(th))
# # inter region constraint
# other_regions_id = [j for j in reg_list if i != j]
# inds1 = np.array(np.argwhere(reg_sm[i]>0))
# sampled_pts_1 = inds1[np.random.choice(inds1.shape[0], min(10, inds1.shape[0]), replace=False), :]
# for j in other_regions_id:
# inds2 = np.array(np.argwhere(reg_sm[j]>0))
# sampled_pts_2 = inds2[np.random.choice(inds2.shape[0], min(1, inds2.shape[0]), replace=False), :]
# if i > j: # make order
# for pt1 in sampled_pts_1:
# for pt2 in sampled_pts_2:
# intersec_edges = castRayBetweenRegions(pt1[::-1], pt2[::-1], ls_list, junctions, reg_sm[i], reg_sm[j])
# sum_in_set = LinExpr(0)
# for e in list(intersec_edges):
# k, l = e
# sum_in_set += ls_var_dict[(k, l)]
# m.addConstr(sum_in_set >= reg_var_ls[i]*reg_var_ls[j], "r3_{}_{}".format(k, l))
#if corner_penalty:
# corner penalty
# for j in js_list:
# obj -= 0.1*js_var_dict[j]
if intersection_constraint:
# intersection constraint
for k, (j0, j1) in enumerate(ls_list):
for l, (j2, j3) in enumerate(ls_list):
if l > k:
p1, q1 = junctions[j0], junctions[j1]
p2, q2 = junctions[j2], junctions[j3]
if doIntersect(p1, q1, p2, q2):
m.addConstr(
ls_var_dict[(j0, j1)] * ls_var_dict[(j2, j3)] == 0,
"i_{}_{}_{}_{}".format(j0, j1, j2, j3))
if use_junctions_with_var or use_junctions:
for j1 in js_list:
# consider only valid degrees
# if len(thetas[j1]) >= 2:
# create list of lines for each junction
lines_sets = [LinExpr(0) for _ in range(len(thetas[j1]) + 1)]
lines_sets_deb = [list() for _ in range(len(thetas[j1]) + 1)]
lines_max_in_sets = [0.0 for _ in range(len(thetas[j1]) + 1)]
for j2 in js_list:
if j1 != j2:
# get line var
if (j1, j2) in ls_var_dict:
ls_var = ls_var_dict[(j1, j2)]
else:
ls_var = ls_var_dict[(j2, j1)]
# check each line angle at junction
in_sets = False
for i, a in enumerate(thetas[j1]):
lb = (a - angle_thresh) if (
a -
angle_thresh) >= 0 else 360.0 + (a - angle_thresh)
up = (a + angle_thresh) % 360.0
pt1 = junctions[j1]
pt2 = junctions[j2]
ajl = getAngle(pt1, pt2)
if inBetween(ajl, lb, up):
lw = getLineWeight(edge_map, pt1, pt2)
lines_max_in_sets[i] = max(lines_max_in_sets[i],
lw)
lines_sets[i] += ls_var
in_sets = True
#print(i, j1, j2, ajl, a, lb, up, inBetween(ajl, lb, up))
# not in any direction set
if not in_sets:
lines_sets[-1] += ls_var
# print(lines_sets_deb)
# # Debug
# x1, y1 = junctions[j1]
# for angle_i, line_set in enumerate(lines_sets_deb):
# im_deb = Image.new('RGB', (256, 256))
# dr = ImageDraw.Draw(im_deb)
# dr.ellipse((x1-2, y1-2, x1+2, y1+2), fill='blue')
# for v in line_set:
# x2, y2 = junctions[v]
# dr.line((x1, y1, x2, y2), fill='green', width=2)
# dr.ellipse((x2-2, y2-2, x2+2, y2+2), fill='red')
# if angle_i < len(thetas[j1]):
# print(thetas[j1][angle_i])
# else:
# print('Others')
# plt.imshow(im_deb)
# plt.show()
# add to constraints
set_sum = QuadExpr(0)
# add all sets
for i in range(len(thetas[j1])):
if use_junctions_with_var:
junc_th_var = m.addVar(vtype=GRB.BINARY,
name="angle_{}".format(j1))
#obj += lines_sets[i] * (np.min([lines_max_in_sets[i], corner_confs[j1], theta_confs[j1][i]]) - theta_threshold)
#obj += (np.prod([lines_max_in_sets[i], theta_confs[j1][i]])-theta_threshold)*junc_th_var
m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1))
set_sum += junc_th_var * lines_sets[i]
else:
m.addConstr(lines_sets[i] <= 1.0, "a_{}_{}".format(i, j1))
#set_sum += junc_th_var*lines_sets[i]
# add not in set
slack_var = m.addVar(vtype=GRB.INTEGER,
name="slack_var_{}_{}".format(i, j1))
m.addConstr(lines_sets[-1] - slack_var == 0,
"a_{}_{}".format(-1, j1))
obj -= 0.1 * slack_var
set_sum += junc_th_var * lines_sets[-1]
# if use_junctions_with_var:
# # final constraint
# m.addConstr(set_sum == junc_th_var*len(thetas[j1]), "a_sum_{}".format(j1))
if corner_suppression:
# junction spatial constraint
junc_sets = set()
for j1 in js_list:
junc_intersec_set = []
for j2 in js_list:
pt1 = np.array(junctions[j1])
pt2 = np.array(junctions[j2])
dist = np.linalg.norm(pt1 - pt2)
if dist < dist_thresh:
junc_intersec_set.append(j2)
junc_intersec_set = tuple(np.sort(junc_intersec_set))
junc_sets.add(junc_intersec_set)
# avoid duplicated constraints
for js_tuple in junc_sets:
junc_expr = LinExpr(0)
for j in np.array(js_tuple):
junc_expr += js_var_dict[j]
m.addConstr(junc_expr <= 1, "s_{}".format(j1))
if corner_min_degree_constraint:
# degree constraint
for j in js_list:
# degree expression
deg_j = QuadExpr(0)
for k, l in ls_list:
if (j == k) or (j == l):
deg_j += ls_var_dict[(k, l)]
# degree constraint - active junctions must have degree >= 2
m.addConstr(deg_j * js_var_dict[j] >= 2 * js_var_dict[j],
"d_1_{}".format(j))
# set optimizer
m.setObjective(obj, GRB.MAXIMIZE)
m.optimize()
# parse solution
juncs_on = []
lines_on = []
regs_sm_on = []
for v in m.getVars():
if 'junc' in v.varName and v.x >= .5:
juncs_on.append(int(v.varName.split('_')[-1]))
elif 'line' in v.varName and v.x >= .5:
lines_on.append(
(int(v.varName.split('_')[-2]), int(v.varName.split('_')[-1])))
elif 'reg' in v.varName and v.x >= .5:
print('REGION ON')
reg_id = int(v.varName.split('_')[-1])
reg = regions[reg_id]
reg_small = Image.fromarray(reg * 255.0)
reg_small = reg_small.filter(ImageFilter.MinFilter(filter_size))
regs_sm_on.append(reg_small)
if not with_corner_variables:
juncs_on = np.array(list(set(sum(lines_on, ()))))
if use_regions:
return junctions, juncs_on, lines_on, regs_sm_on
return junctions, juncs_on, lines_on
def extract_regions_v2(junctions, juncs_on, lines_on):
# is_closed, degree_list = check_polygon(lines_on, degree_list=True)
# print(is_closed, degree_list)
# handle intersections
edges_intersect = set()
for k, (p1, q1) in enumerate(lines_on):
for l, (p2, q2) in enumerate(lines_on):
if k > l:
if doIntersect(junctions[p1], junctions[q1], junctions[p2], junctions[q2]):
edges_intersect.add((p1, q1))
edges_intersect.add((p2, q2))
# remove dangling edges
junctions, juncs_on, lines_on = remove_dangling(junctions, juncs_on, lines_on)
# init variables
lines_on = [tuple([x, y]) for x, y in lines_on]
tracker = set() # (clockwise, anticlockwise)
if len(junctions) == 0:
return [], []
# find contour
tracker = find_contour(junctions, lines_on, tracker)
# extract regions
start_edge = lines_on[0]
regions = []
while True:
# find loop, updating tracker
region, updated_tracker, is_repeated = find_loop(junctions, start_edge, start_edge[1], lines_on, tracker)
if region is None:
region, updated_tracker, is_repeated = find_loop(junctions, start_edge, start_edge[0], lines_on, tracker)
tracker = updated_tracker
if not is_repeated:
regions.append(region)
# find next edge that does not appear twice in tracker
start_edge = None
for (i, j) in lines_on:
if ((i, j) not in tracker):
start_edge = (i, j)
tracker.add((i, j))
break
if ((j, i) not in tracker):
start_edge = (j, i)
tracker.add((j, i))
break
# else stop
if start_edge is None:
break
# mark regions using edges with intersection
false_pos = []
for l, poly in enumerate(regions):
for k in range(len(poly)-1):
i, j = poly[k], poly[k+1]
if ((i, j) in edges_intersect) or ((j, i) in edges_intersect):
false_pos.append(l)
break
# extract region masks
region_mks = []
for poly in regions:
rm = Image.new('L', (256, 256))
dr = ImageDraw.Draw(rm)
poly_coords = [tuple(junctions[x]) for x in poly]
dr.polygon(poly_coords, fill='white')
region_mks.append(np.array(rm)/255.0)
# ### DEBUG
# print(false_pos)
# print(poly_coords)
# plt.imshow(rm)
# plt.show()
# print('test:', len(region_mks))
return region_mks, false_pos