def paintParameterLine(parameterLine, width, height, img):
#PAINTPARAMETERLINE Paint parameterized line
# parameterLine: [n1,n2,n3,planeID,u1,u2], first 3 dims are normal
# direction, 4th dim is the base plane ID for U (1=XY), 5-6th dims are U
# of start and end point.
# width, height: the size of output panorama, width = height x 2
# img: the image on which lines will be drawn. If no img,
# img=zeros(height,width).
lines = parameterLine
if img is None:
panoEdgeC = np.zeros([height, width])
else:
img = np.double(img)
panoEdgeC = imresize(img, [height, width])
if img.shape[2] == 3:
pimg = PIL.Image.fromarray(np.uint8(panoEdgeC))
panoEdgeC = pimg.convert('L')
panoEdgeC = np.array(panoEdgeC) * 0.5
# valid = true(size(lines,1),1);
# uv_vp = xyz2vp([vp;-vp]);
# vpm = min(floor( (uv_vp(:,1)-(-pi)) /(2*pi)*width)+1, width);
# vpn = min(floor( ((pi/2)-uv_vp(:,2))/(pi)*height )+1, height);
# valid = lines(:,4)==1;
# lines = lines(~valid,:); #horizontal
# lines = lines(valid,:); #vertical
num_sample = max(height, width)
for i in np.arange(lines.shape[0]):
# fprintf('#d\n',i);
n = lines[i, 0:3]
sid = lines[i, 4] * 2 * np.pi
eid = lines[i, 5] * 2 * np.pi
if eid < sid:
x = np.linspace(sid, eid + 2 * np.pi, num_sample)
x = x % (2 * np.pi)
# x = sid-1:(eid-1+numBins);
# x = rem(x,numBins) + 1;
else:
x = np.linspace(sid, eid, num_sample)
# u = -pi + (x'-1)*uBinSize + uBinSize/2;
u = -np.pi + x.T
v = CoordsTransform.computeUVN(n, u, lines[i, 3])
xyz = CoordsTransform.uv2xyzN(np.column_stack((u, v)), lines[i, 3])
uv = CoordsTransform.xyz2uvN(xyz, 0)
uv = uv.T
m = np.minimum(np.floor((uv[:, 0] - (-np.pi)) / (2 * np.pi) * width),
width - 1)
n = np.minimum(np.floor(((np.pi / 2) - uv[:, 1]) / (np.pi) * height),
height - 1)
#drawId = sub2ind([height, width], n, m);
panoEdgeC[np.int32(n), np.int32(m)] = 255
return panoEdgeC
def lineFromTwoPoint( pt1, pt2 ):
#LINEFROMTWOPOINT Generate line segment based on two points on panorama
# pt1, pt2: two points on panorama
# lines:
# 1~3-th dim: normal of the line
# 4-th dim: the projection dimension ID
# 5~6-th dim: the u of line segment endpoints in projection plane
# use paintParameterLine to visualize
numLine = pt1.shape[0];
lines = np.zeros([numLine, 6]);
n = np.cross( pt1, pt2);
n = n/repmat( np.sqrt(np.sum(n**2,1)), 3, 1).T;
lines[:,0:3] = n;
areaXY = np.abs(np.sum(n*repmat(np.array([0 ,0 ,1]), numLine, 1),1));
areaYZ = np.abs(np.sum(n*repmat(np.array([1, 0, 0]), numLine, 1),1));
areaZX = np.abs(np.sum(n*repmat(np.array([0, 1, 0]), numLine, 1),1));
planeIDs = np.argmax(np.array([areaXY, areaYZ, areaZX]),0); # 1:XY 2:YZ 3:ZX
lines[:,3] = planeIDs;
for i in np.arange(numLine):
uv = CoordsTransform.xyz2uvN(np.array([pt1[i,:], pt2[i,:]]), lines[i,3]).T;
umax = np.max(uv[:, 0])+np.pi;
umin = np.min(uv[:,0])+np.pi;
if umax-umin>np.pi:
lines[i,4:6] = np.array([umax, umin])/2/np.pi;
else:
lines[i,4:6] = np.array([umin ,umax])/2/np.pi;
return lines
def rotatePanorama(img, vp, R):
#ROTATEPANORAMA Rotate panorama
# if R is given, vp (vanishing point) will be overlooked
# otherwise R is computed from vp
[sphereH, sphereW, C] = img.shape
# rotImg = zeros( sphereH, sphereW, C);
## new uv coordinates
[TX, TY] = np.meshgrid(np.arange(sphereW), np.arange(sphereH))
TX = TX.T.flatten() + 1
TY = TY.T.flatten() + 1
ANGx = (TX - sphereW / 2 - 0.5) / sphereW * np.pi * 2
ANGy = -(TY - sphereH / 2 - 0.5) / sphereH * np.pi
uvNew = np.column_stack((ANGx, ANGy))
xyzNew = CoordsTransform.uv2xyzN(uvNew, 0)
## rotation matrix
if R == None:
R = np.dot(np.diag([1, 1, 1]), np.linalg.pinv(vp.T))
xyzOld = np.dot(np.linalg.pinv(R), xyzNew.T).T
uvOld = CoordsTransform.xyz2uvN(xyzOld, 0).T
# Px = uvOld(:,1)/2/pi*sphereW + 0.5 + sphereW/2;
# Py = -uvOld(:,2)/pi*sphereH + 0.5 + sphereH/2;
Px = (uvOld[:, 0] + np.pi) / (2 * np.pi) * sphereW + 0.5
Py = (-uvOld[:, 1] + np.pi / 2) / np.pi * sphereH + 0.5
Px = np.reshape(Px, [sphereW, sphereH]).T
Py = np.reshape(Py, [sphereW, sphereH]).T
# boundary
imgNew = np.double(np.zeros([sphereH + 2, sphereW + 2, C]))
imgNew[1:-1, 1:-1, :] = img
imgNew[1:-1, 0, :] = img[:, -1, :]
imgNew[1:-1, -1, :] = img[:, 0, :]
halfW = np.int(sphereW / 2)
imgNew[0, 1:halfW + 1, :] = img[0, sphereW:halfW - 1:-1, :]
imgNew[0, halfW + 1:-1, :] = img[0, halfW - 1::-1, :]
imgNew[-1, 1:halfW + 1, :] = img[-1, sphereW:halfW - 1:-1, :]
imgNew[-1, halfW + 1:-1, :] = img[0, halfW:0:-1, :]
imgNew[0, 0, :] = img[0, 0, :]
imgNew[-1, -1, :] = img[-1, -1, :]
imgNew[0, -1, :] = img[0, -1, :]
imgNew[-1, 0, :] = img[-1, 0, :]
rotImg = Projection.warpImageFast(imgNew, Px + 1, Py + 1)
# rotImg = warpImageFast(img, Px, Py);
return rotImg, R
def rotateLines(lines, R):
#ROTATELINES Rotate lines on panorama
# lines: parameterized lines, R: rotation matrix
[numLine, dimLine] = lines.shape
lines_N = np.zeros([numLine, dimLine])
for i in np.arange(numLine):
n = lines[i, 0:3]
sid = lines[i, 4] * 2 * np.pi - np.pi
eid = lines[i, 5] * 2 * np.pi - np.pi
u = np.row_stack((sid, eid))
v = CoordsTransform.computeUVN(n, u, lines[i, 3])
xyz = CoordsTransform.uv2xyzN(np.column_stack((u, v)), lines[i, 3])
n_N = np.dot(R, n.T).T
n_N = n_N / np.linalg.norm(n_N, 2)
xyz_N = np.dot(R, xyz.T).T
lines_N[i, 3] = np.argmax(np.abs(n_N[[2, 0, 1]]))
uv_N = CoordsTransform.xyz2uvN(xyz_N, lines_N[i, 3]).T
umax = max(uv_N[:, 0]) + np.pi
umin = min(uv_N[:, 0]) + np.pi
if umax - umin > np.pi:
lines_N[i, 4:6] = np.array([umax, umin]) / 2 / np.pi
else:
lines_N[i, 4:6] = np.array([umin, umax]) / 2 / np.pi
lines_N[i, 0:3] = n_N
# lines_N(i,5:6) = (uv_N(:,1)'+pi)/2/pi;
if dimLine >= 7:
lines_N[i, 6] = np.arccos(
np.sum(xyz_N[0, :] * xyz_N[1, :]) /
(np.linalg.norm(xyz_N[0, :], 2) *
np.linalg.norm(xyz_N[1, :], 2)))
# lines_N(i,7) = lines(i,7); # this should be ok as well
if dimLine >= 8:
lines_N[i, 7] = lines[i, 7]
return lines_N
def gbPanoSegment(img, sigma, k, minSz):
#GBPANOSEGMENT Graph-based image segmentation on panorama
# Similar as Pedro's algorithm, only the graph is built on sphere, so
# left and right side are considered as attached.
# img: should be in uint8 [0~256]
# sigma, k, minSz: same parameters as in Pedro's algorithm
[height, width, _] = img.shape
#img_smooth = smooth(img, sigma);
#sigma = 10;
img_smooth = np.zeros([height, width, 3])
img_smooth[:, :, 0] = gaussian_filter(img[:, :, 0], sigma)
img_smooth[:, :, 1] = gaussian_filter(img[:, :, 1], sigma)
img_smooth[:, :, 2] = gaussian_filter(img[:, :, 2], sigma)
#SrcImage = './data/rotImg_smooth.mat'
#dict = loadmat(SrcImage)
#img_smooth = dict['img_smooth']
#plt.subplot(2, 1, 1)
#plt.imshow(np.uint8(img))
#plt.subplot(2, 1, 2)
#plt.imshow(np.uint8(img_smooth))
#plt.show()
## uniformly sample vectors on sphere and segment, test later
dict = loadmat('./data/uniformvector_lvl8.mat')
coor = dict['coor']
tri = dict['tri']
#[coor, tri] = getUniformVector(8);
# [ E ] = getSketchTokenEdgemap( img );
# [EE, Ix, Iy] = dt2(double(E), 0.1, 0, 0.1, 0 );
EE = np.zeros([height, width])
xySubs = CoordsTransform.uv2coords(CoordsTransform.xyz2uvN(coor, 0), width,
height, 0)
xySubs = np.int32(xySubs)
#SrcImage = './data/xySubs.mat'
#dict = loadmat(SrcImage)
#xySubs = dict['xySubs']
#xySubs = xySubs -1;
idx = np.where(xySubs[:, 1] < 0)
xySubs[idx, 1] = 0
idx = np.where(xySubs[:, 1] >= 512)
xySubs[idx, 1] = 511
idx = np.where(xySubs[:, 0] >= 1024)
xySubs[idx, 0] = 1023
SubXY = np.array([xySubs[:, 1], xySubs[:, 0]]).T
#xyinds = np.ravel_multi_index(SubXY,(height ,width));
#offset = width*height;
tri = tri - 1
e0 = np.array([tri[:, 0], tri[:, 1]]).T
e1 = np.array([tri[:, 1], tri[:, 2]]).T
e2 = np.array([tri[:, 2], tri[:, 0]]).T
edges = np.row_stack((e0, e1, e2))
invert = edges[:, 1] < edges[:, 0]
edges[invert, :] = edges[invert, 1::-1]
uniEdges, _ = np.unique(edges, return_inverse=True, axis=0)
#eid0 = np.unravel_index(xyinds[uniEdges[:,0]],(height,width,3))
#eid1 = np.unravel_index(xyinds[uniEdges[:,1]],(height,width,3))
eid0 = SubXY[uniEdges[:, 0], :].T
eid1 = SubXY[uniEdges[:, 1], :].T
#eid0offset = np.unravel_index(xyinds[uniEdges[:,0]] + offset,(height,width,3))
#eid1offset = np.unravel_index(xyinds[uniEdges[:,1]] + offset,(height,width,3))
#eid0offset2 = np.unravel_index(xyinds[uniEdges[:,0]] + 2 * offset,(height,width,3))
#eid1offset2 = np.unravel_index(xyinds[uniEdges[:,1]] + 2 * offset,(height,width,3))
if any(eid0[:, 0] >= 512):
print('nono')
weight = (img_smooth[eid0[0], eid0[1], 0] - img_smooth[eid1[0], eid1[1], 0]
)**2 + (img_smooth[eid0[0], eid0[1], 1] -
img_smooth[eid1[0], eid1[1], 1])**2 + (
img_smooth[eid0[0], eid0[1], 2] -
img_smooth[eid1[0], eid1[1], 2])**2
gdweight = (EE[eid0[0], eid0[1]] + EE[eid0[0], eid0[1]]) / 2
panoEdge = np.array([
uniEdges[:, 0], uniEdges[:, 1],
np.sqrt(np.double(weight)) + 10 * np.double(gdweight)
])
maxID = coor.shape[0]
num = uniEdges.shape[0]
edgeLabel = segmentGraphEdge((maxID, num, panoEdge, k, minSz))
L = np.unique(edgeLabel)
temp = np.zeros(len(edgeLabel))
[gridX, gridY] = np.meshgrid(np.arange(width), np.arange(height))
for i in np.arange(len(L)):
temp[edgeLabel == L[i]] = i + 1
pixelvector = CoordsTransform.uv2xyzN(
CoordsTransform.coords2uv([gridX[:] + 1, gridY[:] + 1], width, height),
0)
# k = 1;
# [nnidx, dists] = annsearch( coor', pixelvector', k);
#[nnidx, dists] = knnsearch( coor, pixelvector);
nbrs = NearestNeighbors(n_neighbors=1, algorithm='auto').fit(coor)
dists, nnidx = nbrs.kneighbors(pixelvector)
#from scipy import spatial
#tree = spatial.KDTree(coor);
#dists, nnidx = tree.query(pixelvector,k=1);
#SrcImage = './data/temp.mat'
#dict = loadmat(SrcImage)
#tempm = dict['temp']
#SrcImage = './data/nnidx.mat'
#dict = loadmat(SrcImage)
#nnidxm = dict['nnidx']
#nnidxm = nnidxm - 1
panoSegment = np.reshape(temp[nnidx], [width, height]).T
#pimg = PIL.Image.fromarray(np.uint8(panoSegment))
##pimggray = pimg.convert('L')
#pimg.save('./data/segmentation.jpg');
return panoSegment
def process2(rotImg):
#SrcImage = './data/rotImg.jpg'
#rotImg = imread(SrcImage, mode='RGB')
#rotImg = np.uint8(rotImg)
## image segmentation:
panoSegment = Segmentation.gbPanoSegment(rotImg, 0.5, 200, 50)
#SrcImage = './data/panoSegment.mat'
#dict = loadmat(SrcImage)
#panoSegment = dict['panoSegment']
plt.imshow(panoSegment, cmap='gray')
plt.title('Segmentation: left and right are connected')
plt.show()
## Get region inside a polygon
dict = loadmat('./data/points.mat')
# load room corner
points = dict['points']
dict = loadmat('./data/uniformvector_lvl8.mat')
coor = dict['coor']
tri = dict['tri']
vcs = CoordsTransform.uv2coords(CoordsTransform.xyz2uvN(coor, 0), 1024,
512, 0)
# transfer vectors to image coordinates
coords = CoordsTransform.uv2coords(CoordsTransform.xyz2uvN(points, 0),
1024, 512, 0)
[s_xyz, _] = PolygonRegion.sortXYZ(points[0:4, :])
# define a region with 4 vertices
[inside, _, _] = PolygonRegion.insideCone(s_xyz[-1::-1, :], coor, 0)
# test which vectors are in region
#figure(8);
plt.imshow(rotImg)
#hold on
for i in np.arange(4):
plt.scatter(coords[i, 0], coords[i, 1], 100, 'r', 's')
for i in np.where(inside):
plt.scatter(vcs[i, 0], vcs[i, 1], 1, 'g', 'o')
[s_xyz, I] = PolygonRegion.sortXYZ(points[4:8, :])
[inside, _, _] = PolygonRegion.insideCone(s_xyz[-1::-1, :], coor, 0)
for i in np.arange(4, 8):
plt.scatter(coords[i, 0], coords[i, 1], 100, 'r', 's')
for i in np.where(inside):
plt.scatter(vcs[i, 0], vcs[i, 1], 1, 'b', 'o')
plt.title('Display of two wall regions')
plt.show()
## Reconstruct a box, assuming perfect upperright cuboid
D3point = np.zeros([8, 3])
pointUV = CoordsTransform.xyz2uvN(points, 0).T
floor = -160
floorPtID = np.array([2, 3, 6, 7, 2]) - 1
ceilPtID = np.array([1, 4, 5, 8, 1]) - 1
for i in np.arange(4):
D3point[floorPtID[i], :] = LineFaceIntersection.LineFaceIntersection(
np.array([0, 0, floor]), np.array([0, 0, 1]), np.array([0, 0, 0]),
points[floorPtID[i], :])
D3point[ceilPtID[i], 2] = D3point[floorPtID[i], 2] / np.tan(
pointUV[floorPtID[i], 1]) * np.tan(pointUV[ceilPtID[i], 1])
ceiling = np.mean(D3point[ceilPtID, 2])
for i in np.arange(4):
D3point[ceilPtID[i], :] = LineFaceIntersection.LineFaceIntersection(
np.array([0, 0, ceiling]), np.array([0, 0, 1]),
np.array([0, 0, 0]), points[ceilPtID[i], :])
#figure(9);
#fig = plt.figure()
#ax = fig.add_subplot(111, projection='3d')
#ax.scatter(D3point[floorPtID,0], D3point[floorPtID,1], D3point[floorPtID,2]);
#hold on
#ax.scatter(D3point[ceilPtID,0], D3point[ceilPtID,1], D3point[ceilPtID,2]);
#for i in np.arange(4):
#ax.scatter(D3point[[floorPtID[i], ceilPtID[i]],0], D3point[[floorPtID[i], ceilPtID[i]],1], D3point[[floorPtID[i], ceilPtID[i]],2]);
#plt.title('Basic 3D reconstruction');
#plt.show()
#figure(10);
firstID = np.array([1, 4, 5, 8, 2, 3, 6, 7, 1, 4, 5, 8]) - 1
secndID = np.array([4, 5, 8, 1, 3, 6, 7, 2, 2, 3, 6, 7]) - 1
lines = LineFaceIntersection.lineFromTwoPoint(points[firstID, :],
points[secndID, :])
plt.imshow(Visualization.paintParameterLine(lines, 1024, 512, rotImg))
#hold on
for i in np.arange(8):
plt.scatter(coords[i, 0], coords[i, 1], 100, 'r', 's')
plt.title('Get lines by two points')
plt.show()
def process1():
SrcImage = './data/pano_room.jpg'
panoImg = imread(SrcImage, mode='RGB')
panoImg = np.double(panoImg / 255)
pi = np.pi
#project it to multiple perspective views
cutSize = 320
# size of perspective views
fov = pi / 3
# horizontal field of view of perspective views
xh = np.arange(-pi, 5 / 6 * pi, pi / 6)
yh = np.zeros([1, len(xh)])
xp = [
-3 / 3, -2 / 3, -1 / 3, +0 / 3, +1 / 3, +2 / 3, -3 / 3, -2 / 3, -1 / 3,
+0 / 3, +1 / 3, +2 / 3
]
xp = np.multiply(xp, pi)
yp = [
1 / 4, 1 / 4, 1 / 4, 1 / 4, 1 / 4, 1 / 4, -1 / 4, -1 / 4, -1 / 4,
-1 / 4, -1 / 4, -1 / 4
]
yp = np.multiply(yp, pi)
x = np.append(xh, xp)
x = np.append(x, 0)
x = np.append(x, 0)
y = np.append(yh, yp)
y = np.append(y, +pi / 2)
y = np.append(y, -pi / 2) # viewing direction of perspective views
sepScene = Projection.separatePano(panoImg, fov, x, y, cutSize)
#figure 1
#plt.imshow(np.uint8(sepScene[0].img* 255))
#plt.show()
## Line segment detection on panorama: first on perspective views and project back to panorama
numScene = len(sepScene)
edges = []
for i in np.arange(numScene):
[edgeMap, edgeList] = VpEstimation.lsdWrap(sepScene[i].img, 0.7)
edge = VpEstimation.Edge()
edge.img = edgeMap
edge.edgeLst = edgeList
edge.fov = sepScene[i].fov
edge.vx = sepScene[i].vx
edge.vy = sepScene[i].vy
edge.panoLst = VpEstimation.edgeFromImg2Pano(edge)
edges = np.append(edges, edge)
[lines, olines] = VpEstimation.combineEdgesN(edges)
# combine line segments from views
panoEdge = Visualization.paintParameterLine(lines, 1024, 512, None)
# paint parameterized line segments
##plt.subplot(3, 1, 1)
#plt.imshow(np.uint8(panoEdge),cmap='gray')
#plt.show()
##plt.subplot(3, 1, 2)
#plt.imshow(np.uint8(sepScene[0].img * 255))
#plt.show()
##plt.subplot(3, 1, 3)
#plt.imshow(np.uint8(edges[0].img) * 255,cmap='gray')
#plt.show()
# estimating vanishing point: Hough
[olines, mainDirect, _, _] = VpEstimation.vpEstimationPano(lines)
# mainDirect is vanishing point, in xyz format
vpCoords = CoordsTransform.uv2coords(
CoordsTransform.xyz2uvN(mainDirect, 0), 1024, 512, 0)
# transfer to uv format, then image coords
imgres = imresize(panoImg, [512, 1024])
panoEdge1r = Visualization.paintParameterLine(olines[0].line, 1024, 512,
imgres)
panoEdge2r = Visualization.paintParameterLine(olines[1].line, 1024, 512,
imgres)
panoEdge3r = Visualization.paintParameterLine(olines[2].line, 1024, 512,
imgres)
panoEdgeVP = np.array([panoEdge1r, panoEdge2r, panoEdge3r])
panoEdgeVP = np.zeros((512, 1024, 3), 'uint8')
panoEdgeVP[:, :, 0] = panoEdge1r
panoEdgeVP[:, :, 1] = panoEdge2r
panoEdgeVP[:, :, 2] = panoEdge3r
#panoEdgeVP = np.reshape(panoEdgeVP,[512,1024,3])
plt.imshow(panoEdgeVP)
color = 'rgb'
for i in np.arange(3):
plt.scatter(vpCoords[i, 0], vpCoords[i, 1], 100, color[i], 'o')
plt.scatter(vpCoords[i + 3, 0], vpCoords[i + 3, 1], 100, color[i], 'o')
plt.title('Vanishing points and assigned line segments')
#plt.show()
# rotate panorama to coordinates spanned by vanishing directions
vp = mainDirect[2::-1, :]
[rotImg, R] = Rotation.rotatePanorama(imgres, vp, None)
newMainDirect = Rotation.rotatePoint(mainDirect, R)
panoEdge1r = Visualization.paintParameterLine(
Rotation.rotateLines(olines[0].line, R), 1024, 512, rotImg)
panoEdge2r = Visualization.paintParameterLine(
Rotation.rotateLines(olines[1].line, R), 1024, 512, rotImg)
panoEdge3r = Visualization.paintParameterLine(
Rotation.rotateLines(olines[2].line, R), 1024, 512, rotImg)
newPanoEdgeVP = np.zeros((512, 1024, 3), 'uint8')
newPanoEdgeVP[:, :, 0] = panoEdge1r
newPanoEdgeVP[:, :, 1] = panoEdge2r
newPanoEdgeVP[:, :, 2] = panoEdge3r
plt.imshow(newPanoEdgeVP)
for i in np.arange(3):
plt.scatter(vpCoords[i, 0], vpCoords[i, 1], 100, color[i], 'o')
plt.scatter(vpCoords[i + 3, 0], vpCoords[i + 3, 1], 100, color[i], 'o')
plt.title('Original image')
plt.show()
plt.imshow(newPanoEdgeVP)
newVpCoords = CoordsTransform.uv2coords(
CoordsTransform.xyz2uvN(newMainDirect, 0), 1024, 512, 0)
for i in np.arange(3):
plt.scatter(newVpCoords[i, 0], newVpCoords[i, 1], 100, color[i], 'o')
plt.scatter(newVpCoords[i + 3, 0], newVpCoords[i + 3, 1], 100,
color[i], 'o')
plt.title('Rotated image')
plt.show()
return newPanoEdgeVP
def sphereHoughVote(segNormal, segLength, segScores, binRadius, orthTolerance,
candiSet, force_unempty):
#SPHEREHOUGHVOTE Summary of this function goes here
# Detailed explanation goes here
if 'force_unempty' in locals() == False:
force_unempty = true
## initial guess
# segNormal = arcList(:,1:3);
# segLength = sqrt( sum((arcList(:,4:6)-arcList(:,7:9)).^2, 2));
# segScores = arcList(:,end);
numLinesg = segNormal.shape[0]
# [voteBinPoints tri] = icosahedron2sphere(level);
voteBinPoints = candiSet
voteBinPoints = voteBinPoints[voteBinPoints[:, 2] >= 0, :]
reversValid = segNormal[:, 2] < 0
segNormal[reversValid, :] = -segNormal[reversValid, :]
voteBinUV = CoordsTransform.xyz2uvN(voteBinPoints, 0)
numVoteBin = len(voteBinPoints)
voteBinValues = np.zeros([numVoteBin, 1])
for i in np.arange(numLinesg):
tempNorm = segNormal[i, :]
tempDots = np.sum(voteBinPoints * repmat(tempNorm, numVoteBin, 1), 1)
# tempAngs = acos(abs(tempDots));
# voteBinValues = voteBinValues + normpdf(tempAngs, 0, 0.5*binRadius*pi/180)*segScores(i)*segLength(i);
# voteBinValues = voteBinValues + max(0, (2*binRadius*pi/180-tempAngs)./(2*binRadius*pi/180))*segScores(i)*segLength(i);
valid = np.abs(tempDots) < np.cos((90 - binRadius) * np.pi / 180)
voteBinValues[
valid] = voteBinValues[valid] + segScores[i] * segLength[i]
checkIDs1 = np.where(voteBinUV[1, :] > np.pi / 3)
checkIDs1 = checkIDs1[0]
voteMax = 0
checkID1Max = 0
checkID2Max = 0
checkID3Max = 0
for j in np.arange(len(checkIDs1)):
# fprintf('#d/#d\n', j, length(checkIDs1));
checkID1 = checkIDs1[j]
vote1 = voteBinValues[checkID1]
if voteBinValues[checkID1] == 0 and force_unempty:
continue
checkNormal = voteBinPoints[checkID1, :]
dotProduct = np.sum(
voteBinPoints * repmat(checkNormal, voteBinPoints.shape[0], 1), 1)
checkIDs2 = np.where(
np.abs(dotProduct) < np.cos((90 - orthTolerance) * np.pi / 180))
checkIDs2 = checkIDs2[0]
for i in np.arange(len(checkIDs2)):
checkID2 = checkIDs2[i]
if voteBinValues[checkID2] == 0 and force_unempty:
continue
vote2 = vote1 + voteBinValues[checkID2]
cpv = np.cross(voteBinPoints[checkID1, :],
voteBinPoints[checkID2, :])
cpn = np.sqrt(np.sum(cpv**2))
dotProduct = np.sum(
voteBinPoints * repmat(cpv, voteBinPoints.shape[0], 1),
1) / cpn
checkIDs3 = np.where(
abs(dotProduct) > np.cos(orthTolerance * np.pi / 180))
checkIDs3 = checkIDs3[0]
for k in np.arange(len(checkIDs3)):
checkID3 = checkIDs3[k]
if voteBinValues[checkID3] == 0 and force_unempty:
continue
vote3 = vote2 + voteBinValues[checkID3]
if vote3 > voteMax:
# print('#f\n', vote3);
lastStepCost = vote3 - voteMax
if voteMax != 0:
tmp = np.sum(
voteBinPoints[
[checkID1Max, checkID2Max, checkID3Max], :] *
voteBinPoints[[checkID1, checkID2, checkID3], :],
1)
lastStepAngle = np.arccos(tmp)
else:
lastStepAngle = [0, 0, 0]
checkID1Max = checkID1
checkID2Max = checkID2
checkID3Max = checkID3
voteMax = vote3
# voteBins(checkID1, checkID2, checkID3) = true;
if checkID1Max == 0:
print('Warning: No orthogonal voting exist!!!\n')
refiXYZ = []
lastStepCost = 0
lastStepAngle = 0
return
initXYZ = voteBinPoints[[checkID1Max, checkID2Max, checkID3Max], :]
## refine
# binRadius = binRadius/2;
refiXYZ = np.zeros([3, 3])
dotprod = np.sum(segNormal * repmat(initXYZ[0, :], segNormal.shape[0], 1),
1)
valid = np.abs(dotprod) < np.cos((90 - binRadius) * np.pi / 180)
validNm = segNormal[valid, :]
segL = segLength[valid]
validWt = np.reshape(segL, [segL.shape[0], 1]) * segScores[valid]
validWt = validWt / np.max(validWt)
_, refiNM = curveFitting(validNm, validWt)
refiXYZ[0, :] = refiNM
dotprod = np.sum(segNormal * repmat(initXYZ[1, :], segNormal.shape[0], 1),
1)
valid = np.abs(dotprod) < np.cos((90 - binRadius) * np.pi / 180)
validNm = segNormal[valid, :]
segl = segLength[valid]
validWt = np.reshape(segl, [segl.shape[0], 1]) * segScores[valid]
validWt = validWt / max(validWt)
validNm = np.row_stack((validNm, refiXYZ[0, :]))
validWt = np.row_stack((validWt, sum(validWt) * 0.1))
_, refiNM = curveFitting(validNm, validWt)
refiXYZ[1, :] = refiNM
refiNM = np.cross(refiXYZ[0, :], refiXYZ[1, :])
refiXYZ[2, :] = refiNM / np.linalg.norm(refiNM, 2)
# [~,refiNM] = curveFitting(validNm, validWt);
# refiXYZ(i,:) = refiNM;
#
#
#
# for i = 1:3
# dotprod = dot(segNormal, repmat(initXYZ(i,:), [size(segNormal,1) 1]), 2);
# valid = abs(dotprod)<cos((90-binRadius)*pi/180);
# validNm = segNormal(valid,:);
# validWt = segLength(valid).*segScores(valid);
# [~,refiNM] = curveFitting(validNm, validWt);
# refiXYZ(i,:) = refiNM;
# end
## output
# # [voteBinPoints tri] = icosahedron2sphere(level);
# OBJ.vertices = voteBinPoints;
# # OBJ.vertices_normal = zeros(size(voteBinPoints));
# # OBJ.vertices_normal(:,1) = 1;
# OBJ.vertices_normal = voteBinPoints;
# uv = xyz2uvN(voteBinPoints);
# OBJ.vertices_texture = [(uv(:,1)+pi)/2/pi (uv(:,2)+pi/2)/pi];
#
# OBJ.objects.type = 'f';
# OBJ.objects.data.vertices = tri;
# OBJ.objects.data.texture = tri;
# OBJ.objects.data.normal = tri;
#
# # check boundary
# newVTSID = size(OBJ.vertices_texture,1);
# newFace = 0;
# newAddVT = zeros(1000,2);
# for i = 1:size(OBJ.objects.data.vertices,1)
# texture = OBJ.objects.data.texture(i,:);
# vt = OBJ.vertices_texture(texture,:);
# v = OBJ.vertices(texture,:);
# if (std(vt(:,1)))<0.3
# continue;
# end
#
# newFace = newFace + 1;
#
# modify = (vt(1,1)-vt(:,1))>0.5;
# vt(modify,1) = vt(modify,1)+1;
# modify = (vt(1,1)-vt(:,1))<-0.5;
# vt(modify,1) = vt(modify,1)-1;
#
# newAddVT((newFace-1)*3+1:newFace*3,:) = vt;
# OBJ.objects.data.texture(i,:) = [(newFace-1)*3+1:newFace*3] + newVTSID;
#
# if newFace>300
# fprintf('Warning: pre-assign more memory!/n');
# end
# end
# OBJ.vertices_texture = [OBJ.vertices_texture;newAddVT];
#
# material(1).type='newmtl';
# material(1).data='Textured';
# material(2).type='Ka';
# material(2).data=[1.0 1.0 1.0];
# material(3).type='Kd';
# material(3).data=[1.0 1.0 1.0];
# material(4).type='Ks';
# material(4).data=[1.0 1.0 1.0];
# material(5).type='illum';
# material(5).data=2;
# material(6).type='map_Kd';
# material(6).data='pano_hotel_2.jpg';
# OBJ.material = material;
#
# write_wobj(OBJ, 'exciting_notext.obj');
# uv = xyz2uvN(voteBinPoints);
# textureMap = zeros(512, 1024);
# x = min(round((uv(:,1)+pi)/2/pi*1024+1), 1024);
# y = 512 - min(round((uv(:,2)+pi/2)/pi*512+1), 512) + 1;
# value = voteBinValues./max(voteBinValues);
# value = value.^3;
# # [gridX, gridY] = meshgrid(x,y);
# for i = 1:length(x)
# sx = max(1, x(i)-3);
# ex = min(1024, x(i)+3);
# sy = max(1, y(i)-3);
# ey = min(512, y(i)+3);
# textureMap(sy:ey, sx:ex) = value(i);
# end
#
# # ind = sub2ind([512 1024], y, x);
# # textureMap(ind) = voteBinValues;
# imwrite(textureMap, 'exciting.png');
return [refiXYZ, lastStepCost, lastStepAngle]
def findMainDirectionEMA(lines):
#FINDMAINDIRECTION compute vp from set of lines
# Detailed explanation goes here
print('Computing vanishing point:\n')
# arcList = [];
# for i = 1:length(edge)
# panoLst = edge(i).panoLst;
# if size(panoLst,1) == 0
# continue;
# end
# arcList = [arcList; panoLst];
# end
## initial guess
segNormal = lines[:, 0:3]
segLength = lines[:, 6]
segScores = np.ones([lines.shape[0], 1])
#lines(:,8);
shortSegValid = segLength < 5 * np.pi / 180
segNormal = segNormal[~shortSegValid, :]
segLength = segLength[~shortSegValid]
segScores = segScores[~shortSegValid]
numLinesg = segNormal.shape[0]
[candiSet, tri] = icosahedron2sphere.icosahedron2sphere(3)
ang = np.arccos(np.sum(
candiSet[tri[0, 0], :] * candiSet[tri[0, 1], :])) / np.pi * 180
binRadius = ang / 2
[initXYZ, score, angle] = sphereHoughVote(segNormal, segLength, segScores,
2 * binRadius, 2, candiSet, True)
if len(initXYZ) == 0:
print('Initial Failed\n')
mainDirect = []
return
print('Initial Computation: #d candidates, #d line segments\n',
candiSet.shape[0], numLinesg)
print(
'direction 1: #f #f #f\ndirection 2: #f #f #f\ndirection 3: #f #f #f\n',
initXYZ[0, 0], initXYZ[0, 1], initXYZ[0, 2], initXYZ[1, 0],
initXYZ[1, 1], initXYZ[1, 2], initXYZ[2, 0], initXYZ[2, 1], initXYZ[2,
2])
## iterative refine
iter_max = 3
[candiSet, tri] = icosahedron2sphere.icosahedron2sphere(5)
numCandi = candiSet.shape[0]
angD = np.arccos(np.sum(
candiSet[tri[0, 0], :] * candiSet[tri[0, 1], :])) / np.pi * 180
binRadiusD = angD / 2
curXYZ = initXYZ
tol = np.linspace(4 * binRadius, 4 * binRadiusD, iter_max)
# shrink down #ls and #candi
for iter in np.arange(iter_max):
dot1 = np.abs(np.sum(segNormal * repmat(curXYZ[0, :], numLinesg, 1),
1))
dot2 = np.abs(np.sum(segNormal * repmat(curXYZ[1, :], numLinesg, 1),
1))
dot3 = np.abs(np.sum(segNormal * repmat(curXYZ[2, :], numLinesg, 1),
1))
valid1 = dot1 < np.cos((90 - tol[iter]) * np.pi / 180)
valid2 = dot2 < np.cos((90 - tol[iter]) * np.pi / 180)
valid3 = dot3 < np.cos((90 - tol[iter]) * np.pi / 180)
valid = valid1 | valid2 | valid3
if (sum(valid) == 0):
print('ZERO line segment for voting\n')
break
subSegNormal = segNormal[valid, :]
subSegLength = segLength[valid]
subSegScores = segScores[valid]
dot1 = np.abs(np.sum(candiSet * repmat(curXYZ[0, :], numCandi, 1), 1))
dot2 = np.abs(np.sum(candiSet * repmat(curXYZ[1, :], numCandi, 1), 1))
dot3 = np.abs(np.sum(candiSet * repmat(curXYZ[2, :], numCandi, 1), 1))
valid1 = dot1 > np.cos(tol[iter] * np.pi / 180)
valid2 = dot2 > np.cos(tol[iter] * np.pi / 180)
valid3 = dot3 > np.cos(tol[iter] * np.pi / 180)
valid = valid1 | valid2 | valid3
if (sum(valid) == 0):
print('ZERO candidate for voting\n')
break
subCandiSet = candiSet[valid, :]
[tcurXYZ, _,
_] = sphereHoughVote(subSegNormal, subSegLength, subSegScores,
2 * binRadiusD, 2, subCandiSet, True)
if (len(tcurXYZ) == 0):
print('NO answer found!\n')
break
curXYZ = tcurXYZ
print('#d-th iteration: #d candidates, #d line segments\n', iter,
subCandiSet.shape[0], len(subSegScores))
print(
'direction 1: #f #f #f\ndirection 2: #f #f #f\ndirection 3: #f #f #f\n',
curXYZ[0, 0], curXYZ[0, 1], curXYZ[0, 2], curXYZ[1, 0], curXYZ[1, 1],
curXYZ[1, 2], curXYZ[2, 0], curXYZ[2, 1], curXYZ[2, 2])
mainDirect = curXYZ
mainDirect[0, :] = mainDirect[0, :] * np.sign(mainDirect[0, 2])
mainDirect[1, :] = mainDirect[1, :] * np.sign(mainDirect[1, 2])
mainDirect[2, :] = mainDirect[2, :] * np.sign(mainDirect[2, 2])
uv = CoordsTransform.xyz2uvN(mainDirect, 0)
I1 = np.argmax(uv[1, :])
J = np.setdiff1d([
0,
1,
2,
], I1)
I2 = np.argmin(np.abs(np.sin(uv[0, J])))
I2 = J[I2]
I3 = np.setdiff1d([0, 1, 2], [I1, I2])
mainDirect = np.row_stack(
(mainDirect[I1, :], mainDirect[I2, :], mainDirect[I3, :]))
mainDirect[0, :] = mainDirect[0, :] * np.sign(mainDirect[0, 2])
mainDirect[1, :] = mainDirect[1, :] * np.sign(mainDirect[1, 1])
mainDirect[2, :] = mainDirect[2, :] * np.sign(mainDirect[2, 0])
mainDirect = np.row_stack((mainDirect, -mainDirect))
# score = 0;
return [mainDirect, score, angle]
def combineEdgesN(edges):
#COMBINEEDGES Combine some small line segments, should be very conservative
# lines: combined line segments
# ori_lines: original line segments
# line format: [nx ny nz projectPlaneID umin umax LSfov score]
arcList = []
for i in np.arange(edges.shape[0]):
panoLst = edges[i].panoLst
if panoLst[0].shape[0] == 0:
continue
if (len(arcList)) == 0:
arcList = panoLst
else:
arcList = np.row_stack((arcList, panoLst))
## ori lines
numLine = arcList.shape[0]
ori_lines = np.zeros((numLine, 8))
areaXY = np.abs(
np.sum(arcList[:, 0:3] * repmat([0, 0, 1], arcList.shape[0], 1), 1))
areaYZ = np.abs(
np.sum(arcList[:, 0:3] * repmat([1, 0, 0], arcList.shape[0], 1), 1))
areaZX = np.abs(
np.sum(arcList[:, 0:3] * repmat([0, 1, 0], arcList.shape[0], 1), 1))
vec = [areaXY, areaYZ, areaZX]
#[_, planeIDs] = np.max(vec, 1); # 1:XY 2:YZ 3:ZX
planeIDs = np.argmax(vec, 0)
for i in np.arange(numLine):
ori_lines[i, 0:3] = arcList[i, 0:3]
ori_lines[i, 3] = planeIDs[i]
coord1 = arcList[i, 3:6]
coord2 = arcList[i, 6:9]
uv = CoordsTransform.xyz2uvN(np.row_stack((coord1, coord2)),
planeIDs[i])
umax = np.max(uv[0, :]) + np.pi
umin = np.min(uv[0, :]) + np.pi
if umax - umin > np.pi:
ori_lines[i, 4:6] = np.column_stack((umax, umin)) / 2 / np.pi
# ori_lines(i,7) = umin + 1 - umax;
else:
ori_lines[i, 4:6] = np.column_stack((umin, umax)) / 2 / np.pi
# ori_lines(i,7) = umax - umin;
ori_lines[i, 6] = np.arccos(
np.sum(coord1 * coord2) /
(np.linalg.norm(coord1, 2) * np.linalg.norm(coord2, 2)))
ori_lines[i, 7] = arcList[i, 9]
# valid = ori_lines(:,3)<0;
# ori_lines(valid,1:3) = -ori_lines(valid,1:3);
## additive combination
lines = ori_lines
# panoEdge = paintParameterLine( lines, 1024, 512);
# figure; imshow(panoEdge);
for iter in np.arange(3):
numLine = lines.shape[0]
valid_line = np.ones([numLine], dtype=bool)
for i in np.arange(numLine):
# fprintf('#d/#d\n', i, numLine);
if valid_line[i] == False:
continue
dotProd = np.sum(lines[:, 0:3] * repmat(lines[i, 0:3], numLine, 1),
1)
valid_curr = (np.abs(dotProd) > np.cos(
1 * np.pi / 180)) & valid_line
valid_curr[i] = False
valid_ang = np.where(valid_curr)
for j in valid_ang[0]:
range1 = lines[i, 4:6]
range2 = lines[j, 4:6]
valid_rag = intersection(range1, range2)
if valid_rag == False:
continue
# combine
I = np.argmax(np.abs(lines[i, 0:3]))
if lines[i, I] * lines[j, I] > 0:
nc = lines[i, 0:3] * lines[i, 6] + lines[j, 0:3] * lines[j,
6]
else:
nc = lines[i, 0:3] * lines[i, 6] - lines[j, 0:3] * lines[j,
6]
nc = nc / np.linalg.norm(nc, 2)
if insideRange(range1[0], range2):
nrmin = range2[0]
else:
nrmin = range1[0]
if insideRange(range1[1], range2):
nrmax = range2[1]
else:
nrmax = range1[1]
u = np.array([nrmin, nrmax]) * 2 * np.pi - np.pi
v = CoordsTransform.computeUVN(nc, u, lines[i, 3])
xyz = CoordsTransform.uv2xyzN(np.column_stack((u, v)),
lines[i, 3])
length = np.arccos(np.sum(xyz[0, :] * xyz[1, :]))
scr = (lines[i, 6] * lines[i, 7] +
lines[j, 6] * lines[j, 7]) / (lines[i, 6] + lines[j, 6])
nc = np.append(nc, lines[i, 3])
nc = np.append(nc, nrmin)
nc = np.append(nc, nrmax)
nc = np.append(nc, length)
nc = np.append(nc, scr)
newLine = nc
lines[i, :] = newLine
valid_line[j] = False
lines = lines[valid_line, :]
print('iter: #d, before: #d, after: #d\n', iter, len(valid_line),
sum(valid_line))
return [lines, ori_lines]
'''
def computePanoOmap(scene, edges, xyz):
#COMPUTEPANOOMAP compute orientation map
# edge: line segments in separate views
# xyz: vanishing point
# OUTPUT:
# omap: orientation map of separate views
# panoOmap: project to panorama theta phi coordinate
uv = CoordsTransform.xyz2uvN(xyz, 0).T
omap = edges
numViews = len(edges)
[H, W] = edges[0].img.shape
for i in np.arange(numViews):
edgeLst = edges[i].edgeLst
if edgeLst.shape[0] == 0:
omap[i].img = np.zeros(H, W, 3)
continue
edge = VpEstimation.Edge()
lines = []
for j in np.arange(edgeLst.shape[0]):
line = Line()
line.point1 = edgeLst[j, 0:2]
line.point2 = edgeLst[j, 2:4]
line.length = np.sqrt(
np.sum((edgeLst[j, 0:2] - edgeLst[j, 2:4])**2))
lines = np.append(lines, line)
x = edges[i].vx
y = edges[i].vy
fov = edges[i].fov
ANGx = uv[:, 0]
ANGy = uv[:, 1]
# compute the radius of ball
[imH, imW] = edges[i].img.shape
R = (imW / 2) / np.tan(fov / 2)
# im is the tangent plane, contacting with ball at [x0 y0 z0]
x0 = R * np.cos(y) * np.sin(x)
y0 = R * np.cos(y) * np.cos(x)
z0 = R * np.sin(y)
# plane function: x0(x-x0)+y0(y-y0)+z0(z-z0)=0
# view line: x/alpha=y/belta=z/gamma
# alpha=cos(phi)sin(theta); belta=cos(phi)cos(theta); gamma=sin(phi)
alpha = np.cos(ANGy) * np.sin(ANGx)
belta = np.cos(ANGy) * np.cos(ANGx)
gamma = np.sin(ANGy)
# solve for intersection of plane and viewing line: [x1 y1 z1]
division = x0 * alpha + y0 * belta + z0 * gamma
x1 = R * R * alpha / division
y1 = R * R * belta / division
z1 = R * R * gamma / division
# vector in plane: [x1-x0 y1-y0 z1-z0]
# positive x vector: vecposX = [cos(x) -sin(x) 0]
# positive y vector: vecposY = [x0 y0 z0] x vecposX
vec = np.row_stack([x1 - x0, y1 - y0, z1 - z0]).T
vecposX = np.row_stack([np.cos(x), -np.sin(x), 0]).T
deltaX = np.dot(vecposX, vec.T) / np.sqrt(np.dot(
vecposX, vecposX.T)) + (imW + 1) / 2
vecposY = np.cross(np.column_stack([x0, y0, z0]), vecposX)
deltaY = np.dot(vecposY, vec.T) / np.sqrt(np.dot(
vecposY, vecposY.T)) + (imH + 1) / 2
deltaX = deltaX.flatten()
deltaY = deltaY.reshape([-1])
vp = np.zeros([3]).tolist()
vp[0] = np.array([deltaX[0], deltaY[0]])
vp[1] = np.array([deltaX[1], deltaY[1]])
vp[2] = np.array([deltaX[2], deltaY[2]])
lines_orig = lines
[lines, lines_ex] = VVPR.taglinesvp(vp, lines_orig)
[omapmore, OMAP_FACTOR] = VORM.compute_omap(lines, vp, [H, W, 3])
#plt.imshow( np.uint8(omapmore) * 255)
#plt.show()
omap[i].img = np.double(omapmore)
omap[i].lines_orig = lines_orig
omap[i].lines = lines
omap[i].vp = vp
linesImg = np.zeros([H, W, 3])
for j in np.arange(lines.shape[0]):
lineclass = lines[j].lineclass
if lineclass == 0:
continue
x = np.linspace(lines[j].point1[0] + 1, lines[j].point2[0] + 1,
1000)
y = np.linspace(lines[j].point1[1] + 1, lines[j].point2[1] + 1,
1000)
xx = np.maximum(np.minimum(np.round(x), W - 1), 0)
yy = np.maximum(np.minimum(np.round(y), H - 1), 0)
#index = sub2ind( [H W], yy, xx);
#linesImg(H*W*(lineclass-1)+index) = 1;
omap[i].linesImg = linesImg
# roomhyp = sample_roomhyp(1000, lines_ex, vp, [H W 3]);
# omap(i).roomhyp = roomhyp;
# [ bestHyp ] = evaluateRoomHyp( omap(i) );
# disp_room(roomhyp(randsample(length(roomhyp),10)), scene(i).img, 1); # display some
# cuboidhyp_omap = generate_cuboid_from_omap(omapmore, vp, OMAP_FACTOR);
# disp_cubes(cuboidhyp_omap, scene(i).img, 1); # display all
## original
# img = edge(i).img;
# [lines linesmore] = compute_lines(img);
# if length(lines)<=3
# omap(i).img = zeros(H,W,3);
# continue;
# end
#
# [vp f] = compute_vp(lines, [H W 3]);
# lines_orig = lines;
# [lines lines_ex] = taglinesvp(vp, lines_orig);
# linesmore_orig = linesmore;
# [linesmore linesmore_ex] = taglinesvp(vp, linesmore_orig);
# [omapmore, OMAP_FACTOR] = compute_omap(lines, vp, [H W 3]);
# omap(i).img = double(omapmore);
'''
from scipy.io import loadmat
dict = loadmat('./data/var.mat'); # load room corner
omap = dict['omap']
edgesObjs = []
for i in np.arange(omap.shape[1]):
edge = VpEstimation.Edge()
edge.img = omap[0,i][0];
edge.edgeLst = omap[0,i][1];
edge.fov = omap[0,i][4];
edge.vx = omap[0,i][2];
edge.vy = omap[0,i][3];
edge.panoLst = omap[0,i][5];
edge.lines_orig = omap[0,i][6];
edge.lines = omap[0,i][7];
edge.vp = omap[0,i][8];
edge.linesImg = omap[0,i][9];
edgesObjs = np.append(edgesObjs,edge)
'''
panoOmap = Projection.combineViews(omap, 2048, 1024)
return [omap, panoOmap]