def viewparam(source,tyaw):
# camera calibration matrix
Kcal = geom.cameramat(source.image.size[0], source.image.size[1], source.fov)
# camera orientation (camera to world)
yaw = source.yaw if np.isnan(tyaw) else tyaw
Rot = geom.RfromYPR(yaw, source.pitch, source.roll)
return Kcal, Rot
def genView(panopath, outpath, orientation, viewdir, detail):
# panopath: location of raster, depth and plane_pano images
# outpath: location to put generated view, depth, and plane images
# orientation: [yaw, pitch, roll] of panorama (in degrees)
# viewdir: clock-based view; in set [2,3,4,8,9,10] for database
# detail: 0 = 768x512 with 60d fov, 1 = 2500x1200 with 90d fov
# local constants
start = time.time()
pi = np.pi
width, height, fov = (2500, 1200, 90) if detail else (768, 512, 60)
# view details
Rpano = geom.RfromYPR(orientation[0],orientation[1],orientation[2])
Yview = np.mod( orientation[0] + 30*viewdir, 360 )
Rview = geom.RfromYPR(Yview, 0, 0)
Kview = geom.cameramat(width, height, fov*pi/180)
Kinv = alg.inv(Kview)
# Load image pano, depth pano, and plane pano images
cvIP = cv.LoadImageM( os.path.join(panopath,'raster.jpg'), cv.CV_LOAD_IMAGE_UNCHANGED )
cvDP = cv.fromarray( np.asarray( Image.open( os.path.join(panopath,'depth.png') ) ) )
pp = np.asarray( Image.open( os.path.join(panopath,'plane_pano.png') ) ).copy()
vals = set(list(pp.reshape(-1)))
vals.remove(255)
gnd = max(vals)
pp[pp==gnd] = np.uint8(0)
cvPP = cv.fromarray(pp)
# load pixel map
pix = np.append(np.array(np.meshgrid(range(width),range(height))).reshape(2,-1),np.ones([1,width*height]),0)
midpoint = time.time()
print 'Loading pano images took ' + str(midpoint-start) + ' seconds.'
# Create output openCV matrices
cvI = cv.CreateMat(height,width,cv.CV_8UC3)
cvD = cv.CreateMat(height,width,cv.CV_32SC1)
cvP = cv.CreateMat(height,width,cv.CV_8UC1)
# compute mappings
ray = np.dot( np.transpose(Rpano), np.dot( Rview, np.dot( Kinv, pix ) ) )
yaw, pitch = np.arctan2( ray[0,:] , ray[2,:] ) , np.arctan2( -ray[1,:] , np.sqrt((np.array([ray[0,:],ray[2,:]])**2).sum(0)) )
ix, iy = cv.fromarray(np.array(8192/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(4096/pi*(pi/2-pitch),np.float32).reshape(height,width))
dx, dy = cv.fromarray(np.array(5000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(2500/pi*(pi/2-pitch),np.float32).reshape(height,width))
px, py = cv.fromarray(np.array(1000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array( 500/pi*(pi/2-pitch),np.float32).reshape(height,width))
# call remap function
cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_CUBIC)
cv.Remap(cvDP,cvD,dx,dy,cv.CV_INTER_NN)
cv.Remap(cvPP,cvP,px,py,cv.CV_INTER_NN)
# write images to file
Image.fromarray(np.array(cvI)[:,:,[2,1,0]]).save(os.path.join(outpath,'view.jpg'),'jpeg')
Image.fromarray(np.array(cvD)).save(os.path.join(outpath,'depth.png'),'png')
Image.fromarray(np.array(cvP)).save(os.path.join(outpath,'plane.png'),'png')
print 'Generating views from pano took ' + str(time.time()-midpoint) + ' seconds.'
def genView(panopath, outpath, orientation, viewdir, detail):
# panopath: location of raster, depth and plane_pano images
# outpath: location to put generated view, depth, and plane images
# orientation: [yaw, pitch, roll] of panorama (in degrees)
# viewdir: clock-based view; in set [2,3,4,8,9,10] for database
# detail: 0 = 768x512 with 60d fov, 1 = 2500x1200 with 90d fov
# local constants
start = time.time()
pi = np.pi
width, height, fov = (2500, 1200, 90) if detail else (768, 512, 60)
# view details
Rpano = geom.RfromYPR(orientation[0],orientation[1],orientation[2])
Yview = np.mod( orientation[0] + 30*viewdir, 360 )
Rview = geom.RfromYPR(Yview, 0, 0)
Kview = geom.cameramat(width, height, fov*pi/180)
Kinv = alg.inv(Kview)
# Load image pano, depth pano, and plane pano images
cvIP = cv.LoadImageM( os.path.join(panopath,'raster.jpg'), cv.CV_LOAD_IMAGE_UNCHANGED )
cvDP = cv.fromarray( np.asarray( Image.open( os.path.join(panopath,'depth.png') ) ) )
pp = np.asarray( Image.open( os.path.join(panopath,'plane_pano.png') ) ).copy()
vals = set(list(pp.reshape(-1)))
vals.remove(255)
gnd = max(vals)
pp[pp==gnd] = np.uint8(0)
cvPP = cv.fromarray(pp)
# load pixel map
pix = np.append(np.array(np.meshgrid(range(width),range(height))).reshape(2,-1),np.ones([1,width*height]),0)
midpoint = time.time()
print 'Loading pano images took ' + str(midpoint-start) + ' seconds.'
# Create output openCV matrices
cvI = cv.CreateMat(height,width,cv.CV_8UC3)
cvD = cv.CreateMat(height,width,cv.CV_32SC1)
cvP = cv.CreateMat(height,width,cv.CV_8UC1)
# compute mappings
ray = np.dot( np.transpose(Rpano), np.dot( Rview, np.dot( Kinv, pix ) ) )
yaw, pitch = np.arctan2( ray[0,:] , ray[2,:] ) , np.arctan2( -ray[1,:] , np.sqrt((np.array([ray[0,:],ray[2,:]])**2).sum(0)) )
ix, iy = cv.fromarray(np.array(8192/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(4096/pi*(pi/2-pitch),np.float32).reshape(height,width))
dx, dy = cv.fromarray(np.array(5000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(2500/pi*(pi/2-pitch),np.float32).reshape(height,width))
px, py = cv.fromarray(np.array(1000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array( 500/pi*(pi/2-pitch),np.float32).reshape(height,width))
# call remap function
cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_CUBIC)
cv.Remap(cvDP,cvD,dx,dy,cv.CV_INTER_NN)
cv.Remap(cvPP,cvP,px,py,cv.CV_INTER_NN)
# write images to file
Image.fromarray(np.array(cvI)[:,:,[2,1,0]]).save(os.path.join(outpath,'view.jpg'),'jpeg')
Image.fromarray(np.array(cvD)).save(os.path.join(outpath,'depth.png'),'png')
Image.fromarray(np.array(cvP)).save(os.path.join(outpath,'plane.png'),'png')
print 'Generating views from pano took ' + str(time.time()-midpoint) + ' seconds.'
def estimate_pose(C, Q, dbmatch, gtStatus=None):
# settings
param = C.pose_param
runflag = param['runflag']
np.seterr(all='ignore')
Q.datafile = os.path.join(C.pose_param['resultsdir'],'data_'+Q.name+'.txt')
open(Q.datafile,'w').close()
#####----- PRINT RUN DETAILS -----#####
run_info = os.path.join(param['resultsdir'],param['run_info'])
open(run_info,'w').close()
with open(run_info,'a') as ri:
if runflag == 11: print >>ri, 'Method: Yaw, planes from VPs. Scale computed with homography.'
elif runflag == 10: print >>ri, 'Method: Yaw, planes from VPs. Scale computed after homography.'
if param['cheat']: print >>ri, 'Ground truth yaw and plane used (cheating).'
print >>ri, 'Inlier base error threshold: %.3f' % param['inlier_error']
print >>ri, 'Base iteration scale: %d' % param['ransac_iter']
#####----- PRINT RUN DETAILS -----#####
# get high res db image and sift paths
dbinfo = os.path.join(C.hiresdir, dbmatch + '.info')
dbimg = os.path.join(C.hiresdir,dbmatch+'.jpg')
dbsift = os.path.join(C.hiresdir,dbmatch+'sift.txt')
dbsource = render_tags.EarthmineImageInfo(dbimg, dbinfo)
# Set Kd, wRd, and db position
wx,wy = dbsource.image.size
fov = dbsource.fov
Kd = geom.cameramat(wx, wy, fov)
Kdinv = alg.inv(Kd)
y,p,r = dbsource.yaw, dbsource.pitch, dbsource.roll
wRd = geom.RfromYPR(y,p,r) # db camera orientation (camera to world)
olat,olon,oalt = dbsource.lat,dbsource.lon,dbsource.alt # database location
# get high res query information
qname = Q.name
qimg = os.path.join(C.querydir,'hires',qname+'.jpg')
qsift = os.path.join(C.querydir,'hires',qname+'sift.txt')
qsource = render_tags.QueryImageInfo(Q.datasource)
glat,glon = qsource.lat,qsource.lon
gzx = geom.lltom(olat,olon,glat,glon)
timename = qname[-12:-10]+qname[-9:-7]+qname[-6:-4]#+qname[-3:]
# Set Kq
wx,wy = qsource.image.size
fov = qsource.fov
Kq = geom.cameramat(wx, wy, fov)
Kqinv = alg.inv(Kq)
cyaw,p,r = qsource.yaw, qsource.pitch, qsource.roll # cyaw - cell phone yaw
# get high res sift rematch
matches = highresSift(C, Q, dbmatch)
with open(Q.datafile,'a') as df:
print >>df, 'Number of matches | number of queries | ratio: %.0f | %.0f | %.2f' % (matches['nmat'], matches['numq'], float(matches['nmat'])/matches['numq'])
print >>df, ''
# Get estimated ground truth query location and normal direction
tlat, tlon, tnorm, tyaw = getGTpose(C, Q)
qzx = geom.lltom(olat,olon,tlat,tlon)
# get query yaw and plane yaw from vanishing point anaylsis
yawforvp = tyaw if param['cheat'] else np.nan
vyaw, vnorms = vp_analysis.getQNyaws(C, Q, qimg, dbimg, qsource, yawforvp)
# get dominant planes
dplanes, psizes, planeprms = find_dbplanes(C, Q, dbmatch, Kdinv, wRd)
# match vanishing point planes to database planes
pyaws, planes, pconfs = combine_planes(runflag,vnorms,dplanes,psizes,planeprms)
print 'VP and DB Planes: ' + str(np.int_(pyaws)) + ', ' + str(planes)
with open(Q.datafile,'a') as df:
# print >>df, 'Planes detected with vanishing points:'
for i in range(len(pconfs)):
perr = np.round(np.mod(pyaws[i]-tnorm,360))
perr = perr if perr<180 else 360-perr
print >>df, 'Plane Yaw | DB plane | Confidence | Error : %3.0f | %d | %.2f | %.0f' % (pyaws[i],0 if planes[i]<0 else planes[i],pconfs[i],perr)
yerr = np.round(np.mod(vyaw-tyaw,360))
yerr = yerr if yerr<180 else 360-yerr
print >>df, 'VP Yaw | Confidence | Error : %3.0f | %.2f | %.0f' % (vyaw,np.nan,yerr)
print >>df, 'Cell yaw | True yaw | Plane : %3.0f | %3.0f | %3.0f' % (cyaw,tyaw,tnorm)
print >>df, ''
# Set yaw value to be used
if runflag >= 10: # vanishing point methods
if np.isnan(vyaw): yaw, yawerr = cyaw, 0
else: yaw, yawerr = vyaw, 0
else: yaw, yawerr = cyaw, 0 # set cell phone yaw to use, plane normal
wRq = geom.RfromYPR(yaw,p,r) # camera orientation (camera to world)
### --- THIS IS FOR CHEATING --- ###
if param['cheat']:
if not np.isnan(tnorm):
pyaws, planes, pconfs = np.append(pyaws,tnorm), np.append(planes,-1), np.append(pconfs,1)
yaw, yawerr = tyaw, 0
wRq = geom.RfromYPR(yaw,p,r) # camera orientation (camera to world)
### --- THIS IS FOR CHEATING --- ###
# print pre-homography data to file
vyaw_err = np.round(np.round(np.mod(tyaw-vyaw,360))) if not np.isnan(vyaw) else np.nan
vyaw_err = vyaw_err if vyaw_err<180 else 360-vyaw_err
dbears = np.mod( 180/np.pi*np.arctan2(planeprms[:,0],planeprms[:,2]) , 360 )
print 'Computed / ground truth cell phone yaw: %.0f / %.0f' % (vyaw,tyaw)
with open(os.path.join(param['resultsdir'],param['extras_file']),'a') as extras_file:
print >>extras_file, '\t'.join([timename, '%.0f' % tnorm, '%.0f' % np.round(tyaw), '%.0f' % cyaw, '%.0f' % vyaw, '%.4f'%np.nan, str(vyaw_err)])
print >>extras_file, '\t'.join([ '%.4f' % 0 for vnorm in vnorms ])
print >>extras_file, '\t'.join([ '%.0f' % vnorm for vnorm in vnorms ])
print >>extras_file, '\t'.join([ '%.0f' % plane for plane in planes ])
print >>extras_file, '\t'.join([ '%.0f' % dplane for dplane in dplanes ])
print >>extras_file, '\t'.join([ '%.0f' % dbear for dbear in dbears ])
print >>extras_file, '\t'.join([ '%.3f' % dnerr for dnerr in planeprms[:,4] ])
# Fill out match information
nmat = matches['nmat']
matches['qray'] = geom.normalrows(tp(np.dot(Kqinv,np.append(tp(matches['q2d']),[np.ones(nmat)],0))))
matches['dray'] = geom.normalrows(tp(np.dot(Kdinv,np.append(tp(matches['d2d']),[np.ones(nmat)],0))))
matches = match_info(C, Q, matches, dbmatch, wRd)
matches_start = matches.copy()
# Solve for query pose using constrained image geometry
init_matches = initMatches(matches.copy())
matches['numi'], matches['hconf'] = 0, 0
runflag, ntry, planechose = param['runflag'], 0, 0
parameters = ( wRq, wRd, yaw, np.nan, runflag, param['inlier_error'], param['ransac_iter'], 10, True )
if param['ransac_iter'] == 0:
matches = init_matches
matches['numi'], matches['hconf'] == 0, 0
pose = np.zeros(6)
pose[3:5] = np.nan
elif runflag < 10:
matches, pose = solveGeom(init_matches,parameters,yawerr)
else:
ntry = 1
viter = -np.ones(3)
parameters = ( wRq, wRd, yaw, np.nan, runflag, param['inlier_error'], param['ransac_iter'], 15, True )
matches, pose, planechose = solveNorm(C,Q,dbmatch,pyaws,planes,init_matches,parameters,yawerr)
viter[0] = matches['viter']
if matches['numi'] == 0 or matches['hconf'] == 0:
ntry = 2
parameters = ( wRq, wRd, yaw, np.nan, runflag, 3*param['inlier_error'], param['ransac_iter'], 10, True )
matches, pose, planechose = solveNorm(C,Q,dbmatch,pyaws,planes,init_matches,parameters,yawerr)
viter[1] = matches['viter']
if matches['numi'] == 0 or matches['hconf'] == 0:
ntry, planechose = 3, 0
parameters = ( wRq, wRd, yaw, np.nan, 7, 3*param['inlier_error'], param['ransac_iter'], 10, True )
matches, pose = solveYaw(init_matches,parameters,yawerr)
viter[2] = matches['viter']
if matches['numi'] == 0 or matches['hconf'] == 0:
ntry, planechose = 4, 0
# save matches to disk
matches_file = os.path.join(param['resultsdir'],'matches_'+qname+'.pkl')
matches_out = open(matches_file,'wb')
pickle.dump(matches,matches_out)
matches_out.close()
# extract pose parameters
comp_runflag = matches['runflag']
tray = pose[:3]
comp_yaw = pose[3]
comp_pyaw = pose[4] if runflag>=0 else np.nan
scale = pose[5] if runflag>=0 else np.nan
# Get scaled translation for query location
if np.isnan(scale):
wRq_pr = geom.YPRfromR(wRq)[1:]
comp_wRq = geom.RfromYPR(comp_yaw, wRq_pr[0], wRq_pr[1])
qloc = scalefrom3d(matches, tray, comp_wRq)[0]
else: # scale calculated in RANSAC loop
qloc = scale*tray
# temporarily get yaw error
qyaw_error = np.round(abs(np.mod(tyaw-comp_yaw,360)))
qyaw_error = qyaw_error if qyaw_error<180 else abs(qyaw_error-360)
# compute location errors wrt estimated query locations
loc_err = ( (qloc[0]-qzx[1])**2 + (qloc[2]-qzx[0])**2 )**0.5
gps_err = ( (gzx[1] -qzx[1])**2 + (gzx[0] -qzx[0])**2 )**0.5
# compute the angle difference between T and ground truth translation
tyaw_error = np.round(abs( 180/np.pi * np.arccos( np.abs(qloc[0]*qzx[1]+qloc[2]*qzx[0]) / (alg.norm([qloc[0],qloc[2]])*alg.norm(qzx)) ) ))
# compute the plane normal angle error
nyaw_error = np.nan if np.isnan(comp_pyaw) or np.isnan(tnorm) else np.mod(np.round(abs(comp_pyaw-tnorm)),180)
nyaw_error = nyaw_error if nyaw_error<90 else abs(nyaw_error-180)
# write pose estimation results to file
yaw_err = np.nan
pose_file = os.path.join(param['resultsdir'],param['pose_file'])
with open(pose_file,'a') as pf:
print >>pf, '\t'.join([qname, str(loc_err), str(gps_err), \
str(tyaw_error), str(qyaw_error), str(nyaw_error), str(matches['numi']), \
str(matches['numq']), str(matches['nmat']), str(matches['hconf']), \
str(qloc[0]), str(qloc[2]), str(yaw_err), str(runflag)])
# print post-homography data to file
with open(Q.datafile,'a') as df:
print >>df, ''
print >>df, '------------------'
print >>df, ''
if ntry==1: print >>df, 'Homography solution using low error threshold with restrictions.'
elif ntry==2: print >>df, 'Homography solution using high error threshold with restrictions.'
else: print >>df, 'Solution not found. Setting T=0.'
if planechose==0: print >>df, 'Solution formed with unset plane normal.'
else: 'Solution chosen with plane normal %d chosen.' % planechose
print >>df, 'VP yaw | Computed yaw | Actual Yaw | Error : %3.0f | %3.0f | %3.0f | %3.0f' % (vyaw,comp_yaw,tyaw,qyaw_error)
print >>df, 'Computed Normal | Actual Normal | Error : %3.0f | %3.0f | %3.0f' % (comp_pyaw,tnorm,nyaw_error)
print >>df, 'Translation (x|y|z): %.1f | %.1f | %.1f' % (qloc[0],qloc[1],qloc[2])
print >>df, 'True position (x|-|z): %.1f | - | %.1f' % (qzx[1],qzx[0])
print >>df, 'Angle error | Location error: %.0f | %.1f' % (tyaw_error,loc_err)
print >>df, 'Number of Inliers | Total matches | Ratio: %d | %d | %.2f' % (matches['numi'],matches['nmat'],np.nan if matches['nmat']==0 else float(matches['numi'])/matches['nmat'])
print >>df, 'Reprojection error | Homography confidence: %.3f | %.1f' % (matches['rperr'],matches['hconf'])
print >>df, 'Valid Homographies | Iterations | Ratio: %d | %d | %.3f' % (matches['viter'],matches['niter'],np.nan if matches['niter']==0 else float(matches['viter'])/matches['niter'])
print >>df, ''
print >>df, '------------------'
print >>df, ''
booleans = [ loc_err<5, loc_err<10, not(5<np.mod(vyaw-tyaw,360)<355), \
not(10<np.mod(vyaw-tyaw,360)<350), \
not(5<np.mod(comp_yaw-tyaw,360)<355), ntry==1, ntry!=3, \
planechose!=0, matches['nmat']!=matches_start['nmat'], \
0 if planechose==0 else pconfs[planechose-1]>0, comp_yaw-vyaw ]
print >>df, '|'.join(['%.0f' % (b) for b in booleans])
# draw matches
close = int(loc_err<5) + int(loc_err<10)
yawclose = int(not(5<np.mod(vyaw-tyaw,360)<355)) + int(not(10<np.mod(vyaw-tyaw,360)<350))
imgpath = os.path.join( param['resultsdir'] , qname + ';locerr=%.2f' % (loc_err) + ';locPerf_' + str(close) \
+ ';yawPerf_' + str(yawclose) + ';nplanes_' + str(len(pyaws)) + ';plane_' + str(planes[planechose]) + ';try_' + str(ntry) \
+ ';tAng=%.0f' % (tyaw_error) + ';qAng=%.0f' % (qyaw_error) + ';nAng=%.0f' % (nyaw_error) + ';' + dbmatch + '.jpg')
draw_matches(matches, qimg, dbimg, imgpath)
# imgpath = os.path.join( param['resultsdir'] , 'homography;' + qname + ';' + dbmatch + '.jpg')
# draw_hom(matches, pose, wRq, wRd, Kq, Kd, qimg, dbimg, imgpath)
if C.QUERY == 'oakland1': C.pose_param['draw_tags'] = False
if C.pose_param['draw_tags']: draw_tags(C, Q, matches, pose, dbmatch, olat, olon, Kd, Kq)
print 'Computed yaw / ground truth yaw : %.0f / %.0f' % (comp_yaw,tyaw)
if runflag < 10: print 'Computed normal bearing / ground truth : %.0f / %.0f' % (comp_pyaw,tnorm)
print 'Computed query location relative to db : %.1f, %.1f, %.1f' % tuple(qloc)
print 'Ground truth query location relative to db : %.1f, - , %.1f' % (qzx[1],qzx[0])
input = (wRq, wRd)
return qloc, loc_err, matches, input
def extract_panorama(panopath, outbase, panoinfo, detail):
"""Generates raster, plane, and depth views at rot_degrees"""
print "Processing panorama " + '%d' % panoinfo['pano'][0]
# panopath: location of raster, depth and plane_pano images
# outbase: base name to put generated view, depth, and plane images
# panoinfo: Contains information about the panoramic scene
# detail: 0 = 768x512 with 60d fov, 1 = 2500x1200 with 90d fov
# local constants
pi = np.pi
width, height, fov = (2500, 1200, 90) if detail else (768, 512, 60)
# pano and view details details
orientation = [
panoinfo['yaw'][0], panoinfo['pitch'][0], panoinfo['roll'][0]
]
Rpano = geom.RfromYPR(orientation[0], orientation[1], orientation[2])
Kview = geom.cameramat(width, height, fov * pi / 180)
Kinv = alg.inv(Kview)
# Load image pano, depth pano, and plane pano images
cvIP = cv.LoadImageM(os.path.join(panopath, 'raster.jpg'),
cv.CV_LOAD_IMAGE_UNCHANGED)
cvDP = cv.fromarray(
np.asarray(Image.open(os.path.join(panopath, 'depth.png'))))
pp = np.asarray(Image.open(os.path.join(panopath,
'plane_pano.png'))).copy()
vals = set(list(pp.reshape(-1)))
vals.remove(255)
gnd = max(vals)
pp[pp == gnd] = np.uint8(0)
cvPP = cv.fromarray(pp)
# load pixel map
pix = np.append(
np.array(np.meshgrid(range(width), range(height))).reshape(2, -1),
np.ones([1, width * height]), 0)
# Create output openCV matrices
cvI = cv.CreateMat(height, width, cv.CV_8UC3)
cvD = cv.CreateMat(height, width, cv.CV_32SC1)
cvP = cv.CreateMat(height, width, cv.CV_8UC1)
for viewdir in [2, 3, 4, 8, 9, 10]:
# add to base name and generate info file
viewname = outbase + '%04d' % viewdir
gen_info_file(panoinfo, viewname + '.info', detail, 30 * viewdir)
# generate view orientation
Yview = np.mod(orientation[0] + 30 * viewdir, 360)
Rview = geom.RfromYPR(Yview, 0, 0)
# compute mappings
ray = np.dot(np.transpose(Rpano), np.dot(Rview, np.dot(Kinv, pix)))
yaw, pitch = np.arctan2(ray[0, :], ray[2, :]), np.arctan2(
-ray[1, :], np.sqrt((np.array([ray[0, :], ray[2, :]])**2).sum(0)))
ix, iy = cv.fromarray(
np.array(8192 / (2 * pi) * (pi + yaw),
np.float32).reshape(height, width)), cv.fromarray(
np.array(4096 / pi * (pi / 2 - pitch),
np.float32).reshape(height, width))
dx, dy = cv.fromarray(
np.array(5000 / (2 * pi) * (pi + yaw),
np.float32).reshape(height, width)), cv.fromarray(
np.array(2500 / pi * (pi / 2 - pitch),
np.float32).reshape(height, width))
px, py = cv.fromarray(
np.array(1000 / (2 * pi) * (pi + yaw),
np.float32).reshape(height, width)), cv.fromarray(
np.array(500 / pi * (pi / 2 - pitch),
np.float32).reshape(height, width))
# call remap function
cv.Remap(cvIP, cvI, ix, iy, cv.CV_INTER_CUBIC
) # if detail else cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_AREA)
cv.Remap(cvDP, cvD, dx, dy, cv.CV_INTER_NN)
cv.Remap(cvPP, cvP, px, py, cv.CV_INTER_NN)
# write images to file
Image.fromarray(np.array(cvI)[:, :,
[2, 1, 0]]).save(viewname + '.jpg',
'jpeg')
Image.fromarray(np.array(cvD)).save(viewname + '-depth.png', 'png')
Image.fromarray(np.array(cvP)).save(viewname + '-planes.png', 'png')
def estimate_pose(C, Q, dbmatch, gtStatus=None):
# settings
param = C.pose_param
runflag = param['runflag']
np.seterr(all='ignore')
Q.datafile = os.path.join(C.pose_param['resultsdir'],
'data_' + Q.name + '.txt')
open(Q.datafile, 'w').close()
#####----- PRINT RUN DETAILS -----#####
run_info = os.path.join(param['resultsdir'], param['run_info'])
open(run_info, 'w').close()
with open(run_info, 'a') as ri:
if runflag == 11:
print >> ri, 'Method: Yaw, planes from VPs. Scale computed with homography.'
elif runflag == 10:
print >> ri, 'Method: Yaw, planes from VPs. Scale computed after homography.'
if param['cheat']:
print >> ri, 'Ground truth yaw and plane used (cheating).'
print >> ri, 'Inlier base error threshold: %.3f' % param['inlier_error']
print >> ri, 'Base iteration scale: %d' % param['ransac_iter']
#####----- PRINT RUN DETAILS -----#####
# get high res db image and sift paths
dbinfo = os.path.join(C.hiresdir, dbmatch + '.info')
dbimg = os.path.join(C.hiresdir, dbmatch + '.jpg')
dbsift = os.path.join(C.hiresdir, dbmatch + 'sift.txt')
dbsource = render_tags.EarthmineImageInfo(dbimg, dbinfo)
# Set Kd, wRd, and db position
wx, wy = dbsource.image.size
fov = dbsource.fov
Kd = geom.cameramat(wx, wy, fov)
Kdinv = alg.inv(Kd)
y, p, r = dbsource.yaw, dbsource.pitch, dbsource.roll
wRd = geom.RfromYPR(y, p, r) # db camera orientation (camera to world)
olat, olon, oalt = dbsource.lat, dbsource.lon, dbsource.alt # database location
# get high res query information
qname = Q.name
qimg = os.path.join(C.querydir, 'hires', qname + '.jpg')
qsift = os.path.join(C.querydir, 'hires', qname + 'sift.txt')
qsource = render_tags.QueryImageInfo(Q.datasource)
glat, glon = qsource.lat, qsource.lon
gzx = geom.lltom(olat, olon, glat, glon)
timename = qname[-12:-10] + qname[-9:-7] + qname[-6:-4] #+qname[-3:]
# Set Kq
wx, wy = qsource.image.size
fov = qsource.fov
Kq = geom.cameramat(wx, wy, fov)
Kqinv = alg.inv(Kq)
cyaw, p, r = qsource.yaw, qsource.pitch, qsource.roll # cyaw - cell phone yaw
# get high res sift rematch
matches = highresSift(C, Q, dbmatch)
with open(Q.datafile, 'a') as df:
print >> df, 'Number of matches | number of queries | ratio: %.0f | %.0f | %.2f' % (
matches['nmat'], matches['numq'],
float(matches['nmat']) / matches['numq'])
print >> df, ''
# Get estimated ground truth query location and normal direction
tlat, tlon, tnorm, tyaw = getGTpose(C, Q)
qzx = geom.lltom(olat, olon, tlat, tlon)
# get query yaw and plane yaw from vanishing point anaylsis
yawforvp = tyaw if param['cheat'] else np.nan
vyaw, vnorms = vp_analysis.getQNyaws(C, Q, qimg, dbimg, qsource, yawforvp)
# get dominant planes
dplanes, psizes, planeprms = find_dbplanes(C, Q, dbmatch, Kdinv, wRd)
# match vanishing point planes to database planes
pyaws, planes, pconfs = combine_planes(runflag, vnorms, dplanes, psizes,
planeprms)
print 'VP and DB Planes: ' + str(np.int_(pyaws)) + ', ' + str(planes)
with open(Q.datafile, 'a') as df:
# print >>df, 'Planes detected with vanishing points:'
for i in range(len(pconfs)):
perr = np.round(np.mod(pyaws[i] - tnorm, 360))
perr = perr if perr < 180 else 360 - perr
print >> df, 'Plane Yaw | DB plane | Confidence | Error : %3.0f | %d | %.2f | %.0f' % (
pyaws[i], 0 if planes[i] < 0 else planes[i], pconfs[i], perr)
yerr = np.round(np.mod(vyaw - tyaw, 360))
yerr = yerr if yerr < 180 else 360 - yerr
print >> df, 'VP Yaw | Confidence | Error : %3.0f | %.2f | %.0f' % (
vyaw, np.nan, yerr)
print >> df, 'Cell yaw | True yaw | Plane : %3.0f | %3.0f | %3.0f' % (
cyaw, tyaw, tnorm)
print >> df, ''
# Set yaw value to be used
if runflag >= 10: # vanishing point methods
if np.isnan(vyaw): yaw, yawerr = cyaw, 0
else: yaw, yawerr = vyaw, 0
else: yaw, yawerr = cyaw, 0 # set cell phone yaw to use, plane normal
wRq = geom.RfromYPR(yaw, p, r) # camera orientation (camera to world)
### --- THIS IS FOR CHEATING --- ###
if param['cheat']:
if not np.isnan(tnorm):
pyaws, planes, pconfs = np.append(pyaws, tnorm), np.append(
planes, -1), np.append(pconfs, 1)
yaw, yawerr = tyaw, 0
wRq = geom.RfromYPR(yaw, p, r) # camera orientation (camera to world)
### --- THIS IS FOR CHEATING --- ###
# print pre-homography data to file
vyaw_err = np.round(np.round(np.mod(
tyaw - vyaw, 360))) if not np.isnan(vyaw) else np.nan
vyaw_err = vyaw_err if vyaw_err < 180 else 360 - vyaw_err
dbears = np.mod(180 / np.pi * np.arctan2(planeprms[:, 0], planeprms[:, 2]),
360)
print 'Computed / ground truth cell phone yaw: %.0f / %.0f' % (vyaw, tyaw)
with open(os.path.join(param['resultsdir'], param['extras_file']),
'a') as extras_file:
print >> extras_file, '\t'.join([
timename,
'%.0f' % tnorm,
'%.0f' % np.round(tyaw),
'%.0f' % cyaw,
'%.0f' % vyaw,
'%.4f' % np.nan,
str(vyaw_err)
])
print >> extras_file, '\t'.join(['%.4f' % 0 for vnorm in vnorms])
print >> extras_file, '\t'.join(['%.0f' % vnorm for vnorm in vnorms])
print >> extras_file, '\t'.join(['%.0f' % plane for plane in planes])
print >> extras_file, '\t'.join(
['%.0f' % dplane for dplane in dplanes])
print >> extras_file, '\t'.join(['%.0f' % dbear for dbear in dbears])
print >> extras_file, '\t'.join(
['%.3f' % dnerr for dnerr in planeprms[:, 4]])
# Fill out match information
nmat = matches['nmat']
matches['qray'] = geom.normalrows(
tp(np.dot(Kqinv, np.append(tp(matches['q2d']), [np.ones(nmat)], 0))))
matches['dray'] = geom.normalrows(
tp(np.dot(Kdinv, np.append(tp(matches['d2d']), [np.ones(nmat)], 0))))
matches = match_info(C, Q, matches, dbmatch, wRd)
matches_start = matches.copy()
# Solve for query pose using constrained image geometry
init_matches = initMatches(matches.copy())
matches['numi'], matches['hconf'] = 0, 0
runflag, ntry, planechose = param['runflag'], 0, 0
parameters = (wRq, wRd, yaw, np.nan, runflag, param['inlier_error'],
param['ransac_iter'], 10, True)
if param['ransac_iter'] == 0:
matches = init_matches
matches['numi'], matches['hconf'] == 0, 0
pose = np.zeros(6)
pose[3:5] = np.nan
elif runflag < 10:
matches, pose = solveGeom(init_matches, parameters, yawerr)
else:
ntry = 1
viter = -np.ones(3)
parameters = (wRq, wRd, yaw, np.nan, runflag, param['inlier_error'],
param['ransac_iter'], 15, True)
matches, pose, planechose = solveNorm(C, Q, dbmatch, pyaws, planes,
init_matches, parameters, yawerr)
viter[0] = matches['viter']
if matches['numi'] == 0 or matches['hconf'] == 0:
ntry = 2
parameters = (wRq, wRd, yaw, np.nan, runflag,
3 * param['inlier_error'], param['ransac_iter'], 10,
True)
matches, pose, planechose = solveNorm(C, Q, dbmatch, pyaws, planes,
init_matches, parameters,
yawerr)
viter[1] = matches['viter']
if matches['numi'] == 0 or matches['hconf'] == 0:
ntry, planechose = 3, 0
parameters = (wRq, wRd, yaw, np.nan, 7, 3 * param['inlier_error'],
param['ransac_iter'], 10, True)
matches, pose = solveYaw(init_matches, parameters, yawerr)
viter[2] = matches['viter']
if matches['numi'] == 0 or matches['hconf'] == 0:
ntry, planechose = 4, 0
# save matches to disk
matches_file = os.path.join(param['resultsdir'],
'matches_' + qname + '.pkl')
matches_out = open(matches_file, 'wb')
pickle.dump(matches, matches_out)
matches_out.close()
# extract pose parameters
comp_runflag = matches['runflag']
tray = pose[:3]
comp_yaw = pose[3]
comp_pyaw = pose[4] if runflag >= 0 else np.nan
scale = pose[5] if runflag >= 0 else np.nan
# Get scaled translation for query location
if np.isnan(scale):
wRq_pr = geom.YPRfromR(wRq)[1:]
comp_wRq = geom.RfromYPR(comp_yaw, wRq_pr[0], wRq_pr[1])
qloc = scalefrom3d(matches, tray, comp_wRq)[0]
else: # scale calculated in RANSAC loop
qloc = scale * tray
# temporarily get yaw error
qyaw_error = np.round(abs(np.mod(tyaw - comp_yaw, 360)))
qyaw_error = qyaw_error if qyaw_error < 180 else abs(qyaw_error - 360)
# compute location errors wrt estimated query locations
loc_err = ((qloc[0] - qzx[1])**2 + (qloc[2] - qzx[0])**2)**0.5
gps_err = ((gzx[1] - qzx[1])**2 + (gzx[0] - qzx[0])**2)**0.5
# compute the angle difference between T and ground truth translation
tyaw_error = np.round(
abs(180 / np.pi * np.arccos(
np.abs(qloc[0] * qzx[1] + qloc[2] * qzx[0]) /
(alg.norm([qloc[0], qloc[2]]) * alg.norm(qzx)))))
# compute the plane normal angle error
nyaw_error = np.nan if np.isnan(comp_pyaw) or np.isnan(tnorm) else np.mod(
np.round(abs(comp_pyaw - tnorm)), 180)
nyaw_error = nyaw_error if nyaw_error < 90 else abs(nyaw_error - 180)
# write pose estimation results to file
yaw_err = np.nan
pose_file = os.path.join(param['resultsdir'], param['pose_file'])
with open(pose_file, 'a') as pf:
print >>pf, '\t'.join([qname, str(loc_err), str(gps_err), \
str(tyaw_error), str(qyaw_error), str(nyaw_error), str(matches['numi']), \
str(matches['numq']), str(matches['nmat']), str(matches['hconf']), \
str(qloc[0]), str(qloc[2]), str(yaw_err), str(runflag)])
# print post-homography data to file
with open(Q.datafile, 'a') as df:
print >> df, ''
print >> df, '------------------'
print >> df, ''
if ntry == 1:
print >> df, 'Homography solution using low error threshold with restrictions.'
elif ntry == 2:
print >> df, 'Homography solution using high error threshold with restrictions.'
else:
print >> df, 'Solution not found. Setting T=0.'
if planechose == 0:
print >> df, 'Solution formed with unset plane normal.'
else:
'Solution chosen with plane normal %d chosen.' % planechose
print >> df, 'VP yaw | Computed yaw | Actual Yaw | Error : %3.0f | %3.0f | %3.0f | %3.0f' % (
vyaw, comp_yaw, tyaw, qyaw_error)
print >> df, 'Computed Normal | Actual Normal | Error : %3.0f | %3.0f | %3.0f' % (
comp_pyaw, tnorm, nyaw_error)
print >> df, 'Translation (x|y|z): %.1f | %.1f | %.1f' % (
qloc[0], qloc[1], qloc[2])
print >> df, 'True position (x|-|z): %.1f | - | %.1f' % (qzx[1],
qzx[0])
print >> df, 'Angle error | Location error: %.0f | %.1f' % (tyaw_error,
loc_err)
print >> df, 'Number of Inliers | Total matches | Ratio: %d | %d | %.2f' % (
matches['numi'], matches['nmat'], np.nan if matches['nmat'] == 0
else float(matches['numi']) / matches['nmat'])
print >> df, 'Reprojection error | Homography confidence: %.3f | %.1f' % (
matches['rperr'], matches['hconf'])
print >> df, 'Valid Homographies | Iterations | Ratio: %d | %d | %.3f' % (
matches['viter'], matches['niter'], np.nan if matches['niter'] == 0
else float(matches['viter']) / matches['niter'])
print >> df, ''
print >> df, '------------------'
print >> df, ''
booleans = [ loc_err<5, loc_err<10, not(5<np.mod(vyaw-tyaw,360)<355), \
not(10<np.mod(vyaw-tyaw,360)<350), \
not(5<np.mod(comp_yaw-tyaw,360)<355), ntry==1, ntry!=3, \
planechose!=0, matches['nmat']!=matches_start['nmat'], \
0 if planechose==0 else pconfs[planechose-1]>0, comp_yaw-vyaw ]
print >> df, '|'.join(['%.0f' % (b) for b in booleans])
# draw matches
close = int(loc_err < 5) + int(loc_err < 10)
yawclose = int(not (5 < np.mod(vyaw - tyaw, 360) < 355)) + int(not (
10 < np.mod(vyaw - tyaw, 360) < 350))
imgpath = os.path.join( param['resultsdir'] , qname + ';locerr=%.2f' % (loc_err) + ';locPerf_' + str(close) \
+ ';yawPerf_' + str(yawclose) + ';nplanes_' + str(len(pyaws)) + ';plane_' + str(planes[planechose]) + ';try_' + str(ntry) \
+ ';tAng=%.0f' % (tyaw_error) + ';qAng=%.0f' % (qyaw_error) + ';nAng=%.0f' % (nyaw_error) + ';' + dbmatch + '.jpg')
draw_matches(matches, qimg, dbimg, imgpath)
# imgpath = os.path.join( param['resultsdir'] , 'homography;' + qname + ';' + dbmatch + '.jpg')
# draw_hom(matches, pose, wRq, wRd, Kq, Kd, qimg, dbimg, imgpath)
if C.QUERY == 'oakland1': C.pose_param['draw_tags'] = False
if C.pose_param['draw_tags']:
draw_tags(C, Q, matches, pose, dbmatch, olat, olon, Kd, Kq)
print 'Computed yaw / ground truth yaw : %.0f / %.0f' % (comp_yaw,
tyaw)
if runflag < 10:
print 'Computed normal bearing / ground truth : %.0f / %.0f' % (
comp_pyaw, tnorm)
print 'Computed query location relative to db : %.1f, %.1f, %.1f' % tuple(
qloc)
print 'Ground truth query location relative to db : %.1f, - , %.1f' % (
qzx[1], qzx[0])
input = (wRq, wRd)
return qloc, loc_err, matches, input
def extract_panorama(panopath, outbase, panoinfo, detail):
"""Generates raster, plane, and depth views at rot_degrees"""
print "Processing panorama " + '%d' % panoinfo['pano'][0]
# panopath: location of raster, depth and plane_pano images
# outbase: base name to put generated view, depth, and plane images
# panoinfo: Contains information about the panoramic scene
# detail: 0 = 768x512 with 60d fov, 1 = 2500x1200 with 90d fov
# local constants
pi = np.pi
width, height, fov = (2500, 1200, 90) if detail else (768, 512, 60)
# pano and view details details
orientation = [panoinfo['yaw'][0],panoinfo['pitch'][0],panoinfo['roll'][0]]
Rpano = geom.RfromYPR(orientation[0],orientation[1],orientation[2])
Kview = geom.cameramat(width, height, fov*pi/180)
Kinv = alg.inv(Kview)
# Load image pano, depth pano, and plane pano images
cvIP = cv.LoadImageM( os.path.join(panopath,'raster.jpg'), cv.CV_LOAD_IMAGE_UNCHANGED )
cvDP = cv.fromarray( np.asarray( Image.open( os.path.join(panopath,'depth.png') ) ) )
pp = np.asarray( Image.open( os.path.join(panopath,'plane_pano.png') ) ).copy()
vals = set(list(pp.reshape(-1)))
vals.remove(255)
gnd = max(vals)
pp[pp==gnd] = np.uint8(0)
cvPP = cv.fromarray(pp)
# load pixel map
pix = np.append(np.array(np.meshgrid(range(width),range(height))).reshape(2,-1),np.ones([1,width*height]),0)
# Create output openCV matrices
cvI = cv.CreateMat(height,width,cv.CV_8UC3)
cvD = cv.CreateMat(height,width,cv.CV_32SC1)
cvP = cv.CreateMat(height,width,cv.CV_8UC1)
for viewdir in [2,3,4,8,9,10]:
# add to base name and generate info file
viewname = outbase + '%04d' % viewdir
gen_info_file(panoinfo, viewname + '.info', detail, 30*viewdir)
# generate view orientation
Yview = np.mod( orientation[0] + 30*viewdir, 360 )
Rview = geom.RfromYPR(Yview, 0, 0)
# compute mappings
ray = np.dot( np.transpose(Rpano), np.dot( Rview, np.dot( Kinv, pix ) ) )
yaw, pitch = np.arctan2( ray[0,:] , ray[2,:] ) , np.arctan2( -ray[1,:] , np.sqrt((np.array([ray[0,:],ray[2,:]])**2).sum(0)) )
ix, iy = cv.fromarray(np.array(8192/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(4096/pi*(pi/2-pitch),np.float32).reshape(height,width))
dx, dy = cv.fromarray(np.array(5000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(2500/pi*(pi/2-pitch),np.float32).reshape(height,width))
px, py = cv.fromarray(np.array(1000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array( 500/pi*(pi/2-pitch),np.float32).reshape(height,width))
# call remap function
cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_CUBIC) # if detail else cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_AREA)
cv.Remap(cvDP,cvD,dx,dy,cv.CV_INTER_NN)
cv.Remap(cvPP,cvP,px,py,cv.CV_INTER_NN)
# write images to file
Image.fromarray(np.array(cvI)[:,:,[2,1,0]]).save(viewname+'.jpg','jpeg')
Image.fromarray(np.array(cvD)).save(viewname+'-depth.png','png')
Image.fromarray(np.array(cvP)).save(viewname+'-planes.png','png')
