def viewparam(source,tyaw): # camera calibration matrix Kcal = geom.cameramat(source.image.size[0], source.image.size[1], source.fov) # camera orientation (camera to world) yaw = source.yaw if np.isnan(tyaw) else tyaw Rot = geom.RfromYPR(yaw, source.pitch, source.roll) return Kcal, Rot
def genView(panopath, outpath, orientation, viewdir, detail): # panopath: location of raster, depth and plane_pano images # outpath: location to put generated view, depth, and plane images # orientation: [yaw, pitch, roll] of panorama (in degrees) # viewdir: clock-based view; in set [2,3,4,8,9,10] for database # detail: 0 = 768x512 with 60d fov, 1 = 2500x1200 with 90d fov # local constants start = time.time() pi = np.pi width, height, fov = (2500, 1200, 90) if detail else (768, 512, 60) # view details Rpano = geom.RfromYPR(orientation[0],orientation[1],orientation[2]) Yview = np.mod( orientation[0] + 30*viewdir, 360 ) Rview = geom.RfromYPR(Yview, 0, 0) Kview = geom.cameramat(width, height, fov*pi/180) Kinv = alg.inv(Kview) # Load image pano, depth pano, and plane pano images cvIP = cv.LoadImageM( os.path.join(panopath,'raster.jpg'), cv.CV_LOAD_IMAGE_UNCHANGED ) cvDP = cv.fromarray( np.asarray( Image.open( os.path.join(panopath,'depth.png') ) ) ) pp = np.asarray( Image.open( os.path.join(panopath,'plane_pano.png') ) ).copy() vals = set(list(pp.reshape(-1))) vals.remove(255) gnd = max(vals) pp[pp==gnd] = np.uint8(0) cvPP = cv.fromarray(pp) # load pixel map pix = np.append(np.array(np.meshgrid(range(width),range(height))).reshape(2,-1),np.ones([1,width*height]),0) midpoint = time.time() print 'Loading pano images took ' + str(midpoint-start) + ' seconds.' # Create output openCV matrices cvI = cv.CreateMat(height,width,cv.CV_8UC3) cvD = cv.CreateMat(height,width,cv.CV_32SC1) cvP = cv.CreateMat(height,width,cv.CV_8UC1) # compute mappings ray = np.dot( np.transpose(Rpano), np.dot( Rview, np.dot( Kinv, pix ) ) ) yaw, pitch = np.arctan2( ray[0,:] , ray[2,:] ) , np.arctan2( -ray[1,:] , np.sqrt((np.array([ray[0,:],ray[2,:]])**2).sum(0)) ) ix, iy = cv.fromarray(np.array(8192/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(4096/pi*(pi/2-pitch),np.float32).reshape(height,width)) dx, dy = cv.fromarray(np.array(5000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(2500/pi*(pi/2-pitch),np.float32).reshape(height,width)) px, py = cv.fromarray(np.array(1000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array( 500/pi*(pi/2-pitch),np.float32).reshape(height,width)) # call remap function cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_CUBIC) cv.Remap(cvDP,cvD,dx,dy,cv.CV_INTER_NN) cv.Remap(cvPP,cvP,px,py,cv.CV_INTER_NN) # write images to file Image.fromarray(np.array(cvI)[:,:,[2,1,0]]).save(os.path.join(outpath,'view.jpg'),'jpeg') Image.fromarray(np.array(cvD)).save(os.path.join(outpath,'depth.png'),'png') Image.fromarray(np.array(cvP)).save(os.path.join(outpath,'plane.png'),'png') print 'Generating views from pano took ' + str(time.time()-midpoint) + ' seconds.'
def estimate_pose(C, Q, dbmatch, gtStatus=None): # settings param = C.pose_param runflag = param['runflag'] np.seterr(all='ignore') Q.datafile = os.path.join(C.pose_param['resultsdir'],'data_'+Q.name+'.txt') open(Q.datafile,'w').close() #####----- PRINT RUN DETAILS -----##### run_info = os.path.join(param['resultsdir'],param['run_info']) open(run_info,'w').close() with open(run_info,'a') as ri: if runflag == 11: print >>ri, 'Method: Yaw, planes from VPs. Scale computed with homography.' elif runflag == 10: print >>ri, 'Method: Yaw, planes from VPs. Scale computed after homography.' if param['cheat']: print >>ri, 'Ground truth yaw and plane used (cheating).' print >>ri, 'Inlier base error threshold: %.3f' % param['inlier_error'] print >>ri, 'Base iteration scale: %d' % param['ransac_iter'] #####----- PRINT RUN DETAILS -----##### # get high res db image and sift paths dbinfo = os.path.join(C.hiresdir, dbmatch + '.info') dbimg = os.path.join(C.hiresdir,dbmatch+'.jpg') dbsift = os.path.join(C.hiresdir,dbmatch+'sift.txt') dbsource = render_tags.EarthmineImageInfo(dbimg, dbinfo) # Set Kd, wRd, and db position wx,wy = dbsource.image.size fov = dbsource.fov Kd = geom.cameramat(wx, wy, fov) Kdinv = alg.inv(Kd) y,p,r = dbsource.yaw, dbsource.pitch, dbsource.roll wRd = geom.RfromYPR(y,p,r) # db camera orientation (camera to world) olat,olon,oalt = dbsource.lat,dbsource.lon,dbsource.alt # database location # get high res query information qname = Q.name qimg = os.path.join(C.querydir,'hires',qname+'.jpg') qsift = os.path.join(C.querydir,'hires',qname+'sift.txt') qsource = render_tags.QueryImageInfo(Q.datasource) glat,glon = qsource.lat,qsource.lon gzx = geom.lltom(olat,olon,glat,glon) timename = qname[-12:-10]+qname[-9:-7]+qname[-6:-4]#+qname[-3:] # Set Kq wx,wy = qsource.image.size fov = qsource.fov Kq = geom.cameramat(wx, wy, fov) Kqinv = alg.inv(Kq) cyaw,p,r = qsource.yaw, qsource.pitch, qsource.roll # cyaw - cell phone yaw # get high res sift rematch matches = highresSift(C, Q, dbmatch) with open(Q.datafile,'a') as df: print >>df, 'Number of matches | number of queries | ratio: %.0f | %.0f | %.2f' % (matches['nmat'], matches['numq'], float(matches['nmat'])/matches['numq']) print >>df, '' # Get estimated ground truth query location and normal direction tlat, tlon, tnorm, tyaw = getGTpose(C, Q) qzx = geom.lltom(olat,olon,tlat,tlon) # get query yaw and plane yaw from vanishing point anaylsis yawforvp = tyaw if param['cheat'] else np.nan vyaw, vnorms = vp_analysis.getQNyaws(C, Q, qimg, dbimg, qsource, yawforvp) # get dominant planes dplanes, psizes, planeprms = find_dbplanes(C, Q, dbmatch, Kdinv, wRd) # match vanishing point planes to database planes pyaws, planes, pconfs = combine_planes(runflag,vnorms,dplanes,psizes,planeprms) print 'VP and DB Planes: ' + str(np.int_(pyaws)) + ', ' + str(planes) with open(Q.datafile,'a') as df: # print >>df, 'Planes detected with vanishing points:' for i in range(len(pconfs)): perr = np.round(np.mod(pyaws[i]-tnorm,360)) perr = perr if perr<180 else 360-perr print >>df, 'Plane Yaw | DB plane | Confidence | Error : %3.0f | %d | %.2f | %.0f' % (pyaws[i],0 if planes[i]<0 else planes[i],pconfs[i],perr) yerr = np.round(np.mod(vyaw-tyaw,360)) yerr = yerr if yerr<180 else 360-yerr print >>df, 'VP Yaw | Confidence | Error : %3.0f | %.2f | %.0f' % (vyaw,np.nan,yerr) print >>df, 'Cell yaw | True yaw | Plane : %3.0f | %3.0f | %3.0f' % (cyaw,tyaw,tnorm) print >>df, '' # Set yaw value to be used if runflag >= 10: # vanishing point methods if np.isnan(vyaw): yaw, yawerr = cyaw, 0 else: yaw, yawerr = vyaw, 0 else: yaw, yawerr = cyaw, 0 # set cell phone yaw to use, plane normal wRq = geom.RfromYPR(yaw,p,r) # camera orientation (camera to world) ### --- THIS IS FOR CHEATING --- ### if param['cheat']: if not np.isnan(tnorm): pyaws, planes, pconfs = np.append(pyaws,tnorm), np.append(planes,-1), np.append(pconfs,1) yaw, yawerr = tyaw, 0 wRq = geom.RfromYPR(yaw,p,r) # camera orientation (camera to world) ### --- THIS IS FOR CHEATING --- ### # print pre-homography data to file vyaw_err = np.round(np.round(np.mod(tyaw-vyaw,360))) if not np.isnan(vyaw) else np.nan vyaw_err = vyaw_err if vyaw_err<180 else 360-vyaw_err dbears = np.mod( 180/np.pi*np.arctan2(planeprms[:,0],planeprms[:,2]) , 360 ) print 'Computed / ground truth cell phone yaw: %.0f / %.0f' % (vyaw,tyaw) with open(os.path.join(param['resultsdir'],param['extras_file']),'a') as extras_file: print >>extras_file, '\t'.join([timename, '%.0f' % tnorm, '%.0f' % np.round(tyaw), '%.0f' % cyaw, '%.0f' % vyaw, '%.4f'%np.nan, str(vyaw_err)]) print >>extras_file, '\t'.join([ '%.4f' % 0 for vnorm in vnorms ]) print >>extras_file, '\t'.join([ '%.0f' % vnorm for vnorm in vnorms ]) print >>extras_file, '\t'.join([ '%.0f' % plane for plane in planes ]) print >>extras_file, '\t'.join([ '%.0f' % dplane for dplane in dplanes ]) print >>extras_file, '\t'.join([ '%.0f' % dbear for dbear in dbears ]) print >>extras_file, '\t'.join([ '%.3f' % dnerr for dnerr in planeprms[:,4] ]) # Fill out match information nmat = matches['nmat'] matches['qray'] = geom.normalrows(tp(np.dot(Kqinv,np.append(tp(matches['q2d']),[np.ones(nmat)],0)))) matches['dray'] = geom.normalrows(tp(np.dot(Kdinv,np.append(tp(matches['d2d']),[np.ones(nmat)],0)))) matches = match_info(C, Q, matches, dbmatch, wRd) matches_start = matches.copy() # Solve for query pose using constrained image geometry init_matches = initMatches(matches.copy()) matches['numi'], matches['hconf'] = 0, 0 runflag, ntry, planechose = param['runflag'], 0, 0 parameters = ( wRq, wRd, yaw, np.nan, runflag, param['inlier_error'], param['ransac_iter'], 10, True ) if param['ransac_iter'] == 0: matches = init_matches matches['numi'], matches['hconf'] == 0, 0 pose = np.zeros(6) pose[3:5] = np.nan elif runflag < 10: matches, pose = solveGeom(init_matches,parameters,yawerr) else: ntry = 1 viter = -np.ones(3) parameters = ( wRq, wRd, yaw, np.nan, runflag, param['inlier_error'], param['ransac_iter'], 15, True ) matches, pose, planechose = solveNorm(C,Q,dbmatch,pyaws,planes,init_matches,parameters,yawerr) viter[0] = matches['viter'] if matches['numi'] == 0 or matches['hconf'] == 0: ntry = 2 parameters = ( wRq, wRd, yaw, np.nan, runflag, 3*param['inlier_error'], param['ransac_iter'], 10, True ) matches, pose, planechose = solveNorm(C,Q,dbmatch,pyaws,planes,init_matches,parameters,yawerr) viter[1] = matches['viter'] if matches['numi'] == 0 or matches['hconf'] == 0: ntry, planechose = 3, 0 parameters = ( wRq, wRd, yaw, np.nan, 7, 3*param['inlier_error'], param['ransac_iter'], 10, True ) matches, pose = solveYaw(init_matches,parameters,yawerr) viter[2] = matches['viter'] if matches['numi'] == 0 or matches['hconf'] == 0: ntry, planechose = 4, 0 # save matches to disk matches_file = os.path.join(param['resultsdir'],'matches_'+qname+'.pkl') matches_out = open(matches_file,'wb') pickle.dump(matches,matches_out) matches_out.close() # extract pose parameters comp_runflag = matches['runflag'] tray = pose[:3] comp_yaw = pose[3] comp_pyaw = pose[4] if runflag>=0 else np.nan scale = pose[5] if runflag>=0 else np.nan # Get scaled translation for query location if np.isnan(scale): wRq_pr = geom.YPRfromR(wRq)[1:] comp_wRq = geom.RfromYPR(comp_yaw, wRq_pr[0], wRq_pr[1]) qloc = scalefrom3d(matches, tray, comp_wRq)[0] else: # scale calculated in RANSAC loop qloc = scale*tray # temporarily get yaw error qyaw_error = np.round(abs(np.mod(tyaw-comp_yaw,360))) qyaw_error = qyaw_error if qyaw_error<180 else abs(qyaw_error-360) # compute location errors wrt estimated query locations loc_err = ( (qloc[0]-qzx[1])**2 + (qloc[2]-qzx[0])**2 )**0.5 gps_err = ( (gzx[1] -qzx[1])**2 + (gzx[0] -qzx[0])**2 )**0.5 # compute the angle difference between T and ground truth translation tyaw_error = np.round(abs( 180/np.pi * np.arccos( np.abs(qloc[0]*qzx[1]+qloc[2]*qzx[0]) / (alg.norm([qloc[0],qloc[2]])*alg.norm(qzx)) ) )) # compute the plane normal angle error nyaw_error = np.nan if np.isnan(comp_pyaw) or np.isnan(tnorm) else np.mod(np.round(abs(comp_pyaw-tnorm)),180) nyaw_error = nyaw_error if nyaw_error<90 else abs(nyaw_error-180) # write pose estimation results to file yaw_err = np.nan pose_file = os.path.join(param['resultsdir'],param['pose_file']) with open(pose_file,'a') as pf: print >>pf, '\t'.join([qname, str(loc_err), str(gps_err), \ str(tyaw_error), str(qyaw_error), str(nyaw_error), str(matches['numi']), \ str(matches['numq']), str(matches['nmat']), str(matches['hconf']), \ str(qloc[0]), str(qloc[2]), str(yaw_err), str(runflag)]) # print post-homography data to file with open(Q.datafile,'a') as df: print >>df, '' print >>df, '------------------' print >>df, '' if ntry==1: print >>df, 'Homography solution using low error threshold with restrictions.' elif ntry==2: print >>df, 'Homography solution using high error threshold with restrictions.' else: print >>df, 'Solution not found. Setting T=0.' if planechose==0: print >>df, 'Solution formed with unset plane normal.' else: 'Solution chosen with plane normal %d chosen.' % planechose print >>df, 'VP yaw | Computed yaw | Actual Yaw | Error : %3.0f | %3.0f | %3.0f | %3.0f' % (vyaw,comp_yaw,tyaw,qyaw_error) print >>df, 'Computed Normal | Actual Normal | Error : %3.0f | %3.0f | %3.0f' % (comp_pyaw,tnorm,nyaw_error) print >>df, 'Translation (x|y|z): %.1f | %.1f | %.1f' % (qloc[0],qloc[1],qloc[2]) print >>df, 'True position (x|-|z): %.1f | - | %.1f' % (qzx[1],qzx[0]) print >>df, 'Angle error | Location error: %.0f | %.1f' % (tyaw_error,loc_err) print >>df, 'Number of Inliers | Total matches | Ratio: %d | %d | %.2f' % (matches['numi'],matches['nmat'],np.nan if matches['nmat']==0 else float(matches['numi'])/matches['nmat']) print >>df, 'Reprojection error | Homography confidence: %.3f | %.1f' % (matches['rperr'],matches['hconf']) print >>df, 'Valid Homographies | Iterations | Ratio: %d | %d | %.3f' % (matches['viter'],matches['niter'],np.nan if matches['niter']==0 else float(matches['viter'])/matches['niter']) print >>df, '' print >>df, '------------------' print >>df, '' booleans = [ loc_err<5, loc_err<10, not(5<np.mod(vyaw-tyaw,360)<355), \ not(10<np.mod(vyaw-tyaw,360)<350), \ not(5<np.mod(comp_yaw-tyaw,360)<355), ntry==1, ntry!=3, \ planechose!=0, matches['nmat']!=matches_start['nmat'], \ 0 if planechose==0 else pconfs[planechose-1]>0, comp_yaw-vyaw ] print >>df, '|'.join(['%.0f' % (b) for b in booleans]) # draw matches close = int(loc_err<5) + int(loc_err<10) yawclose = int(not(5<np.mod(vyaw-tyaw,360)<355)) + int(not(10<np.mod(vyaw-tyaw,360)<350)) imgpath = os.path.join( param['resultsdir'] , qname + ';locerr=%.2f' % (loc_err) + ';locPerf_' + str(close) \ + ';yawPerf_' + str(yawclose) + ';nplanes_' + str(len(pyaws)) + ';plane_' + str(planes[planechose]) + ';try_' + str(ntry) \ + ';tAng=%.0f' % (tyaw_error) + ';qAng=%.0f' % (qyaw_error) + ';nAng=%.0f' % (nyaw_error) + ';' + dbmatch + '.jpg') draw_matches(matches, qimg, dbimg, imgpath) # imgpath = os.path.join( param['resultsdir'] , 'homography;' + qname + ';' + dbmatch + '.jpg') # draw_hom(matches, pose, wRq, wRd, Kq, Kd, qimg, dbimg, imgpath) if C.QUERY == 'oakland1': C.pose_param['draw_tags'] = False if C.pose_param['draw_tags']: draw_tags(C, Q, matches, pose, dbmatch, olat, olon, Kd, Kq) print 'Computed yaw / ground truth yaw : %.0f / %.0f' % (comp_yaw,tyaw) if runflag < 10: print 'Computed normal bearing / ground truth : %.0f / %.0f' % (comp_pyaw,tnorm) print 'Computed query location relative to db : %.1f, %.1f, %.1f' % tuple(qloc) print 'Ground truth query location relative to db : %.1f, - , %.1f' % (qzx[1],qzx[0]) input = (wRq, wRd) return qloc, loc_err, matches, input
def extract_panorama(panopath, outbase, panoinfo, detail): """Generates raster, plane, and depth views at rot_degrees""" print "Processing panorama " + '%d' % panoinfo['pano'][0] # panopath: location of raster, depth and plane_pano images # outbase: base name to put generated view, depth, and plane images # panoinfo: Contains information about the panoramic scene # detail: 0 = 768x512 with 60d fov, 1 = 2500x1200 with 90d fov # local constants pi = np.pi width, height, fov = (2500, 1200, 90) if detail else (768, 512, 60) # pano and view details details orientation = [ panoinfo['yaw'][0], panoinfo['pitch'][0], panoinfo['roll'][0] ] Rpano = geom.RfromYPR(orientation[0], orientation[1], orientation[2]) Kview = geom.cameramat(width, height, fov * pi / 180) Kinv = alg.inv(Kview) # Load image pano, depth pano, and plane pano images cvIP = cv.LoadImageM(os.path.join(panopath, 'raster.jpg'), cv.CV_LOAD_IMAGE_UNCHANGED) cvDP = cv.fromarray( np.asarray(Image.open(os.path.join(panopath, 'depth.png')))) pp = np.asarray(Image.open(os.path.join(panopath, 'plane_pano.png'))).copy() vals = set(list(pp.reshape(-1))) vals.remove(255) gnd = max(vals) pp[pp == gnd] = np.uint8(0) cvPP = cv.fromarray(pp) # load pixel map pix = np.append( np.array(np.meshgrid(range(width), range(height))).reshape(2, -1), np.ones([1, width * height]), 0) # Create output openCV matrices cvI = cv.CreateMat(height, width, cv.CV_8UC3) cvD = cv.CreateMat(height, width, cv.CV_32SC1) cvP = cv.CreateMat(height, width, cv.CV_8UC1) for viewdir in [2, 3, 4, 8, 9, 10]: # add to base name and generate info file viewname = outbase + '%04d' % viewdir gen_info_file(panoinfo, viewname + '.info', detail, 30 * viewdir) # generate view orientation Yview = np.mod(orientation[0] + 30 * viewdir, 360) Rview = geom.RfromYPR(Yview, 0, 0) # compute mappings ray = np.dot(np.transpose(Rpano), np.dot(Rview, np.dot(Kinv, pix))) yaw, pitch = np.arctan2(ray[0, :], ray[2, :]), np.arctan2( -ray[1, :], np.sqrt((np.array([ray[0, :], ray[2, :]])**2).sum(0))) ix, iy = cv.fromarray( np.array(8192 / (2 * pi) * (pi + yaw), np.float32).reshape(height, width)), cv.fromarray( np.array(4096 / pi * (pi / 2 - pitch), np.float32).reshape(height, width)) dx, dy = cv.fromarray( np.array(5000 / (2 * pi) * (pi + yaw), np.float32).reshape(height, width)), cv.fromarray( np.array(2500 / pi * (pi / 2 - pitch), np.float32).reshape(height, width)) px, py = cv.fromarray( np.array(1000 / (2 * pi) * (pi + yaw), np.float32).reshape(height, width)), cv.fromarray( np.array(500 / pi * (pi / 2 - pitch), np.float32).reshape(height, width)) # call remap function cv.Remap(cvIP, cvI, ix, iy, cv.CV_INTER_CUBIC ) # if detail else cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_AREA) cv.Remap(cvDP, cvD, dx, dy, cv.CV_INTER_NN) cv.Remap(cvPP, cvP, px, py, cv.CV_INTER_NN) # write images to file Image.fromarray(np.array(cvI)[:, :, [2, 1, 0]]).save(viewname + '.jpg', 'jpeg') Image.fromarray(np.array(cvD)).save(viewname + '-depth.png', 'png') Image.fromarray(np.array(cvP)).save(viewname + '-planes.png', 'png')
def estimate_pose(C, Q, dbmatch, gtStatus=None): # settings param = C.pose_param runflag = param['runflag'] np.seterr(all='ignore') Q.datafile = os.path.join(C.pose_param['resultsdir'], 'data_' + Q.name + '.txt') open(Q.datafile, 'w').close() #####----- PRINT RUN DETAILS -----##### run_info = os.path.join(param['resultsdir'], param['run_info']) open(run_info, 'w').close() with open(run_info, 'a') as ri: if runflag == 11: print >> ri, 'Method: Yaw, planes from VPs. Scale computed with homography.' elif runflag == 10: print >> ri, 'Method: Yaw, planes from VPs. Scale computed after homography.' if param['cheat']: print >> ri, 'Ground truth yaw and plane used (cheating).' print >> ri, 'Inlier base error threshold: %.3f' % param['inlier_error'] print >> ri, 'Base iteration scale: %d' % param['ransac_iter'] #####----- PRINT RUN DETAILS -----##### # get high res db image and sift paths dbinfo = os.path.join(C.hiresdir, dbmatch + '.info') dbimg = os.path.join(C.hiresdir, dbmatch + '.jpg') dbsift = os.path.join(C.hiresdir, dbmatch + 'sift.txt') dbsource = render_tags.EarthmineImageInfo(dbimg, dbinfo) # Set Kd, wRd, and db position wx, wy = dbsource.image.size fov = dbsource.fov Kd = geom.cameramat(wx, wy, fov) Kdinv = alg.inv(Kd) y, p, r = dbsource.yaw, dbsource.pitch, dbsource.roll wRd = geom.RfromYPR(y, p, r) # db camera orientation (camera to world) olat, olon, oalt = dbsource.lat, dbsource.lon, dbsource.alt # database location # get high res query information qname = Q.name qimg = os.path.join(C.querydir, 'hires', qname + '.jpg') qsift = os.path.join(C.querydir, 'hires', qname + 'sift.txt') qsource = render_tags.QueryImageInfo(Q.datasource) glat, glon = qsource.lat, qsource.lon gzx = geom.lltom(olat, olon, glat, glon) timename = qname[-12:-10] + qname[-9:-7] + qname[-6:-4] #+qname[-3:] # Set Kq wx, wy = qsource.image.size fov = qsource.fov Kq = geom.cameramat(wx, wy, fov) Kqinv = alg.inv(Kq) cyaw, p, r = qsource.yaw, qsource.pitch, qsource.roll # cyaw - cell phone yaw # get high res sift rematch matches = highresSift(C, Q, dbmatch) with open(Q.datafile, 'a') as df: print >> df, 'Number of matches | number of queries | ratio: %.0f | %.0f | %.2f' % ( matches['nmat'], matches['numq'], float(matches['nmat']) / matches['numq']) print >> df, '' # Get estimated ground truth query location and normal direction tlat, tlon, tnorm, tyaw = getGTpose(C, Q) qzx = geom.lltom(olat, olon, tlat, tlon) # get query yaw and plane yaw from vanishing point anaylsis yawforvp = tyaw if param['cheat'] else np.nan vyaw, vnorms = vp_analysis.getQNyaws(C, Q, qimg, dbimg, qsource, yawforvp) # get dominant planes dplanes, psizes, planeprms = find_dbplanes(C, Q, dbmatch, Kdinv, wRd) # match vanishing point planes to database planes pyaws, planes, pconfs = combine_planes(runflag, vnorms, dplanes, psizes, planeprms) print 'VP and DB Planes: ' + str(np.int_(pyaws)) + ', ' + str(planes) with open(Q.datafile, 'a') as df: # print >>df, 'Planes detected with vanishing points:' for i in range(len(pconfs)): perr = np.round(np.mod(pyaws[i] - tnorm, 360)) perr = perr if perr < 180 else 360 - perr print >> df, 'Plane Yaw | DB plane | Confidence | Error : %3.0f | %d | %.2f | %.0f' % ( pyaws[i], 0 if planes[i] < 0 else planes[i], pconfs[i], perr) yerr = np.round(np.mod(vyaw - tyaw, 360)) yerr = yerr if yerr < 180 else 360 - yerr print >> df, 'VP Yaw | Confidence | Error : %3.0f | %.2f | %.0f' % ( vyaw, np.nan, yerr) print >> df, 'Cell yaw | True yaw | Plane : %3.0f | %3.0f | %3.0f' % ( cyaw, tyaw, tnorm) print >> df, '' # Set yaw value to be used if runflag >= 10: # vanishing point methods if np.isnan(vyaw): yaw, yawerr = cyaw, 0 else: yaw, yawerr = vyaw, 0 else: yaw, yawerr = cyaw, 0 # set cell phone yaw to use, plane normal wRq = geom.RfromYPR(yaw, p, r) # camera orientation (camera to world) ### --- THIS IS FOR CHEATING --- ### if param['cheat']: if not np.isnan(tnorm): pyaws, planes, pconfs = np.append(pyaws, tnorm), np.append( planes, -1), np.append(pconfs, 1) yaw, yawerr = tyaw, 0 wRq = geom.RfromYPR(yaw, p, r) # camera orientation (camera to world) ### --- THIS IS FOR CHEATING --- ### # print pre-homography data to file vyaw_err = np.round(np.round(np.mod( tyaw - vyaw, 360))) if not np.isnan(vyaw) else np.nan vyaw_err = vyaw_err if vyaw_err < 180 else 360 - vyaw_err dbears = np.mod(180 / np.pi * np.arctan2(planeprms[:, 0], planeprms[:, 2]), 360) print 'Computed / ground truth cell phone yaw: %.0f / %.0f' % (vyaw, tyaw) with open(os.path.join(param['resultsdir'], param['extras_file']), 'a') as extras_file: print >> extras_file, '\t'.join([ timename, '%.0f' % tnorm, '%.0f' % np.round(tyaw), '%.0f' % cyaw, '%.0f' % vyaw, '%.4f' % np.nan, str(vyaw_err) ]) print >> extras_file, '\t'.join(['%.4f' % 0 for vnorm in vnorms]) print >> extras_file, '\t'.join(['%.0f' % vnorm for vnorm in vnorms]) print >> extras_file, '\t'.join(['%.0f' % plane for plane in planes]) print >> extras_file, '\t'.join( ['%.0f' % dplane for dplane in dplanes]) print >> extras_file, '\t'.join(['%.0f' % dbear for dbear in dbears]) print >> extras_file, '\t'.join( ['%.3f' % dnerr for dnerr in planeprms[:, 4]]) # Fill out match information nmat = matches['nmat'] matches['qray'] = geom.normalrows( tp(np.dot(Kqinv, np.append(tp(matches['q2d']), [np.ones(nmat)], 0)))) matches['dray'] = geom.normalrows( tp(np.dot(Kdinv, np.append(tp(matches['d2d']), [np.ones(nmat)], 0)))) matches = match_info(C, Q, matches, dbmatch, wRd) matches_start = matches.copy() # Solve for query pose using constrained image geometry init_matches = initMatches(matches.copy()) matches['numi'], matches['hconf'] = 0, 0 runflag, ntry, planechose = param['runflag'], 0, 0 parameters = (wRq, wRd, yaw, np.nan, runflag, param['inlier_error'], param['ransac_iter'], 10, True) if param['ransac_iter'] == 0: matches = init_matches matches['numi'], matches['hconf'] == 0, 0 pose = np.zeros(6) pose[3:5] = np.nan elif runflag < 10: matches, pose = solveGeom(init_matches, parameters, yawerr) else: ntry = 1 viter = -np.ones(3) parameters = (wRq, wRd, yaw, np.nan, runflag, param['inlier_error'], param['ransac_iter'], 15, True) matches, pose, planechose = solveNorm(C, Q, dbmatch, pyaws, planes, init_matches, parameters, yawerr) viter[0] = matches['viter'] if matches['numi'] == 0 or matches['hconf'] == 0: ntry = 2 parameters = (wRq, wRd, yaw, np.nan, runflag, 3 * param['inlier_error'], param['ransac_iter'], 10, True) matches, pose, planechose = solveNorm(C, Q, dbmatch, pyaws, planes, init_matches, parameters, yawerr) viter[1] = matches['viter'] if matches['numi'] == 0 or matches['hconf'] == 0: ntry, planechose = 3, 0 parameters = (wRq, wRd, yaw, np.nan, 7, 3 * param['inlier_error'], param['ransac_iter'], 10, True) matches, pose = solveYaw(init_matches, parameters, yawerr) viter[2] = matches['viter'] if matches['numi'] == 0 or matches['hconf'] == 0: ntry, planechose = 4, 0 # save matches to disk matches_file = os.path.join(param['resultsdir'], 'matches_' + qname + '.pkl') matches_out = open(matches_file, 'wb') pickle.dump(matches, matches_out) matches_out.close() # extract pose parameters comp_runflag = matches['runflag'] tray = pose[:3] comp_yaw = pose[3] comp_pyaw = pose[4] if runflag >= 0 else np.nan scale = pose[5] if runflag >= 0 else np.nan # Get scaled translation for query location if np.isnan(scale): wRq_pr = geom.YPRfromR(wRq)[1:] comp_wRq = geom.RfromYPR(comp_yaw, wRq_pr[0], wRq_pr[1]) qloc = scalefrom3d(matches, tray, comp_wRq)[0] else: # scale calculated in RANSAC loop qloc = scale * tray # temporarily get yaw error qyaw_error = np.round(abs(np.mod(tyaw - comp_yaw, 360))) qyaw_error = qyaw_error if qyaw_error < 180 else abs(qyaw_error - 360) # compute location errors wrt estimated query locations loc_err = ((qloc[0] - qzx[1])**2 + (qloc[2] - qzx[0])**2)**0.5 gps_err = ((gzx[1] - qzx[1])**2 + (gzx[0] - qzx[0])**2)**0.5 # compute the angle difference between T and ground truth translation tyaw_error = np.round( abs(180 / np.pi * np.arccos( np.abs(qloc[0] * qzx[1] + qloc[2] * qzx[0]) / (alg.norm([qloc[0], qloc[2]]) * alg.norm(qzx))))) # compute the plane normal angle error nyaw_error = np.nan if np.isnan(comp_pyaw) or np.isnan(tnorm) else np.mod( np.round(abs(comp_pyaw - tnorm)), 180) nyaw_error = nyaw_error if nyaw_error < 90 else abs(nyaw_error - 180) # write pose estimation results to file yaw_err = np.nan pose_file = os.path.join(param['resultsdir'], param['pose_file']) with open(pose_file, 'a') as pf: print >>pf, '\t'.join([qname, str(loc_err), str(gps_err), \ str(tyaw_error), str(qyaw_error), str(nyaw_error), str(matches['numi']), \ str(matches['numq']), str(matches['nmat']), str(matches['hconf']), \ str(qloc[0]), str(qloc[2]), str(yaw_err), str(runflag)]) # print post-homography data to file with open(Q.datafile, 'a') as df: print >> df, '' print >> df, '------------------' print >> df, '' if ntry == 1: print >> df, 'Homography solution using low error threshold with restrictions.' elif ntry == 2: print >> df, 'Homography solution using high error threshold with restrictions.' else: print >> df, 'Solution not found. Setting T=0.' if planechose == 0: print >> df, 'Solution formed with unset plane normal.' else: 'Solution chosen with plane normal %d chosen.' % planechose print >> df, 'VP yaw | Computed yaw | Actual Yaw | Error : %3.0f | %3.0f | %3.0f | %3.0f' % ( vyaw, comp_yaw, tyaw, qyaw_error) print >> df, 'Computed Normal | Actual Normal | Error : %3.0f | %3.0f | %3.0f' % ( comp_pyaw, tnorm, nyaw_error) print >> df, 'Translation (x|y|z): %.1f | %.1f | %.1f' % ( qloc[0], qloc[1], qloc[2]) print >> df, 'True position (x|-|z): %.1f | - | %.1f' % (qzx[1], qzx[0]) print >> df, 'Angle error | Location error: %.0f | %.1f' % (tyaw_error, loc_err) print >> df, 'Number of Inliers | Total matches | Ratio: %d | %d | %.2f' % ( matches['numi'], matches['nmat'], np.nan if matches['nmat'] == 0 else float(matches['numi']) / matches['nmat']) print >> df, 'Reprojection error | Homography confidence: %.3f | %.1f' % ( matches['rperr'], matches['hconf']) print >> df, 'Valid Homographies | Iterations | Ratio: %d | %d | %.3f' % ( matches['viter'], matches['niter'], np.nan if matches['niter'] == 0 else float(matches['viter']) / matches['niter']) print >> df, '' print >> df, '------------------' print >> df, '' booleans = [ loc_err<5, loc_err<10, not(5<np.mod(vyaw-tyaw,360)<355), \ not(10<np.mod(vyaw-tyaw,360)<350), \ not(5<np.mod(comp_yaw-tyaw,360)<355), ntry==1, ntry!=3, \ planechose!=0, matches['nmat']!=matches_start['nmat'], \ 0 if planechose==0 else pconfs[planechose-1]>0, comp_yaw-vyaw ] print >> df, '|'.join(['%.0f' % (b) for b in booleans]) # draw matches close = int(loc_err < 5) + int(loc_err < 10) yawclose = int(not (5 < np.mod(vyaw - tyaw, 360) < 355)) + int(not ( 10 < np.mod(vyaw - tyaw, 360) < 350)) imgpath = os.path.join( param['resultsdir'] , qname + ';locerr=%.2f' % (loc_err) + ';locPerf_' + str(close) \ + ';yawPerf_' + str(yawclose) + ';nplanes_' + str(len(pyaws)) + ';plane_' + str(planes[planechose]) + ';try_' + str(ntry) \ + ';tAng=%.0f' % (tyaw_error) + ';qAng=%.0f' % (qyaw_error) + ';nAng=%.0f' % (nyaw_error) + ';' + dbmatch + '.jpg') draw_matches(matches, qimg, dbimg, imgpath) # imgpath = os.path.join( param['resultsdir'] , 'homography;' + qname + ';' + dbmatch + '.jpg') # draw_hom(matches, pose, wRq, wRd, Kq, Kd, qimg, dbimg, imgpath) if C.QUERY == 'oakland1': C.pose_param['draw_tags'] = False if C.pose_param['draw_tags']: draw_tags(C, Q, matches, pose, dbmatch, olat, olon, Kd, Kq) print 'Computed yaw / ground truth yaw : %.0f / %.0f' % (comp_yaw, tyaw) if runflag < 10: print 'Computed normal bearing / ground truth : %.0f / %.0f' % ( comp_pyaw, tnorm) print 'Computed query location relative to db : %.1f, %.1f, %.1f' % tuple( qloc) print 'Ground truth query location relative to db : %.1f, - , %.1f' % ( qzx[1], qzx[0]) input = (wRq, wRd) return qloc, loc_err, matches, input
def extract_panorama(panopath, outbase, panoinfo, detail): """Generates raster, plane, and depth views at rot_degrees""" print "Processing panorama " + '%d' % panoinfo['pano'][0] # panopath: location of raster, depth and plane_pano images # outbase: base name to put generated view, depth, and plane images # panoinfo: Contains information about the panoramic scene # detail: 0 = 768x512 with 60d fov, 1 = 2500x1200 with 90d fov # local constants pi = np.pi width, height, fov = (2500, 1200, 90) if detail else (768, 512, 60) # pano and view details details orientation = [panoinfo['yaw'][0],panoinfo['pitch'][0],panoinfo['roll'][0]] Rpano = geom.RfromYPR(orientation[0],orientation[1],orientation[2]) Kview = geom.cameramat(width, height, fov*pi/180) Kinv = alg.inv(Kview) # Load image pano, depth pano, and plane pano images cvIP = cv.LoadImageM( os.path.join(panopath,'raster.jpg'), cv.CV_LOAD_IMAGE_UNCHANGED ) cvDP = cv.fromarray( np.asarray( Image.open( os.path.join(panopath,'depth.png') ) ) ) pp = np.asarray( Image.open( os.path.join(panopath,'plane_pano.png') ) ).copy() vals = set(list(pp.reshape(-1))) vals.remove(255) gnd = max(vals) pp[pp==gnd] = np.uint8(0) cvPP = cv.fromarray(pp) # load pixel map pix = np.append(np.array(np.meshgrid(range(width),range(height))).reshape(2,-1),np.ones([1,width*height]),0) # Create output openCV matrices cvI = cv.CreateMat(height,width,cv.CV_8UC3) cvD = cv.CreateMat(height,width,cv.CV_32SC1) cvP = cv.CreateMat(height,width,cv.CV_8UC1) for viewdir in [2,3,4,8,9,10]: # add to base name and generate info file viewname = outbase + '%04d' % viewdir gen_info_file(panoinfo, viewname + '.info', detail, 30*viewdir) # generate view orientation Yview = np.mod( orientation[0] + 30*viewdir, 360 ) Rview = geom.RfromYPR(Yview, 0, 0) # compute mappings ray = np.dot( np.transpose(Rpano), np.dot( Rview, np.dot( Kinv, pix ) ) ) yaw, pitch = np.arctan2( ray[0,:] , ray[2,:] ) , np.arctan2( -ray[1,:] , np.sqrt((np.array([ray[0,:],ray[2,:]])**2).sum(0)) ) ix, iy = cv.fromarray(np.array(8192/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(4096/pi*(pi/2-pitch),np.float32).reshape(height,width)) dx, dy = cv.fromarray(np.array(5000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array(2500/pi*(pi/2-pitch),np.float32).reshape(height,width)) px, py = cv.fromarray(np.array(1000/(2*pi)*(pi+yaw),np.float32).reshape(height,width)), cv.fromarray(np.array( 500/pi*(pi/2-pitch),np.float32).reshape(height,width)) # call remap function cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_CUBIC) # if detail else cv.Remap(cvIP,cvI,ix,iy,cv.CV_INTER_AREA) cv.Remap(cvDP,cvD,dx,dy,cv.CV_INTER_NN) cv.Remap(cvPP,cvP,px,py,cv.CV_INTER_NN) # write images to file Image.fromarray(np.array(cvI)[:,:,[2,1,0]]).save(viewname+'.jpg','jpeg') Image.fromarray(np.array(cvD)).save(viewname+'-depth.png','png') Image.fromarray(np.array(cvP)).save(viewname+'-planes.png','png')