def modelMergeCheckLocal(sfm_data_path, sfm_locOut, medThres):
# load sfm_data
sfm_data = FileUtils.loadjson(sfm_data_path)
# collect all image names ad location
imgName = []
imgLoc = []
for filename in os.listdir(sfm_locOut):
if filename[-4:]!="json":
continue
locJsonDict = FileUtils.loadjson(os.path.join(sfm_locOut,filename))
if "t" in locJsonDict:
imgName.append(os.path.basename(locJsonDict["filename"]))
imgLoc.append(locJsonDict["t"])
imgID = imgnameToViewID(imgName, sfm_data)
imgSfMLoc = get3DViewloc(sfm_data, imgID)
# calculate distance and count agreement
countFile = 0
countAgree = 0
for j in range(0,len(imgLoc)):
dist = np.linalg.norm(np.array(imgLoc[j])-np.array(imgSfMLoc[j]))
if dist < float("inf"):
countFile = countFile + 1
if dist < medThres:
countAgree = countAgree + 1
return countFile, countAgree
def modelMergeCheckLocal(sfm_data_path, sfm_locOut, medThres):
# load sfm_data
sfm_data = FileUtils.loadjson(sfm_data_path)
# collect all image names ad location
imgName = []
imgLoc = []
for filename in os.listdir(sfm_locOut):
if filename[-4:]!="json":
continue
locJsonDict = FileUtils.loadjson(os.path.join(sfm_locOut,filename))
imgName.append(os.path.basename(locJsonDict["filename"]))
imgLoc.append(locJsonDict["t"])
imgID = imgnameToViewID(imgName, sfm_data)
imgSfMLoc = get3DViewloc(sfm_data, imgID)
# calculate distance and count agreement
countFile = 0
countAgree = 0
for j in range(0,len(imgLoc)):
dist = np.linalg.norm(np.array(imgLoc[j])-np.array(imgSfMLoc[j]))
if dist < float("inf"):
countFile = countFile + 1
if dist < medThres:
countAgree = countAgree + 1
return countFile, countAgree
def __init__(self,
name,
imgFolLoc,
csvFolLoc,
matchesFolLoc,
locFolLoc,
sfm_dataLoc,
validMergeRansacThres=-1,
validMergeRansacThresK=-1,
ransacStructureThres=-1,
ransacStructureThresK=-1,
mergeStructureThres=-1,
mergeStructureThresK=-1):
if (validMergeRansacThres == -1 and validMergeRansacThresK == -1):
print "error : invalid argument for sfmModel valid merge ransac"
sys.exit()
if (ransacStructureThres == -1 and ransacStructureThresK == -1):
print "error : invalid argument for sfmModel structure ransac"
sys.exit()
if (mergeStructureThres == -1 and mergeStructureThresK == -1):
print "error : invalid argument for sfmModel structure merge"
sys.exit()
self.name = name # folder name
self.mergeOrder = name # structure similar to a tree specifying merge order
self.imgFolLoc = imgFolLoc # folder dir of input image folder
self.csvFolLoc = csvFolLoc # folder dir of csv folder
self.matchesFolLoc = matchesFolLoc # folder of match folder with descriptor
self.locFolLoc = locFolLoc # folder of localization result
self.sfm_dataLoc = sfm_dataLoc # file dir of sfm_data.json
# get list of reconstructed frames
if self.sfm_dataLoc != "":
sfm_data = FileUtils.loadjson(self.sfm_dataLoc)
extKeyTmp = [x["key"] for x in sfm_data["extrinsics"]]
self.reconFrame = [
x["value"]["ptr_wrapper"]["data"]["id_view"]
for x in sfm_data["views"]
if x["value"]["ptr_wrapper"]["data"]["id_pose"] in extKeyTmp
]
if validMergeRansacThresK > 0:
self.validMergeRansacThres = mergeSfM.findMedianThres(
sfm_data, validMergeRansacThresK)
else:
self.validMergeRansacThres = validMergeRansacThres
if ransacStructureThresK > 0:
self.ransacStructureThres = mergeSfM.findMedianStructurePointsThres(
sfm_data, ransacStructureThresK)
else:
self.ransacStructureThres = ransacStructureThres
if mergeStructureThresK > 0:
self.mergeStructureThres = mergeSfM.findMedianStructurePointsThres(
sfm_data, mergeStructureThresK)
else:
self.mergeStructureThres = mergeStructureThres
def __init__(self, name, imgFolLoc, csvFolLoc, matchesFolLoc, locFolLoc, sfm_dataLoc):
self.name = name # folder name
self.mergeOrder = name # structure similar to a tree specifying merge order
self.imgFolLoc = imgFolLoc # folder dir of input image folder
self.csvFolLoc = csvFolLoc # folder dir of csv folder
self.matchesFolLoc = matchesFolLoc # folder of match folder with descriptor
self.locFolLoc = locFolLoc # folder of localization result
self.sfm_dataLoc = sfm_dataLoc # file dir of sfm_data.json
# get list of reconstructed frames
if self.sfm_dataLoc != "":
sfm_data = FileUtils.loadjson(self.sfm_dataLoc)
extKeyTmp = [x["key"] for x in sfm_data["extrinsics"]]
self.reconFrame = [x["value"]["ptr_wrapper"]["data"]["id_view"] for x in sfm_data["views"] if x["value"]["ptr_wrapper"]["data"]["id_pose"] in extKeyTmp]
def calculateAverageBOW(sfmDataFile, matchesFolLoc):
sfmData = FileUtils.loadjson(sfmDataFile)
avgBow = None
for view in sfmData["views"]:
viewImage = view["value"]["ptr_wrapper"]["data"]["filename"]
viewBow = os.path.join(matchesFolLoc, os.path.splitext(viewImage)[0] + ".bow")
bowvec = FileUtils.loadBinMat(viewBow)
if avgBow is None:
avgBow = bowvec
else:
avgBow += bowvec
avgBow /= len(sfmData["views"])
return avgBow
def readMatch(locFolder):
imgname = []
matchlist = []
print "Reading loc output: " + locFolder
for filename in sorted(os.listdir(locFolder)):
if filename[-4:] != "json":
continue
jsondata = FileUtils.loadjson(os.path.join(locFolder, filename))
imgname.append(os.path.basename(jsondata["filename"]))
matchlist.append(jsondata["pair"])
return imgname, matchlist
def readMatch(locFolder):
imgname = []
matchlist = []
print "Reading loc output: " + locFolder
for filename in sorted(os.listdir(locFolder)):
if filename[-4:] != "json":
continue
jsondata = FileUtils.loadjson(os.path.join(locFolder, filename))
imgname.append(os.path.basename(jsondata["filename"]))
matchlist.append(jsondata["pair"])
return imgname, matchlist
def calculateAverageBOW(sfmDataFile, matchesFolLoc):
sfmData = FileUtils.loadjson(sfmDataFile)
avgBow = None
for view in sfmData["views"]:
viewImage = view["value"]["ptr_wrapper"]["data"]["filename"]
viewBow = os.path.join(matchesFolLoc,
os.path.splitext(viewImage)[0] + ".bow")
bowvec = FileUtils.loadBinMat(viewBow)
if avgBow is None:
avgBow = bowvec
else:
avgBow += bowvec
avgBow /= len(sfmData["views"])
return avgBow
def __init__(self, name, imgFolLoc, csvFolLoc, matchesFolLoc, locFolLoc,
sfm_dataLoc):
self.name = name # folder name
self.mergeOrder = name # structure similar to a tree specifying merge order
self.imgFolLoc = imgFolLoc # folder dir of input image folder
self.csvFolLoc = csvFolLoc # folder dir of csv folder
self.matchesFolLoc = matchesFolLoc # folder of match folder with descriptor
self.locFolLoc = locFolLoc # folder of localization result
self.sfm_dataLoc = sfm_dataLoc # file dir of sfm_data.json
# get list of reconstructed frames
if self.sfm_dataLoc != "":
sfm_data = FileUtils.loadjson(self.sfm_dataLoc)
extKeyTmp = [x["key"] for x in sfm_data["extrinsics"]]
self.reconFrame = [
x["value"]["ptr_wrapper"]["data"]["id_view"]
for x in sfm_data["views"]
if x["value"]["ptr_wrapper"]["data"]["id_pose"] in extKeyTmp
]
def mergeModel(sfm_data_dirA, sfm_data_dirB, locFolderB, outfile, ransacThres, mergePointThres, ransacRoundMul=100, inputImgDir="", minLimit=4, svdRatio=1.75):
print "Loading sfm_data"
sfm_dataB = FileUtils.loadjson(sfm_data_dirB)
# read matching pairs from localization result
imgnameB, matchlistB = readMatch(locFolderB)
# get viewID from image name for model B
viewIDB = imgnameToViewID(imgnameB, sfm_dataB)
# get mapping between viewID,featID to 3D point ID
viewFeatMapB = getViewFeatTo3DMap(sfm_dataB)
# find consistent match between 3D of model B to 3D of model A
print "Calculating consistent 3D matches"
match3D_BA = getConsistent3DMatch(viewIDB, matchlistB, viewFeatMapB)
print "Found " + str(len(match3D_BA)) + " consistent matches"
# not enough matches
if len(match3D_BA) <= 4 or len(match3D_BA) <= minLimit:
return len(match3D_BA), len(match3D_BA), np.asarray([])
# move the load of larger model here to reduce time if merging is not possible
sfm_dataA = FileUtils.loadjson(sfm_data_dirA)
# get 3D point. Note that element 0 of each pair in match3D_BA
# is 3D pt ID of model B and element 1 is that of model A
print "Load 3D points"
pointA = get3DPointloc(sfm_dataA, [x[1] for x in match3D_BA])
pointB = get3DPointloc(sfm_dataB, [x[0] for x in match3D_BA])
pointAn = np.asarray(pointA, dtype=np.float).T
pointBn = np.asarray(pointB, dtype=np.float).T
# find robust transformation
print "Find transformation with RANSAC"
ransacRound = len(match3D_BA)*ransacRoundMul
print "Number of RANSAC round : " + str(ransacRound)
M, inliers = ransacTransform(pointAn, pointBn, ransacThres, ransacRound, svdRatio)
# cannot find RANSAC transformation
if (M.size==0):
return len(match3D_BA), len(match3D_BA), np.asarray([])
print M
# stop if not enough inliers
sSvd = np.linalg.svd(M[0:3,0:3],compute_uv=0)
# fixed by T.Ishihara to use minLimit 2016.06.06
#if len(inliers) <= 4 or sSvd[0]/sSvd[-1] > svdRatio:
if len(inliers) <= minLimit or sSvd[0]/sSvd[-1] > svdRatio:
return len(match3D_BA), len(inliers), M
# perform merge
# last argument is map from inliers 3D pt Id of model B to that of model A
print "Merging sfm_data"
# fixed by T. Ishihara, use different parameter to find ransac inlier and merge points inliers
'''
merge_sfm_data(sfm_dataA, sfm_dataB, M, {match3D_BA[x][0]: match3D_BA[x][1] for x in inliers})
'''
mergePointInliers = getInliersByAffineTransform(pointAn, pointBn, M, mergePointThres)
merge_sfm_data(sfm_dataA, sfm_dataB, M, {match3D_BA[x][0]: match3D_BA[x][1] for x in mergePointInliers})
# change input image folder
if inputImgDir != "":
sfm_dataA["root_path"] = inputImgDir
# save json file
print "Saving json file"
FileUtils.savejson(sfm_dataA,outfile)
# return number of inliers for transformation
return len(match3D_BA), len(inliers), M
def mergeModel(sfm_data_dirA,
sfm_data_dirB,
locFolderB,
outfile,
ransacK=1.0,
ransacRound=10000,
inputImgDir="",
minLimit=4,
svdRatio=1.75):
print "Loading sfm_data"
sfm_dataB = FileUtils.loadjson(sfm_data_dirB)
# read matching pairs from localization result
imgnameB, matchlistB = readMatch(locFolderB)
# get viewID from image name for model B
viewIDB = imgnameToViewID(imgnameB, sfm_dataB)
# get mapping between viewID,featID to 3D point ID
viewFeatMapB = getViewFeatTo3DMap(sfm_dataB)
# find consistent match between 3D of model B to 3D of model A
print "Calculating consistent 3D matches"
match3D_BA = getConsistent3DMatch(viewIDB, matchlistB, viewFeatMapB)
print "Found " + str(len(match3D_BA)) + " consistent matches"
# not enough matches
if len(match3D_BA) <= 4 or len(match3D_BA) <= minLimit:
return len(match3D_BA), [0]
# move the load of larger model here to reduce time if merging is not possible
sfm_dataA = FileUtils.loadjson(sfm_data_dirA)
# get 3D point. Note that element 0 of each pair in match3D_BA
# is 3D pt ID of model B and element 1 is that of model A
print "Load 3D points"
pointA = get3DPointloc(sfm_dataA, [x[1] for x in match3D_BA])
pointB = get3DPointloc(sfm_dataB, [x[0] for x in match3D_BA])
pointAn = np.asarray(pointA, dtype=np.float).T
pointBn = np.asarray(pointB, dtype=np.float).T
# calculate ransac threshold
# calculate as 4 times the median of distance between
# 3D pt of A
print "Find transformation with RANSAC"
# modified by T.Ishihara 2016.04.08
# median of camera positions merge too many points, use median of structure points instead
#ransacThres = findMedianThres(sfm_dataA, ransacK)
ransacThres = findMedianStructurePointsThres(sfm_dataA, ransacK)
# TODO : replace with RANSAC similarity transform
# find robust transformation
M, inliers = ransacAffineTransform(pointAn, pointBn, ransacThres,
ransacRound, svdRatio)
# cannot find RANSAC transformation
if (len(inliers) == 0):
return len(match3D_BA), [0]
print M
# stop if not enough inliers
sSvd = np.linalg.svd(M[0:3, 0:3], compute_uv=0)
if len(inliers) <= 4 or sSvd[0] / sSvd[-1] > svdRatio:
return len(inliers), M
# perform merge
# last argument is map from inliers 3D pt Id of model B to that of model A
print "Merging sfm_data"
merge_sfm_data(sfm_dataA, sfm_dataB, M,
{match3D_BA[x][0]: match3D_BA[x][1]
for x in inliers})
# change input image folder
if inputImgDir != "":
sfm_dataA["root_path"] = inputImgDir
# save json file
print "Saving json file"
FileUtils.savejson(sfm_dataA, outfile)
# return number of inliers for transformation
return len(inliers), M
def mergeOneModel(self, model1, model2, reconParam, reconIBeaconParam, reconBOWParam):
sfmOutPath = os.path.join(self.mSfMPath,"global"+str(self.nMergedModel))
# modified by T. IShihara 2016.06.14
# fix file name too long issue
#
# create a temporary folder for reconstructed image of model2
#inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmp"+model2.name)
inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmpModel2")
if os.path.isdir(inputImgTmpFolder):
FileUtils.removedir(inputImgTmpFolder)
# copy reconstructed image fom model2 to tmp folder
sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc)
if not os.path.isdir(inputImgTmpFolder):
listReconFrameName = [sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["filename"] for x in range(0,len(sfm_data2["views"])) if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["id_view"] in model2.reconFrame]
FileUtils.makedir(inputImgTmpFolder)
for reconFrameName in listReconFrameName:
os.system("cp -s " + os.path.join(model2.imgFolLoc,reconFrameName) + " " + inputImgTmpFolder)
# remove all old localization result
FileUtils.removedir(model2.locFolLoc)
FileUtils.makedir(model2.locFolLoc)
# localize the images from model2 on model1
if self.useBow:
os.system(reconIBeaconParam.LOCALIZE_PROJECT_PATH + \
" " + inputImgTmpFolder + \
" " + os.path.dirname(model1.sfm_dataLoc) + \
" " + self.mMatchesPath + \
" " + model2.locFolLoc + \
" -f=" + str(reconParam.locFeatDistRatio) + \
" -r=" + str(reconParam.locRansacRound) + \
" -b=" + model1.beaconFileLoc + \
" -e=" + model2.csvFolLoc + \
" -k=" + str(reconIBeaconParam.locKNNnum) + \
" -c=" + str(reconIBeaconParam.coocThres) + \
" -i=" + str(reconParam.locSkipFrame) + \
" -v=" + str(reconIBeaconParam.locSkipSelKNN) + \
" -n=" + str(reconIBeaconParam.normApproach) + \
" -kb=" + str(reconBOWParam.locKNNnum) + \
" -a=" + os.path.join(self.mMatchesPath, "BOWfile.yml") + \
" -p=" + os.path.join(self.mMatchesPath, "PCAfile.yml"))
else:
os.system(reconIBeaconParam.LOCALIZE_PROJECT_PATH + \
" " + inputImgTmpFolder + \
" " + os.path.dirname(model1.sfm_dataLoc) + \
" " + self.mMatchesPath + \
" " + model2.locFolLoc + \
" -f=" + str(reconParam.locFeatDistRatio) + \
" -r=" + str(reconParam.locRansacRound) + \
" -b=" + model1.beaconFileLoc + \
" -e=" + model2.csvFolLoc + \
" -k=" + str(reconIBeaconParam.locKNNnum) + \
" -c=" + str(reconIBeaconParam.coocThres) + \
" -i=" + str(reconParam.locSkipFrame) + \
" -v=" + str(reconIBeaconParam.locSkipSelKNN) + \
" -n=" + str(reconIBeaconParam.normApproach))
# remove temporary image folder
# removedir(inputImgTmpFolder)
# extract centers from all json file and write to a file
fileLoc = open(os.path.join(model2.locFolLoc,"center.txt"),"w")
countLocFrame = 0
for filename in sorted(os.listdir(model2.locFolLoc)):
if filename[-4:]!="json":
continue
countLocFrame = countLocFrame + 1
with open(os.path.join(model2.locFolLoc,filename)) as locJson:
#print os.path.join(sfm_locOut,filename)
locJsonDict = json.load(locJson)
loc = locJsonDict["t"]
fileLoc.write(str(loc[0]) + " " + str(loc[1]) + " " +str(loc[2]) + " 255 0 0\n" )
fileLoc.close()
# get inlier matches
FileUtils.makedir(sfmOutPath)
resultSfMDataFile = os.path.join(sfmOutPath,"sfm_data.json")
# below also checks if the ratio between first and last svd of M[0:3,0:3]
# is good or not. If not then reject
# TODO : revisit ransacRound parameter, use number of reconstruction frame to determine structure points transform seems small
nMatchPointsTmp, nInlierTmp, M = mergeSfM.mergeModel(model1.sfm_dataLoc,
model2.sfm_dataLoc,
model2.locFolLoc,
resultSfMDataFile,
ransacThres=model1.ransacStructureThres,
mergePointThres=model1.mergeStructureThres,
ransacRoundMul=reconParam.ransacRoundMul,
inputImgDir=self.mInputImgPath,
minLimit=reconParam.min3DnInliers)
ratioInlierMatchPoints = 0.0
if nMatchPointsTmp>0:
ratioInlierMatchPoints = float(nInlierTmp)/nMatchPointsTmp
# 3. perform test whether merge is good
sfm_merge_generated = True
countFileAgree = 0
countFileLoc = 1
if os.path.isfile(resultSfMDataFile):
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + resultSfMDataFile + " " + resultSfMDataFile)
countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal(resultSfMDataFile, model2.locFolLoc, model1.validMergeRansacThres)
else:
sfm_merge_generated = False
ratioAgreeFrameReconFrame = 0.0
if (len(model2.reconFrame)>0):
ratioAgreeFrameReconFrame = float(countFileAgree)/len(model2.reconFrame)
ratioAgreeFrameLocFrame = 0.0
if (countFileLoc>0):
ratioAgreeFrameLocFrame = float(countFileAgree)/countFileLoc
# write log file
with open(os.path.join(self.mSfMPath,"global"+str(self.nMergedModel),"log.txt"),"a") as filelog:
filelog.write(("M1: " + model1.name + "\n" + \
"M2: " + model2.name + "\n" + \
"nMatchedPoints: " + str(nMatchPointsTmp) + "\n" + \
"nInliers: " + str(nInlierTmp) + "\n" + \
"ratioInlierWithMatchedPoints: " + str(ratioInlierMatchPoints) + "\n" + \
"countLocFrame: " + str(countLocFrame) + "\n" + \
"nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \
"countFileAgree: " + str(countFileAgree) + "\n" + \
"countFileLoc: " + str(countFileLoc) + "\n" + \
"not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \
# obsolete condition by T. Ishihara 2015.11.10
#"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \
"countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \
"ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \
"ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \
"ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \
"ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \
str(M) + "\n\n"))
# rename the localization folder to save localization result
'''
if os.path.isdir(model2.locFolLoc+model1.name):
FileUtils.removedir(model2.locFolLoc+model1.name)
os.rename(model2.locFolLoc,model2.locFolLoc+model1.name)
'''
# obsolete merge condition
'''
if not sfm_merge_generated or \
not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \
((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \
((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and
(float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))):
'''
# update merge condition by T. Ishihara 2015.11.10
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \
countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
'''
# update merge condition by T. Ishihara 2016.04.02
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
'''
# update merge condition by T. Ishihara 2016.06.09
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \
nInlierTmp > reconParam.min3DnInliers and \
ratioInlierMatchPoints > reconParam.vldMergeRatioInliersMatchPoints):
'''
# update merge condition by T. Ishihara 2016.06.20
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \
nInlierTmp > reconParam.min3DnInliers):
print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "."
'''
if os.path.isfile(os.path.join(sfmOutPath,"sfm_data.json")):
os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \
os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json"))
'''
# move to next video
return False, sfmModelIBeacon("","","","","","","",validMergeRansacThres=0,validMergeRansacThresK=0,
ransacStructureThres=0, ransacStructureThresK=0,
mergeStructureThres=0, mergeStructureThresK=0)
# generate colorized before bundle adjustment for comparison
os.system("openMVG_main_ComputeSfM_DataColor " +
" -i " + os.path.join(sfmOutPath,"sfm_data.json") +
" -o " + os.path.join(sfmOutPath,"colorized_pre.ply"))
# TODO : try computing structure from know pose here
# https://github.com/openMVG/openMVG/issues/246
# http://openmvg.readthedocs.io/en/latest/software/SfM/ComputeStructureFromKnownPoses/
# TODO : revisit the order of bundle adjustment
# perform bundle adjustment
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "rs,rst,rsti" + " -r=" + "1")
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "rst,rsti" + " -r=" + "1")
os.system("openMVG_main_ComputeSfM_DataColor " +
" -i " + os.path.join(sfmOutPath,"sfm_data.json") +
" -o " + os.path.join(sfmOutPath,"colorized.ply"))
# write new beacon file
IBeaconUtils.exportBeaconDataForSfmImageFrames(self.mCsvPath, resultSfMDataFile, os.path.join(self.mInputPath,"listbeacon.txt"),
os.path.join(sfmOutPath,"beacon.txt"), reconIBeaconParam.normApproach)
return True, sfmModelIBeacon("A" + model1.name + "," + model2.name +"Z", self.mInputImgPath, self.mCsvPath,
os.path.join(sfmOutPath,"beacon.txt"), self.mMatchesPath, os.path.join(sfmOutPath,"loc"),
resultSfMDataFile, validMergeRansacThres=model1.validMergeRansacThres,
ransacStructureThres=model1.ransacStructureThres,
mergeStructureThres=model1.mergeStructureThres)
def mergeOneModel(self, model1, model2, reconParam, reconBOWParam):
sfmOutPath = os.path.join(self.mSfMPath,
"global" + str(self.nMergedModel))
# modified by T. IShihara 2016.06.14
# fix file name too long issue
#
# create a temporary folder for reconstructed image of model2
#inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmp"+model2.name)
inputImgTmpFolder = os.path.join(self.mSfMPath, "inputImgTmp",
"inputImgTmpModel2")
if os.path.isdir(inputImgTmpFolder):
FileUtils.removedir(inputImgTmpFolder)
# copy reconstructed image fom model2 to tmp folder
sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc)
if not os.path.isdir(inputImgTmpFolder):
listReconFrameName = [
sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]
["filename"] for x in range(0, len(sfm_data2["views"]))
if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]
["id_view"] in model2.reconFrame
]
FileUtils.makedir(inputImgTmpFolder)
for reconFrameName in listReconFrameName:
os.system("cp -s " +
os.path.join(model2.imgFolLoc, reconFrameName) +
" " + inputImgTmpFolder)
# remove all old localization result
FileUtils.removedir(model2.locFolLoc)
FileUtils.makedir(model2.locFolLoc)
# localize the images from model2 on model1
guideMatchOption = ""
if reconParam.bGuidedMatchingLocalize:
guideMatchOption = " -gm"
os.system(reconParam.LOCALIZE_PROJECT_PATH + \
" " + inputImgTmpFolder + \
" " + os.path.dirname(model1.sfm_dataLoc) + \
" " + self.mMatchesPath + \
" " + model2.locFolLoc + \
" -f=" + str(reconParam.locFeatDistRatio) + \
" -r=" + str(reconParam.locRansacRound) + \
" -i=" + str(reconParam.locSkipFrame) + \
" -k=" + str(reconBOWParam.locKNNnum) + \
" -a=" + os.path.join(self.mMatchesPath, "BOWfile.yml") + \
" -p=" + os.path.join(self.mMatchesPath, "PCAfile.yml") + \
guideMatchOption)
# remove temporary image folder
# removedir(inputImgTmpFolder)
# extract centers from all json file and write to a file
fileLoc = open(os.path.join(model2.locFolLoc, "center.txt"), "w")
countLocFrame = 0
for filename in sorted(os.listdir(model2.locFolLoc)):
if filename[-4:] != "json":
continue
countLocFrame = countLocFrame + 1
with open(os.path.join(model2.locFolLoc, filename)) as locJson:
#print os.path.join(sfm_locOut,filename)
locJsonDict = json.load(locJson)
loc = locJsonDict["t"]
fileLoc.write(
str(loc[0]) + " " + str(loc[1]) + " " + str(loc[2]) +
" 255 0 0\n")
fileLoc.close()
# get inlier matches
FileUtils.makedir(sfmOutPath)
resultSfMDataFile = os.path.join(sfmOutPath, "sfm_data.json")
# below also checks if the ratio between first and last svd of M[0:3,0:3]
# is good or not. If not then reject
# TODO : revisit ransacRound parameter, use number of reconstruction frame to determine structure points transform seems small
nMatchPointsTmp, nInlierTmp, M = mergeSfM.mergeModel(
model1.sfm_dataLoc,
model2.sfm_dataLoc,
model2.locFolLoc,
resultSfMDataFile,
ransacThres=model1.ransacStructureThres,
mergePointThres=model1.mergeStructureThres,
ransacRoundMul=reconParam.ransacRoundMul,
inputImgDir=self.mInputImgPath,
minLimit=reconParam.min3DnInliers)
ratioInlierMatchPoints = 0.0
if nMatchPointsTmp > 0:
ratioInlierMatchPoints = float(nInlierTmp) / nMatchPointsTmp
# 3. perform test whether merge is good
sfm_merge_generated = True
countFileAgree = 0
countFileLoc = 1
if os.path.isfile(resultSfMDataFile):
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " +
resultSfMDataFile + " " + resultSfMDataFile)
countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal(
resultSfMDataFile, model2.locFolLoc,
model1.validMergeRansacThres)
else:
sfm_merge_generated = False
ratioAgreeFrameReconFrame = 0.0
if (len(model2.reconFrame) > 0):
ratioAgreeFrameReconFrame = float(countFileAgree) / len(
model2.reconFrame)
ratioAgreeFrameLocFrame = 0.0
if (countFileLoc > 0):
ratioAgreeFrameLocFrame = float(countFileAgree) / countFileLoc
# write log file
with open(
os.path.join(self.mSfMPath, "global" + str(self.nMergedModel),
"log.txt"), "a") as filelog:
filelog.write(("M1: " + model1.name + "\n" + \
"M2: " + model2.name + "\n" + \
"nMatchedPoints: " + str(nMatchPointsTmp) + "\n" + \
"nInliers: " + str(nInlierTmp) + "\n" + \
"ratioInlierWithMatchedPoints: " + str(ratioInlierMatchPoints) + "\n" + \
"countLocFrame: " + str(countLocFrame) + "\n" + \
"nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \
"countFileAgree: " + str(countFileAgree) + "\n" + \
"countFileLoc: " + str(countFileLoc) + "\n" + \
"not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \
# obsolete condition by T. Ishihara 2015.11.10
#"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \
"countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \
"ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \
"ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \
"ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \
"ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \
str(M) + "\n\n"))
# rename the localization folder to save localization result
'''
if os.path.isdir(model2.locFolLoc+model1.name):
FileUtils.removedir(model2.locFolLoc+model1.name)
os.rename(model2.locFolLoc,model2.locFolLoc+model1.name)
'''
# obsolete merge condition
'''
if not sfm_merge_generated or \
not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \
((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \
((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and
(float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))):
'''
# update merge condition by T. Ishihara 2015.11.10
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \
countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
'''
# update merge condition by T. Ishihara 2016.04.02
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
'''
# update merge condition by T. Ishihara 2016.06.09
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \
nInlierTmp > reconParam.min3DnInliers and \
ratioInlierMatchPoints > reconParam.vldMergeRatioInliersMatchPoints):
'''
# update merge condition by T. Ishihara 2016.06.20
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \
nInlierTmp > reconParam.min3DnInliers):
print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "."
'''
if os.path.isfile(os.path.join(sfmOutPath,"sfm_data.json")):
os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \
os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json"))
'''
if os.path.isfile(os.path.join(sfmOutPath, "sfm_data.json")):
os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \
os.path.join(sfmOutPath,"sfm_data_fail_merge.json"))
# move to next video
return False, sfmModelBOW("",
"",
"",
"",
"",
"",
validMergeRansacThres=0,
validMergeRansacThresK=0,
ransacStructureThres=0,
ransacStructureThresK=0,
mergeStructureThres=0,
mergeStructureThresK=0)
# generate colorized before bundle adjustment for comparison
os.system("openMVG_main_ComputeSfM_DataColor " + " -i " +
os.path.join(sfmOutPath, "sfm_data.json") + " -o " +
os.path.join(sfmOutPath, "colorized_pre.ply"))
# TODO : try computing structure from know pose here
# https://github.com/openMVG/openMVG/issues/246
# http://openmvg.readthedocs.io/en/latest/software/SfM/ComputeStructureFromKnownPoses/
# TODO : revisit the order of bundle adjustment
# perform bundle adjustment
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "rs,rst,rsti" + " -r=" + "1")
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "rst,rsti" + " -r=" + "1")
os.system("openMVG_main_ComputeSfM_DataColor " + " -i " +
os.path.join(sfmOutPath, "sfm_data.json") + " -o " +
os.path.join(sfmOutPath, "colorized.ply"))
return True, sfmModelBOW(
"A" + model1.name + "," + model2.name + "Z",
self.mInputImgPath,
self.mCsvPath,
self.mMatchesPath,
os.path.join(sfmOutPath, "loc"),
resultSfMDataFile,
validMergeRansacThres=model1.validMergeRansacThres,
ransacStructureThres=model1.ransacStructureThres,
mergeStructureThres=model1.mergeStructureThres)
def mergeModel(sfm_data_dirA, sfm_data_dirB, locFolderB, outfile, ransacThres, mergePointThres, ransacRoundMul=100, inputImgDir="", minLimit=4, svdRatio=1.75):
print "Loading sfm_data"
sfm_dataB = FileUtils.loadjson(sfm_data_dirB)
# read matching pairs from localization result
imgnameB, matchlistB = readMatch(locFolderB)
# get viewID from image name for model B
viewIDB = imgnameToViewID(imgnameB, sfm_dataB)
# get mapping between viewID,featID to 3D point ID
viewFeatMapB = getViewFeatTo3DMap(sfm_dataB)
# find consistent match between 3D of model B to 3D of model A
print "Calculating consistent 3D matches"
match3D_BA = getConsistent3DMatch(viewIDB, matchlistB, viewFeatMapB)
print "Found " + str(len(match3D_BA)) + " consistent matches"
# not enough matches
if len(match3D_BA) <= 4 or len(match3D_BA) <= minLimit:
return len(match3D_BA), len(match3D_BA), np.asarray([])
# move the load of larger model here to reduce time if merging is not possible
sfm_dataA = FileUtils.loadjson(sfm_data_dirA)
# get 3D point. Note that element 0 of each pair in match3D_BA
# is 3D pt ID of model B and element 1 is that of model A
print "Load 3D points"
pointA = get3DPointloc(sfm_dataA, [x[1] for x in match3D_BA])
pointB = get3DPointloc(sfm_dataB, [x[0] for x in match3D_BA])
pointAn = np.asarray(pointA, dtype=np.float).T
pointBn = np.asarray(pointB, dtype=np.float).T
# find robust transformation
print "Find transformation with RANSAC"
ransacRound = len(match3D_BA)*ransacRoundMul
print "Number of RANSAC round : " + str(ransacRound)
M, inliers = ransacTransform(pointAn, pointBn, ransacThres, ransacRound, svdRatio)
# cannot find RANSAC transformation
if (M.size==0):
return len(match3D_BA), len(match3D_BA), np.asarray([])
print M
# stop if not enough inliers
sSvd = np.linalg.svd(M[0:3,0:3],compute_uv=0)
# fixed by T.Ishihara to use minLimit 2016.06.06
#if len(inliers) <= 4 or sSvd[0]/sSvd[-1] > svdRatio:
if len(inliers) <= minLimit or sSvd[0]/sSvd[-1] > svdRatio:
return len(match3D_BA), len(inliers), M
# perform merge
# last argument is map from inliers 3D pt Id of model B to that of model A
print "Merging sfm_data"
# fixed by T. Ishihara, use different parameter to find ransac inlier and merge points inliers
'''
merge_sfm_data(sfm_dataA, sfm_dataB, M, {match3D_BA[x][0]: match3D_BA[x][1] for x in inliers})
'''
mergePointInliers = getInliersByAffineTransform(pointAn, pointBn, M, mergePointThres)
merge_sfm_data(sfm_dataA, sfm_dataB, M, {match3D_BA[x][0]: match3D_BA[x][1] for x in mergePointInliers})
# change input image folder
if inputImgDir != "":
sfm_dataA["root_path"] = inputImgDir
# save json file
print "Saving json file"
FileUtils.savejson(sfm_dataA,outfile)
# return number of inliers for transformation
return len(match3D_BA), len(inliers), M
def cleanSfM(sfm_data_path, matchesFile):
sfm_data = FileUtils.loadjson(sfm_data_path)
if (len(sfm_data["views"]) == 0):
print "No views are used in reconstruction of " + sfm_data_path
return [[], []]
if (len(sfm_data["extrinsics"]) == 0):
print "No extrinsics are used in reconstruction of " + sfm_data_path
return [[], []]
# get map from ID to index
viewMap = {}
for i in range(0, len(sfm_data["views"])):
viewMap[sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]
["id_view"]] = i
extMap = {}
for i in range(0, len(sfm_data["extrinsics"])):
extMap[sfm_data["extrinsics"][i]["key"]] = i
strMap = {}
for i in range(0, len(sfm_data["structure"])):
strMap[sfm_data["structure"][i]["key"]] = i
# find viewIDs of first and last frame used in reconstruction
firstViewID = len(sfm_data["views"])
lastViewID = 0
firstExtID = min(extMap.keys())
for i in range(0, len(sfm_data["views"])):
viewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][
"id_view"]
extID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"]
if extID in extMap:
if firstViewID > viewID:
firstViewID = viewID
if lastViewID < viewID:
lastViewID = viewID
if firstViewID >= lastViewID:
print "No views are used in reconstruction of " + sfm_data_path
return [[], []]
# get list of unused view back to front
# and change the view Index
unusedImgName = [[], []]
for i in range(len(sfm_data["views"]) - 1, lastViewID, -1):
unusedImgName[1].append(
sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"])
sfm_data["views"].pop(i)
for i in range(lastViewID, firstViewID - 1, -1):
newViewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][
"id_view"] - firstViewID
sfm_data["views"][i]["key"] = newViewID
sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][
"id_view"] = newViewID
sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][
"id_pose"] = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][
"id_pose"] - firstExtID
for i in range(firstViewID - 1, -1, -1):
unusedImgName[0].append(
sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"])
sfm_data["views"].pop(i)
# change extrinsics ID
for i in range(0, len(sfm_data["extrinsics"])):
sfm_data["extrinsics"][i][
"key"] = sfm_data["extrinsics"][i]["key"] - firstExtID
# change index of refered view in structure
for i in range(0, len(sfm_data["structure"])):
for j in range(0,
len(sfm_data["structure"][i]["value"]["observations"])):
sfm_data["structure"][i]["value"]["observations"][j]["key"] = \
sfm_data["structure"][i]["value"]["observations"][j]["key"]-firstViewID
# save jsonfile back
FileUtils.savejson(sfm_data, sfm_data_path)
# update matches file
for matchfile in matchesFile:
matchFileName = os.path.basename(matchfile)
matchFileNameTmp = matchFileName.join(
random.choice(string.lowercase) for i in range(10)) #random name
matchDir = os.path.dirname(matchfile)
fout = open(os.path.join(matchDir, matchFileNameTmp), "w")
with open(matchfile, "r") as mfile:
mode = 0
write = False
countLine = 0
for line in mfile:
line = line.strip()
if mode == 0:
line = line.split(" ")
view1 = int(line[0])
view2 = int(line[1])
if view1 < firstViewID or view1 > lastViewID or \
view2 < firstViewID or view2 > lastViewID:
write = False
else:
write = True
if write:
# update viewID and write out
fout.write(str(int(line[0]) - firstViewID))
fout.write(" ")
fout.write(str(int(line[1]) - firstViewID))
fout.write("\n")
countLine = 0
mode = 1
elif mode == 1:
numMatch = int(line)
if write:
# get number of matches and write out
fout.write(line + "\n")
mode = 2
elif mode == 2:
if write:
# write out matches
fout.write(line + "\n")
countLine = countLine + 1
if countLine == numMatch:
mode = 0
os.rename(os.path.join(matchDir, matchFileName),
os.path.join(matchDir, matchFileName + "_old"))
os.rename(os.path.join(matchDir, matchFileNameTmp),
os.path.join(matchDir, matchFileName))
return unusedImgName
def mergeOneModel(self, model1, model2, reconParam):
sfmOutPath = os.path.join(self.mSfMPath,"global"+str(self.nMergedModel))
# create a temporary folder for reconstructed image of model2
inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmp"+model2.name)
# copy reconstructed image fom model2 to tmp folder
sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc)
if not os.path.isdir(inputImgTmpFolder):
listReconFrameName = [sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["filename"] for x in range(0,len(sfm_data2["views"])) if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["id_view"] in model2.reconFrame]
FileUtils.makedir(inputImgTmpFolder)
for reconFrameName in listReconFrameName:
os.system("cp -s " + os.path.join(model2.imgFolLoc,reconFrameName) + " " + inputImgTmpFolder)
# remove all old localization result
FileUtils.removedir(model2.locFolLoc)
FileUtils.makedir(model2.locFolLoc)
# localize the images from model2 on model1
os.system(reconParam.LOCALIZE_PROJECT_PATH + \
" " + inputImgTmpFolder + \
" " + os.path.dirname(model1.sfm_dataLoc) + \
" " + self.mMatchesPath + \
" " + model2.locFolLoc + \
" -f=" + str(reconParam.locFeatDistRatio) + \
" -r=" + str(reconParam.locRansacRound) + \
" -e=" + model2.csvFolLoc + \
" -i=" + str(reconParam.locSkipFrame))
# remove temporary image folder
# removedir(inputImgTmpFolder)
# extract centers from all json file and write to a file
fileLoc = open(os.path.join(model2.locFolLoc,"center.txt"),"w")
countLocFrame = 0
for filename in sorted(os.listdir(model2.locFolLoc)):
if filename[-4:]!="json":
continue
countLocFrame = countLocFrame + 1
with open(os.path.join(model2.locFolLoc,filename)) as locJson:
#print os.path.join(sfm_locOut,filename)
locJsonDict = json.load(locJson)
loc = locJsonDict["t"]
fileLoc.write(str(loc[0]) + " " + str(loc[1]) + " " +str(loc[2]) + " 255 0 0\n" )
fileLoc.close()
# get inlier matches
FileUtils.makedir(sfmOutPath)
resultSfMDataFile = os.path.join(sfmOutPath,"sfm_data.json")
# below also checks if the ratio between first and last svd of M[0:3,0:3]
# is good or not. If not then reject
nInlierTmp, M = mergeSfM.mergeModel(model1.sfm_dataLoc,
model2.sfm_dataLoc,
model2.locFolLoc,
resultSfMDataFile,
ransacK=reconParam.ransacStructureThresMul,
ransacRound=reconParam.ransacRoundMul*len(model1.reconFrame),
inputImgDir=self.mInputImgPath,
minLimit=reconParam.min3DnInliers)
# 3. perform test whether merge is good
sfm_merge_generated = True
countFileAgree = 0
countFileLoc = 1
if os.path.isfile(resultSfMDataFile):
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + resultSfMDataFile + " " + resultSfMDataFile)
countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal(resultSfMDataFile, model2.locFolLoc, reconParam.vldMergeAgrFrameThresK)
else:
sfm_merge_generated = False
ratioAgreeFrameReconFrame = 0.0
if (len(model2.reconFrame)>0):
ratioAgreeFrameReconFrame = float(countFileAgree)/len(model2.reconFrame)
ratioAgreeFrameLocFrame = 0.0
if (countFileLoc>0):
ratioAgreeFrameLocFrame = float(countFileAgree)/countFileLoc
# write log file
with open(os.path.join(self.mSfMPath,"global"+str(self.nMergedModel),"log.txt"),"a") as filelog:
filelog.write(("M1: " + model1.name + "\n" + \
"M2: " + model2.name + "\n" + \
"nInliers: " + str(nInlierTmp) + "\n" + \
"countLocFrame: " + str(countLocFrame) + "\n" + \
"nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \
"countFileAgree: " + str(countFileAgree) + "\n" + \
"countFileLoc: " + str(countFileLoc) + "\n" + \
"not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \
# obsolete condition by T. Ishihara 2015.11.10
#"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \
"countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \
"ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \
"ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \
"ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \
"ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \
str(M) + "\n\n"))
# rename the localization folder to save localization result
if os.path.isdir(model2.locFolLoc+model1.name):
FileUtils.removedir(model2.locFolLoc+model1.name)
os.rename(model2.locFolLoc,model2.locFolLoc+model1.name)
# obsolete merge condition
'''
if not sfm_merge_generated or \
not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \
((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \
((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and
(float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))):
'''
# update merge condition by T. Ishihara 2015.11.10
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \
countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
'''
# update merge condition by T. Ishihara 2016.04.02
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "."
if os.path.isfile(os.path.join(sfmOutPath,"sfm_data.json")):
os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \
os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json"))
# move to next video
return False, sfmModel("","","","","","")
# generate colorized before bundle adjustment for comparison
os.system("openMVG_main_ComputeSfM_DataColor " +
" -i " + os.path.join(sfmOutPath,"sfm_data.json") +
" -o " + os.path.join(sfmOutPath,"colorized_pre.ply"))
# perform bundle adjustment
# modified by T.Ishihara 2016.04.08
# fix only translation at first
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "rs,rst,rsti" + " -r=" + "1")
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "st,rst,rsti" + " -r=" + "1")
os.system("openMVG_main_ComputeSfM_DataColor " +
" -i " + os.path.join(sfmOutPath,"sfm_data.json") +
" -o " + os.path.join(sfmOutPath,"colorized.ply"))
return True, sfmModel("A" + model1.name + "," + model2.name +"Z", self.mInputImgPath, self.mCsvPath, self.mMatchesPath, os.path.join(sfmOutPath,"loc"), resultSfMDataFile)
def cleanSfM(sfm_data_path,matchesFile):
sfm_data = FileUtils.loadjson(sfm_data_path)
if (len(sfm_data["views"])==0):
print "No views are used in reconstruction of " + sfm_data_path
return [[],[]]
if (len(sfm_data["extrinsics"])==0):
print "No extrinsics are used in reconstruction of " + sfm_data_path
return [[],[]]
# get map from ID to index
viewMap = {}
for i in range(0,len(sfm_data["views"])):
viewMap[sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"]] = i
extMap = {}
for i in range(0,len(sfm_data["extrinsics"])):
extMap[sfm_data["extrinsics"][i]["key"]] = i
strMap = {}
for i in range(0,len(sfm_data["structure"])):
strMap[sfm_data["structure"][i]["key"]] = i
# find viewIDs of first and last frame used in reconstruction
firstViewID = len(sfm_data["views"])
lastViewID = 0
firstExtID = min(extMap.keys())
for i in range(0,len(sfm_data["views"])):
viewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"]
extID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"]
if extID in extMap:
if firstViewID > viewID:
firstViewID = viewID
if lastViewID < viewID:
lastViewID = viewID
if firstViewID >= lastViewID:
print "No views are used in reconstruction of " + sfm_data_path
return [[],[]]
# get list of unused view back to front
# and change the view Index
unusedImgName = [[],[]]
for i in range(len(sfm_data["views"])-1,lastViewID,-1):
unusedImgName[1].append(sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"])
sfm_data["views"].pop(i)
for i in range(lastViewID,firstViewID-1,-1):
newViewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"]-firstViewID
sfm_data["views"][i]["key"] = newViewID
sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"] = newViewID
sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"] = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"] - firstExtID
for i in range(firstViewID-1,-1,-1):
unusedImgName[0].append(sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"])
sfm_data["views"].pop(i)
# change extrinsics ID
for i in range(0,len(sfm_data["extrinsics"])):
sfm_data["extrinsics"][i]["key"] = sfm_data["extrinsics"][i]["key"]-firstExtID
# change index of refered view in structure
for i in range(0,len(sfm_data["structure"])):
for j in range(0,len(sfm_data["structure"][i]["value"]["observations"])):
sfm_data["structure"][i]["value"]["observations"][j]["key"] = \
sfm_data["structure"][i]["value"]["observations"][j]["key"]-firstViewID
# save jsonfile back
FileUtils.savejson(sfm_data,sfm_data_path)
# update matches file
for matchfile in matchesFile:
matchFileName = os.path.basename(matchfile)
matchFileNameTmp = matchFileName.join(random.choice(string.lowercase) for i in range(10)) #random name
matchDir = os.path.dirname(matchfile)
fout = open(os.path.join(matchDir,matchFileNameTmp),"w")
with open(matchfile,"r") as mfile:
mode = 0
write = False
countLine = 0
for line in mfile:
line = line.strip()
if mode == 0:
line = line.split(" ")
view1 = int(line[0])
view2 = int(line[1])
if view1 < firstViewID or view1 > lastViewID or \
view2 < firstViewID or view2 > lastViewID:
write = False
else:
write = True
if write:
# update viewID and write out
fout.write(str(int(line[0])-firstViewID))
fout.write(" ")
fout.write(str(int(line[1])-firstViewID))
fout.write("\n")
countLine = 0
mode = 1
elif mode == 1:
numMatch= int(line)
if write:
# get number of matches and write out
fout.write(line + "\n")
mode = 2
elif mode == 2:
if write:
# write out matches
fout.write(line + "\n")
countLine = countLine + 1
if countLine == numMatch:
mode = 0
os.rename(os.path.join(matchDir,matchFileName),os.path.join(matchDir,matchFileName+"_old"))
os.rename(os.path.join(matchDir,matchFileNameTmp),os.path.join(matchDir,matchFileName))
return unusedImgName
def mergeOneModel(self, model1, model2, reconParam):
sfmOutPath = os.path.join(self.mSfMPath,
"global" + str(self.nMergedModel))
# create a temporary folder for reconstructed image of model2
inputImgTmpFolder = os.path.join(self.mSfMPath, "inputImgTmp",
"inputImgTmp" + model2.name)
# copy reconstructed image fom model2 to tmp folder
sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc)
if not os.path.isdir(inputImgTmpFolder):
listReconFrameName = [
sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]
["filename"] for x in range(0, len(sfm_data2["views"]))
if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]
["id_view"] in model2.reconFrame
]
FileUtils.makedir(inputImgTmpFolder)
for reconFrameName in listReconFrameName:
os.system("cp -s " +
os.path.join(model2.imgFolLoc, reconFrameName) +
" " + inputImgTmpFolder)
# remove all old localization result
FileUtils.removedir(model2.locFolLoc)
FileUtils.makedir(model2.locFolLoc)
# localize the images from model2 on model1
os.system(reconParam.LOCALIZE_PROJECT_PATH + \
" " + inputImgTmpFolder + \
" " + os.path.dirname(model1.sfm_dataLoc) + \
" " + self.mMatchesPath + \
" " + model2.locFolLoc + \
" -f=" + str(reconParam.locFeatDistRatio) + \
" -r=" + str(reconParam.locRansacRound) + \
" -e=" + model2.csvFolLoc + \
" -i=" + str(reconParam.locSkipFrame))
# remove temporary image folder
# removedir(inputImgTmpFolder)
# extract centers from all json file and write to a file
fileLoc = open(os.path.join(model2.locFolLoc, "center.txt"), "w")
countLocFrame = 0
for filename in sorted(os.listdir(model2.locFolLoc)):
if filename[-4:] != "json":
continue
countLocFrame = countLocFrame + 1
with open(os.path.join(model2.locFolLoc, filename)) as locJson:
#print os.path.join(sfm_locOut,filename)
locJsonDict = json.load(locJson)
loc = locJsonDict["t"]
fileLoc.write(
str(loc[0]) + " " + str(loc[1]) + " " + str(loc[2]) +
" 255 0 0\n")
fileLoc.close()
# get inlier matches
FileUtils.makedir(sfmOutPath)
resultSfMDataFile = os.path.join(sfmOutPath, "sfm_data.json")
# below also checks if the ratio between first and last svd of M[0:3,0:3]
# is good or not. If not then reject
nInlierTmp, M = mergeSfM.mergeModel(
model1.sfm_dataLoc,
model2.sfm_dataLoc,
model2.locFolLoc,
resultSfMDataFile,
ransacK=reconParam.ransacStructureThresMul,
ransacRound=reconParam.ransacRoundMul * len(model1.reconFrame),
inputImgDir=self.mInputImgPath,
minLimit=reconParam.min3DnInliers)
# 3. perform test whether merge is good
sfm_merge_generated = True
countFileAgree = 0
countFileLoc = 1
if os.path.isfile(resultSfMDataFile):
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " +
resultSfMDataFile + " " + resultSfMDataFile)
countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal(
resultSfMDataFile, model2.locFolLoc,
reconParam.vldMergeAgrFrameThresK)
else:
sfm_merge_generated = False
ratioAgreeFrameReconFrame = 0.0
if (len(model2.reconFrame) > 0):
ratioAgreeFrameReconFrame = float(countFileAgree) / len(
model2.reconFrame)
ratioAgreeFrameLocFrame = 0.0
if (countFileLoc > 0):
ratioAgreeFrameLocFrame = float(countFileAgree) / countFileLoc
# write log file
with open(
os.path.join(self.mSfMPath, "global" + str(self.nMergedModel),
"log.txt"), "a") as filelog:
filelog.write(("M1: " + model1.name + "\n" + \
"M2: " + model2.name + "\n" + \
"nInliers: " + str(nInlierTmp) + "\n" + \
"countLocFrame: " + str(countLocFrame) + "\n" + \
"nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \
"countFileAgree: " + str(countFileAgree) + "\n" + \
"countFileLoc: " + str(countFileLoc) + "\n" + \
"not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \
# obsolete condition by T. Ishihara 2015.11.10
#"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \
"countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \
# obsolete condition by T. Ishihara 2016.04.02
#"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \
"ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \
"ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \
"ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \
"ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \
str(M) + "\n\n"))
# rename the localization folder to save localization result
if os.path.isdir(model2.locFolLoc + model1.name):
FileUtils.removedir(model2.locFolLoc + model1.name)
os.rename(model2.locFolLoc, model2.locFolLoc + model1.name)
# obsolete merge condition
'''
if not sfm_merge_generated or \
not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \
((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \
((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and
(float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))):
'''
# update merge condition by T. Ishihara 2015.11.10
'''
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \
countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
'''
# update merge condition by T. Ishihara 2016.04.02
if not sfm_merge_generated or \
not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \
((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \
ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \
ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF):
print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "."
if os.path.isfile(os.path.join(sfmOutPath, "sfm_data.json")):
os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \
os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json"))
# move to next video
return False, sfmModel("", "", "", "", "", "")
# generate colorized before bundle adjustment for comparison
os.system("openMVG_main_ComputeSfM_DataColor " + " -i " +
os.path.join(sfmOutPath, "sfm_data.json") + " -o " +
os.path.join(sfmOutPath, "colorized_pre.ply"))
# perform bundle adjustment
# modified by T.Ishihara 2016.04.08
# fix only translation at first
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "rs,rst,rsti" + " -r=" + "1")
'''
os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \
" -c=" + "st,rst,rsti" + " -r=" + "1")
os.system("openMVG_main_ComputeSfM_DataColor " + " -i " +
os.path.join(sfmOutPath, "sfm_data.json") + " -o " +
os.path.join(sfmOutPath, "colorized.ply"))
return True, sfmModel("A" + model1.name + "," + model2.name + "Z",
self.mInputImgPath,
self.mCsvPath, self.mMatchesPath,
os.path.join(sfmOutPath,
"loc"), resultSfMDataFile)
