def modelMergeCheckLocal(sfm_data_path, sfm_locOut, medThres): # load sfm_data sfm_data = FileUtils.loadjson(sfm_data_path) # collect all image names ad location imgName = [] imgLoc = [] for filename in os.listdir(sfm_locOut): if filename[-4:]!="json": continue locJsonDict = FileUtils.loadjson(os.path.join(sfm_locOut,filename)) if "t" in locJsonDict: imgName.append(os.path.basename(locJsonDict["filename"])) imgLoc.append(locJsonDict["t"]) imgID = imgnameToViewID(imgName, sfm_data) imgSfMLoc = get3DViewloc(sfm_data, imgID) # calculate distance and count agreement countFile = 0 countAgree = 0 for j in range(0,len(imgLoc)): dist = np.linalg.norm(np.array(imgLoc[j])-np.array(imgSfMLoc[j])) if dist < float("inf"): countFile = countFile + 1 if dist < medThres: countAgree = countAgree + 1 return countFile, countAgree
def modelMergeCheckLocal(sfm_data_path, sfm_locOut, medThres): # load sfm_data sfm_data = FileUtils.loadjson(sfm_data_path) # collect all image names ad location imgName = [] imgLoc = [] for filename in os.listdir(sfm_locOut): if filename[-4:]!="json": continue locJsonDict = FileUtils.loadjson(os.path.join(sfm_locOut,filename)) imgName.append(os.path.basename(locJsonDict["filename"])) imgLoc.append(locJsonDict["t"]) imgID = imgnameToViewID(imgName, sfm_data) imgSfMLoc = get3DViewloc(sfm_data, imgID) # calculate distance and count agreement countFile = 0 countAgree = 0 for j in range(0,len(imgLoc)): dist = np.linalg.norm(np.array(imgLoc[j])-np.array(imgSfMLoc[j])) if dist < float("inf"): countFile = countFile + 1 if dist < medThres: countAgree = countAgree + 1 return countFile, countAgree
def __init__(self, name, imgFolLoc, csvFolLoc, matchesFolLoc, locFolLoc, sfm_dataLoc, validMergeRansacThres=-1, validMergeRansacThresK=-1, ransacStructureThres=-1, ransacStructureThresK=-1, mergeStructureThres=-1, mergeStructureThresK=-1): if (validMergeRansacThres == -1 and validMergeRansacThresK == -1): print "error : invalid argument for sfmModel valid merge ransac" sys.exit() if (ransacStructureThres == -1 and ransacStructureThresK == -1): print "error : invalid argument for sfmModel structure ransac" sys.exit() if (mergeStructureThres == -1 and mergeStructureThresK == -1): print "error : invalid argument for sfmModel structure merge" sys.exit() self.name = name # folder name self.mergeOrder = name # structure similar to a tree specifying merge order self.imgFolLoc = imgFolLoc # folder dir of input image folder self.csvFolLoc = csvFolLoc # folder dir of csv folder self.matchesFolLoc = matchesFolLoc # folder of match folder with descriptor self.locFolLoc = locFolLoc # folder of localization result self.sfm_dataLoc = sfm_dataLoc # file dir of sfm_data.json # get list of reconstructed frames if self.sfm_dataLoc != "": sfm_data = FileUtils.loadjson(self.sfm_dataLoc) extKeyTmp = [x["key"] for x in sfm_data["extrinsics"]] self.reconFrame = [ x["value"]["ptr_wrapper"]["data"]["id_view"] for x in sfm_data["views"] if x["value"]["ptr_wrapper"]["data"]["id_pose"] in extKeyTmp ] if validMergeRansacThresK > 0: self.validMergeRansacThres = mergeSfM.findMedianThres( sfm_data, validMergeRansacThresK) else: self.validMergeRansacThres = validMergeRansacThres if ransacStructureThresK > 0: self.ransacStructureThres = mergeSfM.findMedianStructurePointsThres( sfm_data, ransacStructureThresK) else: self.ransacStructureThres = ransacStructureThres if mergeStructureThresK > 0: self.mergeStructureThres = mergeSfM.findMedianStructurePointsThres( sfm_data, mergeStructureThresK) else: self.mergeStructureThres = mergeStructureThres
def __init__(self, name, imgFolLoc, csvFolLoc, matchesFolLoc, locFolLoc, sfm_dataLoc): self.name = name # folder name self.mergeOrder = name # structure similar to a tree specifying merge order self.imgFolLoc = imgFolLoc # folder dir of input image folder self.csvFolLoc = csvFolLoc # folder dir of csv folder self.matchesFolLoc = matchesFolLoc # folder of match folder with descriptor self.locFolLoc = locFolLoc # folder of localization result self.sfm_dataLoc = sfm_dataLoc # file dir of sfm_data.json # get list of reconstructed frames if self.sfm_dataLoc != "": sfm_data = FileUtils.loadjson(self.sfm_dataLoc) extKeyTmp = [x["key"] for x in sfm_data["extrinsics"]] self.reconFrame = [x["value"]["ptr_wrapper"]["data"]["id_view"] for x in sfm_data["views"] if x["value"]["ptr_wrapper"]["data"]["id_pose"] in extKeyTmp]
def calculateAverageBOW(sfmDataFile, matchesFolLoc): sfmData = FileUtils.loadjson(sfmDataFile) avgBow = None for view in sfmData["views"]: viewImage = view["value"]["ptr_wrapper"]["data"]["filename"] viewBow = os.path.join(matchesFolLoc, os.path.splitext(viewImage)[0] + ".bow") bowvec = FileUtils.loadBinMat(viewBow) if avgBow is None: avgBow = bowvec else: avgBow += bowvec avgBow /= len(sfmData["views"]) return avgBow
def readMatch(locFolder): imgname = [] matchlist = [] print "Reading loc output: " + locFolder for filename in sorted(os.listdir(locFolder)): if filename[-4:] != "json": continue jsondata = FileUtils.loadjson(os.path.join(locFolder, filename)) imgname.append(os.path.basename(jsondata["filename"])) matchlist.append(jsondata["pair"]) return imgname, matchlist
def __init__(self, name, imgFolLoc, csvFolLoc, matchesFolLoc, locFolLoc, sfm_dataLoc): self.name = name # folder name self.mergeOrder = name # structure similar to a tree specifying merge order self.imgFolLoc = imgFolLoc # folder dir of input image folder self.csvFolLoc = csvFolLoc # folder dir of csv folder self.matchesFolLoc = matchesFolLoc # folder of match folder with descriptor self.locFolLoc = locFolLoc # folder of localization result self.sfm_dataLoc = sfm_dataLoc # file dir of sfm_data.json # get list of reconstructed frames if self.sfm_dataLoc != "": sfm_data = FileUtils.loadjson(self.sfm_dataLoc) extKeyTmp = [x["key"] for x in sfm_data["extrinsics"]] self.reconFrame = [ x["value"]["ptr_wrapper"]["data"]["id_view"] for x in sfm_data["views"] if x["value"]["ptr_wrapper"]["data"]["id_pose"] in extKeyTmp ]
def mergeModel(sfm_data_dirA, sfm_data_dirB, locFolderB, outfile, ransacThres, mergePointThres, ransacRoundMul=100, inputImgDir="", minLimit=4, svdRatio=1.75): print "Loading sfm_data" sfm_dataB = FileUtils.loadjson(sfm_data_dirB) # read matching pairs from localization result imgnameB, matchlistB = readMatch(locFolderB) # get viewID from image name for model B viewIDB = imgnameToViewID(imgnameB, sfm_dataB) # get mapping between viewID,featID to 3D point ID viewFeatMapB = getViewFeatTo3DMap(sfm_dataB) # find consistent match between 3D of model B to 3D of model A print "Calculating consistent 3D matches" match3D_BA = getConsistent3DMatch(viewIDB, matchlistB, viewFeatMapB) print "Found " + str(len(match3D_BA)) + " consistent matches" # not enough matches if len(match3D_BA) <= 4 or len(match3D_BA) <= minLimit: return len(match3D_BA), len(match3D_BA), np.asarray([]) # move the load of larger model here to reduce time if merging is not possible sfm_dataA = FileUtils.loadjson(sfm_data_dirA) # get 3D point. Note that element 0 of each pair in match3D_BA # is 3D pt ID of model B and element 1 is that of model A print "Load 3D points" pointA = get3DPointloc(sfm_dataA, [x[1] for x in match3D_BA]) pointB = get3DPointloc(sfm_dataB, [x[0] for x in match3D_BA]) pointAn = np.asarray(pointA, dtype=np.float).T pointBn = np.asarray(pointB, dtype=np.float).T # find robust transformation print "Find transformation with RANSAC" ransacRound = len(match3D_BA)*ransacRoundMul print "Number of RANSAC round : " + str(ransacRound) M, inliers = ransacTransform(pointAn, pointBn, ransacThres, ransacRound, svdRatio) # cannot find RANSAC transformation if (M.size==0): return len(match3D_BA), len(match3D_BA), np.asarray([]) print M # stop if not enough inliers sSvd = np.linalg.svd(M[0:3,0:3],compute_uv=0) # fixed by T.Ishihara to use minLimit 2016.06.06 #if len(inliers) <= 4 or sSvd[0]/sSvd[-1] > svdRatio: if len(inliers) <= minLimit or sSvd[0]/sSvd[-1] > svdRatio: return len(match3D_BA), len(inliers), M # perform merge # last argument is map from inliers 3D pt Id of model B to that of model A print "Merging sfm_data" # fixed by T. Ishihara, use different parameter to find ransac inlier and merge points inliers ''' merge_sfm_data(sfm_dataA, sfm_dataB, M, {match3D_BA[x][0]: match3D_BA[x][1] for x in inliers}) ''' mergePointInliers = getInliersByAffineTransform(pointAn, pointBn, M, mergePointThres) merge_sfm_data(sfm_dataA, sfm_dataB, M, {match3D_BA[x][0]: match3D_BA[x][1] for x in mergePointInliers}) # change input image folder if inputImgDir != "": sfm_dataA["root_path"] = inputImgDir # save json file print "Saving json file" FileUtils.savejson(sfm_dataA,outfile) # return number of inliers for transformation return len(match3D_BA), len(inliers), M
def mergeModel(sfm_data_dirA, sfm_data_dirB, locFolderB, outfile, ransacK=1.0, ransacRound=10000, inputImgDir="", minLimit=4, svdRatio=1.75): print "Loading sfm_data" sfm_dataB = FileUtils.loadjson(sfm_data_dirB) # read matching pairs from localization result imgnameB, matchlistB = readMatch(locFolderB) # get viewID from image name for model B viewIDB = imgnameToViewID(imgnameB, sfm_dataB) # get mapping between viewID,featID to 3D point ID viewFeatMapB = getViewFeatTo3DMap(sfm_dataB) # find consistent match between 3D of model B to 3D of model A print "Calculating consistent 3D matches" match3D_BA = getConsistent3DMatch(viewIDB, matchlistB, viewFeatMapB) print "Found " + str(len(match3D_BA)) + " consistent matches" # not enough matches if len(match3D_BA) <= 4 or len(match3D_BA) <= minLimit: return len(match3D_BA), [0] # move the load of larger model here to reduce time if merging is not possible sfm_dataA = FileUtils.loadjson(sfm_data_dirA) # get 3D point. Note that element 0 of each pair in match3D_BA # is 3D pt ID of model B and element 1 is that of model A print "Load 3D points" pointA = get3DPointloc(sfm_dataA, [x[1] for x in match3D_BA]) pointB = get3DPointloc(sfm_dataB, [x[0] for x in match3D_BA]) pointAn = np.asarray(pointA, dtype=np.float).T pointBn = np.asarray(pointB, dtype=np.float).T # calculate ransac threshold # calculate as 4 times the median of distance between # 3D pt of A print "Find transformation with RANSAC" # modified by T.Ishihara 2016.04.08 # median of camera positions merge too many points, use median of structure points instead #ransacThres = findMedianThres(sfm_dataA, ransacK) ransacThres = findMedianStructurePointsThres(sfm_dataA, ransacK) # TODO : replace with RANSAC similarity transform # find robust transformation M, inliers = ransacAffineTransform(pointAn, pointBn, ransacThres, ransacRound, svdRatio) # cannot find RANSAC transformation if (len(inliers) == 0): return len(match3D_BA), [0] print M # stop if not enough inliers sSvd = np.linalg.svd(M[0:3, 0:3], compute_uv=0) if len(inliers) <= 4 or sSvd[0] / sSvd[-1] > svdRatio: return len(inliers), M # perform merge # last argument is map from inliers 3D pt Id of model B to that of model A print "Merging sfm_data" merge_sfm_data(sfm_dataA, sfm_dataB, M, {match3D_BA[x][0]: match3D_BA[x][1] for x in inliers}) # change input image folder if inputImgDir != "": sfm_dataA["root_path"] = inputImgDir # save json file print "Saving json file" FileUtils.savejson(sfm_dataA, outfile) # return number of inliers for transformation return len(inliers), M
def mergeOneModel(self, model1, model2, reconParam, reconIBeaconParam, reconBOWParam): sfmOutPath = os.path.join(self.mSfMPath,"global"+str(self.nMergedModel)) # modified by T. IShihara 2016.06.14 # fix file name too long issue # # create a temporary folder for reconstructed image of model2 #inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmp"+model2.name) inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmpModel2") if os.path.isdir(inputImgTmpFolder): FileUtils.removedir(inputImgTmpFolder) # copy reconstructed image fom model2 to tmp folder sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc) if not os.path.isdir(inputImgTmpFolder): listReconFrameName = [sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["filename"] for x in range(0,len(sfm_data2["views"])) if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["id_view"] in model2.reconFrame] FileUtils.makedir(inputImgTmpFolder) for reconFrameName in listReconFrameName: os.system("cp -s " + os.path.join(model2.imgFolLoc,reconFrameName) + " " + inputImgTmpFolder) # remove all old localization result FileUtils.removedir(model2.locFolLoc) FileUtils.makedir(model2.locFolLoc) # localize the images from model2 on model1 if self.useBow: os.system(reconIBeaconParam.LOCALIZE_PROJECT_PATH + \ " " + inputImgTmpFolder + \ " " + os.path.dirname(model1.sfm_dataLoc) + \ " " + self.mMatchesPath + \ " " + model2.locFolLoc + \ " -f=" + str(reconParam.locFeatDistRatio) + \ " -r=" + str(reconParam.locRansacRound) + \ " -b=" + model1.beaconFileLoc + \ " -e=" + model2.csvFolLoc + \ " -k=" + str(reconIBeaconParam.locKNNnum) + \ " -c=" + str(reconIBeaconParam.coocThres) + \ " -i=" + str(reconParam.locSkipFrame) + \ " -v=" + str(reconIBeaconParam.locSkipSelKNN) + \ " -n=" + str(reconIBeaconParam.normApproach) + \ " -kb=" + str(reconBOWParam.locKNNnum) + \ " -a=" + os.path.join(self.mMatchesPath, "BOWfile.yml") + \ " -p=" + os.path.join(self.mMatchesPath, "PCAfile.yml")) else: os.system(reconIBeaconParam.LOCALIZE_PROJECT_PATH + \ " " + inputImgTmpFolder + \ " " + os.path.dirname(model1.sfm_dataLoc) + \ " " + self.mMatchesPath + \ " " + model2.locFolLoc + \ " -f=" + str(reconParam.locFeatDistRatio) + \ " -r=" + str(reconParam.locRansacRound) + \ " -b=" + model1.beaconFileLoc + \ " -e=" + model2.csvFolLoc + \ " -k=" + str(reconIBeaconParam.locKNNnum) + \ " -c=" + str(reconIBeaconParam.coocThres) + \ " -i=" + str(reconParam.locSkipFrame) + \ " -v=" + str(reconIBeaconParam.locSkipSelKNN) + \ " -n=" + str(reconIBeaconParam.normApproach)) # remove temporary image folder # removedir(inputImgTmpFolder) # extract centers from all json file and write to a file fileLoc = open(os.path.join(model2.locFolLoc,"center.txt"),"w") countLocFrame = 0 for filename in sorted(os.listdir(model2.locFolLoc)): if filename[-4:]!="json": continue countLocFrame = countLocFrame + 1 with open(os.path.join(model2.locFolLoc,filename)) as locJson: #print os.path.join(sfm_locOut,filename) locJsonDict = json.load(locJson) loc = locJsonDict["t"] fileLoc.write(str(loc[0]) + " " + str(loc[1]) + " " +str(loc[2]) + " 255 0 0\n" ) fileLoc.close() # get inlier matches FileUtils.makedir(sfmOutPath) resultSfMDataFile = os.path.join(sfmOutPath,"sfm_data.json") # below also checks if the ratio between first and last svd of M[0:3,0:3] # is good or not. If not then reject # TODO : revisit ransacRound parameter, use number of reconstruction frame to determine structure points transform seems small nMatchPointsTmp, nInlierTmp, M = mergeSfM.mergeModel(model1.sfm_dataLoc, model2.sfm_dataLoc, model2.locFolLoc, resultSfMDataFile, ransacThres=model1.ransacStructureThres, mergePointThres=model1.mergeStructureThres, ransacRoundMul=reconParam.ransacRoundMul, inputImgDir=self.mInputImgPath, minLimit=reconParam.min3DnInliers) ratioInlierMatchPoints = 0.0 if nMatchPointsTmp>0: ratioInlierMatchPoints = float(nInlierTmp)/nMatchPointsTmp # 3. perform test whether merge is good sfm_merge_generated = True countFileAgree = 0 countFileLoc = 1 if os.path.isfile(resultSfMDataFile): os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + resultSfMDataFile + " " + resultSfMDataFile) countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal(resultSfMDataFile, model2.locFolLoc, model1.validMergeRansacThres) else: sfm_merge_generated = False ratioAgreeFrameReconFrame = 0.0 if (len(model2.reconFrame)>0): ratioAgreeFrameReconFrame = float(countFileAgree)/len(model2.reconFrame) ratioAgreeFrameLocFrame = 0.0 if (countFileLoc>0): ratioAgreeFrameLocFrame = float(countFileAgree)/countFileLoc # write log file with open(os.path.join(self.mSfMPath,"global"+str(self.nMergedModel),"log.txt"),"a") as filelog: filelog.write(("M1: " + model1.name + "\n" + \ "M2: " + model2.name + "\n" + \ "nMatchedPoints: " + str(nMatchPointsTmp) + "\n" + \ "nInliers: " + str(nInlierTmp) + "\n" + \ "ratioInlierWithMatchedPoints: " + str(ratioInlierMatchPoints) + "\n" + \ "countLocFrame: " + str(countLocFrame) + "\n" + \ "nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \ "countFileAgree: " + str(countFileAgree) + "\n" + \ "countFileLoc: " + str(countFileLoc) + "\n" + \ "not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \ # obsolete condition by T. Ishihara 2015.11.10 #"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \ "countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \ "ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \ "ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \ "ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \ "ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \ str(M) + "\n\n")) # rename the localization folder to save localization result ''' if os.path.isdir(model2.locFolLoc+model1.name): FileUtils.removedir(model2.locFolLoc+model1.name) os.rename(model2.locFolLoc,model2.locFolLoc+model1.name) ''' # obsolete merge condition ''' if not sfm_merge_generated or \ not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \ ((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \ ((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and (float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))): ''' # update merge condition by T. Ishihara 2015.11.10 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \ countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): ''' # update merge condition by T. Ishihara 2016.04.02 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): ''' # update merge condition by T. Ishihara 2016.06.09 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \ nInlierTmp > reconParam.min3DnInliers and \ ratioInlierMatchPoints > reconParam.vldMergeRatioInliersMatchPoints): ''' # update merge condition by T. Ishihara 2016.06.20 if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \ nInlierTmp > reconParam.min3DnInliers): print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "." ''' if os.path.isfile(os.path.join(sfmOutPath,"sfm_data.json")): os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \ os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json")) ''' # move to next video return False, sfmModelIBeacon("","","","","","","",validMergeRansacThres=0,validMergeRansacThresK=0, ransacStructureThres=0, ransacStructureThresK=0, mergeStructureThres=0, mergeStructureThresK=0) # generate colorized before bundle adjustment for comparison os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath,"sfm_data.json") + " -o " + os.path.join(sfmOutPath,"colorized_pre.ply")) # TODO : try computing structure from know pose here # https://github.com/openMVG/openMVG/issues/246 # http://openmvg.readthedocs.io/en/latest/software/SfM/ComputeStructureFromKnownPoses/ # TODO : revisit the order of bundle adjustment # perform bundle adjustment ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "rs,rst,rsti" + " -r=" + "1") ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "rst,rsti" + " -r=" + "1") os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath,"sfm_data.json") + " -o " + os.path.join(sfmOutPath,"colorized.ply")) # write new beacon file IBeaconUtils.exportBeaconDataForSfmImageFrames(self.mCsvPath, resultSfMDataFile, os.path.join(self.mInputPath,"listbeacon.txt"), os.path.join(sfmOutPath,"beacon.txt"), reconIBeaconParam.normApproach) return True, sfmModelIBeacon("A" + model1.name + "," + model2.name +"Z", self.mInputImgPath, self.mCsvPath, os.path.join(sfmOutPath,"beacon.txt"), self.mMatchesPath, os.path.join(sfmOutPath,"loc"), resultSfMDataFile, validMergeRansacThres=model1.validMergeRansacThres, ransacStructureThres=model1.ransacStructureThres, mergeStructureThres=model1.mergeStructureThres)
def mergeOneModel(self, model1, model2, reconParam, reconBOWParam): sfmOutPath = os.path.join(self.mSfMPath, "global" + str(self.nMergedModel)) # modified by T. IShihara 2016.06.14 # fix file name too long issue # # create a temporary folder for reconstructed image of model2 #inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmp"+model2.name) inputImgTmpFolder = os.path.join(self.mSfMPath, "inputImgTmp", "inputImgTmpModel2") if os.path.isdir(inputImgTmpFolder): FileUtils.removedir(inputImgTmpFolder) # copy reconstructed image fom model2 to tmp folder sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc) if not os.path.isdir(inputImgTmpFolder): listReconFrameName = [ sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"] ["filename"] for x in range(0, len(sfm_data2["views"])) if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"] ["id_view"] in model2.reconFrame ] FileUtils.makedir(inputImgTmpFolder) for reconFrameName in listReconFrameName: os.system("cp -s " + os.path.join(model2.imgFolLoc, reconFrameName) + " " + inputImgTmpFolder) # remove all old localization result FileUtils.removedir(model2.locFolLoc) FileUtils.makedir(model2.locFolLoc) # localize the images from model2 on model1 guideMatchOption = "" if reconParam.bGuidedMatchingLocalize: guideMatchOption = " -gm" os.system(reconParam.LOCALIZE_PROJECT_PATH + \ " " + inputImgTmpFolder + \ " " + os.path.dirname(model1.sfm_dataLoc) + \ " " + self.mMatchesPath + \ " " + model2.locFolLoc + \ " -f=" + str(reconParam.locFeatDistRatio) + \ " -r=" + str(reconParam.locRansacRound) + \ " -i=" + str(reconParam.locSkipFrame) + \ " -k=" + str(reconBOWParam.locKNNnum) + \ " -a=" + os.path.join(self.mMatchesPath, "BOWfile.yml") + \ " -p=" + os.path.join(self.mMatchesPath, "PCAfile.yml") + \ guideMatchOption) # remove temporary image folder # removedir(inputImgTmpFolder) # extract centers from all json file and write to a file fileLoc = open(os.path.join(model2.locFolLoc, "center.txt"), "w") countLocFrame = 0 for filename in sorted(os.listdir(model2.locFolLoc)): if filename[-4:] != "json": continue countLocFrame = countLocFrame + 1 with open(os.path.join(model2.locFolLoc, filename)) as locJson: #print os.path.join(sfm_locOut,filename) locJsonDict = json.load(locJson) loc = locJsonDict["t"] fileLoc.write( str(loc[0]) + " " + str(loc[1]) + " " + str(loc[2]) + " 255 0 0\n") fileLoc.close() # get inlier matches FileUtils.makedir(sfmOutPath) resultSfMDataFile = os.path.join(sfmOutPath, "sfm_data.json") # below also checks if the ratio between first and last svd of M[0:3,0:3] # is good or not. If not then reject # TODO : revisit ransacRound parameter, use number of reconstruction frame to determine structure points transform seems small nMatchPointsTmp, nInlierTmp, M = mergeSfM.mergeModel( model1.sfm_dataLoc, model2.sfm_dataLoc, model2.locFolLoc, resultSfMDataFile, ransacThres=model1.ransacStructureThres, mergePointThres=model1.mergeStructureThres, ransacRoundMul=reconParam.ransacRoundMul, inputImgDir=self.mInputImgPath, minLimit=reconParam.min3DnInliers) ratioInlierMatchPoints = 0.0 if nMatchPointsTmp > 0: ratioInlierMatchPoints = float(nInlierTmp) / nMatchPointsTmp # 3. perform test whether merge is good sfm_merge_generated = True countFileAgree = 0 countFileLoc = 1 if os.path.isfile(resultSfMDataFile): os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + resultSfMDataFile + " " + resultSfMDataFile) countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal( resultSfMDataFile, model2.locFolLoc, model1.validMergeRansacThres) else: sfm_merge_generated = False ratioAgreeFrameReconFrame = 0.0 if (len(model2.reconFrame) > 0): ratioAgreeFrameReconFrame = float(countFileAgree) / len( model2.reconFrame) ratioAgreeFrameLocFrame = 0.0 if (countFileLoc > 0): ratioAgreeFrameLocFrame = float(countFileAgree) / countFileLoc # write log file with open( os.path.join(self.mSfMPath, "global" + str(self.nMergedModel), "log.txt"), "a") as filelog: filelog.write(("M1: " + model1.name + "\n" + \ "M2: " + model2.name + "\n" + \ "nMatchedPoints: " + str(nMatchPointsTmp) + "\n" + \ "nInliers: " + str(nInlierTmp) + "\n" + \ "ratioInlierWithMatchedPoints: " + str(ratioInlierMatchPoints) + "\n" + \ "countLocFrame: " + str(countLocFrame) + "\n" + \ "nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \ "countFileAgree: " + str(countFileAgree) + "\n" + \ "countFileLoc: " + str(countFileLoc) + "\n" + \ "not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \ # obsolete condition by T. Ishihara 2015.11.10 #"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \ "countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \ "ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \ "ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \ "ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \ "ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \ str(M) + "\n\n")) # rename the localization folder to save localization result ''' if os.path.isdir(model2.locFolLoc+model1.name): FileUtils.removedir(model2.locFolLoc+model1.name) os.rename(model2.locFolLoc,model2.locFolLoc+model1.name) ''' # obsolete merge condition ''' if not sfm_merge_generated or \ not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \ ((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \ ((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and (float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))): ''' # update merge condition by T. Ishihara 2015.11.10 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \ countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): ''' # update merge condition by T. Ishihara 2016.04.02 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): ''' # update merge condition by T. Ishihara 2016.06.09 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \ nInlierTmp > reconParam.min3DnInliers and \ ratioInlierMatchPoints > reconParam.vldMergeRatioInliersMatchPoints): ''' # update merge condition by T. Ishihara 2016.06.20 if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF and \ nInlierTmp > reconParam.min3DnInliers): print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "." ''' if os.path.isfile(os.path.join(sfmOutPath,"sfm_data.json")): os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \ os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json")) ''' if os.path.isfile(os.path.join(sfmOutPath, "sfm_data.json")): os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \ os.path.join(sfmOutPath,"sfm_data_fail_merge.json")) # move to next video return False, sfmModelBOW("", "", "", "", "", "", validMergeRansacThres=0, validMergeRansacThresK=0, ransacStructureThres=0, ransacStructureThresK=0, mergeStructureThres=0, mergeStructureThresK=0) # generate colorized before bundle adjustment for comparison os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath, "sfm_data.json") + " -o " + os.path.join(sfmOutPath, "colorized_pre.ply")) # TODO : try computing structure from know pose here # https://github.com/openMVG/openMVG/issues/246 # http://openmvg.readthedocs.io/en/latest/software/SfM/ComputeStructureFromKnownPoses/ # TODO : revisit the order of bundle adjustment # perform bundle adjustment ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "rs,rst,rsti" + " -r=" + "1") ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "rst,rsti" + " -r=" + "1") os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath, "sfm_data.json") + " -o " + os.path.join(sfmOutPath, "colorized.ply")) return True, sfmModelBOW( "A" + model1.name + "," + model2.name + "Z", self.mInputImgPath, self.mCsvPath, self.mMatchesPath, os.path.join(sfmOutPath, "loc"), resultSfMDataFile, validMergeRansacThres=model1.validMergeRansacThres, ransacStructureThres=model1.ransacStructureThres, mergeStructureThres=model1.mergeStructureThres)
def cleanSfM(sfm_data_path, matchesFile): sfm_data = FileUtils.loadjson(sfm_data_path) if (len(sfm_data["views"]) == 0): print "No views are used in reconstruction of " + sfm_data_path return [[], []] if (len(sfm_data["extrinsics"]) == 0): print "No extrinsics are used in reconstruction of " + sfm_data_path return [[], []] # get map from ID to index viewMap = {} for i in range(0, len(sfm_data["views"])): viewMap[sfm_data["views"][i]["value"]["ptr_wrapper"]["data"] ["id_view"]] = i extMap = {} for i in range(0, len(sfm_data["extrinsics"])): extMap[sfm_data["extrinsics"][i]["key"]] = i strMap = {} for i in range(0, len(sfm_data["structure"])): strMap[sfm_data["structure"][i]["key"]] = i # find viewIDs of first and last frame used in reconstruction firstViewID = len(sfm_data["views"]) lastViewID = 0 firstExtID = min(extMap.keys()) for i in range(0, len(sfm_data["views"])): viewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][ "id_view"] extID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"] if extID in extMap: if firstViewID > viewID: firstViewID = viewID if lastViewID < viewID: lastViewID = viewID if firstViewID >= lastViewID: print "No views are used in reconstruction of " + sfm_data_path return [[], []] # get list of unused view back to front # and change the view Index unusedImgName = [[], []] for i in range(len(sfm_data["views"]) - 1, lastViewID, -1): unusedImgName[1].append( sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"]) sfm_data["views"].pop(i) for i in range(lastViewID, firstViewID - 1, -1): newViewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][ "id_view"] - firstViewID sfm_data["views"][i]["key"] = newViewID sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][ "id_view"] = newViewID sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][ "id_pose"] = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"][ "id_pose"] - firstExtID for i in range(firstViewID - 1, -1, -1): unusedImgName[0].append( sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"]) sfm_data["views"].pop(i) # change extrinsics ID for i in range(0, len(sfm_data["extrinsics"])): sfm_data["extrinsics"][i][ "key"] = sfm_data["extrinsics"][i]["key"] - firstExtID # change index of refered view in structure for i in range(0, len(sfm_data["structure"])): for j in range(0, len(sfm_data["structure"][i]["value"]["observations"])): sfm_data["structure"][i]["value"]["observations"][j]["key"] = \ sfm_data["structure"][i]["value"]["observations"][j]["key"]-firstViewID # save jsonfile back FileUtils.savejson(sfm_data, sfm_data_path) # update matches file for matchfile in matchesFile: matchFileName = os.path.basename(matchfile) matchFileNameTmp = matchFileName.join( random.choice(string.lowercase) for i in range(10)) #random name matchDir = os.path.dirname(matchfile) fout = open(os.path.join(matchDir, matchFileNameTmp), "w") with open(matchfile, "r") as mfile: mode = 0 write = False countLine = 0 for line in mfile: line = line.strip() if mode == 0: line = line.split(" ") view1 = int(line[0]) view2 = int(line[1]) if view1 < firstViewID or view1 > lastViewID or \ view2 < firstViewID or view2 > lastViewID: write = False else: write = True if write: # update viewID and write out fout.write(str(int(line[0]) - firstViewID)) fout.write(" ") fout.write(str(int(line[1]) - firstViewID)) fout.write("\n") countLine = 0 mode = 1 elif mode == 1: numMatch = int(line) if write: # get number of matches and write out fout.write(line + "\n") mode = 2 elif mode == 2: if write: # write out matches fout.write(line + "\n") countLine = countLine + 1 if countLine == numMatch: mode = 0 os.rename(os.path.join(matchDir, matchFileName), os.path.join(matchDir, matchFileName + "_old")) os.rename(os.path.join(matchDir, matchFileNameTmp), os.path.join(matchDir, matchFileName)) return unusedImgName
def mergeOneModel(self, model1, model2, reconParam): sfmOutPath = os.path.join(self.mSfMPath,"global"+str(self.nMergedModel)) # create a temporary folder for reconstructed image of model2 inputImgTmpFolder = os.path.join(self.mSfMPath,"inputImgTmp","inputImgTmp"+model2.name) # copy reconstructed image fom model2 to tmp folder sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc) if not os.path.isdir(inputImgTmpFolder): listReconFrameName = [sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["filename"] for x in range(0,len(sfm_data2["views"])) if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"]["id_view"] in model2.reconFrame] FileUtils.makedir(inputImgTmpFolder) for reconFrameName in listReconFrameName: os.system("cp -s " + os.path.join(model2.imgFolLoc,reconFrameName) + " " + inputImgTmpFolder) # remove all old localization result FileUtils.removedir(model2.locFolLoc) FileUtils.makedir(model2.locFolLoc) # localize the images from model2 on model1 os.system(reconParam.LOCALIZE_PROJECT_PATH + \ " " + inputImgTmpFolder + \ " " + os.path.dirname(model1.sfm_dataLoc) + \ " " + self.mMatchesPath + \ " " + model2.locFolLoc + \ " -f=" + str(reconParam.locFeatDistRatio) + \ " -r=" + str(reconParam.locRansacRound) + \ " -e=" + model2.csvFolLoc + \ " -i=" + str(reconParam.locSkipFrame)) # remove temporary image folder # removedir(inputImgTmpFolder) # extract centers from all json file and write to a file fileLoc = open(os.path.join(model2.locFolLoc,"center.txt"),"w") countLocFrame = 0 for filename in sorted(os.listdir(model2.locFolLoc)): if filename[-4:]!="json": continue countLocFrame = countLocFrame + 1 with open(os.path.join(model2.locFolLoc,filename)) as locJson: #print os.path.join(sfm_locOut,filename) locJsonDict = json.load(locJson) loc = locJsonDict["t"] fileLoc.write(str(loc[0]) + " " + str(loc[1]) + " " +str(loc[2]) + " 255 0 0\n" ) fileLoc.close() # get inlier matches FileUtils.makedir(sfmOutPath) resultSfMDataFile = os.path.join(sfmOutPath,"sfm_data.json") # below also checks if the ratio between first and last svd of M[0:3,0:3] # is good or not. If not then reject nInlierTmp, M = mergeSfM.mergeModel(model1.sfm_dataLoc, model2.sfm_dataLoc, model2.locFolLoc, resultSfMDataFile, ransacK=reconParam.ransacStructureThresMul, ransacRound=reconParam.ransacRoundMul*len(model1.reconFrame), inputImgDir=self.mInputImgPath, minLimit=reconParam.min3DnInliers) # 3. perform test whether merge is good sfm_merge_generated = True countFileAgree = 0 countFileLoc = 1 if os.path.isfile(resultSfMDataFile): os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + resultSfMDataFile + " " + resultSfMDataFile) countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal(resultSfMDataFile, model2.locFolLoc, reconParam.vldMergeAgrFrameThresK) else: sfm_merge_generated = False ratioAgreeFrameReconFrame = 0.0 if (len(model2.reconFrame)>0): ratioAgreeFrameReconFrame = float(countFileAgree)/len(model2.reconFrame) ratioAgreeFrameLocFrame = 0.0 if (countFileLoc>0): ratioAgreeFrameLocFrame = float(countFileAgree)/countFileLoc # write log file with open(os.path.join(self.mSfMPath,"global"+str(self.nMergedModel),"log.txt"),"a") as filelog: filelog.write(("M1: " + model1.name + "\n" + \ "M2: " + model2.name + "\n" + \ "nInliers: " + str(nInlierTmp) + "\n" + \ "countLocFrame: " + str(countLocFrame) + "\n" + \ "nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \ "countFileAgree: " + str(countFileAgree) + "\n" + \ "countFileLoc: " + str(countFileLoc) + "\n" + \ "not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \ # obsolete condition by T. Ishihara 2015.11.10 #"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \ "countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \ "ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \ "ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \ "ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \ "ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \ str(M) + "\n\n")) # rename the localization folder to save localization result if os.path.isdir(model2.locFolLoc+model1.name): FileUtils.removedir(model2.locFolLoc+model1.name) os.rename(model2.locFolLoc,model2.locFolLoc+model1.name) # obsolete merge condition ''' if not sfm_merge_generated or \ not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \ ((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \ ((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and (float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))): ''' # update merge condition by T. Ishihara 2015.11.10 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \ countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): ''' # update merge condition by T. Ishihara 2016.04.02 if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "." if os.path.isfile(os.path.join(sfmOutPath,"sfm_data.json")): os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \ os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json")) # move to next video return False, sfmModel("","","","","","") # generate colorized before bundle adjustment for comparison os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath,"sfm_data.json") + " -o " + os.path.join(sfmOutPath,"colorized_pre.ply")) # perform bundle adjustment # modified by T.Ishihara 2016.04.08 # fix only translation at first ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "rs,rst,rsti" + " -r=" + "1") ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "st,rst,rsti" + " -r=" + "1") os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath,"sfm_data.json") + " -o " + os.path.join(sfmOutPath,"colorized.ply")) return True, sfmModel("A" + model1.name + "," + model2.name +"Z", self.mInputImgPath, self.mCsvPath, self.mMatchesPath, os.path.join(sfmOutPath,"loc"), resultSfMDataFile)
def cleanSfM(sfm_data_path,matchesFile): sfm_data = FileUtils.loadjson(sfm_data_path) if (len(sfm_data["views"])==0): print "No views are used in reconstruction of " + sfm_data_path return [[],[]] if (len(sfm_data["extrinsics"])==0): print "No extrinsics are used in reconstruction of " + sfm_data_path return [[],[]] # get map from ID to index viewMap = {} for i in range(0,len(sfm_data["views"])): viewMap[sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"]] = i extMap = {} for i in range(0,len(sfm_data["extrinsics"])): extMap[sfm_data["extrinsics"][i]["key"]] = i strMap = {} for i in range(0,len(sfm_data["structure"])): strMap[sfm_data["structure"][i]["key"]] = i # find viewIDs of first and last frame used in reconstruction firstViewID = len(sfm_data["views"]) lastViewID = 0 firstExtID = min(extMap.keys()) for i in range(0,len(sfm_data["views"])): viewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"] extID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"] if extID in extMap: if firstViewID > viewID: firstViewID = viewID if lastViewID < viewID: lastViewID = viewID if firstViewID >= lastViewID: print "No views are used in reconstruction of " + sfm_data_path return [[],[]] # get list of unused view back to front # and change the view Index unusedImgName = [[],[]] for i in range(len(sfm_data["views"])-1,lastViewID,-1): unusedImgName[1].append(sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"]) sfm_data["views"].pop(i) for i in range(lastViewID,firstViewID-1,-1): newViewID = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"]-firstViewID sfm_data["views"][i]["key"] = newViewID sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_view"] = newViewID sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"] = sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["id_pose"] - firstExtID for i in range(firstViewID-1,-1,-1): unusedImgName[0].append(sfm_data["views"][i]["value"]["ptr_wrapper"]["data"]["filename"]) sfm_data["views"].pop(i) # change extrinsics ID for i in range(0,len(sfm_data["extrinsics"])): sfm_data["extrinsics"][i]["key"] = sfm_data["extrinsics"][i]["key"]-firstExtID # change index of refered view in structure for i in range(0,len(sfm_data["structure"])): for j in range(0,len(sfm_data["structure"][i]["value"]["observations"])): sfm_data["structure"][i]["value"]["observations"][j]["key"] = \ sfm_data["structure"][i]["value"]["observations"][j]["key"]-firstViewID # save jsonfile back FileUtils.savejson(sfm_data,sfm_data_path) # update matches file for matchfile in matchesFile: matchFileName = os.path.basename(matchfile) matchFileNameTmp = matchFileName.join(random.choice(string.lowercase) for i in range(10)) #random name matchDir = os.path.dirname(matchfile) fout = open(os.path.join(matchDir,matchFileNameTmp),"w") with open(matchfile,"r") as mfile: mode = 0 write = False countLine = 0 for line in mfile: line = line.strip() if mode == 0: line = line.split(" ") view1 = int(line[0]) view2 = int(line[1]) if view1 < firstViewID or view1 > lastViewID or \ view2 < firstViewID or view2 > lastViewID: write = False else: write = True if write: # update viewID and write out fout.write(str(int(line[0])-firstViewID)) fout.write(" ") fout.write(str(int(line[1])-firstViewID)) fout.write("\n") countLine = 0 mode = 1 elif mode == 1: numMatch= int(line) if write: # get number of matches and write out fout.write(line + "\n") mode = 2 elif mode == 2: if write: # write out matches fout.write(line + "\n") countLine = countLine + 1 if countLine == numMatch: mode = 0 os.rename(os.path.join(matchDir,matchFileName),os.path.join(matchDir,matchFileName+"_old")) os.rename(os.path.join(matchDir,matchFileNameTmp),os.path.join(matchDir,matchFileName)) return unusedImgName
def mergeOneModel(self, model1, model2, reconParam): sfmOutPath = os.path.join(self.mSfMPath, "global" + str(self.nMergedModel)) # create a temporary folder for reconstructed image of model2 inputImgTmpFolder = os.path.join(self.mSfMPath, "inputImgTmp", "inputImgTmp" + model2.name) # copy reconstructed image fom model2 to tmp folder sfm_data2 = FileUtils.loadjson(model2.sfm_dataLoc) if not os.path.isdir(inputImgTmpFolder): listReconFrameName = [ sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"] ["filename"] for x in range(0, len(sfm_data2["views"])) if sfm_data2["views"][x]["value"]["ptr_wrapper"]["data"] ["id_view"] in model2.reconFrame ] FileUtils.makedir(inputImgTmpFolder) for reconFrameName in listReconFrameName: os.system("cp -s " + os.path.join(model2.imgFolLoc, reconFrameName) + " " + inputImgTmpFolder) # remove all old localization result FileUtils.removedir(model2.locFolLoc) FileUtils.makedir(model2.locFolLoc) # localize the images from model2 on model1 os.system(reconParam.LOCALIZE_PROJECT_PATH + \ " " + inputImgTmpFolder + \ " " + os.path.dirname(model1.sfm_dataLoc) + \ " " + self.mMatchesPath + \ " " + model2.locFolLoc + \ " -f=" + str(reconParam.locFeatDistRatio) + \ " -r=" + str(reconParam.locRansacRound) + \ " -e=" + model2.csvFolLoc + \ " -i=" + str(reconParam.locSkipFrame)) # remove temporary image folder # removedir(inputImgTmpFolder) # extract centers from all json file and write to a file fileLoc = open(os.path.join(model2.locFolLoc, "center.txt"), "w") countLocFrame = 0 for filename in sorted(os.listdir(model2.locFolLoc)): if filename[-4:] != "json": continue countLocFrame = countLocFrame + 1 with open(os.path.join(model2.locFolLoc, filename)) as locJson: #print os.path.join(sfm_locOut,filename) locJsonDict = json.load(locJson) loc = locJsonDict["t"] fileLoc.write( str(loc[0]) + " " + str(loc[1]) + " " + str(loc[2]) + " 255 0 0\n") fileLoc.close() # get inlier matches FileUtils.makedir(sfmOutPath) resultSfMDataFile = os.path.join(sfmOutPath, "sfm_data.json") # below also checks if the ratio between first and last svd of M[0:3,0:3] # is good or not. If not then reject nInlierTmp, M = mergeSfM.mergeModel( model1.sfm_dataLoc, model2.sfm_dataLoc, model2.locFolLoc, resultSfMDataFile, ransacK=reconParam.ransacStructureThresMul, ransacRound=reconParam.ransacRoundMul * len(model1.reconFrame), inputImgDir=self.mInputImgPath, minLimit=reconParam.min3DnInliers) # 3. perform test whether merge is good sfm_merge_generated = True countFileAgree = 0 countFileLoc = 1 if os.path.isfile(resultSfMDataFile): os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + resultSfMDataFile + " " + resultSfMDataFile) countFileLoc, countFileAgree = mergeSfM.modelMergeCheckLocal( resultSfMDataFile, model2.locFolLoc, reconParam.vldMergeAgrFrameThresK) else: sfm_merge_generated = False ratioAgreeFrameReconFrame = 0.0 if (len(model2.reconFrame) > 0): ratioAgreeFrameReconFrame = float(countFileAgree) / len( model2.reconFrame) ratioAgreeFrameLocFrame = 0.0 if (countFileLoc > 0): ratioAgreeFrameLocFrame = float(countFileAgree) / countFileLoc # write log file with open( os.path.join(self.mSfMPath, "global" + str(self.nMergedModel), "log.txt"), "a") as filelog: filelog.write(("M1: " + model1.name + "\n" + \ "M2: " + model2.name + "\n" + \ "nInliers: " + str(nInlierTmp) + "\n" + \ "countLocFrame: " + str(countLocFrame) + "\n" + \ "nReconFrame M2: " + str(len(model2.reconFrame)) + "\n" + \ "countFileAgree: " + str(countFileAgree) + "\n" + \ "countFileLoc: " + str(countFileLoc) + "\n" + \ "not sfm_merge_generated: " + str(not sfm_merge_generated) + "\n" + \ # obsolete condition by T. Ishihara 2015.11.10 #"nInlierTmp > "+str(reconParam.vldMergeRatioInliersFileagree)+"*countFileAgree: " + str(nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree) + "\n" + \ "countFileAgree > "+str(reconParam.vldMergeMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileAgree > "+str(reconParam.vldMergeSmallMinCountFileAgree)+": " + str(countFileAgree > reconParam.vldMergeSmallMinCountFileAgree) + "\n" + \ # obsolete condition by T. Ishihara 2016.04.02 #"countFileLoc < countFileAgree*" +str(reconParam.vldMergeShortRatio)+ ": " + str(countFileLoc < countFileAgree*reconParam.vldMergeShortRatio) + "\n" + \ "ratioLocAgreeWithReconFrame: " + str(ratioAgreeFrameReconFrame) + "\n" + \ "ratioLocAgreeWithReconFrame > " + str(reconParam.vldMergeRatioAgrFReconF) + ": " + str(ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) + "\n" + \ "ratioLocAgreeWithLocFrame: " + str(ratioAgreeFrameLocFrame) + "\n" + \ "ratioLocAgreeWithLocFrame > " + str(reconParam.vldMergeRatioAgrFLocF) + ": " + str(ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF) + "\n" + \ str(M) + "\n\n")) # rename the localization folder to save localization result if os.path.isdir(model2.locFolLoc + model1.name): FileUtils.removedir(model2.locFolLoc + model1.name) os.rename(model2.locFolLoc, model2.locFolLoc + model1.name) # obsolete merge condition ''' if not sfm_merge_generated or \ not (nInlierTmp > reconParam.vldMergeRatioInliersFileagree*countFileAgree and \ ((countFileAgree > reconParam.vldMergeMinCountFileAgree or (countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and countFileLoc < countFileAgree*reconParam.vldMergeShortRatio)) and \ ((nInlierTmp > reconParam.vldMergeNInliers and float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconFNInliers) or float(countFileAgree)/countFileLoc > reconParam.vldMergeRatioAgrFLocF) and (float(countFileAgree)/len(model2.reconFrame) > reconParam.vldMergeRatioAgrFReconF))): ''' # update merge condition by T. Ishihara 2015.11.10 ''' if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ countFileAgree > reconParam.vldMergeSmallMinCountFileAgree and \ countFileLoc < countFileAgree*reconParam.vldMergeShortRatio and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): ''' # update merge condition by T. Ishihara 2016.04.02 if not sfm_merge_generated or \ not (countFileAgree > reconParam.vldMergeMinCountFileAgree and \ ((nInlierTmp > reconParam.vldMergeNInliers and ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconFNInliers) or \ ratioAgreeFrameReconFrame > reconParam.vldMergeRatioAgrFReconF) and \ ratioAgreeFrameLocFrame > reconParam.vldMergeRatioAgrFLocF): print "Transformed locations do not agree with localization. Skip merge between " + model1.name + " and " + model2.name + "." if os.path.isfile(os.path.join(sfmOutPath, "sfm_data.json")): os.rename(os.path.join(sfmOutPath,"sfm_data.json"), \ os.path.join(sfmOutPath,"sfm_data_("+model1.name + "," + model2.name+").json")) # move to next video return False, sfmModel("", "", "", "", "", "") # generate colorized before bundle adjustment for comparison os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath, "sfm_data.json") + " -o " + os.path.join(sfmOutPath, "colorized_pre.ply")) # perform bundle adjustment # modified by T.Ishihara 2016.04.08 # fix only translation at first ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "rs,rst,rsti" + " -r=" + "1") ''' os.system(reconParam.BUNDLE_ADJUSTMENT_PROJECT_PATH + " " + os.path.join(sfmOutPath,"sfm_data.json") + " " + os.path.join(sfmOutPath,"sfm_data.json") + \ " -c=" + "st,rst,rsti" + " -r=" + "1") os.system("openMVG_main_ComputeSfM_DataColor " + " -i " + os.path.join(sfmOutPath, "sfm_data.json") + " -o " + os.path.join(sfmOutPath, "colorized.ply")) return True, sfmModel("A" + model1.name + "," + model2.name + "Z", self.mInputImgPath, self.mCsvPath, self.mMatchesPath, os.path.join(sfmOutPath, "loc"), resultSfMDataFile)