def conversionIsFinished(self, startFrame, stopFrame, verbose=False):
'''Return true if this run is present and conversion has finished running on it'''
logger = logging.getLogger(__name__)
# Make sure that there is a camera file for input image file.
# - This could be a more expansive check.
cameraFolder = self.getCameraFolder()
imageList = self.getImageList()
for imageFile in imageList:
camFile = os.path.join(
cameraFolder, icebridge_common.getCameraFileName(imageFile))
# Check only within range
# TODO: Actually we need the cameras to go a bit beyond
frame = icebridge_common.getFrameNumberFromFilename(camFile)
if frame < startFrame or frame >= stopFrame:
continue
if not os.path.exists(camFile):
if verbose:
logger.error('Missing file ' + camFile)
return False
# Do a simple check of the converted lidar files
prependFolder = True
lidarFolder = self.getLidarFolder()
convLidarFile = icebridge_common.getConvertedLidarIndexFile(
lidarFolder)
(lidarDict, dummyUrlDict) = icebridge_common.readIndexFile(
convLidarFile, prependFolder)
pairedLidarFolder = icebridge_common.getPairedLidarFolder(lidarFolder)
pairedLidarFile = icebridge_common.getPairedIndexFile(
pairedLidarFolder)
(pairedLidarDict, dummyUrlDict) = icebridge_common.readIndexFile(
pairedLidarFile, prependFolder)
numLidar = len(lidarDict.values())
numPairedLidar = len(pairedLidarDict.values())
if numLidar != (numPairedLidar + 1):
logger.error('Not enough paired lidar files found')
return False
# Make sure the lidar files are not empty
success = True
for f in lidarDict.values() + pairedLidarDict.values():
if not asp_file_utils.fileIsNonZero(f):
logger.error('lidar file ' + f + ' is empty!')
os.system('rm -f ' + f) # Remove bad files
success = False
return success
def conversionIsFinished(self, startFrame, stopFrame, verbose=False):
'''Return true if this run is present and conversion has finished running on it'''
logger = logging.getLogger(__name__)
# Make sure that there is a camera file for input image file.
# - This could be a more expansive check.
cameraFolder = self.getCameraFolder()
imageList = self.getImageList()
for imageFile in imageList:
camFile = os.path.join(cameraFolder,
icebridge_common.getCameraFileName(imageFile))
# Check only within range
# TODO: Actually we need the cameras to go a bit beyond
frame = icebridge_common.getFrameNumberFromFilename(camFile)
if frame < startFrame or frame >= stopFrame:
continue
if not os.path.exists(camFile):
if verbose:
logger.error('Missing file ' + camFile)
return False
# Do a simple check of the converted lidar files
prependFolder = True
lidarFolder = self.getLidarFolder()
convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder)
(lidarDict, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile,
prependFolder)
pairedLidarFolder = icebridge_common.getPairedLidarFolder(lidarFolder)
pairedLidarFile = icebridge_common.getPairedIndexFile(pairedLidarFolder)
(pairedLidarDict, dummyUrlDict) = icebridge_common.readIndexFile(pairedLidarFile,
prependFolder)
numLidar = len(lidarDict.values())
numPairedLidar = len(pairedLidarDict.values())
if numLidar != (numPairedLidar+1):
logger.error('Not enough paired lidar files found')
return False
# Make sure the lidar files are not empty
success = True
for f in lidarDict.values() + pairedLidarDict.values():
if not asp_file_utils.fileIsNonZero(f):
logger.error('lidar file ' + f + ' is empty!')
os.system('rm -f ' + f) # Remove bad files
success = False
return success
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder,
inputCalCamera, cameraLookupPath, noNav,
navCameraFolder, yyyymmdd, site, refDemPath,
cameraFolder, simpleCameras, startFrame,
stopFrame, framesFile, numProcesses, numThreads,
logger):
'''Generate camera models from the ortho files.
Returns false if any files were not generated.'''
logger.info('Generating camera models from ortho images...')
imageFiles = icebridge_common.getTifs(imageFolder)
orthoFiles = icebridge_common.getTifs(orthoFolder)
if navCameraFolder != "":
estimateFiles = icebridge_common.getByExtension(
navCameraFolder, '.tsai')
else:
estimateFiles = []
# See if to process frames from file
filesSet = set()
if framesFile != "":
filesSet = icebridge_common.readLinesInSet(framesFile)
# Make a dictionary of ortho files by frame
# - The orthoFiles list contains _gray.tif as well as the original
# images. Prefer the gray versions because it saves a bit of time
# in the ortho2pinhole process.
orthoFrames = {}
for f in orthoFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
# Record this file if it is the first of this frame or
# if it is the gray version of this frame.
if (frame not in orthoFrames) or ('_gray.tif' in f):
orthoFrames[frame] = f
# Make a dictionary of estimated camera files by frame
estimatedFrames = {}
for f in estimateFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
estimatedFrames[frame] = f
imageFiles.sort()
logger.info('Starting ortho processing pool with ' + str(numProcesses) +
' processes.')
pool = multiprocessing.Pool(numProcesses)
# Loop through all input images
taskHandles = []
outputFiles = []
for imageFile in imageFiles:
# Skip non-image files (including _sub images made by stereo_gui)
# TODO: Use a function here from icebridge_common. Also replace
# all similar locations.
ext = os.path.splitext(imageFile)[1]
if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif'
in imageFile):
continue
# Get associated orthofile
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
if not frame in orthoFrames.keys():
continue
# Find the estimated camera file to use with this ortho frame.
orthoFile = orthoFrames[frame]
try:
estimatedCameraFile = estimatedFrames[frame]
estimatedCameraPath = os.path.join(navCameraFolder,
estimatedCameraFile)
except:
# For now treat this as an error, a missing nav file suggests
# that something is going wrong with the flight!
if not noNav:
logger.error('Missing nav estimated camera for frame ' +
str(frame))
continue
else:
estimatedCameraFile = None
estimatedCameraPath = None
#estimatedCameraFile = None
#estimatedCameraPath = None
# Check output file
inputPath = os.path.join(imageFolder, imageFile)
orthoPath = os.path.join(orthoFolder, orthoFile)
outputCamFile = os.path.join(
cameraFolder, icebridge_common.getCameraFileName(imageFile))
outputFiles.append(outputCamFile)
if os.path.exists(outputCamFile):
logger.info("File exists, skipping: " + outputCamFile)
os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs
continue
# Determine which input camera file will be used for this frame
if inputCalCamera == "":
inputCamFile = getCalibrationFileForFrame(cameraLookupPath,
inputCalFolder, frame,
yyyymmdd, site)
else:
# This logic will force to use a given camera rather than
# looking it up. This is not the usual way of doing things.
inputCamFile = inputCalCamera
if not os.path.exists(inputCalCamera):
raise Exception("Could not find: " + inputCalCamera)
orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath,
outputCamFile, refDemPath, simpleCameras, numThreads)
if numProcesses > 1:
# Add ortho2pinhole command to the task pool
taskHandles.append(
pool.apply_async(cameraFromOrthoWrapper, orthoArgs))
else:
# Single process, more logging info this way
cameraFromOrthoWrapper(*orthoArgs)
# Wait for all the tasks to complete
logger.info('Finished adding ' + str(len(taskHandles)) +
' tasks to the pool.')
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
logger,
interactive=False,
quitKey='q')
# All tasks should be finished
icebridge_common.stopTaskPool(pool)
logger.info('Finished ortho processing.')
# Run a check to see if we got all the output files
for f in outputFiles:
if not os.path.exists(f):
return False
return True
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder,
cameraLookupPath, yyyymmdd, site, refDemPath,
cameraFolder, numProcesses, numThreads):
'''Generate camera models from the ortho files'''
logger = logging.getLogger(__name__)
logger.info('Generating camera models from ortho images...')
imageFiles = os.listdir(imageFolder)
orthoFiles = os.listdir(orthoFolder)
# Make a dictionary of ortho files by frame
orthoFrames = {}
for f in orthoFiles:
# Skip non-image files
ext = os.path.splitext(f)[1]
if (ext != '.tif') or ('_sub' in f):
continue
frame = icebridge_common.getFrameNumberFromFilename(f)
orthoFrames[frame] = f
logger.info('Starting ortho processing pool with ' + str(numProcesses) +
' processes.')
pool = multiprocessing.Pool(numProcesses)
# Loop through all input images
taskHandles = []
for imageFile in imageFiles:
# Skip non-image files (including junk from stereo_gui)
ext = os.path.splitext(imageFile)[1]
if (ext != '.tif') or ('_sub' in imageFile):
continue
# Get associated orthofile
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
orthoFile = orthoFrames[frame]
# Check output file
inputPath = os.path.join(imageFolder, imageFile)
orthoPath = os.path.join(orthoFolder, orthoFile)
outputCamFile = os.path.join(
cameraFolder, icebridge_common.getCameraFileName(imageFile))
if os.path.exists(outputCamFile):
logger.info("File exists, skipping: " + outputCamFile)
continue
# Determine which input camera file will be used for this frame
inputCamFile = getCalibrationFileForFrame(cameraLookupPath,
inputCalFolder, frame,
yyyymmdd, site)
# Add ortho2pinhole command to the task pool
taskHandles.append(
pool.apply_async(cameraFromOrthoWrapper,
(inputPath, orthoPath, inputCamFile,
outputCamFile, refDemPath, numThreads)))
# Wait for all the tasks to complete
logger.info('Finished adding ' + str(len(taskHandles)) +
' tasks to the pool.')
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
interactive=False,
quitKey='q')
# All tasks should be finished, clean up the processing pool
logger.info('Cleaning up the ortho processing pool...')
icebridge_common.stopTaskPool(pool)
logger.info('Finished cleaning up the ortho processing pool')
def getCameraModelsFromOrtho(imageFolder, orthoFolder,
inputCalFolder, inputCalCamera,
cameraLookupFile,
noNav,
navCameraFolder,
yyyymmdd, site,
refDemPath, cameraFolder,
simpleCameras,
startFrame, stopFrame,
framesFile,
numProcesses, numThreads, logger):
'''Generate camera models from the ortho files.
Returns false if any files were not generated.'''
logger.info('Generating camera models from ortho images...')
imageFiles = icebridge_common.getTifs(imageFolder)
orthoFiles = icebridge_common.getTifs(orthoFolder)
if navCameraFolder != "":
estimateFiles = icebridge_common.getByExtension(navCameraFolder, '.tsai')
else:
estimateFiles = []
# See if to process frames from file
filesSet = set()
if framesFile != "":
filesSet = icebridge_common.readLinesInSet(framesFile)
# Make a dictionary of ortho files by frame
# - The orthoFiles list contains _gray.tif as well as the original
# images. Prefer the gray versions because it saves a bit of time
# in the ortho2pinhole process.
orthoFrames = {}
for f in orthoFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
# Record this file if it is the first of this frame or
# if it is the gray version of this frame.
if (frame not in orthoFrames) or ('_gray.tif' in f):
orthoFrames[frame] = f
# Make a dictionary of estimated camera files by frame
estimatedFrames = {}
for f in estimateFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
estimatedFrames[frame] = f
imageFiles.sort()
logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.')
pool = multiprocessing.Pool(numProcesses)
# Loop through all input images
taskHandles = []
outputFiles = []
for imageFile in imageFiles:
# Skip non-image files (including _sub images made by stereo_gui)
# TODO: Use a function here from icebridge_common. Also replace
# all similar locations.
ext = os.path.splitext(imageFile)[1]
if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile):
continue
# Get associated orthofile
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
if not frame in orthoFrames.keys():
continue
# Find the estimated camera file to use with this ortho frame.
orthoFile = orthoFrames[frame]
try:
estimatedCameraFile = estimatedFrames[frame]
estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile)
except:
# For now treat this as an error, a missing nav file suggests
# that something is going wrong with the flight!
if not noNav:
logger.error('Missing nav estimated camera for frame ' + str(frame))
continue
else:
estimatedCameraFile = None
estimatedCameraPath = None
#estimatedCameraFile = None
#estimatedCameraPath = None
# Check output file
inputPath = os.path.join(imageFolder, imageFile)
orthoPath = os.path.join(orthoFolder, orthoFile)
outputCamFile = os.path.join(cameraFolder,
icebridge_common.getCameraFileName(imageFile))
outputFiles.append(outputCamFile)
if os.path.exists(outputCamFile):
logger.info("File exists, skipping: " + outputCamFile)
os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs
continue
# Determine which input camera file will be used for this frame
if inputCalCamera == "":
inputCamFile = getCalibrationFileForFrame(cameraLookupFile, inputCalFolder,
frame, yyyymmdd, site, logger)
else:
# This logic will force to use a given camera rather than
# looking it up. This is not the usual way of doing things.
inputCamFile = inputCalCamera
if not os.path.exists(inputCalCamera):
raise Exception("Could not find: " + inputCalCamera)
orthoArgs = (inputPath, orthoPath, inputCamFile,
estimatedCameraPath, outputCamFile,
refDemPath, simpleCameras, numThreads)
if numProcesses > 1:
# Add ortho2pinhole command to the task pool
taskHandles.append(pool.apply_async(cameraFromOrthoWrapper, orthoArgs))
else:
# Single process, more logging info this way
cameraFromOrthoWrapper(*orthoArgs)
# Wait for all the tasks to complete
logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.')
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False,
quitKey='q')
# All tasks should be finished
icebridge_common.stopTaskPool(pool)
logger.info('Finished ortho processing.')
# Run a check to see if we got all the output files
for f in outputFiles:
if not os.path.exists(f):
return False
return True
