def mapfile( cube, prefix, suffix ):
'''???'''
items = cube.split('.')
mapname = '.'.join([items[0], suffix, 'cub'])
# If the file exists, or it starts with a dot (say ../file.cub)
if( os.path.exists(mapname) or mapname[0] == "."):
items.insert(len(items)-1, suffix)
mapname = '.'.join(items)
# Make somedir/file.cub into myprefix-file.map.cub
if prefix != "":
mapname = prefix + "-" + os.path.basename(mapname)
# Ensure that any output directory exists
dirname = os.path.dirname(mapname)
mkdir_p(dirname)
if( os.path.exists(mapname) ):
raise MapExists( mapname )
return mapname
def mapfile( cube, prefix, suffix ):
'''???'''
items = cube.split('.')
mapname = '.'.join([items[0], suffix, 'cub'])
# If the file exists, or it starts with a dot (say ../file.cub)
if( os.path.exists(mapname) or mapname[0] == "."):
items.insert(len(items)-1, suffix)
mapname = '.'.join(items)
# Make somedir/file.cub into myprefix-file.map.cub
if prefix != "":
mapname = prefix + "-" + os.path.basename(mapname)
# Ensure that any output directory exists
dirname = os.path.dirname(mapname)
mkdir_p(dirname)
if( os.path.exists(mapname) ):
raise MapExists( mapname )
return mapname
def main(argsIn):
try:
usage = '''usage: process_icebridge_run.py <image_folder> <camera_folder>
<lidar_folder> <output_folder>'''
parser = optparse.OptionParser(usage=usage)
# Data selection optios
parser.add_option('--start-frame', dest='startFrame', default=-1,
type='int', help='The frame number to start processing with.')
parser.add_option('--stop-frame', dest='stopFrame', default=-1,
type='int', help='The frame number to finish processing with.')
parser.add_option('--south', action='store_true', default=False, dest='isSouth',
help='MUST be set if the images are in the southern hemisphere.')
# Processing options
parser.add_option('--stereo-arguments', dest='stereoArgs', default='',
help='Additional argument string to be passed to the stereo command.')
parser.add_option('--bundle-length', dest='bundleLength', default=2,
type='int', help='Number of images to bundle adjust and process at once.')
parser.add_option('--image-stereo-interval', dest='imageStereoInterval', default=None,
type='int', help='Advance this many frames to get the stereo pair. Default is auto-calculate')
parser.add_option('--solve-intrinsics', action='store_true', default=False,
dest='solve_intr',
help='If to float the intrinsics params.')
#parser.add_option('--dem-resolution', dest='demResolution', default=0.4,
# type='float', help='Generate output DEMs at this resolution.')
parser.add_option('--max-displacement', dest='maxDisplacement', default=20,
type='float', help='Max displacement value passed to pc_align.')
# Performance options
parser.add_option('--num-processes', dest='numProcesses', default=1,
type='int', help='The number of simultaneous processes to run.')
parser.add_option('--num-threads', dest='numThreads', default=None,
type='int', help='The number threads to use per process.')
# Action options
parser.add_option('--interactive', action='store_true', default=False, dest='interactive',
help='If to wait on user input to terminate the jobs.')
parser.add_option('--log-batches', action='store_true', default=False, dest='logBatches',
help="Just log the batch commands to a file.")
parser.add_option('--cleanup', action='store_true', default=False, dest='cleanup',
help='If the final result is produced delete intermediate files.')
parser.add_option('--many-ip', action='store_true', default=False, dest='manyip',
help='If to use a lot of IP in bundle adjustment from the beginning.')
parser.add_option('--dry-run', action='store_true', default=False, dest='dryRun',
help="Print but don't launch the processing jobs.")
parser.add_option('--ortho-folder', dest='orthoFolder', default=None,
help='Use ortho files to adjust processing to the image spacing.')
parser.add_option('--fireball-folder', dest='fireballFolder', default=None,
help='Location of fireball DEMs for comparison.')
parser.add_option('--reference-dem', dest='referenceDem', default=None,
help='Reference DEM used to calculate the expected GSD.')
(options, args) = parser.parse_args(argsIn)
if len(args) < 4:
print(usage)
return 0
imageFolder = args[0]
cameraFolder = args[1]
lidarFolder = args[2]
outputFolder = args[3]
except optparse.OptionError as msg:
raise Usage(msg)
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
# Check the inputs
for f in [imageFolder, cameraFolder, lidarFolder]:
if not os.path.exists(f):
logger.error('Input folder '+ f +' does not exist!')
return 0
asp_system_utils.mkdir_p(outputFolder)
suppressOutput = False
redo = False
logger.info('\nStarting processing...')
# Get a list of all the input files
imageCameraPairs = icebridge_common.getImageCameraPairs(imageFolder, cameraFolder,
options.startFrame, options.stopFrame,
logger)
numFiles = len(list(imageCameraPairs))
if numFiles < 2:
raise Exception('Failed to find any image camera pairs!')
# Check that the files are properly aligned
lastFrame = -1
availableFrames = []
for (image, camera) in imageCameraPairs:
frameNumber = icebridge_common.getFrameNumberFromFilename(image)
availableFrames.append(frameNumber)
if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber):
logger.error('Error: input files do not align!\n' + str((image, camera)))
return -1
if frameNumber <= lastFrame:
logger.error('Error: input frames not sorted properly!\n')
return -1
lastFrame = frameNumber
# Do not compute output resolution. Will be overwritten anyway per frame.
## Set the output resolution as the computed mean GSD
## TODO: This should be cashed, and recomputed only when the batch file changes!
#NUM_GSD_FRAMES = 20
#logger.info('Computing GSD with ' + str(NUM_GSD_FRAMES) + ' frames.')
#gsdFrameSkip = len(imageCameraPairs) / NUM_GSD_FRAMES
#if gsdFrameSkip < 1:
# gsdFrameSkip = 1
#medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth,
# gsdFrameSkip)
#outputResolution = icebridge_common.gsdToDemRes(medianGsd)
#logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution))
outputResolution = 0.4
# Generate a map of initial camera positions
orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml')
vizString = ''
for (image, camera) in imageCameraPairs:
vizString += camera+' '
cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString
logger.info('Running orbitviz on input files...')
# Suppress (potentially long) output
asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo)
# Set up options for process_icebridge_batch
extraOptions = ''
if options.numThreads:
extraOptions += ' --num-threads ' + str(options.numThreads)
if options.solve_intr:
extraOptions += ' --solve-intrinsics '
if options.isSouth:
extraOptions += ' --south '
if options.maxDisplacement:
extraOptions += ' --max-displacement ' + str(options.maxDisplacement)
if options.fireballFolder:
extraOptions += ' --fireball-folder ' + str(options.fireballFolder)
if options.cleanup:
extraOptions += ' --cleanup '
if options.manyip:
extraOptions += ' --many-ip '
# We ran this before, as part of fetching, so hopefully all the data is cached
forceAllFramesInRange = False
(breaks, largeSkips) = getImageSpacing(options.orthoFolder, availableFrames,
options.startFrame, options.stopFrame,
forceAllFramesInRange)
if options.imageStereoInterval:
logger.info('Using manually specified image stereo interval: ' +
str(options.imageStereoInterval))
largeSkips = [] # Always use a manually specified skip interval
else:
options.imageStereoInterval = 1
sleepTime = 20
# If all we are doing is logging commands then one process is sufficient.
# - Wipe the output file while we are at it.
batchLogPath = ''
batchNum = 0
if options.logBatches:
options.numProcesses = 1
sleepTime = 1
batchLogPath = os.path.join(outputFolder, BATCH_COMMAND_LOG_FILE)
os.system('rm -f ' + batchLogPath)
logger.info('Just generating batch log file '+batchLogPath+', no processing will occur.')
logger.info('Starting processing pool with ' + str(options.numProcesses) +' processes.')
pool = NonDaemonPool(options.numProcesses)
# Call process_icebridge_batch on each batch of images.
# - Batch size should be the largest number of images which can be effectively bundle-adjusted.
taskHandles = []
batchImageCameraPairs = []
frameNumbers = []
i = 0 # The frame index that starts the current batch
while True: # Loop for adding batches
# Bugfix: arrived at the last feasible frame (maybe there are more but
# they lack cameras).
if i >= len(list(imageCameraPairs)):
break
firstBundleFrame = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[i][0])
# Determine the frame skip amount for this batch (set by the first frame)
thisSkipInterval = options.imageStereoInterval
if firstBundleFrame in largeSkips:
#print(" ", firstBundleFrame, " in largeskips!" )
thisSkipInterval = largeSkips[firstBundleFrame]
thisBatchSize = options.bundleLength + thisSkipInterval - 1
# Keep adding frames until we have enough or run out of frames
j = i # The frame index that ends the batch
while True:
if j >= len(imageCameraPairs):
# Bugfix: arrived at the last feasible frame.
break
frameNumber = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[j][0])
# Useful debugging code
#print("Framenumber is ", frameNumber)
#if int(frameNumber) > 8531:
# # pass
# for t in range(8340, 8360):
# print("i frame is ", t, imageCameraPairs[t][0])
# print("breaks are ", breaks)
# sys.exit(1)
# Update conditions
hitBreakFrame = (frameNumber in breaks)
lastFrame = (frameNumber > options.stopFrame) or (j >= numFiles)
endBatch = ( len(frameNumbers) >= thisBatchSize )
if lastFrame or endBatch:
break # The new frame is too much, don't add it to the batch
# Add frame to the list for the current batch
batchImageCameraPairs.append(imageCameraPairs[j])
frameNumbers.append(frameNumber)
if hitBreakFrame:
logger.info("Hit a break, won't start a batch with frame: " + str(frameNumber))
break # Break after this frame, it is the last one added to the batch.
j = j + 1
# Done adding frames to this batch
if len(frameNumbers) <= thisSkipInterval:
logger.info('Batch from frame: ' + str(firstBundleFrame) +
' is too small to run. Skipping.')
else:
# Submit the batch
if not options.logBatches:
logger.info('Processing frame number: ' + str(firstBundleFrame))
# The output folder is named after the first and last frame in the batch.
# We count on this convention in blend_dems.py.
batchFolderName = icebridge_common.batchFolderName(frameNumbers[0], frameNumbers[-1],
options.bundleLength)
thisOutputFolder = os.path.join(outputFolder, batchFolderName)
if not options.logBatches:
logger.info('Running processing batch in output folder: ' + \
thisOutputFolder + '\n' + 'with options: ' + \
extraOptions + ' --stereo-arguments ' + \
options.stereoArgs)
if not options.dryRun:
# Generate the command call
taskHandles.append(pool.apply_async(processBatch,
(batchImageCameraPairs, lidarFolder, options.referenceDem, thisSkipInterval,
thisOutputFolder, extraOptions,
outputResolution, options.stereoArgs, batchNum, batchLogPath)))
batchNum += 1
# Reset these lists
batchImageCameraPairs = []
frameNumbers = []
# Advance to the frame that starts the next batch
if hitBreakFrame:
# When we hit a break in the frames we need to start the
# next batch after the break frame
#print("Hit break frame for i, j, frameNumber", i, j, frameNumber)
i = j + 1
else:
# Start in the next frame that was not used as a "left" stereo image.
i = i + options.bundleLength - 1
if lastFrame:
break # Quit the main loop if we hit the end of the frame list.
# End of loop through input file pairs
logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.')
# Wait for all the tasks to complete
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, options.interactive,
quitKey='q', sleepTime=sleepTime)
# Either all the tasks are finished or the user requested a cancel.
# Clean up the processing pool
icebridge_common.stopTaskPool(pool)
logger.info('Finished process_icebridge_run.') # to avoid ending a log with 'waiting ...'
lidarFolder = args[2]
outputFolder = args[3]
except optparse.OptionError, msg:
raise Usage(msg)
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
# Check the inputs
for f in [imageFolder, cameraFolder, lidarFolder]:
if not os.path.exists(f):
logger.error('Input folder ' + f + ' does not exist!')
return 0
asp_system_utils.mkdir_p(outputFolder)
suppressOutput = False
redo = False
logger.info('\nStarting processing...')
# Get a list of all the input files
imageCameraPairs = icebridge_common.getImageCameraPairs(
imageFolder, cameraFolder, options.startFrame, options.stopFrame,
logger)
numFiles = len(imageCameraPairs)
if numFiles < 2:
raise Exception('Failed to find any image camera pairs!')
# Check that the files are properly aligned