def mapfile( cube, prefix, suffix ): '''???''' items = cube.split('.') mapname = '.'.join([items[0], suffix, 'cub']) # If the file exists, or it starts with a dot (say ../file.cub) if( os.path.exists(mapname) or mapname[0] == "."): items.insert(len(items)-1, suffix) mapname = '.'.join(items) # Make somedir/file.cub into myprefix-file.map.cub if prefix != "": mapname = prefix + "-" + os.path.basename(mapname) # Ensure that any output directory exists dirname = os.path.dirname(mapname) mkdir_p(dirname) if( os.path.exists(mapname) ): raise MapExists( mapname ) return mapname
def mapfile( cube, prefix, suffix ): '''???''' items = cube.split('.') mapname = '.'.join([items[0], suffix, 'cub']) # If the file exists, or it starts with a dot (say ../file.cub) if( os.path.exists(mapname) or mapname[0] == "."): items.insert(len(items)-1, suffix) mapname = '.'.join(items) # Make somedir/file.cub into myprefix-file.map.cub if prefix != "": mapname = prefix + "-" + os.path.basename(mapname) # Ensure that any output directory exists dirname = os.path.dirname(mapname) mkdir_p(dirname) if( os.path.exists(mapname) ): raise MapExists( mapname ) return mapname
def main(argsIn): try: usage = '''usage: process_icebridge_run.py <image_folder> <camera_folder> <lidar_folder> <output_folder>''' parser = optparse.OptionParser(usage=usage) # Data selection optios parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=-1, type='int', help='The frame number to finish processing with.') parser.add_option('--south', action='store_true', default=False, dest='isSouth', help='MUST be set if the images are in the southern hemisphere.') # Processing options parser.add_option('--stereo-arguments', dest='stereoArgs', default='', help='Additional argument string to be passed to the stereo command.') parser.add_option('--bundle-length', dest='bundleLength', default=2, type='int', help='Number of images to bundle adjust and process at once.') parser.add_option('--image-stereo-interval', dest='imageStereoInterval', default=None, type='int', help='Advance this many frames to get the stereo pair. Default is auto-calculate') parser.add_option('--solve-intrinsics', action='store_true', default=False, dest='solve_intr', help='If to float the intrinsics params.') #parser.add_option('--dem-resolution', dest='demResolution', default=0.4, # type='float', help='Generate output DEMs at this resolution.') parser.add_option('--max-displacement', dest='maxDisplacement', default=20, type='float', help='Max displacement value passed to pc_align.') # Performance options parser.add_option('--num-processes', dest='numProcesses', default=1, type='int', help='The number of simultaneous processes to run.') parser.add_option('--num-threads', dest='numThreads', default=None, type='int', help='The number threads to use per process.') # Action options parser.add_option('--interactive', action='store_true', default=False, dest='interactive', help='If to wait on user input to terminate the jobs.') parser.add_option('--log-batches', action='store_true', default=False, dest='logBatches', help="Just log the batch commands to a file.") parser.add_option('--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_option('--many-ip', action='store_true', default=False, dest='manyip', help='If to use a lot of IP in bundle adjustment from the beginning.') parser.add_option('--dry-run', action='store_true', default=False, dest='dryRun', help="Print but don't launch the processing jobs.") parser.add_option('--ortho-folder', dest='orthoFolder', default=None, help='Use ortho files to adjust processing to the image spacing.') parser.add_option('--fireball-folder', dest='fireballFolder', default=None, help='Location of fireball DEMs for comparison.') parser.add_option('--reference-dem', dest='referenceDem', default=None, help='Reference DEM used to calculate the expected GSD.') (options, args) = parser.parse_args(argsIn) if len(args) < 4: print(usage) return 0 imageFolder = args[0] cameraFolder = args[1] lidarFolder = args[2] outputFolder = args[3] except optparse.OptionError as msg: raise Usage(msg) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Check the inputs for f in [imageFolder, cameraFolder, lidarFolder]: if not os.path.exists(f): logger.error('Input folder '+ f +' does not exist!') return 0 asp_system_utils.mkdir_p(outputFolder) suppressOutput = False redo = False logger.info('\nStarting processing...') # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs(imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) numFiles = len(list(imageCameraPairs)) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') # Check that the files are properly aligned lastFrame = -1 availableFrames = [] for (image, camera) in imageCameraPairs: frameNumber = icebridge_common.getFrameNumberFromFilename(image) availableFrames.append(frameNumber) if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber): logger.error('Error: input files do not align!\n' + str((image, camera))) return -1 if frameNumber <= lastFrame: logger.error('Error: input frames not sorted properly!\n') return -1 lastFrame = frameNumber # Do not compute output resolution. Will be overwritten anyway per frame. ## Set the output resolution as the computed mean GSD ## TODO: This should be cashed, and recomputed only when the batch file changes! #NUM_GSD_FRAMES = 20 #logger.info('Computing GSD with ' + str(NUM_GSD_FRAMES) + ' frames.') #gsdFrameSkip = len(imageCameraPairs) / NUM_GSD_FRAMES #if gsdFrameSkip < 1: # gsdFrameSkip = 1 #medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth, # gsdFrameSkip) #outputResolution = icebridge_common.gsdToDemRes(medianGsd) #logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution)) outputResolution = 0.4 # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in imageCameraPairs: vizString += camera+' ' cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString logger.info('Running orbitviz on input files...') # Suppress (potentially long) output asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo) # Set up options for process_icebridge_batch extraOptions = '' if options.numThreads: extraOptions += ' --num-threads ' + str(options.numThreads) if options.solve_intr: extraOptions += ' --solve-intrinsics ' if options.isSouth: extraOptions += ' --south ' if options.maxDisplacement: extraOptions += ' --max-displacement ' + str(options.maxDisplacement) if options.fireballFolder: extraOptions += ' --fireball-folder ' + str(options.fireballFolder) if options.cleanup: extraOptions += ' --cleanup ' if options.manyip: extraOptions += ' --many-ip ' # We ran this before, as part of fetching, so hopefully all the data is cached forceAllFramesInRange = False (breaks, largeSkips) = getImageSpacing(options.orthoFolder, availableFrames, options.startFrame, options.stopFrame, forceAllFramesInRange) if options.imageStereoInterval: logger.info('Using manually specified image stereo interval: ' + str(options.imageStereoInterval)) largeSkips = [] # Always use a manually specified skip interval else: options.imageStereoInterval = 1 sleepTime = 20 # If all we are doing is logging commands then one process is sufficient. # - Wipe the output file while we are at it. batchLogPath = '' batchNum = 0 if options.logBatches: options.numProcesses = 1 sleepTime = 1 batchLogPath = os.path.join(outputFolder, BATCH_COMMAND_LOG_FILE) os.system('rm -f ' + batchLogPath) logger.info('Just generating batch log file '+batchLogPath+', no processing will occur.') logger.info('Starting processing pool with ' + str(options.numProcesses) +' processes.') pool = NonDaemonPool(options.numProcesses) # Call process_icebridge_batch on each batch of images. # - Batch size should be the largest number of images which can be effectively bundle-adjusted. taskHandles = [] batchImageCameraPairs = [] frameNumbers = [] i = 0 # The frame index that starts the current batch while True: # Loop for adding batches # Bugfix: arrived at the last feasible frame (maybe there are more but # they lack cameras). if i >= len(list(imageCameraPairs)): break firstBundleFrame = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[i][0]) # Determine the frame skip amount for this batch (set by the first frame) thisSkipInterval = options.imageStereoInterval if firstBundleFrame in largeSkips: #print(" ", firstBundleFrame, " in largeskips!" ) thisSkipInterval = largeSkips[firstBundleFrame] thisBatchSize = options.bundleLength + thisSkipInterval - 1 # Keep adding frames until we have enough or run out of frames j = i # The frame index that ends the batch while True: if j >= len(imageCameraPairs): # Bugfix: arrived at the last feasible frame. break frameNumber = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[j][0]) # Useful debugging code #print("Framenumber is ", frameNumber) #if int(frameNumber) > 8531: # # pass # for t in range(8340, 8360): # print("i frame is ", t, imageCameraPairs[t][0]) # print("breaks are ", breaks) # sys.exit(1) # Update conditions hitBreakFrame = (frameNumber in breaks) lastFrame = (frameNumber > options.stopFrame) or (j >= numFiles) endBatch = ( len(frameNumbers) >= thisBatchSize ) if lastFrame or endBatch: break # The new frame is too much, don't add it to the batch # Add frame to the list for the current batch batchImageCameraPairs.append(imageCameraPairs[j]) frameNumbers.append(frameNumber) if hitBreakFrame: logger.info("Hit a break, won't start a batch with frame: " + str(frameNumber)) break # Break after this frame, it is the last one added to the batch. j = j + 1 # Done adding frames to this batch if len(frameNumbers) <= thisSkipInterval: logger.info('Batch from frame: ' + str(firstBundleFrame) + ' is too small to run. Skipping.') else: # Submit the batch if not options.logBatches: logger.info('Processing frame number: ' + str(firstBundleFrame)) # The output folder is named after the first and last frame in the batch. # We count on this convention in blend_dems.py. batchFolderName = icebridge_common.batchFolderName(frameNumbers[0], frameNumbers[-1], options.bundleLength) thisOutputFolder = os.path.join(outputFolder, batchFolderName) if not options.logBatches: logger.info('Running processing batch in output folder: ' + \ thisOutputFolder + '\n' + 'with options: ' + \ extraOptions + ' --stereo-arguments ' + \ options.stereoArgs) if not options.dryRun: # Generate the command call taskHandles.append(pool.apply_async(processBatch, (batchImageCameraPairs, lidarFolder, options.referenceDem, thisSkipInterval, thisOutputFolder, extraOptions, outputResolution, options.stereoArgs, batchNum, batchLogPath))) batchNum += 1 # Reset these lists batchImageCameraPairs = [] frameNumbers = [] # Advance to the frame that starts the next batch if hitBreakFrame: # When we hit a break in the frames we need to start the # next batch after the break frame #print("Hit break frame for i, j, frameNumber", i, j, frameNumber) i = j + 1 else: # Start in the next frame that was not used as a "left" stereo image. i = i + options.bundleLength - 1 if lastFrame: break # Quit the main loop if we hit the end of the frame list. # End of loop through input file pairs logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') # Wait for all the tasks to complete icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, options.interactive, quitKey='q', sleepTime=sleepTime) # Either all the tasks are finished or the user requested a cancel. # Clean up the processing pool icebridge_common.stopTaskPool(pool) logger.info('Finished process_icebridge_run.') # to avoid ending a log with 'waiting ...'
lidarFolder = args[2] outputFolder = args[3] except optparse.OptionError, msg: raise Usage(msg) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Check the inputs for f in [imageFolder, cameraFolder, lidarFolder]: if not os.path.exists(f): logger.error('Input folder ' + f + ' does not exist!') return 0 asp_system_utils.mkdir_p(outputFolder) suppressOutput = False redo = False logger.info('\nStarting processing...') # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs( imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) numFiles = len(imageCameraPairs) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') # Check that the files are properly aligned