def main(argsIn): try: usage = '''label_images.py <options>''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--training", dest="trainingPath", required=True, help="Path to the training file.") parser.add_argument('--num-processes', dest='numProcesses', default=8, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=1, type=int, help='Used for mapproject.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) if not os.path.exists(options.trainingPath): print 'Error: Input training file ' + options.trainingPath + ' does not exist!' return -1 # TODO: Everything should use the RunHelper class for this! if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) # Set up a processing tool to handle the frames, this will be more efficient # than using the built-in mulithreading support. pool = multiprocessing.Pool(options.numProcesses) taskHandles = [] for i in range(options.startFrame, options.stopFrame+1): # Run on a single frame with one thread. #label_images(options.outputFolder, i, options.trainingPath, options.site, options.yyyymmdd, options.numThreads) taskHandles.append(pool.apply_async(label_images, (options.outputFolder, i, options.trainingPath, options.site, options.yyyymmdd, options.numThreads))) # Wait for all the tasks to complete print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False) # All tasks should be finished, clean up the processing pool icebridge_common.stopTaskPool(pool) print('Jobs finished.')
def main(argsIn): try: usage = '''label_images.py <options>''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--training", dest="trainingPath", required=True, help="Path to the training file.") parser.add_argument( '--num-processes', dest='numProcesses', default=8, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=1, type=int, help='Used for mapproject.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) if not os.path.exists(options.trainingPath): print 'Error: Input training file ' + options.trainingPath + ' does not exist!' return -1 # TODO: Everything should use the RunHelper class for this! if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) # Set up a processing tool to handle the frames, this will be more efficient # than using the built-in mulithreading support. pool = multiprocessing.Pool(options.numProcesses) taskHandles = [] for i in range(options.startFrame, options.stopFrame + 1): # Run on a single frame with one thread. #label_images(options.outputFolder, i, options.trainingPath, options.site, options.yyyymmdd, options.numThreads) taskHandles.append( pool.apply_async( label_images, (options.outputFolder, i, options.trainingPath, options.site, options.yyyymmdd, options.numThreads))) # Wait for all the tasks to complete print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False) # All tasks should be finished, clean up the processing pool icebridge_common.stopTaskPool(pool) print('Jobs finished.')
if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool) # Run main function if file used from shell if __name__ == "__main__": sys.exit(main(sys.argv[1:]))
def main(argsIn): try: usage = '''usage: process_icebridge_run.py <image_folder> <camera_folder> <lidar_folder> <output_folder>''' parser = optparse.OptionParser(usage=usage) # Data selection optios parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=-1, type='int', help='The frame number to finish processing with.') parser.add_option('--south', action='store_true', default=False, dest='isSouth', help='MUST be set if the images are in the southern hemisphere.') # Processing options parser.add_option('--stereo-arguments', dest='stereoArgs', default='', help='Additional argument string to be passed to the stereo command.') parser.add_option('--bundle-length', dest='bundleLength', default=2, type='int', help='Number of images to bundle adjust and process at once.') parser.add_option('--image-stereo-interval', dest='imageStereoInterval', default=None, type='int', help='Advance this many frames to get the stereo pair. Default is auto-calculate') parser.add_option('--solve-intrinsics', action='store_true', default=False, dest='solve_intr', help='If to float the intrinsics params.') #parser.add_option('--dem-resolution', dest='demResolution', default=0.4, # type='float', help='Generate output DEMs at this resolution.') parser.add_option('--max-displacement', dest='maxDisplacement', default=20, type='float', help='Max displacement value passed to pc_align.') # Performance options parser.add_option('--num-processes', dest='numProcesses', default=1, type='int', help='The number of simultaneous processes to run.') parser.add_option('--num-threads', dest='numThreads', default=None, type='int', help='The number threads to use per process.') # Action options parser.add_option('--interactive', action='store_true', default=False, dest='interactive', help='If to wait on user input to terminate the jobs.') parser.add_option('--log-batches', action='store_true', default=False, dest='logBatches', help="Just log the batch commands to a file.") parser.add_option('--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_option('--many-ip', action='store_true', default=False, dest='manyip', help='If to use a lot of IP in bundle adjustment from the beginning.') parser.add_option('--dry-run', action='store_true', default=False, dest='dryRun', help="Print but don't launch the processing jobs.") parser.add_option('--ortho-folder', dest='orthoFolder', default=None, help='Use ortho files to adjust processing to the image spacing.') parser.add_option('--fireball-folder', dest='fireballFolder', default=None, help='Location of fireball DEMs for comparison.') parser.add_option('--reference-dem', dest='referenceDem', default=None, help='Reference DEM used to calculate the expected GSD.') (options, args) = parser.parse_args(argsIn) if len(args) < 4: print(usage) return 0 imageFolder = args[0] cameraFolder = args[1] lidarFolder = args[2] outputFolder = args[3] except optparse.OptionError as msg: raise Usage(msg) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Check the inputs for f in [imageFolder, cameraFolder, lidarFolder]: if not os.path.exists(f): logger.error('Input folder '+ f +' does not exist!') return 0 asp_system_utils.mkdir_p(outputFolder) suppressOutput = False redo = False logger.info('\nStarting processing...') # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs(imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) numFiles = len(list(imageCameraPairs)) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') # Check that the files are properly aligned lastFrame = -1 availableFrames = [] for (image, camera) in imageCameraPairs: frameNumber = icebridge_common.getFrameNumberFromFilename(image) availableFrames.append(frameNumber) if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber): logger.error('Error: input files do not align!\n' + str((image, camera))) return -1 if frameNumber <= lastFrame: logger.error('Error: input frames not sorted properly!\n') return -1 lastFrame = frameNumber # Do not compute output resolution. Will be overwritten anyway per frame. ## Set the output resolution as the computed mean GSD ## TODO: This should be cashed, and recomputed only when the batch file changes! #NUM_GSD_FRAMES = 20 #logger.info('Computing GSD with ' + str(NUM_GSD_FRAMES) + ' frames.') #gsdFrameSkip = len(imageCameraPairs) / NUM_GSD_FRAMES #if gsdFrameSkip < 1: # gsdFrameSkip = 1 #medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth, # gsdFrameSkip) #outputResolution = icebridge_common.gsdToDemRes(medianGsd) #logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution)) outputResolution = 0.4 # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in imageCameraPairs: vizString += camera+' ' cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString logger.info('Running orbitviz on input files...') # Suppress (potentially long) output asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo) # Set up options for process_icebridge_batch extraOptions = '' if options.numThreads: extraOptions += ' --num-threads ' + str(options.numThreads) if options.solve_intr: extraOptions += ' --solve-intrinsics ' if options.isSouth: extraOptions += ' --south ' if options.maxDisplacement: extraOptions += ' --max-displacement ' + str(options.maxDisplacement) if options.fireballFolder: extraOptions += ' --fireball-folder ' + str(options.fireballFolder) if options.cleanup: extraOptions += ' --cleanup ' if options.manyip: extraOptions += ' --many-ip ' # We ran this before, as part of fetching, so hopefully all the data is cached forceAllFramesInRange = False (breaks, largeSkips) = getImageSpacing(options.orthoFolder, availableFrames, options.startFrame, options.stopFrame, forceAllFramesInRange) if options.imageStereoInterval: logger.info('Using manually specified image stereo interval: ' + str(options.imageStereoInterval)) largeSkips = [] # Always use a manually specified skip interval else: options.imageStereoInterval = 1 sleepTime = 20 # If all we are doing is logging commands then one process is sufficient. # - Wipe the output file while we are at it. batchLogPath = '' batchNum = 0 if options.logBatches: options.numProcesses = 1 sleepTime = 1 batchLogPath = os.path.join(outputFolder, BATCH_COMMAND_LOG_FILE) os.system('rm -f ' + batchLogPath) logger.info('Just generating batch log file '+batchLogPath+', no processing will occur.') logger.info('Starting processing pool with ' + str(options.numProcesses) +' processes.') pool = NonDaemonPool(options.numProcesses) # Call process_icebridge_batch on each batch of images. # - Batch size should be the largest number of images which can be effectively bundle-adjusted. taskHandles = [] batchImageCameraPairs = [] frameNumbers = [] i = 0 # The frame index that starts the current batch while True: # Loop for adding batches # Bugfix: arrived at the last feasible frame (maybe there are more but # they lack cameras). if i >= len(list(imageCameraPairs)): break firstBundleFrame = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[i][0]) # Determine the frame skip amount for this batch (set by the first frame) thisSkipInterval = options.imageStereoInterval if firstBundleFrame in largeSkips: #print(" ", firstBundleFrame, " in largeskips!" ) thisSkipInterval = largeSkips[firstBundleFrame] thisBatchSize = options.bundleLength + thisSkipInterval - 1 # Keep adding frames until we have enough or run out of frames j = i # The frame index that ends the batch while True: if j >= len(imageCameraPairs): # Bugfix: arrived at the last feasible frame. break frameNumber = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[j][0]) # Useful debugging code #print("Framenumber is ", frameNumber) #if int(frameNumber) > 8531: # # pass # for t in range(8340, 8360): # print("i frame is ", t, imageCameraPairs[t][0]) # print("breaks are ", breaks) # sys.exit(1) # Update conditions hitBreakFrame = (frameNumber in breaks) lastFrame = (frameNumber > options.stopFrame) or (j >= numFiles) endBatch = ( len(frameNumbers) >= thisBatchSize ) if lastFrame or endBatch: break # The new frame is too much, don't add it to the batch # Add frame to the list for the current batch batchImageCameraPairs.append(imageCameraPairs[j]) frameNumbers.append(frameNumber) if hitBreakFrame: logger.info("Hit a break, won't start a batch with frame: " + str(frameNumber)) break # Break after this frame, it is the last one added to the batch. j = j + 1 # Done adding frames to this batch if len(frameNumbers) <= thisSkipInterval: logger.info('Batch from frame: ' + str(firstBundleFrame) + ' is too small to run. Skipping.') else: # Submit the batch if not options.logBatches: logger.info('Processing frame number: ' + str(firstBundleFrame)) # The output folder is named after the first and last frame in the batch. # We count on this convention in blend_dems.py. batchFolderName = icebridge_common.batchFolderName(frameNumbers[0], frameNumbers[-1], options.bundleLength) thisOutputFolder = os.path.join(outputFolder, batchFolderName) if not options.logBatches: logger.info('Running processing batch in output folder: ' + \ thisOutputFolder + '\n' + 'with options: ' + \ extraOptions + ' --stereo-arguments ' + \ options.stereoArgs) if not options.dryRun: # Generate the command call taskHandles.append(pool.apply_async(processBatch, (batchImageCameraPairs, lidarFolder, options.referenceDem, thisSkipInterval, thisOutputFolder, extraOptions, outputResolution, options.stereoArgs, batchNum, batchLogPath))) batchNum += 1 # Reset these lists batchImageCameraPairs = [] frameNumbers = [] # Advance to the frame that starts the next batch if hitBreakFrame: # When we hit a break in the frames we need to start the # next batch after the break frame #print("Hit break frame for i, j, frameNumber", i, j, frameNumber) i = j + 1 else: # Start in the next frame that was not used as a "left" stereo image. i = i + options.bundleLength - 1 if lastFrame: break # Quit the main loop if we hit the end of the frame list. # End of loop through input file pairs logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') # Wait for all the tasks to complete icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, options.interactive, quitKey='q', sleepTime=sleepTime) # Either all the tasks are finished or the user requested a cancel. # Clean up the processing pool icebridge_common.stopTaskPool(pool) logger.info('Finished process_icebridge_run.') # to avoid ending a log with 'waiting ...'
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 10 usage = '''blend_dems.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") parser.add_argument("--compute-diff-to-prev-dem", action="store_true", dest="computeDiffToPrev", default=False, help="Compute the absolute difference between the current DEM " + "and the one before it.") parser.add_argument("--blend-to-fireball-footprint", action="store_true", dest="blendToFireball", default=False, help="Create additional blended DEMs having the same " + \ "footprint as Fireball DEMs.") # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_blend_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) logger.info(str(argsIn)) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if options.blendToFireball: fireballFrameDict = icebridge_common.getCorrectedFireballDems(options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(orthoFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No ortho files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in orthoFrameDict: logger.info("Error: Missing ortho file for frame: " + str(frame) + ".") continue orthoFile = orthoFrameDict[frame] try: lidarFile = icebridge_common.findMatchingLidarFile(orthoFile, lidarFolder) except: # Skip if no lidar file matches this frame continue fireballDEM = "" if options.blendToFireball: if frame in fireballFrameDict: fireballDEM = fireballFrameDict[frame] else: logger.info("No fireball DEM for frame: " + str(frame)) args = (frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runBlend, args)) else: runBlend(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument( '--camera-mounting', dest='cameraMounting', default=0, type=int, help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def main(argsIn): try: usage = '''usage: multi_process_command_runner.py ...''' parser = argparse.ArgumentParser(usage=usage) # Data selection optios parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This last one will not be processed.') parser.add_argument('--num-processes', dest='numProcesses', type=int, default=-1, help='How many processes to start at the same time.') parser.add_argument("--command-file-path", dest="commandFilePath", default=None, help="The file from where to read the commands to process.") parser.add_argument("--force-redo-these-frames", dest="redoFrameList", default="", help="For each frame in this file (stored one per line) within the current frame range, delete the batch folder and redo the batch.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others if not os.path.exists(options.commandFilePath): print('Error: File ' + options.commandFilePath + ' does not exist!') return -1 # TODO: Write to a log? print('Starting processing pool with ' + str(options.numProcesses) +' processes.') pool = multiprocessing.Pool(options.numProcesses) taskHandles = [] framesToDo = set() if options.redoFrameList != "" and os.path.exists(options.redoFrameList): with open(options.redoFrameList, 'r') as f: text = f.read() for line in text.split('\n'): line = line.strip() if line == "": continue framesToDo.add(int(line)) # Open the file and loop through all the lines # - Count the lines as we go so we only process the desired lines print('Opening command file ' + options.commandFilePath) text = '' with open(options.commandFilePath, 'r') as f: text = f.read() for line in text.split('\n'): if line == "": continue (begFrame, endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line) # Check line indices if begFrame >= options.startFrame and begFrame < options.stopFrame: if options.redoFrameList != "": if begFrame in framesToDo: folderName = icebridge_common.getBatchFolderFromBatchLine(line) if os.path.exists(folderName): print("will wipe " + folderName) cmd = "rm -rf " + folderName print(cmd) try: os.system(cmd) except Exception as e: pass else: print("Could not find " + folderName) else: print("Will skip frame: " + str(begFrame)) continue # Add the command to the task pool taskHandles.append(pool.apply_async(runCommand, (line,))) # Wait for all the tasks to complete print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False) # All tasks should be finished, clean up the processing pool icebridge_common.stopTaskPool(pool) print('Jobs finished.')
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, inputCalCamera, cameraLookupPath, noNav, navCameraFolder, yyyymmdd, site, refDemPath, cameraFolder, simpleCameras, startFrame, stopFrame, framesFile, numProcesses, numThreads, logger): '''Generate camera models from the ortho files. Returns false if any files were not generated.''' logger.info('Generating camera models from ortho images...') imageFiles = icebridge_common.getTifs(imageFolder) orthoFiles = icebridge_common.getTifs(orthoFolder) if navCameraFolder != "": estimateFiles = icebridge_common.getByExtension( navCameraFolder, '.tsai') else: estimateFiles = [] # See if to process frames from file filesSet = set() if framesFile != "": filesSet = icebridge_common.readLinesInSet(framesFile) # Make a dictionary of ortho files by frame # - The orthoFiles list contains _gray.tif as well as the original # images. Prefer the gray versions because it saves a bit of time # in the ortho2pinhole process. orthoFrames = {} for f in orthoFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue # Record this file if it is the first of this frame or # if it is the gray version of this frame. if (frame not in orthoFrames) or ('_gray.tif' in f): orthoFrames[frame] = f # Make a dictionary of estimated camera files by frame estimatedFrames = {} for f in estimateFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue estimatedFrames[frame] = f imageFiles.sort() logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] outputFiles = [] for imageFile in imageFiles: # Skip non-image files (including _sub images made by stereo_gui) # TODO: Use a function here from icebridge_common. Also replace # all similar locations. ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue if not frame in orthoFrames.keys(): continue # Find the estimated camera file to use with this ortho frame. orthoFile = orthoFrames[frame] try: estimatedCameraFile = estimatedFrames[frame] estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile) except: # For now treat this as an error, a missing nav file suggests # that something is going wrong with the flight! if not noNav: logger.error('Missing nav estimated camera for frame ' + str(frame)) continue else: estimatedCameraFile = None estimatedCameraPath = None #estimatedCameraFile = None #estimatedCameraPath = None # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join( cameraFolder, icebridge_common.getCameraFileName(imageFile)) outputFiles.append(outputCamFile) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs continue # Determine which input camera file will be used for this frame if inputCalCamera == "": inputCamFile = getCalibrationFileForFrame(cameraLookupPath, inputCalFolder, frame, yyyymmdd, site) else: # This logic will force to use a given camera rather than # looking it up. This is not the usual way of doing things. inputCamFile = inputCalCamera if not os.path.exists(inputCalCamera): raise Exception("Could not find: " + inputCalCamera) orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCameras, numThreads) if numProcesses > 1: # Add ortho2pinhole command to the task pool taskHandles.append( pool.apply_async(cameraFromOrthoWrapper, orthoArgs)) else: # Single process, more logging info this way cameraFromOrthoWrapper(*orthoArgs) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q') # All tasks should be finished icebridge_common.stopTaskPool(pool) logger.info('Finished ortho processing.') # Run a check to see if we got all the output files for f in outputFiles: if not os.path.exists(f): return False return True
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, cameraLookupPath, yyyymmdd, site, refDemPath, cameraFolder, numProcesses, numThreads): '''Generate camera models from the ortho files''' logger = logging.getLogger(__name__) logger.info('Generating camera models from ortho images...') imageFiles = os.listdir(imageFolder) orthoFiles = os.listdir(orthoFolder) # Make a dictionary of ortho files by frame orthoFrames = {} for f in orthoFiles: # Skip non-image files ext = os.path.splitext(f)[1] if (ext != '.tif') or ('_sub' in f): continue frame = icebridge_common.getFrameNumberFromFilename(f) orthoFrames[frame] = f logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] for imageFile in imageFiles: # Skip non-image files (including junk from stereo_gui) ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) orthoFile = orthoFrames[frame] # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join( cameraFolder, icebridge_common.getCameraFileName(imageFile)) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) continue # Determine which input camera file will be used for this frame inputCamFile = getCalibrationFileForFrame(cameraLookupPath, inputCalFolder, frame, yyyymmdd, site) # Add ortho2pinhole command to the task pool taskHandles.append( pool.apply_async(cameraFromOrthoWrapper, (inputPath, orthoPath, inputCamFile, outputCamFile, refDemPath, numThreads))) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False, quitKey='q') # All tasks should be finished, clean up the processing pool logger.info('Cleaning up the ortho processing pool...') icebridge_common.stopTaskPool(pool) logger.info('Finished cleaning up the ortho processing pool')
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument('--camera-mounting', dest='cameraMounting', default=0, type=int, help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, inputCalCamera, cameraLookupFile, noNav, navCameraFolder, yyyymmdd, site, refDemPath, cameraFolder, simpleCameras, startFrame, stopFrame, framesFile, numProcesses, numThreads, logger): '''Generate camera models from the ortho files. Returns false if any files were not generated.''' logger.info('Generating camera models from ortho images...') imageFiles = icebridge_common.getTifs(imageFolder) orthoFiles = icebridge_common.getTifs(orthoFolder) if navCameraFolder != "": estimateFiles = icebridge_common.getByExtension(navCameraFolder, '.tsai') else: estimateFiles = [] # See if to process frames from file filesSet = set() if framesFile != "": filesSet = icebridge_common.readLinesInSet(framesFile) # Make a dictionary of ortho files by frame # - The orthoFiles list contains _gray.tif as well as the original # images. Prefer the gray versions because it saves a bit of time # in the ortho2pinhole process. orthoFrames = {} for f in orthoFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue # Record this file if it is the first of this frame or # if it is the gray version of this frame. if (frame not in orthoFrames) or ('_gray.tif' in f): orthoFrames[frame] = f # Make a dictionary of estimated camera files by frame estimatedFrames = {} for f in estimateFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue estimatedFrames[frame] = f imageFiles.sort() logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] outputFiles = [] for imageFile in imageFiles: # Skip non-image files (including _sub images made by stereo_gui) # TODO: Use a function here from icebridge_common. Also replace # all similar locations. ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue if not frame in orthoFrames.keys(): continue # Find the estimated camera file to use with this ortho frame. orthoFile = orthoFrames[frame] try: estimatedCameraFile = estimatedFrames[frame] estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile) except: # For now treat this as an error, a missing nav file suggests # that something is going wrong with the flight! if not noNav: logger.error('Missing nav estimated camera for frame ' + str(frame)) continue else: estimatedCameraFile = None estimatedCameraPath = None #estimatedCameraFile = None #estimatedCameraPath = None # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join(cameraFolder, icebridge_common.getCameraFileName(imageFile)) outputFiles.append(outputCamFile) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs continue # Determine which input camera file will be used for this frame if inputCalCamera == "": inputCamFile = getCalibrationFileForFrame(cameraLookupFile, inputCalFolder, frame, yyyymmdd, site, logger) else: # This logic will force to use a given camera rather than # looking it up. This is not the usual way of doing things. inputCamFile = inputCalCamera if not os.path.exists(inputCalCamera): raise Exception("Could not find: " + inputCalCamera) orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCameras, numThreads) if numProcesses > 1: # Add ortho2pinhole command to the task pool taskHandles.append(pool.apply_async(cameraFromOrthoWrapper, orthoArgs)) else: # Single process, more logging info this way cameraFromOrthoWrapper(*orthoArgs) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q') # All tasks should be finished icebridge_common.stopTaskPool(pool) logger.info('Finished ortho processing.') # Run a check to see if we got all the output files for f in outputFiles: if not os.path.exists(f): return False return True
def main(argsIn): try: usage = '''usage: multi_process_command_runner.py ...''' parser = argparse.ArgumentParser(usage=usage) # Data selection options parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames. Set both to None to blindly run all commands.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This last one will not be processed.') parser.add_argument( '--num-processes', dest='numProcesses', type=int, default=-1, help='How many processes to start at the same time.') parser.add_argument( "--command-file-path", dest="commandFilePath", default=None, help="The file from where to read the commands to process.") parser.add_argument( "--force-redo-these-frames", dest="redoFrameList", default="", help= "For each frame in this file (stored one per line) within the current frame range, delete the batch folder and redo the batch." ) options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others if not os.path.exists(options.commandFilePath): print('Error: File ' + options.commandFilePath + ' does not exist!') return -1 # TODO: Write to a log? print('Starting processing pool with ' + str(options.numProcesses) + ' processes.') pool = multiprocessing.Pool(options.numProcesses) taskHandles = [] framesToDo = set() if options.redoFrameList != "" and os.path.exists(options.redoFrameList): with open(options.redoFrameList, 'r') as f: text = f.read() for line in text.split('\n'): line = line.strip() if line == "": continue framesToDo.add(int(line)) # Open the file and loop through all the lines # - Count the lines as we go so we only process the desired lines print('Opening command file ' + options.commandFilePath) text = '' with open(options.commandFilePath, 'r') as f: text = f.read() for line in text.split('\n'): if line == "": continue # If the frame range is turned off, just run the commands as-is. if (options.startFrame == None and options.stopFrame == None): # Add the command to the task pool taskHandles.append(pool.apply_async(runCommand, (line, ))) continue (begFrame, endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line) # Check line indices if begFrame >= options.startFrame and begFrame < options.stopFrame: if options.redoFrameList != "": if begFrame in framesToDo: folderName = icebridge_common.getBatchFolderFromBatchLine( line) if os.path.exists(folderName): print("will wipe " + folderName) cmd = "rm -rf " + folderName print(cmd) try: os.system(cmd) except Exception as e: pass else: print("Could not find " + folderName) else: print("Will skip frame: " + str(begFrame)) continue # Add the command to the task pool taskHandles.append(pool.apply_async(runCommand, (line, ))) # Wait for all the tasks to complete print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False) # All tasks should be finished, clean up the processing pool icebridge_common.stopTaskPool(pool) print('Jobs finished.')
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 10 usage = '''blend_dems.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") parser.add_argument( "--compute-diff-to-prev-dem", action="store_true", dest="computeDiffToPrev", default=False, help="Compute the absolute difference between the current DEM " + "and the one before it.") parser.add_argument("--blend-to-fireball-footprint", action="store_true", dest="blendToFireball", default=False, help="Create additional blended DEMs having the same " + \ "footprint as Fireball DEMs.") # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_blend_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) logger.info(str(argsIn)) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if options.blendToFireball: fireballFrameDict = icebridge_common.getCorrectedFireballDems( options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(orthoFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No ortho files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in orthoFrameDict: logger.info("Error: Missing ortho file for frame: " + str(frame) + ".") continue orthoFile = orthoFrameDict[frame] try: lidarFile = icebridge_common.findMatchingLidarFile( orthoFile, lidarFolder) except: # Skip if no lidar file matches this frame continue fireballDEM = "" if options.blendToFireball: if frame in fireballFrameDict: fireballDEM = fireballFrameDict[frame] else: logger.info("No fireball DEM for frame: " + str(frame)) args = (frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runBlend, args)) else: runBlend(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)