def main(argsIn): try: usage = '''label_image.py <options>''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--training", dest="trainingPath", required=True, help="Path to the training file.") parser.add_argument('--num-processes', dest='numProcesses', default=8, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=1, type=int, help='IGNORED.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) if not os.path.exists(options.trainingPath): print 'Error: Input training file ' + options.trainingPath + ' does not exist!' return -1 # TODO: Everything should use the RunHelper class for this! if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) # Input is raw jpeg files. inputFolder = icebridge_common.getJpegFolder(options.outputFolder) # Write all tool output to this folder. outputFolder = icebridge_common.getLabelFolder(options.outputFolder) # Do the work label_images(inputFolder, outputFolder, options.startFrame, options.stopFrame, options.trainingPath, options.numProcesses)
def getFrameRange(self): '''Return the min and max frame currently stored for the run''' jpegFolder = icebridge_common.getJpegFolder(self.getFolder()) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath) frames = sorted(jpegFrameDict.keys()) if len(frames) == 0: raise Exception("Empty folder: " + jpegFolder) return (frames[0], frames[-1])
def massRename(self, startFrame, stopFrame, logger): '''We changed how the timestamp for images and cameras is computed. Make all existing converted images, cameras, nav cameras, and aligned cameras conform.''' logger.info("Renaming files with timestamp. This is slow.") # Need to do a mass rename for incorrect timestamp for: # converted images, nav cameras, cameras, and bundle aligned cameras outputFolder = self.getFolder() cameraFolder = icebridge_common.getCameraFolder(outputFolder) imageFolder = icebridge_common.getImageFolder(outputFolder) jpegFolder = icebridge_common.getJpegFolder(outputFolder) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) processedFolder = icebridge_common.getProcessedFolder(outputFolder) navFolder = icebridge_common.getNavFolder(outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder( os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) logger.info('Renaming camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(cameraFolder, '*DMS*tsai'), logger) logger.info('Renaming nav camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(navCameraFolder, '*DMS*tsai'), logger) logger.info('Renaming converted images...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(imageFolder, '*DMS*tif'), logger) logger.info('Renaming aligned cameras...') self.massRenameByGlob( startFrame, stopFrame, orthoFrameDict, os.path.join(processedFolder, 'batch*', icebridge_common.alignedBundleStr() + '*DMS*tsai'), logger)
def getFrameRange(self): '''Return the min and max frame currently stored for the run''' jpegFolder = icebridge_common.getJpegFolder(self.getFolder()) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath) frames = sorted(jpegFrameDict.keys()) if len(frames) == 0: raise Exception("Empty folder: " + jpegFolder) return (frames[0], frames[-1])
def massRename(self, startFrame, stopFrame, logger): '''We changed how the timestamp for images and cameras is computed. Make all existing converted images, cameras, nav cameras, and aligned cameras conform.''' logger.info("Renaming files with timestamp. This is slow.") # Need to do a mass rename for incorrect timestamp for: # converted images, nav cameras, cameras, and bundle aligned cameras outputFolder = self.getFolder() cameraFolder = icebridge_common.getCameraFolder(outputFolder) imageFolder = icebridge_common.getImageFolder(outputFolder) jpegFolder = icebridge_common.getJpegFolder(outputFolder) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) processedFolder = icebridge_common.getProcessedFolder(outputFolder) navFolder = icebridge_common.getNavFolder(outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) logger.info('Renaming camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(cameraFolder, '*DMS*tsai'), logger) logger.info('Renaming nav camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(navCameraFolder, '*DMS*tsai'), logger) logger.info('Renaming converted images...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(imageFolder, '*DMS*tif'), logger) logger.info('Renaming aligned cameras...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(processedFolder, 'batch*', icebridge_common.alignedBundleStr() + '*DMS*tsai'), logger)
def label_images(outputFolder, frameNum, trainingPath, site): '''Apply the labeling algorithm to a single image, then map project the result.''' # Get required paths inputFolder = icebridge_common.getJpegFolder (options.outputFolder) cameraFolder = icebridge_common.getCameraFolder (options.outputFolder) labelFolder = icebridge_common.getLabelFolder (options.outputFolder) orthoFolder = icebridge_common.getLabelOrthoFolder(options.outputFolder) # Run the label tool toolPath = 'python ~/repo/OSSP/ossp_process.py' NO_SPLITTING = 1 # Plenty of RAM to load these images ONE_PROCESS = 1 cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' % (toolPath, inputFolder, labelFolder, frameNum, trainingPath, NO_SPLITTING, ONE_PROCESS)) print cmd os.system(cmd) # Also generate the map projected version of the image # Figure out the camera and output path (labelPath, cameraPath) = get_label_and_camera(labelFolder, cameraFolder, frameNum) fname = os.path.basename(labelPath).replace('classified', 'classified_ortho') mapProjPath = os.path.join(orthoFolder, fname) # Set map projection parameters toolPath = 'mapproject' # TODO: Locate isSouth = (site == 'AN') srs = projString = icebridge_common.getEpsgCode(isSouth, asString=True) demPath = 'WGS84' # Map project on to a flat surface # Mapproject cmd = ('%s %s %s %s %s -t nadirpinhole -t_srs %s' % (toolPath, demPath, labelPath, cameraPath, mapProjPath srs)) print cmd os.system(cmd)
def label_images(outputFolder, frameNum, trainingPath, site, yyyymmdd, numThreads): '''Apply the labeling algorithm to a single image, then map project the result.''' print 'Running label for frame: ' + str(frameNum) # Get required paths inputFolder = icebridge_common.getJpegFolder(outputFolder) cameraFolder = icebridge_common.getCameraFolder(outputFolder) labelFolder = icebridge_common.getLabelFolder(outputFolder) orthoFolder = icebridge_common.getLabelOrthoFolder(outputFolder) # Hardcoded paths!!! toolPath = 'python ~/repo/OSSP/ossp_process.py' refDemFolder = '/nobackup/smcmich1/icebridge/reference_dems/' # Run the label tool NO_SPLITTING = 1 # Plenty of RAM to load these images ONE_PROCESS = 1 # Figure out the label path run = run_helper.RunHelper(site, yyyymmdd) labelName = icebridge_common.makeLabelFileName(run, frameNum) labelPath = os.path.join(labelFolder, labelName) #cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' % # (toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath, NO_SPLITTING, ONE_PROCESS)) cmd = ( 'time %s %s --output_dir %s --min_frame %d --max_frame %d srgb %s' % (toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath)) print cmd if icebridge_common.isValidImage(labelPath): print 'Skipping completed file: ' + labelPath else: os.system(cmd) # Also generate the map projected version of the image # Figure out the camera and output path cameraPath = get_camera(cameraFolder, frameNum) fname = os.path.basename(labelPath).replace('classified', 'classified_ortho') mapProjPath = os.path.join(orthoFolder, fname) if not icebridge_common.isValidImage(labelPath): print 'ERROR: Failed to generate label file: ' + labelPath return # Set map projection parameters toolPath = 'mapproject' isSouth = (site == 'AN') srs = projString = icebridge_common.getEpsgCode(isSouth, asString=True) demPath = 'WGS84' # Map project on to a flat surface, elevation zero. #demPath = os.path.join(refDemFolder, icebridge_common.getReferenceDemName(site)) # The NSIDC ortho DEM # Mapproject cmd = ( 'time %s %s %s %s %s -t nadirpinhole --t_srs %s --threads %d --num-processes 1 --ot Byte --nearest-neighbor' % (toolPath, demPath, labelPath, cameraPath, mapProjPath, srs, numThreads)) print cmd if icebridge_common.isValidImage(mapProjPath): print 'Skipping existing file: ' + mapProjPath else: os.system(cmd) if not os.path.exists(mapProjPath): print 'ERROR: Failed to generate map projected label file: ' + mapProjPath return
def label_images(outputFolder, frameNum, trainingPath, site, yyyymmdd, numThreads): '''Apply the labeling algorithm to a single image, then map project the result.''' print 'Running label for frame: ' + str(frameNum) # Get required paths inputFolder = icebridge_common.getJpegFolder (outputFolder) cameraFolder = icebridge_common.getCameraFolder (outputFolder) labelFolder = icebridge_common.getLabelFolder (outputFolder) orthoFolder = icebridge_common.getLabelOrthoFolder(outputFolder) # Hardcoded paths!!! toolPath = 'python ~/repo/OSSP/ossp_process.py' refDemFolder = '/nobackup/smcmich1/icebridge/reference_dems/' # Run the label tool NO_SPLITTING = 1 # Plenty of RAM to load these images ONE_PROCESS = 1 # Figure out the label path run = run_helper.RunHelper(site, yyyymmdd) labelName = icebridge_common.makeLabelFileName(run, frameNum) labelPath = os.path.join(labelFolder, labelName) #cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' % # (toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath, NO_SPLITTING, ONE_PROCESS)) cmd = ('time %s %s --output_dir %s --min_frame %d --max_frame %d srgb %s' % (toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath)) print cmd if icebridge_common.isValidImage(labelPath): print 'Skipping completed file: ' + labelPath else: os.system(cmd) # Also generate the map projected version of the image # Figure out the camera and output path cameraPath = get_camera(cameraFolder, frameNum) fname = os.path.basename(labelPath).replace('classified', 'classified_ortho') mapProjPath = os.path.join(orthoFolder, fname) if not icebridge_common.isValidImage(labelPath): print 'ERROR: Failed to generate label file: ' + labelPath return # Set map projection parameters toolPath = 'mapproject' isSouth = (site == 'AN') srs = projString = icebridge_common.getEpsgCode(isSouth, asString=True) demPath = 'WGS84' # Map project on to a flat surface, elevation zero. #demPath = os.path.join(refDemFolder, icebridge_common.getReferenceDemName(site)) # The NSIDC ortho DEM # Mapproject cmd = ('time %s %s %s %s %s -t nadirpinhole --t_srs %s --threads %d --num-processes 1 --ot Byte --nearest-neighbor' % (toolPath, demPath, labelPath, cameraPath, mapProjPath, srs, numThreads)) print cmd if icebridge_common.isValidImage(mapProjPath): print 'Skipping existing file: ' + mapProjPath else: os.system(cmd) if not os.path.exists(mapProjPath): print 'ERROR: Failed to generate map projected label file: ' + mapProjPath return
'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = []
def main(argsIn): try: usage = '''label_image.py <options>''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--training", dest="trainingPath", required=True, help="Path to the training file.") parser.add_argument( '--num-processes', dest='numProcesses', default=8, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=1, type=int, help='IGNORED.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) if not os.path.exists(options.trainingPath): print 'Error: Input training file ' + options.trainingPath + ' does not exist!' return -1 # TODO: Everything should use the RunHelper class for this! if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) # Input is raw jpeg files. inputFolder = icebridge_common.getJpegFolder(options.outputFolder) # Write all tool output to this folder. outputFolder = icebridge_common.getLabelFolder(options.outputFolder) # Do the work label_images(inputFolder, outputFolder, options.startFrame, options.stopFrame, options.trainingPath, options.numProcesses)
def main(argsIn): try: # Sample usage: # python full_processing_script.py \ # --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \ # --start-frame 350 --stop-frame 353 --skip-validate \ # --camera-calibration-folder camera_calib \ # --reference-dem-folder ref_dem_folder # An output folder will be crated automatically (with a name like # AN_20091016), or its name can be specified via the --output-folder # option. usage = '''full_processing_script.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None, help="The file to use to find which camera was used for which " + \ "flight. By default it is in the same directory as this script " + \ "and named camera_lookup.txt.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") # TODO: Compute this automatically?? parser.add_argument('--overlap-limit', dest='overlapLimit', default=2, type=int, help="The number of images to treat as overlapping for " + \ "bundle adjustment.") parser.add_argument('--stereo-arguments', dest='stereoArgs', # set --min-xcorr-level 0 to do the left-to-right # and right-to-left consistency check at the lowest level. default='--stereo-algorithm 2 --min-xcorr-level 0', help='Extra arguments to pass to stereo.') parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on.') parser.add_argument('--frames-file', dest='framesFile', default="", help='Specific frames to run ortho2pinhole on within this frame range.') parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None, type=int, help="The maximum number of lidar files to fetch. " + \ "This is used in debugging.") parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None, help="The folder containing camera calibration.") parser.add_argument("--input-calibration-camera", dest="inputCalCamera", default="", help="Instead of looking up the calibrated camera in the calibration folder, use this one.") parser.add_argument("--output-calibration-camera", dest="outputCalCamera", default="", help="If specified, float the intrinsics and write the optimized model here.") parser.add_argument("--output-model-type", dest="outputModelType", default="RPC", help="Generate a distortion model of type RPC, RPC5, or RPC6.") parser.add_argument("--reference-dem-folder", dest="refDemFolder", default=None, help="The folder containing DEMs that created orthoimages.") parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. " + \ "fault is no additional folder") parser.add_argument("--simple-cameras", action="store_true", dest="simpleCameras", default=False, help="Don't use orthoimages to refine the camera models.") # This option is only needed when generating camera models from the nav files. parser.add_argument('--camera-mounting', default=0, dest='cameraMounting', type=int, help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.') # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-ortho-processes', dest='numOrthoProcesses', default=-1, type=int, help='The number of simultaneous ortho processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') # Action control parser.add_argument("--skip-fetch", action="store_true", dest="noFetch", default=False, help="Skip data fetching.") parser.add_argument("--skip-convert", action="store_true", dest="noConvert", default=False, help="Skip data conversion.") parser.add_argument("--stop-after-fetch", action="store_true", dest="stopAfterFetch", default=False, help="Stop program after data fetching.") parser.add_argument("--stop-after-convert", action="store_true", dest="stopAfterConvert", default=False, help="Stop program after data conversion.") parser.add_argument("--skip-validate", action="store_true", dest="skipValidate", default=False, help="Skip input data validation.") parser.add_argument("--ignore-missing-lidar", action="store_true", dest="ignoreMissingLidar", default=False, help="Keep going if the lidar is missing.") parser.add_argument("--log-batches", action="store_true", dest="logBatches", default=False, help="Log the required batch commands without running them.") parser.add_argument('--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_argument('--many-ip', action='store_true', default=False, dest='manyip', help='If to use a lot of IP in bundle adjustment from the beginning.') parser.add_argument("--dry-run", action="store_true", dest="dryRun", default=False, help="Set up the input directories but do not fetch/process any imagery.") parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False, help="Try fetching again if some files turned out invalid " + \ "during conversions.") parser.add_argument("--refetch-index", action="store_true", dest="refetchIndex", default=False, help="Force refetch of the index file.") parser.add_argument("--refetch-nav", action="store_true", dest="refetchNav", default=False, help="Force refetch of the nav file.") parser.add_argument("--stop-after-index-fetch", action="store_true", dest="stopAfterIndexFetch", default=False, help="Stop after fetching the indices.") parser.add_argument("--no-nav", action="store_true", dest="noNavFetch", default=False, help="Don't fetch or convert the nav data.") parser.add_argument("--no-lidar-convert", action="store_true", dest="noLidarConvert", default=False, help="Skip lidar files in the conversion step.") parser.add_argument("--no-ortho-convert", action="store_true", dest="noOrthoConvert", default=False, help="Skip generating camera models in the conversion step.") parser.add_argument("--skip-fast-conversions", action="store_true", dest="skipFastConvert", default=False, help="Skips all non-ortho conversions.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if options.numOrthoProcesses < 0: options.numOrthoProcesses = options.numProcesses isSouth = icebridge_common.checkSite(options.site) # Turned off elevation limits here since they are being set from LIDAR data. ## Add the site based elevation limits to the stereoArgs option #altLimits = icebridge_common.getElevationLimits(options.site) #options.stereoArgs = (' %s --elevation-limit %f %f ' # % (options.stereoArgs, altLimits[0], altLimits[1])) options.stereoArgs = (' %s ' % (options.stereoArgs)) if options.cameraLookupFile is None: options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt') if not os.path.isfile(options.cameraLookupFile): raise Exception("Can't find camera file: " + options.cameraLookupFile) if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) if options.stopAfterIndexFetch: options.stopAfterFetch = True os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Record everything logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_processing_log_frames_' + \ str(options.startFrame) + "_" + str(options.stopFrame)) # Make sure we later know what we were doing logger.info("full_processing_script.py " + " ".join(argsIn)) (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) logger.info("Work dir is " + os.getcwd()) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Perform some input checks and initializations # These are not needed unless cameras are initialized if options.inputCalFolder is None or not os.path.exists(options.inputCalFolder): raise Exception("Missing camera calibration folder.") if options.refDemFolder is None or not os.path.exists(options.refDemFolder): raise Exception("Missing reference DEM folder.") refDemName = icebridge_common.getReferenceDemName(options.site) refDemPath = os.path.join(options.refDemFolder, refDemName) if not os.path.exists(refDemPath): raise Exception("Missing reference DEM: " + refDemPath) # TODO: CLEAN UP!!! # Set up the output folders cameraFolder = icebridge_common.getCameraFolder(options.outputFolder) imageFolder = icebridge_common.getImageFolder(options.outputFolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) fireballFolder = icebridge_common.getFireballFolder(options.outputFolder) corrFireballFolder = icebridge_common.getCorrFireballFolder(options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) navFolder = icebridge_common.getNavFolder(options.outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder) processedFolder = icebridge_common.getProcessedFolder(options.outputFolder) if options.outputCalCamera != "": # Prepare to solve for intrinsics. Note that this modifies some things along the way. (options, cameraFolder, navCameraFolder, processedFolder) = \ solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder, processedFolder, logger) # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processedFolder = os.path.join(processedFolder, options.processingSubfolder) logger.info('Will write to processing subfolder: ' + options.processingSubfolder) # If something failed in the first attempt either in fetch or in # convert, we will wipe bad files, and try to refetch/re-convert. numAttempts = 1 if options.reFetch and (not options.noFetch): numAttempts = 2 for attempt in range(numAttempts): if numAttempts > 1: logger.info("Fetch/convert attempt: " + str(attempt+1)) ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder, jpegFolder, orthoFolder, fireballFolder, corrFireballFolder, lidarFolder, processedFolder, navFolder, navCameraFolder, refDemPath, logger) if ans == 0: break if options.stopAfterFetch or options.dryRun or options.stopAfterConvert: logger.info('Fetch/convert finished!') return 0 # Call the processing routine processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, corrFireballFolder, processedFolder, isSouth, refDemPath) if options.outputCalCamera != "": # Finish solving for intrinscs. solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, processedFolder, isSouth, logger)
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder): '''Create a list of all files that must be fetched unless done already.''' # For AN 20091112, etc, some of the ortho images are stored at the # beginning of the next day's flight. Need to sort this out, and # it is tricky. More comments within the code. fetchNextDay = True separateByLat = (options.type == 'ortho' and isInSeparateByLatTable(options.yyyymmdd)) if separateByLat: # Here we won't fetch the next day, we will just separate by latitude within # a given day fetchNextDay = False orthoOrFireball = ((options.type == 'ortho') or (options.type == 'fireball')) if fetchNextDay: # Normally we fetch for next day only for ortho or fireball. However, # for one single special flight, we do it for jpeg too, as then # the jpegs are also split. if orthoOrFireball or \ ((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)): fetchNextDay = True else: fetchNextDay = False # If we need to parse the next flight day as well, as expected in some runs, # we will fetch two html files, but create a single index out of them. dayVals = [0] if fetchNextDay: dayVals.append(1) indexPath = icebridge_common.htmlIndexFile(outputFolder) currIndexPath = indexPath parsedIndexPath = icebridge_common.csvIndexFile(outputFolder) if options.refetchIndex: os.system('rm -f ' + indexPath) os.system('rm -f ' + parsedIndexPath) if icebridge_common.fileNonEmpty(parsedIndexPath): logger.info('Already have the index file ' + parsedIndexPath + ', keeping it.') return parsedIndexPath frameDict = {} urlDict = {} # We need the list of jpeg frames. Sometimes when fetching ortho images, # and we have to fetch from the next day, don't fetch unless # in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': jpegFolder = icebridge_common.getJpegFolder( os.path.dirname(outputFolder)) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath) orthoStamp = {} if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. orthoFolder = icebridge_common.getOrthoFolder( os.path.dirname(outputFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) for frame in sorted(orthoFrameDict.keys()): filename = orthoFrameDict[frame] [imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename) orthoStamp[frame] = imageTimeString for dayVal in dayVals: if len(dayVals) > 1: currIndexPath = indexPath + '.day' + str(dayVal) if options.refetchIndex: os.system('rm -f ' + currIndexPath) # Find folderUrl which contains all of the files if options.type in LIDAR_TYPES: options.allFrames = True # For lidar, always get all the frames! # For lidar, the data can come from one of three sources. # Unfortunately sometimes there is more than one source, and then # we need to pick by latitude. folderUrls = [] lidar_types = [] for lidar in LIDAR_TYPES: folderUrl = getFolderUrl( options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, lidar) logger.info('Checking lidar URL: ' + folderUrl) if checkIfUrlExists(folderUrl): logger.info('Found match with lidar type: ' + lidar) folderUrls.append(folderUrl) lidar_types.append(lidar) if len(folderUrls) == 0: logger.info( 'WARNING: Could not find any lidar data for the given date!' ) elif len(folderUrls) == 1: # Unique solution folderUrl = folderUrls[0] options.type = lidar_types[0] elif len(folderUrls) >= 2: # Multiple solutions. Pick the good one by latitude. logger.info("Multiples URLs to search: " + " ".join(folderUrls)) count = -1 isGood = False for folderUrl in folderUrls: count += 1 (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, lidar_types[count]) for frame in sorted(localFrameDict.keys()): filename = localFrameDict[frame] xmlFile = icebridge_common.xmlFile(filename) url = os.path.join(folderUrl, xmlFile) # Download the file curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile logger.info(curlCmd) p = subprocess.Popen(curlCmd, shell=True) os.waitpid(p.pid, 0) latitude = icebridge_common.parseLatitude(xmlFile) if os.path.exists(xmlFile): os.remove(xmlFile) if hasGoodLat(latitude, isSouth): isGood = True options.type = lidar_types[count] logger.info("Good latitude " + str(latitude) + ", will use " + folderUrl + " of type " + lidar_types[count]) else: logger.info("Bad latitude " + str(latitude) + ", will not use " + folderUrl + " of type " + lidar_types[count]) # Stop at first file no matter what break if isGood: break if not isGood: if options.type in LIDAR_TYPES and options.ignoreMissingLidar: logger.info("No lidar. None of these URLs are good: " + " ".join(folderUrls)) else: raise Exception("None of these URLs are good: " + " ".join(folderUrls)) else: # Other cases are simpler folderUrl = getFolderUrl( options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, options.type) logger.info('Fetching from URL: ' + folderUrl) (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, options.type) # Append to the main index for frame in sorted(localFrameDict.keys()): if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. # Otherwise we may accidentally getting one from next day. [imageDateString, imageTimeString] = \ icebridge_common.parseTimeStamps(localFrameDict[frame]) if frame not in orthoStamp: #logger.info("Missing ortho for fireball: " + localFrameDict[frame]) continue if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000: # Apparently a tolerance is needed. Use 10 seconds, so the number 1000. #logger.info("Will not use fireball DEM whose timestamp differs from ortho.") #logger.info("Fireball is: " + localFrameDict[frame]) #logger.info("Ortho is: " + orthoFrameDict[frame]) continue # Fetch from next day, unless already have a value. And don't fetch # frames not in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': if not frame in jpegFrameDict.keys(): continue if frame in frameDict.keys(): continue frameDict[frame] = localFrameDict[frame] urlDict[frame] = localUrlDict[frame] # Write the combined index file icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict) return parsedIndexPath
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument( '--camera-mounting', dest='cameraMounting', default=0, type=int, help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder): '''Create a list of all files that must be fetched unless done already.''' # For AN 20091112, etc, some of the ortho images are stored at the # beginning of the next day's flight. Need to sort this out, and # it is tricky. More comments within the code. fetchNextDay = True separateByLat = (options.type == 'ortho' and isInSeparateByLatTable(options.yyyymmdd)) if separateByLat: # Here we won't fetch the next day, we will just separate by latitude within # a given day fetchNextDay = False orthoOrFireball = ( (options.type == 'ortho') or (options.type == 'fireball') ) if fetchNextDay: # Normally we fetch for next day only for ortho or fireball. However, # for one single special flight, we do it for jpeg too, as then # the jpegs are also split. if orthoOrFireball or \ ((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)): fetchNextDay = True else: fetchNextDay = False # If we need to parse the next flight day as well, as expected in some runs, # we will fetch two html files, but create a single index out of them. dayVals = [0] if fetchNextDay: dayVals.append(1) indexPath = icebridge_common.htmlIndexFile(outputFolder) currIndexPath = indexPath parsedIndexPath = icebridge_common.csvIndexFile(outputFolder) if options.refetchIndex: os.system('rm -f ' + indexPath) os.system('rm -f ' + parsedIndexPath) if icebridge_common.fileNonEmpty(parsedIndexPath): logger.info('Already have the index file ' + parsedIndexPath + ', keeping it.') return parsedIndexPath frameDict = {} urlDict = {} # We need the list of jpeg frames. Sometimes when fetching ortho images, # and we have to fetch from the next day, don't fetch unless # in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': jpegFolder = icebridge_common.getJpegFolder(os.path.dirname(outputFolder)) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath) orthoStamp = {} if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(outputFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) for frame in sorted(orthoFrameDict.keys()): filename = orthoFrameDict[frame] [imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename) orthoStamp[frame] = imageTimeString for dayVal in dayVals: if len(dayVals) > 1: currIndexPath = indexPath + '.day' + str(dayVal) if options.refetchIndex: os.system('rm -f ' + currIndexPath) # Find folderUrl which contains all of the files if options.type in LIDAR_TYPES: options.allFrames = True # For lidar, always get all the frames! # For lidar, the data can come from one of three sources. # Unfortunately sometimes there is more than one source, and then # we need to pick by latitude. folderUrls = [] lidar_types = [] for lidar in LIDAR_TYPES: folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, lidar) logger.info('Checking lidar URL: ' + folderUrl) if checkIfUrlExists(folderUrl, baseCurlCmd): logger.info('Found match with lidar type: ' + lidar) folderUrls.append(folderUrl) lidar_types.append(lidar) if len(folderUrls) == 0: logger.info('WARNING: Could not find any lidar data for the given date!') elif len(folderUrls) == 1: # Unique solution folderUrl = folderUrls[0] options.type = lidar_types[0] elif len(folderUrls) >= 2: # Multiple solutions. Pick the good one by latitude. logger.info("Multiples URLs to search: " + " ".join(folderUrls)) count = -1 isGood = False for folderUrl in folderUrls: count += 1 (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, lidar_types[count]) for frame in sorted(localFrameDict.keys()): filename = localFrameDict[frame] xmlFile = icebridge_common.xmlFile(filename) url = os.path.join(folderUrl, xmlFile) # Download the file curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile logger.info(curlCmd) p = subprocess.Popen(curlCmd, shell=True, universal_newlines=True) os.waitpid(p.pid, 0) latitude = icebridge_common.parseLatitude(xmlFile) if os.path.exists(xmlFile): os.remove(xmlFile) if hasGoodLat(latitude, isSouth): isGood = True options.type = lidar_types[count] logger.info("Good latitude " + str(latitude) + ", will use " + folderUrl + " of type " + lidar_types[count]) else: logger.info("Bad latitude " + str(latitude) + ", will not use " + folderUrl + " of type " + lidar_types[count]) # Stop at first file no matter what break if isGood: break if not isGood: if options.type in LIDAR_TYPES and options.ignoreMissingLidar: logger.info("No lidar. None of these URLs are good: " + " ".join(folderUrls)) else: raise Exception("None of these URLs are good: " + " ".join(folderUrls)) else: # Other cases are simpler folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, options.type) logger.info('Fetching from URL: ' + folderUrl) (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, options.type) # Append to the main index for frame in sorted(localFrameDict.keys()): if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. # Otherwise we may accidentally getting one from next day. [imageDateString, imageTimeString] = \ icebridge_common.parseTimeStamps(localFrameDict[frame]) if frame not in orthoStamp: #logger.info("Missing ortho for fireball: " + localFrameDict[frame]) continue if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000: # Apparently a tolerance is needed. Use 10 seconds, so the number 1000. #logger.info("Will not use fireball DEM whose timestamp differs from ortho.") #logger.info("Fireball is: " + localFrameDict[frame]) #logger.info("Ortho is: " + orthoFrameDict[frame]) continue # Fetch from next day, unless already have a value. And don't fetch # frames not in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': if not frame in jpegFrameDict.keys(): continue if frame in frameDict.keys(): continue frameDict[frame] = localFrameDict[frame] urlDict[frame] = localUrlDict[frame] # Write the combined index file icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict) return parsedIndexPath
def main(argsIn): try: # Sample usage: # python full_processing_script.py \ # --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \ # --start-frame 350 --stop-frame 353 --skip-validate \ # --camera-calibration-folder camera_calib \ # --reference-dem-folder ref_dem_folder # An output folder will be crated automatically (with a name like # AN_20091016), or its name can be specified via the --output-folder # option. usage = '''full_processing_script.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None, help="The file to use to find which camera was used for which " + \ "flight. By default it is in the same directory as this script " + \ "and named camera_lookup.txt.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") # TODO: Compute this automatically?? parser.add_argument('--overlap-limit', dest='overlapLimit', default=2, type=int, help="The number of images to treat as overlapping for " + \ "bundle adjustment.") parser.add_argument( '--max-overlap-ratio', dest='maxOverlapRatio', default=0.85, type=float, help= 'The maximum ratio of overlap between images to be accepted as part of a stereo pair. When floating intrinsics, this will be set to 1, to not upset some bookkeeping.' ) parser.add_argument( '--stereo-arguments', dest='stereoArgs', # set --min-xcorr-level 0 to do the left-to-right # and right-to-left consistency check at the lowest level. default='--stereo-algorithm 2 --min-xcorr-level 0', help='Extra arguments to pass to stereo.') parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on.') parser.add_argument( '--frames-file', dest='framesFile', default="", help= 'Specific frames to run ortho2pinhole on within this frame range.') parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None, type=int, help="The maximum number of lidar files to fetch. " + \ "This is used in debugging.") parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None, help="The folder containing camera calibration.") parser.add_argument( "--input-calibration-camera", dest="inputCalCamera", default="", help= "Instead of looking up the calibrated camera in the calibration folder, use this one." ) parser.add_argument( "--output-calibration-camera", dest="outputCalCamera", default="", help= "If specified, float the intrinsics and write the optimized model here." ) parser.add_argument( "--output-model-type", dest="outputModelType", default="RPC", help="Generate a distortion model of type RPC, RPC5, or RPC6.") parser.add_argument( "--reference-dem-folder", dest="refDemFolder", default=None, help="The folder containing DEMs that created orthoimages.") parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. " + \ "fault is no additional folder") parser.add_argument( "--simple-cameras", action="store_true", dest="simpleCameras", default=False, help="Don't use orthoimages to refine the camera models.") # This option is only needed when generating camera models from the nav files. parser.add_argument( '--camera-mounting', default=0, dest='cameraMounting', type=int, help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument( '--num-ortho-processes', dest='numOrthoProcesses', default=-1, type=int, help='The number of simultaneous ortho processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') # Action control parser.add_argument("--skip-fetch", action="store_true", dest="noFetch", default=False, help="Skip data fetching.") parser.add_argument("--skip-convert", action="store_true", dest="noConvert", default=False, help="Skip data conversion.") parser.add_argument("--stop-after-fetch", action="store_true", dest="stopAfterFetch", default=False, help="Stop program after data fetching.") parser.add_argument("--stop-after-convert", action="store_true", dest="stopAfterConvert", default=False, help="Stop program after data conversion.") parser.add_argument("--skip-validate", action="store_true", dest="skipValidate", default=False, help="Skip input data validation.") parser.add_argument("--ignore-missing-lidar", action="store_true", dest="ignoreMissingLidar", default=False, help="Keep going if the lidar is missing.") parser.add_argument( "--log-batches", action="store_true", dest="logBatches", default=False, help="Log the required batch commands without running them.") parser.add_argument( '--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_argument( '--many-ip', action='store_true', default=False, dest='manyip', help= 'If to use a lot of IP in bundle adjustment from the beginning.') parser.add_argument( "--dry-run", action="store_true", dest="dryRun", default=False, help= "Set up the input directories but do not fetch/process any imagery." ) parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False, help="Try fetching again if some files turned out invalid " + \ "during conversions.") parser.add_argument("--refetch-index", action="store_true", dest="refetchIndex", default=False, help="Force refetch of the index file.") parser.add_argument("--refetch-nav", action="store_true", dest="refetchNav", default=False, help="Force refetch of the nav file.") parser.add_argument("--stop-after-index-fetch", action="store_true", dest="stopAfterIndexFetch", default=False, help="Stop after fetching the indices.") parser.add_argument("--no-nav", action="store_true", dest="noNavFetch", default=False, help="Don't fetch or convert the nav data.") parser.add_argument("--no-lidar-convert", action="store_true", dest="noLidarConvert", default=False, help="Skip lidar files in the conversion step.") parser.add_argument( "--no-ortho-convert", action="store_true", dest="noOrthoConvert", default=False, help="Skip generating camera models in the conversion step.") parser.add_argument("--skip-fast-conversions", action="store_true", dest="skipFastConvert", default=False, help="Skips all non-ortho conversions.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if options.numOrthoProcesses < 0: options.numOrthoProcesses = options.numProcesses isSouth = icebridge_common.checkSite(options.site) # Turned off elevation limits here since they are being set from LIDAR data. ## Add the site based elevation limits to the stereoArgs option #altLimits = icebridge_common.getElevationLimits(options.site) #options.stereoArgs = (' %s --elevation-limit %f %f ' # % (options.stereoArgs, altLimits[0], altLimits[1])) options.stereoArgs = (' %s ' % (options.stereoArgs)) if options.cameraLookupFile is None: options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt') if not os.path.isfile(options.cameraLookupFile): raise Exception("Can't find camera file: " + options.cameraLookupFile) if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) if options.stopAfterIndexFetch: options.stopAfterFetch = True os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Record everything logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_processing_log_frames_' + \ str(options.startFrame) + "_" + str(options.stopFrame)) # Make sure we later know what we were doing logger.info("full_processing_script.py " + " ".join(argsIn)) (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) logger.info("Work dir is " + os.getcwd()) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Perform some input checks and initializations # These are not needed unless cameras are initialized if options.inputCalFolder is None or not os.path.exists( options.inputCalFolder): raise Exception("Missing camera calibration folder.") if options.refDemFolder is None or not os.path.exists( options.refDemFolder): raise Exception("Missing reference DEM folder.") refDemName = icebridge_common.getReferenceDemName(options.site) refDemPath = os.path.join(options.refDemFolder, refDemName) if not os.path.exists(refDemPath): raise Exception("Missing reference DEM: " + refDemPath) # TODO: CLEAN UP!!! # Set up the output folders cameraFolder = icebridge_common.getCameraFolder(options.outputFolder) imageFolder = icebridge_common.getImageFolder(options.outputFolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) fireballFolder = icebridge_common.getFireballFolder(options.outputFolder) corrFireballFolder = icebridge_common.getCorrFireballFolder( options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) navFolder = icebridge_common.getNavFolder(options.outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder) processedFolder = icebridge_common.getProcessedFolder(options.outputFolder) if options.outputCalCamera != "": if options.maxOverlapRatio < 1: raise Exception ("For optimizing intrinsics, must set --max-overlap-ratio to 1, " + \ "to always use consecutive frames.") # Prepare to solve for intrinsics. Note that this modifies some things along the way. (options, cameraFolder, navCameraFolder, processedFolder) = \ solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder, processedFolder, logger) # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processedFolder = os.path.join(processedFolder, options.processingSubfolder) logger.info('Will write to processing subfolder: ' + options.processingSubfolder) # If something failed in the first attempt either in fetch or in # convert, we will wipe bad files, and try to refetch/re-convert. numAttempts = 1 if options.reFetch and (not options.noFetch): numAttempts = 2 for attempt in range(numAttempts): if numAttempts > 1: logger.info("Fetch/convert attempt: " + str(attempt + 1)) ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder, jpegFolder, orthoFolder, fireballFolder, corrFireballFolder, lidarFolder, processedFolder, navFolder, navCameraFolder, refDemPath, logger) if ans == 0: break if options.stopAfterFetch or options.dryRun or options.stopAfterConvert: logger.info('Fetch/convert finished!') return 0 # Call the processing routine processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, corrFireballFolder, processedFolder, isSouth, refDemPath) if options.outputCalCamera != "": # Finish solving for intrinscs. solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, processedFolder, isSouth, logger)
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument('--camera-mounting', dest='cameraMounting', default=0, type=int, help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)