def lidarFilesInRange(lidarDict, lidarFolder, startFrame, stopFrame): '''Fetch only lidar files for the given frame range. Do that as follows. For each ortho frame in [startFrame, stopFrame], find the lidar file with the closest timestamp. Collect them all. Add the two neighboring ones, to help with finding lidar pairs later.''' lidarList = [] for frame in sorted(lidarDict.keys()): lidarList.append(lidarDict[frame]) # If we requested all frames, also get all the lidar files. if ((startFrame == icebridge_common.getSmallestFrame()) and (stopFrame == icebridge_common.getLargestFrame())): minLidarIndex = 0 maxLidarIndex = len(lidarList) - 1 else: minLidarIndex = len(lidarList) maxLidarIndex = 0 # Build up a list of lidar files that match the requested input frames orthoFolder = icebridge_common.getOrthoFolder( os.path.dirname(lidarFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) for frame in sorted(orthoFrameDict.keys()): if ((frame < startFrame) or (frame > stopFrame)): continue orthoFrame = orthoFrameDict[frame] try: matchingLidar = icebridge_common.findMatchingLidarFileFromList( orthoFrame, lidarList) except: # Some image files don't have a matching lidar file, just keep going. continue for index in range(len(lidarList)): if lidarList[index] == matchingLidar: if minLidarIndex > index: minLidarIndex = index if maxLidarIndex < index: maxLidarIndex = index # We will fetch neighboring lidar files as well if minLidarIndex > 0: minLidarIndex = minLidarIndex - 1 if maxLidarIndex + 1 < len(lidarList): maxLidarIndex = maxLidarIndex + 1 lidarsToFetch = set() if lidarList: for index in range(minLidarIndex, maxLidarIndex + 1): # Fetch only the requested lidar files. lidarsToFetch.add(lidarList[index]) return lidarsToFetch
def lidarFilesInRange(lidarDict, lidarFolder, startFrame, stopFrame): '''Fetch only lidar files for the given frame range. Do that as follows. For each ortho frame in [startFrame, stopFrame], find the lidar file with the closest timestamp. Collect them all. Add the two neighboring ones, to help with finding lidar pairs later.''' lidarList = [] for frame in sorted(lidarDict.keys()): lidarList.append(lidarDict[frame]) # If we requested all frames, also get all the lidar files. if ((startFrame == icebridge_common.getSmallestFrame()) and (stopFrame == icebridge_common.getLargestFrame() ) ): minLidarIndex = 0 maxLidarIndex = len(lidarList)-1 else: minLidarIndex = len(lidarList) maxLidarIndex = 0 # Build up a list of lidar files that match the requested input frames orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(lidarFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) for frame in sorted(orthoFrameDict.keys()): if ((frame < startFrame) or (frame > stopFrame) ): continue orthoFrame = orthoFrameDict[frame] try: matchingLidar = icebridge_common.findMatchingLidarFileFromList(orthoFrame, lidarList) except: # Some image files don't have a matching lidar file, just keep going. continue for index in range(len(lidarList)): if lidarList[index] == matchingLidar: if minLidarIndex > index: minLidarIndex = index if maxLidarIndex < index: maxLidarIndex = index # We will fetch neighboring lidar files as well if minLidarIndex > 0: minLidarIndex = minLidarIndex -1 if maxLidarIndex + 1 < len(lidarList): maxLidarIndex = maxLidarIndex + 1 lidarsToFetch = set() if lidarList: for index in range(minLidarIndex, maxLidarIndex+1): # Fetch only the requested lidar files. lidarsToFetch.add(lidarList[index]) return lidarsToFetch
def allSourceDataFetched(self, noNav, verbose=False): '''Return true if all the required source data has been downloaded''' logger = logging.getLogger(__name__) # Old file collections use a different naming scheme altIndexName = 'image_index.html.csv' # Verify these input folders subFolders = [ self.getJpegFolder(), self.getLidarFolder(), self.getOrthoFolder() ] for folder in subFolders: # Make sure all the files specified in the parsed index file are present indexFile = icebridge_common.csvIndexFile(folder) if not os.path.exists(indexFile): # See if the older naming scheme was used altPath = os.path.join(folder, altIndexName) if os.path.exists(altPath): indexFile = altPath (fileDict, urlDict) = icebridge_common.readIndexFile(indexFile) logger.info("Checking all files listed in: " + indexFile) num = len(fileDict.keys()) count = 0 for f in fileDict.itervalues(): # Add the progress, as this operation can be terribly slow # when the filesystem is not doing too well, especially on mfe. count = count + 1 if (count - 1) % 1000 == 0: logger.info('Progress: ' + str(count) + '/' + str(num)) path = os.path.join(folder, f) if not os.path.exists(path): logger.error('Missing file ' + path) return False if not noNav: # Simple nav file check navFiles = os.listdir(self.getNavFolder()) navFiles = [x for x in navFiles if '.out' in navFiles] if (len(navFiles) == 0) or (len(navFiles) > 2): logger.error('Wrong number of nav files detected!') return False return True # Success!
def getFrameRange(self): '''Return the min and max frame currently stored for the run''' jpegFolder = icebridge_common.getJpegFolder(self.getFolder()) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath) frames = sorted(jpegFrameDict.keys()) if len(frames) == 0: raise Exception("Empty folder: " + jpegFolder) return (frames[0], frames[-1])
def allSourceDataFetched(self, noNav, verbose=False): '''Return true if all the required source data has been downloaded''' logger = logging.getLogger(__name__) # Old file collections use a different naming scheme altIndexName = 'image_index.html.csv' # Verify these input folders subFolders = [self.getJpegFolder(), self.getLidarFolder(), self.getOrthoFolder()] for folder in subFolders: # Make sure all the files specified in the parsed index file are present indexFile = icebridge_common.csvIndexFile(folder) if not os.path.exists(indexFile): # See if the older naming scheme was used altPath = os.path.join(folder, altIndexName) if os.path.exists(altPath): indexFile = altPath (fileDict, urlDict) = icebridge_common.readIndexFile(indexFile) logger.info("Checking all files listed in: " + indexFile) num = len(fileDict.keys()) count = 0 for f in fileDict.itervalues(): # Add the progress, as this operation can be terribly slow # when the filesystem is not doing too well, especially on mfe. count = count + 1 if (count - 1) % 1000 == 0: logger.info('Progress: ' + str(count) + '/' + str(num)) path = os.path.join(folder, f) if not os.path.exists(path): logger.error('Missing file ' + path) return False if not noNav: # Simple nav file check navFiles = os.listdir(self.getNavFolder()) navFiles = [x for x in navFiles if '.out' in navFiles] if (len(navFiles) == 0) or (len(navFiles) > 2): logger.error('Wrong number of nav files detected!') return False return True # Success!
def massRename(self, startFrame, stopFrame, logger): '''We changed how the timestamp for images and cameras is computed. Make all existing converted images, cameras, nav cameras, and aligned cameras conform.''' logger.info("Renaming files with timestamp. This is slow.") # Need to do a mass rename for incorrect timestamp for: # converted images, nav cameras, cameras, and bundle aligned cameras outputFolder = self.getFolder() cameraFolder = icebridge_common.getCameraFolder(outputFolder) imageFolder = icebridge_common.getImageFolder(outputFolder) jpegFolder = icebridge_common.getJpegFolder(outputFolder) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) processedFolder = icebridge_common.getProcessedFolder(outputFolder) navFolder = icebridge_common.getNavFolder(outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder( os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) logger.info('Renaming camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(cameraFolder, '*DMS*tsai'), logger) logger.info('Renaming nav camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(navCameraFolder, '*DMS*tsai'), logger) logger.info('Renaming converted images...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(imageFolder, '*DMS*tif'), logger) logger.info('Renaming aligned cameras...') self.massRenameByGlob( startFrame, stopFrame, orthoFrameDict, os.path.join(processedFolder, 'batch*', icebridge_common.alignedBundleStr() + '*DMS*tsai'), logger)
def massRename(self, startFrame, stopFrame, logger): '''We changed how the timestamp for images and cameras is computed. Make all existing converted images, cameras, nav cameras, and aligned cameras conform.''' logger.info("Renaming files with timestamp. This is slow.") # Need to do a mass rename for incorrect timestamp for: # converted images, nav cameras, cameras, and bundle aligned cameras outputFolder = self.getFolder() cameraFolder = icebridge_common.getCameraFolder(outputFolder) imageFolder = icebridge_common.getImageFolder(outputFolder) jpegFolder = icebridge_common.getJpegFolder(outputFolder) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) processedFolder = icebridge_common.getProcessedFolder(outputFolder) navFolder = icebridge_common.getNavFolder(outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) logger.info('Renaming camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(cameraFolder, '*DMS*tsai'), logger) logger.info('Renaming nav camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(navCameraFolder, '*DMS*tsai'), logger) logger.info('Renaming converted images...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(imageFolder, '*DMS*tif'), logger) logger.info('Renaming aligned cameras...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(processedFolder, 'batch*', icebridge_common.alignedBundleStr() + '*DMS*tsai'), logger)
def lidarFilesInRange(lidarDict, lidarFolder, startFrame, stopFrame): '''Fetch only lidar files for the given frame range. Do that as follows. ''' '''For each ortho frame in [startFrame, stopFrame], find the lidar ''' '''file with the closest timestamp. Collect them all. ''' '''Add the two neighboring ones, to help with finding lidar pairs later. ''' lidarList = [] for frame in sorted(lidarDict.keys()): lidarList.append(lidarDict[frame]) orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(lidarFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) minLidarIndex = len(lidarList) maxLidarIndex = 0 for frame in sorted(orthoFrameDict.keys()): if ((frame < startFrame) or (frame > stopFrame)): continue orthoFrame = orthoFrameDict[frame] matchingLidar = icebridge_common.findMatchingLidarFileFromList( orthoFrame, lidarList) for index in range(len(lidarList)): if lidarList[index] == matchingLidar: if minLidarIndex > index: minLidarIndex = index if maxLidarIndex < index: maxLidarIndex = index # We will fetch neighboring lidar files as well if minLidarIndex > 0: minLidarIndex = minLidarIndex - 1 if maxLidarIndex + 1 < len(lidarList): maxLidarIndex = maxLidarIndex + 1 lidarsToFetch = set() for index in range(minLidarIndex, maxLidarIndex + 1): lidarsToFetch.add(lidarList[index]) return lidarsToFetch
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList(os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder = True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame]) except Exception as e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90 ' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90 ' cmd = ('%s %s -colorspace Gray %s%s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE|re.MULTILINE|re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) if badFiles: logger.error("Converstion of JPEGs failed. If any files were corrupted, " + "they were removed, and need to be re-fetched.") return (not badFiles)
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 10 usage = '''blend_dems.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") parser.add_argument("--compute-diff-to-prev-dem", action="store_true", dest="computeDiffToPrev", default=False, help="Compute the absolute difference between the current DEM " + "and the one before it.") parser.add_argument("--blend-to-fireball-footprint", action="store_true", dest="blendToFireball", default=False, help="Create additional blended DEMs having the same " + \ "footprint as Fireball DEMs.") # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_blend_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) logger.info(str(argsIn)) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if options.blendToFireball: fireballFrameDict = icebridge_common.getCorrectedFireballDems(options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(orthoFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No ortho files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in orthoFrameDict: logger.info("Error: Missing ortho file for frame: " + str(frame) + ".") continue orthoFile = orthoFrameDict[frame] try: lidarFile = icebridge_common.findMatchingLidarFile(orthoFile, lidarFolder) except: # Skip if no lidar file matches this frame continue fireballDEM = "" if options.blendToFireball: if frame in fireballFrameDict: fireballDEM = fireballFrameDict[frame] else: logger.info("No fireball DEM for frame: " + str(frame)) args = (frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runBlend, args)) else: runBlend(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def pushByType(run, options, logger, dataType): # Fetch the ortho index from NSIDC if missing outputFolder = run.getFolder() logger.info("Output folder is " + outputFolder) os.system("mkdir -p " + outputFolder) # Current directory. It is important to go from /u to the real dir which is /nobackup... unpackDir = os.path.realpath(os.getcwd()) logger.info("Unpack directory is " + unpackDir) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): fetchIndices(options, logger) logger.info("Reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) # Fetch unarchived folder if missing if dataType == 'DEM': unarchivedFolder = run.getAssemblyFolder() elif dataType == 'ORTHO': unarchivedFolder = run.getProcessFolder() else: raise Exception("Unknown data type: " + dataType) logger.info("Unarchived data folder is " + unarchivedFolder) # Especially for ortho, force-fetch each time, as there is no good way # of checking if we fetched well before. start_time() if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType): return stop_time("fetching archived data by type: " + dataType, logger) # Make the output directory at NSIDC m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd) if m: outDir = options.site + "_" + m.group(1) + "." + m.group( 2) + "." + m.group(3) else: raise Exception("Could not parse: " + options.yyyymmdd) # Keep the output directory locally here localDirPath = os.path.join(outputFolder, dataType, outDir) os.system("mkdir -p " + localDirPath) logger.info("Storing the renamed " + dataType + " files in " + localDirPath) logger.info("Directory name at NSIDC: " + outDir) # Read the DEMs and orthos, and copy them to outDir according to the final convention if dataType == 'DEM': dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True) else: dataFiles = glob.glob( os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif')) for dataFile in dataFiles: # Here we use the convention from archive_functions.py for DEMs and from how we store orthos. if dataType == 'DEM': m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \ "\.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) else: m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \ "out-ortho.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) if frameNumber < options.startFrame or frameNumber > options.stopFrame: continue # For each data file, copy from the ortho its meta info if not frameNumber in orthoFrameDict.keys(): # Bugfix: Ortho fetching failed, try again fetchIndices(options, logger) logger.info("Re-reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if not frameNumber in orthoFrameDict.keys(): # This time there is nothing we can do raise Exception("Cannot find ortho for frame: " + str(frameNumber)) orthoFile = orthoFrameDict[frameNumber] [dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile) # It is always possible that the ortho file date will be the next day # after the current flight date, if the flight goes after midnight. # So it is not unreasonable that options.yyyymmdd != dateString. if dataType == 'DEM': outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber)) else: # TODO: Need to think more of the naming convention. outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber)) cmd = "/bin/cp -fv " + dataFile + " " + os.path.join( localDirPath, outFile) logger.info(cmd) os.system(cmd) # Push the directory to NSIDC remoteDirPath = os.path.join( os.path.basename(os.path.dirname(localDirPath)), os.path.basename(localDirPath)) remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath) logger.info("Storing at NSIDC in: " + remoteDirPath) cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \ ' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo logger.info(cmd) start_time() (output, err, status) = asp_system_utils.executeCommand(cmd, suppressOutput=True) #status = os.system(cmd) logger.info("LFTP output and error: " + output + ' ' + err) logger.info("LFTP status: " + str(status)) #if status != 0: # raise Exception("Problem pushing") stop_time("push to NSIDC", logger)
def convertLidarDataToCsv(lidarFolder, startFrame, stopFrame, skipValidate, logger): '''Make sure all lidar data is available in a readable text format. Returns false if any files failed to convert.''' logger.info('Converting LIDAR files...') lidarIndexPath = icebridge_common.csvIndexFile(lidarFolder) (frameDict, urlDict) = icebridge_common.readIndexFile(lidarIndexPath) if not skipValidate: validFilesList = icebridge_common.validFilesList( os.path.dirname(lidarFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) convDict = {} # Loop through all files in the folder badFiles = False for frame in sorted(frameDict.keys()): f = frameDict[frame] extension = icebridge_common.fileExtension(f) # Only interested in a few file types if (extension != '.qi') and (extension != '.hdf5') and (extension != '.h5'): convDict[frame] = f # these are already in plain text continue convDict[frame] = os.path.splitext(f)[0] + '.csv' outputPath = os.path.join(lidarFolder, convDict[frame]) # Handle paths fullPath = os.path.join(lidarFolder, f) if not os.path.exists(fullPath): logger.info("Cannot convert missing file: " + fullPath) continue # If the input is invalid, wipe both it, its xml, and the output # Hopefully there will be a subsquent fetch step where it will get # refetched. if not icebridge_common.hasValidChkSum(fullPath, logger): logger.info("Will wipe invalid file: " + fullPath) xmlFile = icebridge_common.xmlFile(fullPath) os.system('rm -f ' + fullPath) # will not throw os.system('rm -f ' + xmlFile) # will not throw os.system('rm -f ' + outputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # verbose continue if icebridge_common.isValidLidarCSV(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) continue # Call the conversion logger.info("Process " + fullPath) extract_icebridge_ATM_points.main([fullPath]) # Check the result if not icebridge_common.isValidLidarCSV(outputPath): logger.error('Failed to parse LIDAR file, will wipe: ' + outputPath) os.system('rm -f ' + outputPath) # will not throw badFiles = True else: if not skipValidate: validFilesSet.add(outputPath) # mark it as validated convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder) willWriteConvFile = False if not os.path.exists(convLidarFile): willWriteConvFile = True else: # Bugfix: Sometimes the written converted file has the wrong size, maybe # something got interrupted earlier. (lidarDictIn, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile) if lidarDictIn != convDict: willWriteConvFile = True if willWriteConvFile: logger.info("Writing: " + convLidarFile) icebridge_common.writeIndexFile(convLidarFile, convDict, {}) if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
def correctFireballDems(fireballFolder, corrFireballFolder, startFrame, stopFrame, isNorth, skipValidate, logger): '''Fix the header problem in Fireball DEMs''' logger.info('Correcting Fireball DEMs ...') # Read the existing DEMs fireballIndexPath = icebridge_common.csvIndexFile(fireballFolder) if not os.path.exists(fireballIndexPath): raise Exception("Error: Missing fireball index file: " + fireballIndexPath + ".") (fireballFrameDict, fireballUrlDict) = \ icebridge_common.readIndexFile(fireballIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList(os.path.dirname(fireballFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Loop through all the input images os.system('mkdir -p ' + corrFireballFolder) badFiles = False for frame in sorted(fireballFrameDict.keys()): # Skip if outside the frame range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue inputPath = fireballFrameDict[frame] if not icebridge_common.isDEM(inputPath): continue outputPath = os.path.join(corrFireballFolder, os.path.basename(inputPath)) # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very vebose continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) validFilesSet.add(outputPath) # mark it as validated continue # Run the correction script execPath = asp_system_utils.which('correct_icebridge_l3_dem') cmd = (('%s %s %s %d') % (execPath, inputPath, outputPath, isNorth)) logger.info(cmd) # TODO: Run this as a subprocess and check the return code os.system(cmd) # Check if the output file is good if not icebridge_common.isValidImage(outputPath): logger.error('Failed to convert dem file, wiping: ' + inputPath + ' ' + outputPath) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw badFiles = True else: if not skipValidate: validFilesSet.add(outputPath) # mark it as validated if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
def validateOrthosAndFireball(options, fileType, logger): '''Validate ortho and fireball files within the current frame range. This is expected to be in called in parallel for smaller chunks. Lidar files will be validated serially. Jpegs get validated when converted to tif. Return True if all is good.''' badFiles = False logger.info("Validating files of type: " + fileType) if fileType == 'ortho': dataFolder = icebridge_common.getOrthoFolder(options.outputFolder) elif fileType == 'fireball': dataFolder = icebridge_common.getFireballFolder(options.outputFolder) else: raise Exception("Unknown file type: " + fileType) indexPath = icebridge_common.csvIndexFile(dataFolder) if not os.path.exists(indexPath): # The issue of what to do when the index does not exist should # have been settled by now. return (not badFiles) # Fetch from disk the set of already validated files, if any validFilesList = icebridge_common.validFilesList(options.outputFolder, options.startFrame, options.stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) (frameDict, urlDict) = icebridge_common.readIndexFile(indexPath, prependFolder = True) for frame in frameDict.keys(): if frame < options.startFrame or frame > options.stopFrame: continue outputPath = frameDict[frame] xmlFile = icebridge_common.xmlFile(outputPath) if outputPath in validFilesSet and os.path.exists(outputPath) and \ xmlFile in validFilesSet and os.path.exists(xmlFile): #logger.info('Previously validated: ' + outputPath + ' ' + xmlFile) continue else: isGood = icebridge_common.hasValidChkSum(outputPath, logger) if not isGood: logger.info('Found invalid data. Will wipe: ' + outputPath + ' ' + xmlFile) os.system('rm -f ' + outputPath) # will not throw os.system('rm -f ' + xmlFile) # will not throw badFiles = True else: logger.info('Valid file: ' + outputPath) validFilesSet.add(outputPath) validFilesSet.add(xmlFile) if fileType != 'fireball': continue # Also validate tfw tfwFile = icebridge_common.tfwFile(outputPath) xmlFile = icebridge_common.xmlFile(tfwFile) if tfwFile in validFilesSet and os.path.exists(tfwFile) and \ xmlFile in validFilesSet and os.path.exists(xmlFile): #logger.info('Previously validated: ' + tfwFile + ' ' + xmlFile) continue else: isGood = icebridge_common.isValidTfw(tfwFile, logger) if not isGood: logger.info('Found invalid tfw. Will wipe: ' + tfwFile + ' ' + xmlFile) os.system('rm -f ' + tfwFile) # will not throw os.system('rm -f ' + xmlFile) # will not throw badFiles = True else: logger.info('Valid tfw file: ' + tfwFile) validFilesSet.add(tfwFile) validFilesSet.add(xmlFile) # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = \ icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
def convertLidarDataToCsv(lidarFolder, startFrame, stopFrame, skipValidate, logger): '''Make sure all lidar data is available in a readable text format. Returns false if any files failed to convert.''' logger.info('Converting LIDAR files...') lidarIndexPath = icebridge_common.csvIndexFile(lidarFolder) (frameDict, urlDict) = icebridge_common.readIndexFile(lidarIndexPath) if not skipValidate: validFilesList = icebridge_common.validFilesList(os.path.dirname(lidarFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) convDict = {} # Loop through all files in the folder badFiles = False for frame in sorted(frameDict.keys()): f = frameDict[frame] extension = icebridge_common.fileExtension(f) # Only interested in a few file types if (extension != '.qi') and (extension != '.hdf5') and (extension != '.h5'): convDict[frame] = f # these are already in plain text continue convDict[frame] = os.path.splitext(f)[0] + '.csv' outputPath = os.path.join(lidarFolder, convDict[frame]) # Handle paths fullPath = os.path.join(lidarFolder, f) if not os.path.exists(fullPath): logger.info("Cannot convert missing file: " + fullPath) continue # If the input is invalid, wipe both it, its xml, and the output # Hopefully there will be a subsquent fetch step where it will get # refetched. if not icebridge_common.hasValidChkSum(fullPath, logger): logger.info("Will wipe invalid file: " + fullPath) xmlFile = icebridge_common.xmlFile(fullPath) os.system('rm -f ' + fullPath) # will not throw os.system('rm -f ' + xmlFile) # will not throw os.system('rm -f ' + outputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # verbose continue if icebridge_common.isValidLidarCSV(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) continue # Call the conversion logger.info("Process " + fullPath) extract_icebridge_ATM_points.main([fullPath]) # Check the result if not icebridge_common.isValidLidarCSV(outputPath): logger.error('Failed to parse LIDAR file, will wipe: ' + outputPath) os.system('rm -f ' + outputPath) # will not throw badFiles = True else: if not skipValidate: validFilesSet.add(outputPath) # mark it as validated convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder) willWriteConvFile = False if not os.path.exists(convLidarFile): willWriteConvFile = True else: # Bugfix: Sometimes the written converted file has the wrong size, maybe # something got interrupted earlier. (lidarDictIn, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile) if lidarDictIn != convDict: willWriteConvFile = True if willWriteConvFile: logger.info("Writing: " + convLidarFile) icebridge_common.writeIndexFile(convLidarFile, convDict, {}) if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
def solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder, processedFolder, logger): '''Some preliminary work before solving for intrinsics. Here we look up the default calibration file, and generate an RPC approximation of its distortion model with polynomials of degree 4. We will then create cameras and stereo DEMs using this initial camera file with RPC distortion.''' # Sanity checks if options.startFrame == icebridge_common.getSmallestFrame() or \ options.stopFrame == icebridge_common.getLargestFrame(): raise Exception( "When solving for intrinsics, must specify a frame range.") if options.bundleLength != 2: raise Exception( "When solving for intrinsics, we assume bundle length of 2.") if (options.stopFrame - options.startFrame) % 2 == 0: raise Exception( "When solving for intrinsics, must have an even number of frames, " + " so stopFrame - startFrame must be odd.") if options.processingSubfolder: raise Exception( "Processing subfolder not supported when solving for intrinsics.") # Generate extra data we will use later to float intrinsics options.stereoArgs += " --num-matches-from-disp-triplets 10000 --unalign-disparity " # --enable-fill-holes " # Create separate directories for cameras and processed data, # as these will be distinct than what we will finally be # using to do the full run. suff = "_camgen" cameraFolder += suff navCameraFolder += suff processedFolder += suff # Get the input calibration file defaultCalibFile = "" for frame in range(options.startFrame, options.stopFrame + 1): currCalibFile = input_conversions.getCalibrationFileForFrame( options.cameraLookupFile, options.inputCalFolder, frame, options.yyyymmdd, options.site) if defaultCalibFile == "": defaultCalibFile = currCalibFile if defaultCalibFile != currCalibFile: # This is important, the calibration file must be unique raise Exception("Found two distinct calibration files: " + defaultCalibFile + \ " and " + currCalibFile) logger.info("Default calibration file: " + defaultCalibFile) if options.inputCalCamera != "": defaultCalibFile = options.inputCalCamera logger.info("Using instead the user-provided: " + defaultCalibFile) # Find the first image in the range jpegIndex = icebridge_common.csvIndexFile(jpegFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex, prependFolder=True) if options.startFrame not in jpegFrameDict.keys(): raise Exception("Could not find jpeg image for frame: " + options.startFrame) firstImage = jpegFrameDict[options.startFrame] # Create the RPC file before optimization rpcCalibFile = os.path.join(processedFolder, os.path.basename(defaultCalibFile)) rpcCalibFile = rpcCalibFile.replace(".tsai", "_INIT_RPC.tsai") logger.info("Will approximate camera model " + defaultCalibFile + " with " + \ options.outputModelType + " model " + rpcCalibFile) os.system("mkdir -p " + os.path.dirname(rpcCalibFile)) cmd = "convert_pinhole_model --input-file " + firstImage + ' --camera-file ' + \ defaultCalibFile + ' --output-type ' + options.outputModelType + \ ' --sample-spacing 50 -o ' + rpcCalibFile logger.info(cmd) os.system(cmd) # Use this one from now on options.inputCalCamera = rpcCalibFile # Return the modified values return (options, cameraFolder, navCameraFolder, processedFolder)
def pushByType(run, options, logger, dataType): # Fetch the ortho index from NSIDC if missing outputFolder = run.getFolder() logger.info("Output folder is " + outputFolder) os.system("mkdir -p " + outputFolder) # Current directory. It is important to go from /u to the real dir which is /nobackup... unpackDir = os.path.realpath(os.getcwd()) logger.info("Unpack directory is " + unpackDir) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): fetchIndices(options, logger) logger.info("Reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) # Fetch unarchived folder if missing if dataType == 'DEM': unarchivedFolder = run.getAssemblyFolder() elif dataType == 'ORTHO': unarchivedFolder = run.getProcessFolder() else: raise Exception("Unknown data type: " + dataType) logger.info("Unarchived data folder is " + unarchivedFolder) # Especially for ortho, force-fetch each time, as there is no good way # of checking if we fetched well before. start_time() if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType): return stop_time("fetching archived data by type: " + dataType, logger) # Make the output directory at NSIDC m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd) if m: outDir = options.site + "_" + m.group(1) + "." + m.group(2) + "." + m.group(3) else: raise Exception("Could not parse: " + options.yyyymmdd) # Keep the output directory locally here localDirPath = os.path.join(outputFolder, dataType, outDir) os.system("mkdir -p " + localDirPath) logger.info("Storing the renamed " + dataType + " files in " + localDirPath) logger.info("Directory name at NSIDC: " + outDir) # Read the DEMs and orthos, and copy them to outDir according to the final convention if dataType == 'DEM': dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True) else: dataFiles = glob.glob(os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif')) for dataFile in dataFiles: # Here we use the convention from archive_functions.py for DEMs and from how we store orthos. if dataType == 'DEM': m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \ "\.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) else: m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \ "out-ortho.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) if frameNumber < options.startFrame or frameNumber > options.stopFrame: continue # For each data file, copy from the ortho its meta info if not frameNumber in orthoFrameDict.keys(): # Bugfix: Ortho fetching failed, try again fetchIndices(options, logger) logger.info("Re-reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if not frameNumber in orthoFrameDict.keys(): # This time there is nothing we can do raise Exception("Cannot find ortho for frame: " + str(frameNumber)) orthoFile = orthoFrameDict[frameNumber] [dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile) # It is always possible that the ortho file date will be the next day # after the current flight date, if the flight goes after midnight. # So it is not unreasonable that options.yyyymmdd != dateString. if dataType == 'DEM': outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber)) else: # TODO: Need to think more of the naming convention. outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber)) cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(localDirPath, outFile) logger.info(cmd) os.system(cmd) # Push the directory to NSIDC remoteDirPath = os.path.join(os.path.basename(os.path.dirname(localDirPath)), os.path.basename(localDirPath)) remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath) logger.info("Storing at NSIDC in: " + remoteDirPath) cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \ ' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo logger.info(cmd) start_time() (output, err, status) = asp_system_utils.executeCommand(cmd, suppressOutput = True) #status = os.system(cmd) logger.info("LFTP output and error: " + output + ' ' + err) logger.info("LFTP status: " + str(status)) #if status != 0: # raise Exception("Problem pushing") stop_time("push to NSIDC", logger)
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument('--camera-mounting', dest='cameraMounting', default=0, type=int, help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder): '''Create a list of all files that must be fetched unless done already.''' # For AN 20091112, etc, some of the ortho images are stored at the # beginning of the next day's flight. Need to sort this out, and # it is tricky. More comments within the code. fetchNextDay = True separateByLat = (options.type == 'ortho' and isInSeparateByLatTable(options.yyyymmdd)) if separateByLat: # Here we won't fetch the next day, we will just separate by latitude within # a given day fetchNextDay = False orthoOrFireball = ( (options.type == 'ortho') or (options.type == 'fireball') ) if fetchNextDay: # Normally we fetch for next day only for ortho or fireball. However, # for one single special flight, we do it for jpeg too, as then # the jpegs are also split. if orthoOrFireball or \ ((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)): fetchNextDay = True else: fetchNextDay = False # If we need to parse the next flight day as well, as expected in some runs, # we will fetch two html files, but create a single index out of them. dayVals = [0] if fetchNextDay: dayVals.append(1) indexPath = icebridge_common.htmlIndexFile(outputFolder) currIndexPath = indexPath parsedIndexPath = icebridge_common.csvIndexFile(outputFolder) if options.refetchIndex: os.system('rm -f ' + indexPath) os.system('rm -f ' + parsedIndexPath) if icebridge_common.fileNonEmpty(parsedIndexPath): logger.info('Already have the index file ' + parsedIndexPath + ', keeping it.') return parsedIndexPath frameDict = {} urlDict = {} # We need the list of jpeg frames. Sometimes when fetching ortho images, # and we have to fetch from the next day, don't fetch unless # in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': jpegFolder = icebridge_common.getJpegFolder(os.path.dirname(outputFolder)) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath) orthoStamp = {} if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(outputFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) for frame in sorted(orthoFrameDict.keys()): filename = orthoFrameDict[frame] [imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename) orthoStamp[frame] = imageTimeString for dayVal in dayVals: if len(dayVals) > 1: currIndexPath = indexPath + '.day' + str(dayVal) if options.refetchIndex: os.system('rm -f ' + currIndexPath) # Find folderUrl which contains all of the files if options.type in LIDAR_TYPES: options.allFrames = True # For lidar, always get all the frames! # For lidar, the data can come from one of three sources. # Unfortunately sometimes there is more than one source, and then # we need to pick by latitude. folderUrls = [] lidar_types = [] for lidar in LIDAR_TYPES: folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, lidar) logger.info('Checking lidar URL: ' + folderUrl) if checkIfUrlExists(folderUrl, baseCurlCmd): logger.info('Found match with lidar type: ' + lidar) folderUrls.append(folderUrl) lidar_types.append(lidar) if len(folderUrls) == 0: logger.info('WARNING: Could not find any lidar data for the given date!') elif len(folderUrls) == 1: # Unique solution folderUrl = folderUrls[0] options.type = lidar_types[0] elif len(folderUrls) >= 2: # Multiple solutions. Pick the good one by latitude. logger.info("Multiples URLs to search: " + " ".join(folderUrls)) count = -1 isGood = False for folderUrl in folderUrls: count += 1 (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, lidar_types[count]) for frame in sorted(localFrameDict.keys()): filename = localFrameDict[frame] xmlFile = icebridge_common.xmlFile(filename) url = os.path.join(folderUrl, xmlFile) # Download the file curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile logger.info(curlCmd) p = subprocess.Popen(curlCmd, shell=True, universal_newlines=True) os.waitpid(p.pid, 0) latitude = icebridge_common.parseLatitude(xmlFile) if os.path.exists(xmlFile): os.remove(xmlFile) if hasGoodLat(latitude, isSouth): isGood = True options.type = lidar_types[count] logger.info("Good latitude " + str(latitude) + ", will use " + folderUrl + " of type " + lidar_types[count]) else: logger.info("Bad latitude " + str(latitude) + ", will not use " + folderUrl + " of type " + lidar_types[count]) # Stop at first file no matter what break if isGood: break if not isGood: if options.type in LIDAR_TYPES and options.ignoreMissingLidar: logger.info("No lidar. None of these URLs are good: " + " ".join(folderUrls)) else: raise Exception("None of these URLs are good: " + " ".join(folderUrls)) else: # Other cases are simpler folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, options.type) logger.info('Fetching from URL: ' + folderUrl) (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, options.type) # Append to the main index for frame in sorted(localFrameDict.keys()): if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. # Otherwise we may accidentally getting one from next day. [imageDateString, imageTimeString] = \ icebridge_common.parseTimeStamps(localFrameDict[frame]) if frame not in orthoStamp: #logger.info("Missing ortho for fireball: " + localFrameDict[frame]) continue if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000: # Apparently a tolerance is needed. Use 10 seconds, so the number 1000. #logger.info("Will not use fireball DEM whose timestamp differs from ortho.") #logger.info("Fireball is: " + localFrameDict[frame]) #logger.info("Ortho is: " + orthoFrameDict[frame]) continue # Fetch from next day, unless already have a value. And don't fetch # frames not in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': if not frame in jpegFrameDict.keys(): continue if frame in frameDict.keys(): continue frameDict[frame] = localFrameDict[frame] urlDict[frame] = localUrlDict[frame] # Write the combined index file icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict) return parsedIndexPath
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument( '--camera-mounting', dest='cameraMounting', default=0, type=int, help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def fetchAllRunData(options, startFrame, stopFrame, jpegFolder, orthoFolder, fireballFolder, lidarFolder, navFolder): '''Download all data needed to process a run''' logger = logging.getLogger(__name__) logger.info('Downloading data for the run...') baseCommand = (('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') % (options.yyyymmdd, options.site, startFrame, stopFrame)) if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0: baseCommand += ' --max-num-lidar-to-fetch ' + str(options.maxNumLidarToFetch) if options.refetchIndex: baseCommand += ' --refetch-index' # this was not right in older fetched runs if options.refetchNav: baseCommand += ' --refetch-nav' # sometimes this was corrupted if options.stopAfterIndexFetch: baseCommand += ' --stop-after-index-fetch' if options.skipValidate: baseCommand += ' --skip-validate' if options.ignoreMissingLidar: baseCommand += ' --ignore-missing-lidar' if options.dryRun: baseCommand += ' --dry-run' jpegCommand = baseCommand + ' ' + jpegFolder orthoCommand = baseCommand + ' ' + orthoFolder fireballCommand = baseCommand + ' ' + fireballFolder lidarCommand = baseCommand + ' ' + lidarFolder navCommand = baseCommand + ' ' + navFolder # Try to do all the downloads one after another # - On a failure the error message should already be printed. # - The fetching tool will not redownload existing data. if fetch_icebridge_data.main(jpegCommand.split()) < 0: return -1 if fetch_icebridge_data.main(orthoCommand.split()) < 0: return -1 if fetch_icebridge_data.main(fireballCommand.split()) < 0: logger.info('Fireball DEM data is optional, continuing run.') if not options.noNavFetch: if fetch_icebridge_data.main(navCommand.split()) < 0: return -1 # Skip the lidar fetch if the user requested no lidar files if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch > 0): if fetch_icebridge_data.main(lidarCommand.split()) < 0: return -1 # jpeg and ortho indices must be consistent if not options.skipValidate: logger.info("Check for consistency between raw and ortho images.") jpegIndex = icebridge_common.csvIndexFile(jpegFolder) orthoIndex = icebridge_common.csvIndexFile(orthoFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex) for jpegFrame in jpegFrameDict.keys(): if jpegFrame < startFrame or jpegFrame > stopFrame: continue if jpegFrame not in orthoFrameDict.keys(): logger.info("Found jpeg frame missing from ortho: " + str(jpegFrame)) #raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame)) for orthoFrame in orthoFrameDict.keys(): if orthoFrame < startFrame or orthoFrame > stopFrame: continue if orthoFrame not in jpegFrameDict.keys(): # This can happen, don't die because of it logger.info("Found ortho frame missing from jpeg: " + str(orthoFrame)) #raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame)) # TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it. return 0
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder): '''Create a list of all files that must be fetched unless done already.''' # For AN 20091112, etc, some of the ortho images are stored at the # beginning of the next day's flight. Need to sort this out, and # it is tricky. More comments within the code. fetchNextDay = True separateByLat = (options.type == 'ortho' and isInSeparateByLatTable(options.yyyymmdd)) if separateByLat: # Here we won't fetch the next day, we will just separate by latitude within # a given day fetchNextDay = False orthoOrFireball = ((options.type == 'ortho') or (options.type == 'fireball')) if fetchNextDay: # Normally we fetch for next day only for ortho or fireball. However, # for one single special flight, we do it for jpeg too, as then # the jpegs are also split. if orthoOrFireball or \ ((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)): fetchNextDay = True else: fetchNextDay = False # If we need to parse the next flight day as well, as expected in some runs, # we will fetch two html files, but create a single index out of them. dayVals = [0] if fetchNextDay: dayVals.append(1) indexPath = icebridge_common.htmlIndexFile(outputFolder) currIndexPath = indexPath parsedIndexPath = icebridge_common.csvIndexFile(outputFolder) if options.refetchIndex: os.system('rm -f ' + indexPath) os.system('rm -f ' + parsedIndexPath) if icebridge_common.fileNonEmpty(parsedIndexPath): logger.info('Already have the index file ' + parsedIndexPath + ', keeping it.') return parsedIndexPath frameDict = {} urlDict = {} # We need the list of jpeg frames. Sometimes when fetching ortho images, # and we have to fetch from the next day, don't fetch unless # in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': jpegFolder = icebridge_common.getJpegFolder( os.path.dirname(outputFolder)) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath) orthoStamp = {} if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. orthoFolder = icebridge_common.getOrthoFolder( os.path.dirname(outputFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) for frame in sorted(orthoFrameDict.keys()): filename = orthoFrameDict[frame] [imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename) orthoStamp[frame] = imageTimeString for dayVal in dayVals: if len(dayVals) > 1: currIndexPath = indexPath + '.day' + str(dayVal) if options.refetchIndex: os.system('rm -f ' + currIndexPath) # Find folderUrl which contains all of the files if options.type in LIDAR_TYPES: options.allFrames = True # For lidar, always get all the frames! # For lidar, the data can come from one of three sources. # Unfortunately sometimes there is more than one source, and then # we need to pick by latitude. folderUrls = [] lidar_types = [] for lidar in LIDAR_TYPES: folderUrl = getFolderUrl( options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, lidar) logger.info('Checking lidar URL: ' + folderUrl) if checkIfUrlExists(folderUrl): logger.info('Found match with lidar type: ' + lidar) folderUrls.append(folderUrl) lidar_types.append(lidar) if len(folderUrls) == 0: logger.info( 'WARNING: Could not find any lidar data for the given date!' ) elif len(folderUrls) == 1: # Unique solution folderUrl = folderUrls[0] options.type = lidar_types[0] elif len(folderUrls) >= 2: # Multiple solutions. Pick the good one by latitude. logger.info("Multiples URLs to search: " + " ".join(folderUrls)) count = -1 isGood = False for folderUrl in folderUrls: count += 1 (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, lidar_types[count]) for frame in sorted(localFrameDict.keys()): filename = localFrameDict[frame] xmlFile = icebridge_common.xmlFile(filename) url = os.path.join(folderUrl, xmlFile) # Download the file curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile logger.info(curlCmd) p = subprocess.Popen(curlCmd, shell=True) os.waitpid(p.pid, 0) latitude = icebridge_common.parseLatitude(xmlFile) if os.path.exists(xmlFile): os.remove(xmlFile) if hasGoodLat(latitude, isSouth): isGood = True options.type = lidar_types[count] logger.info("Good latitude " + str(latitude) + ", will use " + folderUrl + " of type " + lidar_types[count]) else: logger.info("Bad latitude " + str(latitude) + ", will not use " + folderUrl + " of type " + lidar_types[count]) # Stop at first file no matter what break if isGood: break if not isGood: if options.type in LIDAR_TYPES and options.ignoreMissingLidar: logger.info("No lidar. None of these URLs are good: " + " ".join(folderUrls)) else: raise Exception("None of these URLs are good: " + " ".join(folderUrls)) else: # Other cases are simpler folderUrl = getFolderUrl( options.yyyymmdd, options.year, options.month, options.day, dayVal, # note here the dayVal options.site, options.type) logger.info('Fetching from URL: ' + folderUrl) (localFrameDict, localUrlDict) = \ fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, currIndexPath, options.type) # Append to the main index for frame in sorted(localFrameDict.keys()): if options.type == 'fireball': # This is a bugfix. Ensure that the fireball DEM has not just # the same frame number, but also same timestamp as the ortho. # Otherwise we may accidentally getting one from next day. [imageDateString, imageTimeString] = \ icebridge_common.parseTimeStamps(localFrameDict[frame]) if frame not in orthoStamp: #logger.info("Missing ortho for fireball: " + localFrameDict[frame]) continue if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000: # Apparently a tolerance is needed. Use 10 seconds, so the number 1000. #logger.info("Will not use fireball DEM whose timestamp differs from ortho.") #logger.info("Fireball is: " + localFrameDict[frame]) #logger.info("Ortho is: " + orthoFrameDict[frame]) continue # Fetch from next day, unless already have a value. And don't fetch # frames not in the jpeg index. if len(dayVals) > 1 and options.type != 'jpeg': if not frame in jpegFrameDict.keys(): continue if frame in frameDict.keys(): continue frameDict[frame] = localFrameDict[frame] urlDict[frame] = localUrlDict[frame] # Write the combined index file icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict) return parsedIndexPath
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1:
def validateOrthosAndFireball(options, fileType, logger): '''Validate ortho and fireball files within the current frame range. This is expected to be in called in parallel for smaller chunks. Lidar files will be validated serially. Jpegs get validated when converted to tif. Return True if all is good.''' badFiles = False logger.info("Validating files of type: " + fileType) if fileType == 'ortho': dataFolder = icebridge_common.getOrthoFolder(options.outputFolder) elif fileType == 'fireball': dataFolder = icebridge_common.getFireballFolder(options.outputFolder) else: raise Exception("Unknown file type: " + fileType) indexPath = icebridge_common.csvIndexFile(dataFolder) if not os.path.exists(indexPath): # The issue of what to do when the index does not exist should # have been settled by now. return (not badFiles) # Fetch from disk the set of already validated files, if any validFilesList = icebridge_common.validFilesList(options.outputFolder, options.startFrame, options.stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) (frameDict, urlDict) = icebridge_common.readIndexFile(indexPath, prependFolder=True) for frame in frameDict.keys(): if frame < options.startFrame or frame > options.stopFrame: continue outputPath = frameDict[frame] xmlFile = icebridge_common.xmlFile(outputPath) if outputPath in validFilesSet and os.path.exists(outputPath) and \ xmlFile in validFilesSet and os.path.exists(xmlFile): #logger.info('Previously validated: ' + outputPath + ' ' + xmlFile) continue else: isGood = icebridge_common.hasValidChkSum(outputPath, logger) if not isGood: logger.info('Found invalid data. Will wipe: ' + outputPath + ' ' + xmlFile) os.system('rm -f ' + outputPath) # will not throw os.system('rm -f ' + xmlFile) # will not throw badFiles = True else: logger.info('Valid file: ' + outputPath) validFilesSet.add(outputPath) validFilesSet.add(xmlFile) if fileType != 'fireball': continue # Also validate tfw tfwFile = icebridge_common.tfwFile(outputPath) xmlFile = icebridge_common.xmlFile(tfwFile) if tfwFile in validFilesSet and os.path.exists(tfwFile) and \ xmlFile in validFilesSet and os.path.exists(xmlFile): #logger.info('Previously validated: ' + tfwFile + ' ' + xmlFile) continue else: isGood = icebridge_common.isValidTfw(tfwFile, logger) if not isGood: logger.info('Found invalid tfw. Will wipe: ' + tfwFile + ' ' + xmlFile) os.system('rm -f ' + tfwFile) # will not throw os.system('rm -f ' + xmlFile) # will not throw badFiles = True else: logger.info('Valid tfw file: ' + tfwFile) validFilesSet.add(tfwFile) validFilesSet.add(xmlFile) # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = \ icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
def checkForImages(self, startFrame, stopFrame, logger): '''Return true if all the images have been converted from jpeg.''' logger.info("Checking if all jpegs have been converted.") jpegFolder = self.getJpegFolder() imageFolder = self.getImageFolder() orthoFolder = self.getOrthoFolder() if not os.path.exists(jpegFolder): logger.info("Missing: " + jpegFolder) return False if not os.path.exists(imageFolder): logger.info("Missing: " + imageFolder) return False jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): logger.info("Missing: " + jpegIndexPath) return False (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) # Need the orthos to get the timestamp orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) # Thorough check for missing images. It is very slow. num = len(jpegFrameDict.keys()) allGood = True count = 0 for frame in sorted(jpegFrameDict.keys()): if frame < startFrame or frame > stopFrame: continue # Add the progress, as this operation can be terribly slow # when the filesystem is not doing too well, especially on mfe. count = count + 1 if (count - 1) % 1000 == 0: logger.info('Progress: ' + str(count) + '/' + str(num)) inputPath = jpegFrameDict[frame] if not frame in orthoFrameDict: logger.info("Missing ortho for frame: " + frame) continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame]) except Exception as e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) if os.path.exists(inputPath): os.remove(inputPath) allGood = False continue if not os.path.exists(outputPath): logger.info("Missing image file: " + outputPath) allGood = False return allGood
def fetchAllRunData(options, startFrame, stopFrame, jpegFolder, orthoFolder, fireballFolder, lidarFolder, navFolder): '''Download all data needed to process a run''' logger = logging.getLogger(__name__) logger.info('Downloading data for the run...') baseCommand = ( ('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') % (options.yyyymmdd, options.site, startFrame, stopFrame)) if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0: baseCommand += ' --max-num-lidar-to-fetch ' + str( options.maxNumLidarToFetch) if options.refetchIndex: baseCommand += ' --refetch-index' # this was not right in older fetched runs if options.refetchNav: baseCommand += ' --refetch-nav' # sometimes this was corrupted if options.stopAfterIndexFetch: baseCommand += ' --stop-after-index-fetch' if options.skipValidate: baseCommand += ' --skip-validate' if options.ignoreMissingLidar: baseCommand += ' --ignore-missing-lidar' if options.dryRun: baseCommand += ' --dry-run' jpegCommand = baseCommand + ' ' + jpegFolder orthoCommand = baseCommand + ' ' + orthoFolder fireballCommand = baseCommand + ' ' + fireballFolder lidarCommand = baseCommand + ' ' + lidarFolder navCommand = baseCommand + ' ' + navFolder # Try to do all the downloads one after another # - On a failure the error message should already be printed. # - The fetching tool will not redownload existing data. if fetch_icebridge_data.main(jpegCommand.split()) < 0: return -1 if fetch_icebridge_data.main(orthoCommand.split()) < 0: return -1 if fetch_icebridge_data.main(fireballCommand.split()) < 0: logger.info('Fireball DEM data is optional, continuing run.') if not options.noNavFetch: if fetch_icebridge_data.main(navCommand.split()) < 0: return -1 # Skip the lidar fetch if the user requested no lidar files if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch > 0): if fetch_icebridge_data.main(lidarCommand.split()) < 0: return -1 # jpeg and ortho indices must be consistent if not options.skipValidate: logger.info("Check for consistency between raw and ortho images.") jpegIndex = icebridge_common.csvIndexFile(jpegFolder) orthoIndex = icebridge_common.csvIndexFile(orthoFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex) for jpegFrame in jpegFrameDict.keys(): if jpegFrame < startFrame or jpegFrame > stopFrame: continue if jpegFrame not in orthoFrameDict.keys(): logger.info("Found jpeg frame missing from ortho: " + str(jpegFrame)) #raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame)) for orthoFrame in orthoFrameDict.keys(): if orthoFrame < startFrame or orthoFrame > stopFrame: continue if orthoFrame not in jpegFrameDict.keys(): # This can happen, don't die because of it logger.info("Found ortho frame missing from jpeg: " + str(orthoFrame)) #raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame)) # TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it. return (jpegFolder, orthoFolder, fireballFolder, lidarFolder)
def checkForImages(self, startFrame, stopFrame, logger): '''Return true if all the images have been converted from jpeg.''' logger.info("Checking if all jpegs have been converted.") jpegFolder = self.getJpegFolder() imageFolder = self.getImageFolder() orthoFolder = self.getOrthoFolder() if not os.path.exists(jpegFolder): logger.info("Missing: " + jpegFolder) return False if not os.path.exists(imageFolder): logger.info("Missing: " + imageFolder) return False jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): logger.info("Missing: " + jpegIndexPath) return False (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) # Need the orthos to get the timestamp orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) # Thorough check for missing images. It is very slow. num = len(jpegFrameDict.keys()) allGood = True count = 0 for frame in sorted(jpegFrameDict.keys()): if frame < startFrame or frame > stopFrame: continue # Add the progress, as this operation can be terribly slow # when the filesystem is not doing too well, especially on mfe. count = count + 1 if (count - 1) % 1000 == 0: logger.info('Progress: ' + str(count) + '/' + str(num)) inputPath = jpegFrameDict[frame] if not frame in orthoFrameDict: logger.info("Missing ortho for frame: " + frame) continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile( inputPath, orthoFrameDict[frame]) except Exception as e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) if os.path.exists(inputPath): os.remove(inputPath) allGood = False continue if not os.path.exists(outputPath): logger.info("Missing image file: " + outputPath) allGood = False return allGood
def solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder, processedFolder, logger): '''Some preliminary work before solving for intrinsics. Here we look up the default calibration file, and generate an RPC approximation of its distortion model with polynomials of degree 4. We will then create cameras and stereo DEMs using this initial camera file with RPC distortion.''' # Sanity checks if options.startFrame == icebridge_common.getSmallestFrame() or \ options.stopFrame == icebridge_common.getLargestFrame(): raise Exception("When solving for intrinsics, must specify a frame range.") if options.bundleLength != 2: raise Exception("When solving for intrinsics, we assume bundle length of 2.") if (options.stopFrame - options.startFrame) % 2 == 0: raise Exception("When solving for intrinsics, must have an even number of frames, " + " so stopFrame - startFrame must be odd.") if options.processingSubfolder: raise Exception("Processing subfolder not supported when solving for intrinsics.") # Generate extra data we will use later to float intrinsics options.stereoArgs += " --num-matches-from-disp-triplets 10000 --unalign-disparity " # --enable-fill-holes " # Create separate directories for cameras and processed data, # as these will be distinct than what we will finally be # using to do the full run. suff = "_camgen" cameraFolder += suff navCameraFolder += suff processedFolder += suff # Get the input calibration file defaultCalibFile = "" for frame in range(options.startFrame, options.stopFrame+1): currCalibFile = input_conversions.getCalibrationFileForFrame(options.cameraLookupFile, options.inputCalFolder, frame, options.yyyymmdd, options.site, logger) if defaultCalibFile == "": defaultCalibFile = currCalibFile if defaultCalibFile != currCalibFile: # This is important, the calibration file must be unique raise Exception("Found two distinct calibration files: " + defaultCalibFile + \ " and " + currCalibFile) logger.info("Default calibration file: " + defaultCalibFile) if options.inputCalCamera != "": defaultCalibFile = options.inputCalCamera logger.info("Using instead the user-provided: " + defaultCalibFile) # Find the first image in the range jpegIndex = icebridge_common.csvIndexFile(jpegFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex, prependFolder = True) if options.startFrame not in jpegFrameDict.keys(): raise Exception("Could not find jpeg image for frame: " + options.startFrame) firstImage = jpegFrameDict[options.startFrame] # Create the RPC file before optimization rpcCalibFile = os.path.join(processedFolder, os.path.basename(defaultCalibFile)) rpcCalibFile = rpcCalibFile.replace(".tsai", "_INIT_RPC.tsai") logger.info("Will approximate camera model " + defaultCalibFile + " with " + \ options.outputModelType + " model " + rpcCalibFile) os.system("mkdir -p " + os.path.dirname(rpcCalibFile)) cmd = "convert_pinhole_model --input-file " + firstImage + ' --camera-file ' + \ defaultCalibFile + ' --output-type ' + options.outputModelType + \ ' --sample-spacing 50 -o ' + rpcCalibFile logger.info(cmd) os.system(cmd) # Use this one from now on options.inputCalCamera = rpcCalibFile # Return the modified values return (options, cameraFolder, navCameraFolder, processedFolder)
def correctFireballDems(fireballFolder, corrFireballFolder, startFrame, stopFrame, isNorth, skipValidate, logger): '''Fix the header problem in Fireball DEMs''' logger.info('Correcting Fireball DEMs ...') # Read the existing DEMs fireballIndexPath = icebridge_common.csvIndexFile(fireballFolder) if not os.path.exists(fireballIndexPath): raise Exception("Error: Missing fireball index file: " + fireballIndexPath + ".") (fireballFrameDict, fireballUrlDict) = \ icebridge_common.readIndexFile(fireballIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList( os.path.dirname(fireballFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Loop through all the input images os.system('mkdir -p ' + corrFireballFolder) badFiles = False for frame in sorted(fireballFrameDict.keys()): # Skip if outside the frame range if not ((frame >= startFrame) and (frame <= stopFrame)): continue inputPath = fireballFrameDict[frame] if not icebridge_common.isDEM(inputPath): continue outputPath = os.path.join(corrFireballFolder, os.path.basename(inputPath)) # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very vebose continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) validFilesSet.add(outputPath) # mark it as validated continue # Run the correction script execPath = asp_system_utils.which('correct_icebridge_l3_dem') cmd = (('%s %s %s %d') % (execPath, inputPath, outputPath, isNorth)) logger.info(cmd) # TODO: Run this as a subprocess and check the return code os.system(cmd) # Check if the output file is good if not icebridge_common.isValidImage(outputPath): logger.error('Failed to convert dem file, wiping: ' + inputPath + ' ' + outputPath) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw badFiles = True else: if not skipValidate: validFilesSet.add(outputPath) # mark it as validated if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if options.blendToFireball: fireballFrameDict = icebridge_common.getCorrectedFireballDems( options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads)
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) if not skipValidate: validFilesList = icebridge_common.validFilesList( os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder=True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ((frame >= startFrame) and (frame <= stopFrame)): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile( inputPath, orthoFrameDict[frame]) except Exception, e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90' cmd = ('%s %s -colorspace Gray %s %s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE | re.MULTILINE | re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw
def getImageSpacing(orthoFolder, availableFrames, startFrame, stopFrame, forceAllFramesInRange): '''Find a good image stereo spacing interval that gives us a good balance between coverage and baseline width. Also detect all frames where this is a large break after the current frame.''' logger.info('Computing optimal image stereo interval...') ## With very few cameras this is the only possible way to process them #if len(availableFrames) < 3 and not forceAllFramesInRange: # return ([], {}) # No skip, no breaks # Retrieve a list of the ortho files orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) # From the dictionary create a sorted list of ortho files in the frame range breaks = [] largeSkips = {} orthoFiles = [] for frame in sorted(orthoFrameDict.keys()): # Only process frames within the range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue orthoPath = orthoFrameDict[frame] frame = icebridge_common.getFrameNumberFromFilename(orthoPath) if not forceAllFramesInRange: if frame not in availableFrames: # Skip frames we did not compute a camera for continue orthoFiles.append(orthoPath) orthoFiles.sort() numOrthos = len(orthoFiles) # First load whatever boxes are there projectionIndexFile = icebridge_common.projectionBoundsFile(os.path.dirname(orthoFolder)) logger.info("Reading: " + projectionIndexFile) boundsDict = icebridge_common.readProjectionBounds(projectionIndexFile) # Get the bounding box and frame number of each ortho image logger.info('Loading bounding boxes...') frames = [] updatedBounds = False # will be true if some computation got done count = 0 for i in range(0, numOrthos): # This can be slow, so add a progress dialong count = count + 1 if (count - 1) % 1000 == 0: logger.info('Progress: ' + str(count) + '/' + str(numOrthos)) thisFrame = icebridge_common.getFrameNumberFromFilename(orthoFiles[i]) if thisFrame not in boundsDict: imageGeoInfo = asp_geo_utils.getImageGeoInfo(orthoFiles[i], getStats=False) thisBox = imageGeoInfo['projection_bounds'] boundsDict[thisFrame] = thisBox updatedBounds = True frames.append(thisFrame) # Read this file again, in case some other process modified it in the meantime. # This won't happen in production mode, but can during testing with partial sequences. boundsDictRecent = icebridge_common.readProjectionBounds(projectionIndexFile) for frame in sorted(boundsDictRecent.keys()): if not frame in boundsDict.keys(): boundsDict[frame] = boundsDictRecent[frame] updatedBounds = True # Save the bounds. There is always the danger that two processes will # do that at the same time, but this is rare, as hopefully we get here # only once from the manager. It is not a big loss if this file gets messed up. if updatedBounds: logger.info("Writing: " + projectionIndexFile) icebridge_common.writeProjectionBounds(projectionIndexFile, boundsDict) # Since we are only comparing the image bounding boxes, not their exact corners, # these ratios are only estimates. MAX_RATIO = 0.85 # Increase skip until we get below this... MIN_RATIO = 0.75 # ... but don't go below this value! NOTRY_RATIO = 0.0001 # Don't bother with overlap amounts less than this (small on purpose) def getBboxArea(bbox): '''Return the area of a bounding box in form of (minX, maxX, minY, maxY)''' width = bbox[1] - bbox[0] height = bbox[3] - bbox[2] if (width < 0) or (height < 0): return 0 return width*height # Iterate over the frames and find the best stereo frame for each for i in range(0, numOrthos-1): thisFrame = frames[i] thisBox = boundsDict[thisFrame] thisArea = getBboxArea(thisBox) interval = 1 while(True): # Compute intersection area between this and next image nextFrame = frames[i+interval] nextBox = boundsDict[nextFrame] intersect = [max(thisBox[0], nextBox[0]), # Min X min(thisBox[1], nextBox[1]), # Max X max(thisBox[2], nextBox[2]), # Min Y min(thisBox[3], nextBox[3])] # Max Y area = getBboxArea(intersect) ratio = 0 if area > 0: ratio = area / thisArea if interval == 1: # Cases for the smallest interval... if ratio < NOTRY_RATIO: breaks.append(thisFrame) # No match for this frame logger.info('Detected large break after frame ' + str(thisFrame)) break if ratio < MIN_RATIO: break # No reason to try increasing skip amounts for this frame else: # interval > 1 if ratio < MIN_RATIO: # Went too small, walk back the interval. interval = interval - 1 break if ratio > MAX_RATIO: # Too much overlap, increase interval interval = interval + 1 else: # Overlap is fine, keep this interval. break # Handle the case where we go past the end of frames looking for a match. if i+interval >= len(frames): interval = interval - 1 break if interval > 1: # Only record larger than normal intervals. largeSkips[thisFrame] = interval logger.info('Detected ' + str(len(breaks)) + ' breaks in image coverage.') logger.info('Detected ' + str(len(largeSkips)) + ' images with interval > 1.') return (breaks, largeSkips)
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 10 usage = '''blend_dems.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") parser.add_argument( "--compute-diff-to-prev-dem", action="store_true", dest="computeDiffToPrev", default=False, help="Compute the absolute difference between the current DEM " + "and the one before it.") parser.add_argument("--blend-to-fireball-footprint", action="store_true", dest="blendToFireball", default=False, help="Create additional blended DEMs having the same " + \ "footprint as Fireball DEMs.") # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_blend_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) logger.info(str(argsIn)) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if options.blendToFireball: fireballFrameDict = icebridge_common.getCorrectedFireballDems( options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(orthoFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No ortho files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in orthoFrameDict: logger.info("Error: Missing ortho file for frame: " + str(frame) + ".") continue orthoFile = orthoFrameDict[frame] try: lidarFile = icebridge_common.findMatchingLidarFile( orthoFile, lidarFolder) except: # Skip if no lidar file matches this frame continue fireballDEM = "" if options.blendToFireball: if frame in fireballFrameDict: fireballDEM = fireballFrameDict[frame] else: logger.info("No fireball DEM for frame: " + str(frame)) args = (frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runBlend, args)) else: runBlend(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)