def convertJpegs(jpegFolder, imageFolder): '''Convert jpeg images from RGB to single channel''' logger = logging.getLogger(__name__) logger.info('Converting input images to grayscale...') # Loop through all the input images os.system('mkdir -p ' + imageFolder) jpegFiles = os.listdir(jpegFolder) for jpegFile in jpegFiles: inputPath = os.path.join(jpegFolder, jpegFile) # Skip non-image files ext = os.path.splitext(jpegFile)[1] if ext != '.JPG': continue # Make sure the timestamp and frame number are in the output file name (dateStr, timeStr) = getJpegDateTime(inputPath) frameNumber = icebridge_common.getFrameNumberFromFilename(inputPath) outputName = ('DMS_%s_%s_%05d.tif') % (dateStr, timeStr, frameNumber) outputPath = os.path.join(imageFolder, outputName) # Skip existing files if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale cmd = (('convert %s -colorspace Gray %s') % (inputPath, outputPath)) logger.info(cmd) os.system(cmd) if not os.path.exists(outputPath): raise Exception('Failed to convert jpeg file: ' + jpegFile)
def get_camera(cameraFolder, frame): '''Get the label file and the camera file for a given frame''' # Get a list of all the input files # allImageFiles = icebridge_common.getTifs(labelFolder) allCameraFiles = icebridge_common.getByExtension(cameraFolder, '.tsai') # Keep only the images and cameras within the given range # imageFile = None # for image in allImageFiles: # thisFrame = icebridge_common.getFrameNumberFromFilename(image) # if thisFrame != frame: # continue # imageFile = os.path.join(labelFolder, image) # break cameraFile = None for camera in allCameraFiles: thisFrame = icebridge_common.getFrameNumberFromFilename(camera) if thisFrame != frame: continue cameraFile = os.path.join(cameraFolder, camera) break return cameraFile
def conversionIsFinished(self, startFrame, stopFrame, verbose=False): '''Return true if this run is present and conversion has finished running on it''' logger = logging.getLogger(__name__) # Make sure that there is a camera file for input image file. # - This could be a more expansive check. cameraFolder = self.getCameraFolder() imageList = self.getImageList() for imageFile in imageList: camFile = os.path.join( cameraFolder, icebridge_common.getCameraFileName(imageFile)) # Check only within range # TODO: Actually we need the cameras to go a bit beyond frame = icebridge_common.getFrameNumberFromFilename(camFile) if frame < startFrame or frame >= stopFrame: continue if not os.path.exists(camFile): if verbose: logger.error('Missing file ' + camFile) return False # Do a simple check of the converted lidar files prependFolder = True lidarFolder = self.getLidarFolder() convLidarFile = icebridge_common.getConvertedLidarIndexFile( lidarFolder) (lidarDict, dummyUrlDict) = icebridge_common.readIndexFile( convLidarFile, prependFolder) pairedLidarFolder = icebridge_common.getPairedLidarFolder(lidarFolder) pairedLidarFile = icebridge_common.getPairedIndexFile( pairedLidarFolder) (pairedLidarDict, dummyUrlDict) = icebridge_common.readIndexFile( pairedLidarFile, prependFolder) numLidar = len(lidarDict.values()) numPairedLidar = len(pairedLidarDict.values()) if numLidar != (numPairedLidar + 1): logger.error('Not enough paired lidar files found') return False # Make sure the lidar files are not empty success = True for f in lidarDict.values() + pairedLidarDict.values(): if not asp_file_utils.fileIsNonZero(f): logger.error('lidar file ' + f + ' is empty!') os.system('rm -f ' + f) # Remove bad files success = False return success
def conversionIsFinished(self, startFrame, stopFrame, verbose=False): '''Return true if this run is present and conversion has finished running on it''' logger = logging.getLogger(__name__) # Make sure that there is a camera file for input image file. # - This could be a more expansive check. cameraFolder = self.getCameraFolder() imageList = self.getImageList() for imageFile in imageList: camFile = os.path.join(cameraFolder, icebridge_common.getCameraFileName(imageFile)) # Check only within range # TODO: Actually we need the cameras to go a bit beyond frame = icebridge_common.getFrameNumberFromFilename(camFile) if frame < startFrame or frame >= stopFrame: continue if not os.path.exists(camFile): if verbose: logger.error('Missing file ' + camFile) return False # Do a simple check of the converted lidar files prependFolder = True lidarFolder = self.getLidarFolder() convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder) (lidarDict, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile, prependFolder) pairedLidarFolder = icebridge_common.getPairedLidarFolder(lidarFolder) pairedLidarFile = icebridge_common.getPairedIndexFile(pairedLidarFolder) (pairedLidarDict, dummyUrlDict) = icebridge_common.readIndexFile(pairedLidarFile, prependFolder) numLidar = len(lidarDict.values()) numPairedLidar = len(pairedLidarDict.values()) if numLidar != (numPairedLidar+1): logger.error('Not enough paired lidar files found') return False # Make sure the lidar files are not empty success = True for f in lidarDict.values() + pairedLidarDict.values(): if not asp_file_utils.fileIsNonZero(f): logger.error('lidar file ' + f + ' is empty!') os.system('rm -f ' + f) # Remove bad files success = False return success
def main(argsIn): try: usage = '''usage: process_icebridge_run.py <image_folder> <camera_folder> <lidar_folder> <output_folder>''' parser = optparse.OptionParser(usage=usage) # Data selection optios parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=-1, type='int', help='The frame number to finish processing with.') parser.add_option('--south', action='store_true', default=False, dest='isSouth', help='MUST be set if the images are in the southern hemisphere.') # Processing options parser.add_option('--stereo-arguments', dest='stereoArgs', default='', help='Additional argument string to be passed to the stereo command.') parser.add_option('--bundle-length', dest='bundleLength', default=2, type='int', help='Number of images to bundle adjust and process at once.') parser.add_option('--image-stereo-interval', dest='imageStereoInterval', default=None, type='int', help='Advance this many frames to get the stereo pair. Default is auto-calculate') parser.add_option('--solve-intrinsics', action='store_true', default=False, dest='solve_intr', help='If to float the intrinsics params.') #parser.add_option('--dem-resolution', dest='demResolution', default=0.4, # type='float', help='Generate output DEMs at this resolution.') parser.add_option('--max-displacement', dest='maxDisplacement', default=20, type='float', help='Max displacement value passed to pc_align.') # Performance options parser.add_option('--num-processes', dest='numProcesses', default=1, type='int', help='The number of simultaneous processes to run.') parser.add_option('--num-threads', dest='numThreads', default=None, type='int', help='The number threads to use per process.') # Action options parser.add_option('--interactive', action='store_true', default=False, dest='interactive', help='If to wait on user input to terminate the jobs.') parser.add_option('--log-batches', action='store_true', default=False, dest='logBatches', help="Just log the batch commands to a file.") parser.add_option('--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_option('--many-ip', action='store_true', default=False, dest='manyip', help='If to use a lot of IP in bundle adjustment from the beginning.') parser.add_option('--dry-run', action='store_true', default=False, dest='dryRun', help="Print but don't launch the processing jobs.") parser.add_option('--ortho-folder', dest='orthoFolder', default=None, help='Use ortho files to adjust processing to the image spacing.') parser.add_option('--fireball-folder', dest='fireballFolder', default=None, help='Location of fireball DEMs for comparison.') parser.add_option('--reference-dem', dest='referenceDem', default=None, help='Reference DEM used to calculate the expected GSD.') (options, args) = parser.parse_args(argsIn) if len(args) < 4: print(usage) return 0 imageFolder = args[0] cameraFolder = args[1] lidarFolder = args[2] outputFolder = args[3] except optparse.OptionError as msg: raise Usage(msg) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Check the inputs for f in [imageFolder, cameraFolder, lidarFolder]: if not os.path.exists(f): logger.error('Input folder '+ f +' does not exist!') return 0 asp_system_utils.mkdir_p(outputFolder) suppressOutput = False redo = False logger.info('\nStarting processing...') # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs(imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) numFiles = len(list(imageCameraPairs)) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') # Check that the files are properly aligned lastFrame = -1 availableFrames = [] for (image, camera) in imageCameraPairs: frameNumber = icebridge_common.getFrameNumberFromFilename(image) availableFrames.append(frameNumber) if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber): logger.error('Error: input files do not align!\n' + str((image, camera))) return -1 if frameNumber <= lastFrame: logger.error('Error: input frames not sorted properly!\n') return -1 lastFrame = frameNumber # Do not compute output resolution. Will be overwritten anyway per frame. ## Set the output resolution as the computed mean GSD ## TODO: This should be cashed, and recomputed only when the batch file changes! #NUM_GSD_FRAMES = 20 #logger.info('Computing GSD with ' + str(NUM_GSD_FRAMES) + ' frames.') #gsdFrameSkip = len(imageCameraPairs) / NUM_GSD_FRAMES #if gsdFrameSkip < 1: # gsdFrameSkip = 1 #medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth, # gsdFrameSkip) #outputResolution = icebridge_common.gsdToDemRes(medianGsd) #logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution)) outputResolution = 0.4 # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in imageCameraPairs: vizString += camera+' ' cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString logger.info('Running orbitviz on input files...') # Suppress (potentially long) output asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo) # Set up options for process_icebridge_batch extraOptions = '' if options.numThreads: extraOptions += ' --num-threads ' + str(options.numThreads) if options.solve_intr: extraOptions += ' --solve-intrinsics ' if options.isSouth: extraOptions += ' --south ' if options.maxDisplacement: extraOptions += ' --max-displacement ' + str(options.maxDisplacement) if options.fireballFolder: extraOptions += ' --fireball-folder ' + str(options.fireballFolder) if options.cleanup: extraOptions += ' --cleanup ' if options.manyip: extraOptions += ' --many-ip ' # We ran this before, as part of fetching, so hopefully all the data is cached forceAllFramesInRange = False (breaks, largeSkips) = getImageSpacing(options.orthoFolder, availableFrames, options.startFrame, options.stopFrame, forceAllFramesInRange) if options.imageStereoInterval: logger.info('Using manually specified image stereo interval: ' + str(options.imageStereoInterval)) largeSkips = [] # Always use a manually specified skip interval else: options.imageStereoInterval = 1 sleepTime = 20 # If all we are doing is logging commands then one process is sufficient. # - Wipe the output file while we are at it. batchLogPath = '' batchNum = 0 if options.logBatches: options.numProcesses = 1 sleepTime = 1 batchLogPath = os.path.join(outputFolder, BATCH_COMMAND_LOG_FILE) os.system('rm -f ' + batchLogPath) logger.info('Just generating batch log file '+batchLogPath+', no processing will occur.') logger.info('Starting processing pool with ' + str(options.numProcesses) +' processes.') pool = NonDaemonPool(options.numProcesses) # Call process_icebridge_batch on each batch of images. # - Batch size should be the largest number of images which can be effectively bundle-adjusted. taskHandles = [] batchImageCameraPairs = [] frameNumbers = [] i = 0 # The frame index that starts the current batch while True: # Loop for adding batches # Bugfix: arrived at the last feasible frame (maybe there are more but # they lack cameras). if i >= len(list(imageCameraPairs)): break firstBundleFrame = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[i][0]) # Determine the frame skip amount for this batch (set by the first frame) thisSkipInterval = options.imageStereoInterval if firstBundleFrame in largeSkips: #print(" ", firstBundleFrame, " in largeskips!" ) thisSkipInterval = largeSkips[firstBundleFrame] thisBatchSize = options.bundleLength + thisSkipInterval - 1 # Keep adding frames until we have enough or run out of frames j = i # The frame index that ends the batch while True: if j >= len(imageCameraPairs): # Bugfix: arrived at the last feasible frame. break frameNumber = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[j][0]) # Useful debugging code #print("Framenumber is ", frameNumber) #if int(frameNumber) > 8531: # # pass # for t in range(8340, 8360): # print("i frame is ", t, imageCameraPairs[t][0]) # print("breaks are ", breaks) # sys.exit(1) # Update conditions hitBreakFrame = (frameNumber in breaks) lastFrame = (frameNumber > options.stopFrame) or (j >= numFiles) endBatch = ( len(frameNumbers) >= thisBatchSize ) if lastFrame or endBatch: break # The new frame is too much, don't add it to the batch # Add frame to the list for the current batch batchImageCameraPairs.append(imageCameraPairs[j]) frameNumbers.append(frameNumber) if hitBreakFrame: logger.info("Hit a break, won't start a batch with frame: " + str(frameNumber)) break # Break after this frame, it is the last one added to the batch. j = j + 1 # Done adding frames to this batch if len(frameNumbers) <= thisSkipInterval: logger.info('Batch from frame: ' + str(firstBundleFrame) + ' is too small to run. Skipping.') else: # Submit the batch if not options.logBatches: logger.info('Processing frame number: ' + str(firstBundleFrame)) # The output folder is named after the first and last frame in the batch. # We count on this convention in blend_dems.py. batchFolderName = icebridge_common.batchFolderName(frameNumbers[0], frameNumbers[-1], options.bundleLength) thisOutputFolder = os.path.join(outputFolder, batchFolderName) if not options.logBatches: logger.info('Running processing batch in output folder: ' + \ thisOutputFolder + '\n' + 'with options: ' + \ extraOptions + ' --stereo-arguments ' + \ options.stereoArgs) if not options.dryRun: # Generate the command call taskHandles.append(pool.apply_async(processBatch, (batchImageCameraPairs, lidarFolder, options.referenceDem, thisSkipInterval, thisOutputFolder, extraOptions, outputResolution, options.stereoArgs, batchNum, batchLogPath))) batchNum += 1 # Reset these lists batchImageCameraPairs = [] frameNumbers = [] # Advance to the frame that starts the next batch if hitBreakFrame: # When we hit a break in the frames we need to start the # next batch after the break frame #print("Hit break frame for i, j, frameNumber", i, j, frameNumber) i = j + 1 else: # Start in the next frame that was not used as a "left" stereo image. i = i + options.bundleLength - 1 if lastFrame: break # Quit the main loop if we hit the end of the frame list. # End of loop through input file pairs logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') # Wait for all the tasks to complete icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, options.interactive, quitKey='q', sleepTime=sleepTime) # Either all the tasks are finished or the user requested a cancel. # Clean up the processing pool icebridge_common.stopTaskPool(pool) logger.info('Finished process_icebridge_run.') # to avoid ending a log with 'waiting ...'
def getImageSpacing(orthoFolder, availableFrames, startFrame, stopFrame, forceAllFramesInRange): '''Find a good image stereo spacing interval that gives us a good balance between coverage and baseline width. Also detect all frames where this is a large break after the current frame.''' logger.info('Computing optimal image stereo interval...') ## With very few cameras this is the only possible way to process them #if len(availableFrames) < 3 and not forceAllFramesInRange: # return ([], {}) # No skip, no breaks # Retrieve a list of the ortho files orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) # From the dictionary create a sorted list of ortho files in the frame range breaks = [] largeSkips = {} orthoFiles = [] for frame in sorted(orthoFrameDict.keys()): # Only process frames within the range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue orthoPath = orthoFrameDict[frame] frame = icebridge_common.getFrameNumberFromFilename(orthoPath) if not forceAllFramesInRange: if frame not in availableFrames: # Skip frames we did not compute a camera for continue orthoFiles.append(orthoPath) orthoFiles.sort() numOrthos = len(orthoFiles) # First load whatever boxes are there projectionIndexFile = icebridge_common.projectionBoundsFile(os.path.dirname(orthoFolder)) logger.info("Reading: " + projectionIndexFile) boundsDict = icebridge_common.readProjectionBounds(projectionIndexFile) # Get the bounding box and frame number of each ortho image logger.info('Loading bounding boxes...') frames = [] updatedBounds = False # will be true if some computation got done count = 0 for i in range(0, numOrthos): # This can be slow, so add a progress dialong count = count + 1 if (count - 1) % 1000 == 0: logger.info('Progress: ' + str(count) + '/' + str(numOrthos)) thisFrame = icebridge_common.getFrameNumberFromFilename(orthoFiles[i]) if thisFrame not in boundsDict: imageGeoInfo = asp_geo_utils.getImageGeoInfo(orthoFiles[i], getStats=False) thisBox = imageGeoInfo['projection_bounds'] boundsDict[thisFrame] = thisBox updatedBounds = True frames.append(thisFrame) # Read this file again, in case some other process modified it in the meantime. # This won't happen in production mode, but can during testing with partial sequences. boundsDictRecent = icebridge_common.readProjectionBounds(projectionIndexFile) for frame in sorted(boundsDictRecent.keys()): if not frame in boundsDict.keys(): boundsDict[frame] = boundsDictRecent[frame] updatedBounds = True # Save the bounds. There is always the danger that two processes will # do that at the same time, but this is rare, as hopefully we get here # only once from the manager. It is not a big loss if this file gets messed up. if updatedBounds: logger.info("Writing: " + projectionIndexFile) icebridge_common.writeProjectionBounds(projectionIndexFile, boundsDict) # Since we are only comparing the image bounding boxes, not their exact corners, # these ratios are only estimates. MAX_RATIO = 0.85 # Increase skip until we get below this... MIN_RATIO = 0.75 # ... but don't go below this value! NOTRY_RATIO = 0.0001 # Don't bother with overlap amounts less than this (small on purpose) def getBboxArea(bbox): '''Return the area of a bounding box in form of (minX, maxX, minY, maxY)''' width = bbox[1] - bbox[0] height = bbox[3] - bbox[2] if (width < 0) or (height < 0): return 0 return width*height # Iterate over the frames and find the best stereo frame for each for i in range(0, numOrthos-1): thisFrame = frames[i] thisBox = boundsDict[thisFrame] thisArea = getBboxArea(thisBox) interval = 1 while(True): # Compute intersection area between this and next image nextFrame = frames[i+interval] nextBox = boundsDict[nextFrame] intersect = [max(thisBox[0], nextBox[0]), # Min X min(thisBox[1], nextBox[1]), # Max X max(thisBox[2], nextBox[2]), # Min Y min(thisBox[3], nextBox[3])] # Max Y area = getBboxArea(intersect) ratio = 0 if area > 0: ratio = area / thisArea if interval == 1: # Cases for the smallest interval... if ratio < NOTRY_RATIO: breaks.append(thisFrame) # No match for this frame logger.info('Detected large break after frame ' + str(thisFrame)) break if ratio < MIN_RATIO: break # No reason to try increasing skip amounts for this frame else: # interval > 1 if ratio < MIN_RATIO: # Went too small, walk back the interval. interval = interval - 1 break if ratio > MAX_RATIO: # Too much overlap, increase interval interval = interval + 1 else: # Overlap is fine, keep this interval. break # Handle the case where we go past the end of frames looking for a match. if i+interval >= len(frames): interval = interval - 1 break if interval > 1: # Only record larger than normal intervals. largeSkips[thisFrame] = interval logger.info('Detected ' + str(len(breaks)) + ' breaks in image coverage.') logger.info('Detected ' + str(len(largeSkips)) + ' images with interval > 1.') return (breaks, largeSkips)
if 'fu' in line: goodFile = True break if goodFile: break # Get the ortho list orthoFiles = icebridge_common.getTifs(orthoFolder) logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.') # Look up the frame numbers for each ortho file infoDict = {} for ortho in orthoFiles: if ('gray' in ortho) or ('sub' in ortho): continue frame = icebridge_common.getFrameNumberFromFilename(ortho) if frame < options.startFrame or frame > options.stopFrame: continue infoDict[frame] = [ortho, ''] # Get the image file list try: imageFiles = icebridge_common.getTifs(imageFolder) except Exception, e: raise Exception("Cannot continue with nav generation, will resume later when images are created. This is not a fatal error. " + str(e)) logger.info('Found ' + str(len(imageFiles)) + ' image files.') # Update the second part of each dictionary object for image in imageFiles: if ('gray' in image) or ('sub' in image):
def solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, processedFolder, isSouth, logger): '''Create a camera model with optimized intrinsics. By now we processed a bunch of images and created bundle-adjusted and pc_aligned cameras and DEMs while using a camera model with distortion implemented using RPC coefficients which was obtained from the photometrics model. We now use the obtained cameras as inputs to a bundle adjust problem where we will optimize the intrinsics, including the distortion RPC coefficients, using the lidar as an external constraint, and many dense IP pairs and triplets (no quadruplets yet, even if 4 images overlap).''' # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs(imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) # The paired lidar file for the first image should be huge enough to contain # all images. lidarFile = icebridge_common.findMatchingLidarFile(imageCameraPairs[0][0], lidarFolder) logger.info('Found matching lidar file ' + lidarFile) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) numFiles = len(imageCameraPairs) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') if numFiles % 2 != 0: raise Exception("When solving for intrinsics, must have an even number of frames to use.") # Collect pc_align-ed cameras, unaligned disparities, and dense match files images = [] cameras = [] dispFiles = [] for it in range(numFiles/2): begFrame = options.startFrame + 2*it endFrame = begFrame + 1 batchFolderName = icebridge_common.batchFolderName(begFrame, endFrame, options.bundleLength) thisOutputFolder = os.path.join(processedFolder, batchFolderName) # Find all the cameras after bundle adjustment and pc_align. pattern = icebridge_common.getAlignedBundlePrefix(thisOutputFolder) + '*.tsai' alignedCameras = glob.glob(pattern) if len(alignedCameras) != options.bundleLength: raise Exception("Expected " + str(options.bundleLength) + " cameras, here's what " + " was obtained instead: " + " ".join(alignedCameras)) img0 = ""; cam0 = ""; img1 = ""; cam1 = "" for cam in alignedCameras: frame = icebridge_common.getFrameNumberFromFilename(cam) if begFrame == frame: img0 = imageCameraPairs[2*it][0] cam0 = cam if endFrame == frame: img1 = imageCameraPairs[2*it+1][0] cam1 = cam images.append(img0); images.append(img1) cameras.append(cam0); cameras.append(cam1) # Unaligned disparity stereoFolder = os.path.join(thisOutputFolder, 'stereo_pair_'+str(0)) currDispFiles = glob.glob(os.path.join(stereoFolder, '*unaligned-D.tif')) if len(currDispFiles) != 1: raise Exception("Expecting a single unaligned disparity file in " + stereoFolder) dispFiles.append(currDispFiles[0]) # Match files matchFiles = [] for it in range(numFiles-1): begFrame = options.startFrame + it endFrame = begFrame + 1 batchFolderName = icebridge_common.batchFolderName(begFrame, endFrame, options.bundleLength) thisOutputFolder = os.path.join(processedFolder, batchFolderName) stereoFolder = os.path.join(thisOutputFolder, 'stereo_pair_'+str(0)) DISP_PREFIX = "disp-" currMatchFiles = glob.glob(os.path.join(stereoFolder, '*' + DISP_PREFIX + '*.match')) if len(currMatchFiles) != 1: raise Exception("Expecting a single dense match file in " + stereoFolder) matchFiles.append(currMatchFiles[0]) # Create output directory for bundle adjustment and copy there the match files baDir = os.path.join(processedFolder, "bundle_intrinsics") baPrefix = os.path.join(baDir, "out") os.system("mkdir -p " + baDir) for matchFile in matchFiles: dstFile = os.path.basename(matchFile) dstFile = dstFile.replace(DISP_PREFIX, '') dstFile = os.path.join(baDir, dstFile) cmd = "cp -f " + matchFile + " " + dstFile logger.info(cmd) os.system(cmd) # The bundle adjustment cmd = "bundle_adjust " + " ".join(images) + " " + " ".join(cameras) + \ ' --reference-terrain ' + lidarFile + \ ' --disparity-list "' + " ".join(dispFiles) + '"' + \ ' --datum wgs84 -t nadirpinhole --create-pinhole-cameras --robust-threshold 2' + \ ' --camera-weight 1 --solve-intrinsics --csv-format ' + lidarCsvFormatString + \ ' --overlap-limit 1 --max-disp-error 10 --max-iterations 100 ' + \ ' --parameter-tolerance 1e-12 -o ' + baPrefix logger.info(cmd) os.system(cmd) # Generate DEMs of residuals before and after optimization projString = icebridge_common.getEpsgCode(isSouth, asString=True) for val in ['initial', 'final']: cmd = 'point2dem --t_srs ' + projString + ' --tr 2' + \ ' --csv-format 1:lon,2:lat,4:height_above_datum' + \ ' ' + baPrefix + '-' + val + '_residuals_no_loss_function_pointmap_point_log.csv' logger.info(cmd) os.system(cmd) cmd = 'point2dem --t_srs ' + projString + ' --tr 2' + \ ' --csv-format 1:lon,2:lat,4:height_above_datum' + \ ' ' + baPrefix + '-' + val +'_residuals_no_loss_function_reference_terrain.txt' logger.info(cmd) os.system(cmd) # Look at the latest written tsai file, that will be the optimized distortion file. # Force the initial rotation and translation to be the identity, this is # expected by ortho2pinhole. outFiles = filter(os.path.isfile, glob.glob(baPrefix + '*.tsai')) outFiles.sort(key=lambda x: os.path.getmtime(x)) optFile = outFiles[-1] logger.info("Reading optimized file: " + optFile) with open(optFile, 'r') as f: lines = f.readlines() for it in range(len(lines)): lines[it] = lines[it].strip() if re.match("^C\s*=\s*", lines[it]): lines[it] = "C = 0 0 0" if re.match("^R\s*=\s*", lines[it]): lines[it] = "R = 1 0 0 0 1 0 0 0 1" # Write the final desired optimized RPC file logger.info("Writing final optimized file: " + options.outputCalCamera) # Below is a bugfix, must take full path to find the dir, otherwise it may fail. os.system("mkdir -p " + os.path.dirname(os.path.abspath(options.outputCalCamera))) with open(options.outputCalCamera, 'w') as f: for line in lines: f.write(line + "\n")
def solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, processedFolder, isSouth, logger): '''Create a camera model with optimized intrinsics. By now we processed a bunch of images and created bundle-adjusted and pc_aligned cameras and DEMs while using a camera model with distortion implemented using RPC coefficients which was obtained from the photometrics model. We now use the obtained cameras as inputs to a bundle adjust problem where we will optimize the intrinsics, including the distortion RPC coefficients, using the lidar as an external constraint, and many dense IP pairs and triplets (no quadruplets yet, even if 4 images overlap).''' # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs( imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) # The paired lidar file for the first image should be huge enough to contain # all images. lidarFile = icebridge_common.findMatchingLidarFile(imageCameraPairs[0][0], lidarFolder) logger.info('Found matching lidar file ' + lidarFile) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) numFiles = len(imageCameraPairs) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') if numFiles % 2 != 0: raise Exception( "When solving for intrinsics, must have an even number of frames to use." ) # Collect pc_align-ed cameras, unaligned disparities, and dense match files images = [] cameras = [] dispFiles = [] for it in range(numFiles / 2): begFrame = options.startFrame + 2 * it endFrame = begFrame + 1 batchFolderName = icebridge_common.batchFolderName( begFrame, endFrame, options.bundleLength) thisOutputFolder = os.path.join(processedFolder, batchFolderName) # Find all the cameras after bundle adjustment and pc_align. pattern = icebridge_common.getAlignedBundlePrefix( thisOutputFolder) + '*.tsai' alignedCameras = glob.glob(pattern) if len(alignedCameras) != options.bundleLength: raise Exception("Expected " + str(options.bundleLength) + " cameras, here's what " + " was obtained instead: " + " ".join(alignedCameras)) img0 = "" cam0 = "" img1 = "" cam1 = "" for cam in alignedCameras: frame = icebridge_common.getFrameNumberFromFilename(cam) if begFrame == frame: img0 = imageCameraPairs[2 * it][0] cam0 = cam if endFrame == frame: img1 = imageCameraPairs[2 * it + 1][0] cam1 = cam images.append(img0) images.append(img1) cameras.append(cam0) cameras.append(cam1) # Unaligned disparity stereoFolder = os.path.join(thisOutputFolder, 'stereo_pair_' + str(0)) currDispFiles = glob.glob( os.path.join(stereoFolder, '*unaligned-D.tif')) if len(currDispFiles) != 1: raise Exception("Expecting a single unaligned disparity file in " + stereoFolder) dispFiles.append(currDispFiles[0]) # Match files matchFiles = [] for it in range(numFiles - 1): begFrame = options.startFrame + it endFrame = begFrame + 1 batchFolderName = icebridge_common.batchFolderName( begFrame, endFrame, options.bundleLength) thisOutputFolder = os.path.join(processedFolder, batchFolderName) stereoFolder = os.path.join(thisOutputFolder, 'stereo_pair_' + str(0)) DISP_PREFIX = "disp-" currMatchFiles = glob.glob( os.path.join(stereoFolder, '*' + DISP_PREFIX + '*.match')) if len(currMatchFiles) != 1: raise Exception("Expecting a single dense match file in " + stereoFolder) matchFiles.append(currMatchFiles[0]) # Create output directory for bundle adjustment and copy there the match files baDir = os.path.join(processedFolder, "bundle_intrinsics") baPrefix = os.path.join(baDir, "out") os.system("mkdir -p " + baDir) for matchFile in matchFiles: dstFile = os.path.basename(matchFile) dstFile = dstFile.replace(DISP_PREFIX, '') dstFile = os.path.join(baDir, dstFile) cmd = "cp -f " + matchFile + " " + dstFile logger.info(cmd) os.system(cmd) # The bundle adjustment cmd = "bundle_adjust " + " ".join(images) + " " + " ".join(cameras) + \ ' --reference-terrain ' + lidarFile + \ ' --disparity-list "' + " ".join(dispFiles) + '"' + \ ' --datum wgs84 -t nadirpinhole --local-pinhole --robust-threshold 2' + \ ' --camera-weight 1 --solve-intrinsics --csv-format ' + lidarCsvFormatString + \ ' --overlap-limit 1 --max-disp-error 10 --max-iterations 100 ' + \ ' --parameter-tolerance 1e-12 -o ' + baPrefix logger.info(cmd) os.system(cmd) # Generate DEMs of residuals before and after optimization projString = icebridge_common.getEpsgCode(isSouth, asString=True) for val in ['initial', 'final']: cmd = 'point2dem --t_srs ' + projString + ' --tr 2' + \ ' --csv-format 1:lon,2:lat,4:height_above_datum' + \ ' ' + baPrefix + '-' + val + '_residuals_no_loss_function_pointmap_point_log.csv' logger.info(cmd) os.system(cmd) cmd = 'point2dem --t_srs ' + projString + ' --tr 2' + \ ' --csv-format 1:lon,2:lat,4:height_above_datum' + \ ' ' + baPrefix + '-' + val +'_residuals_no_loss_function_reference_terrain.txt' logger.info(cmd) os.system(cmd) # Look at the latest written tsai file, that will be the optimized distortion file. # Force the initial rotation and translation to be the identity, this is # expected by ortho2pinhole. outFiles = filter(os.path.isfile, glob.glob(baPrefix + '*.tsai')) outFiles.sort(key=lambda x: os.path.getmtime(x)) optFile = outFiles[-1] logger.info("Reading optimized file: " + optFile) with open(optFile, 'r') as f: lines = f.readlines() for it in range(len(lines)): lines[it] = lines[it].strip() if re.match("^C\s*=\s*", lines[it]): lines[it] = "C = 0 0 0" if re.match("^R\s*=\s*", lines[it]): lines[it] = "R = 1 0 0 0 1 0 0 0 1" # Write the final desired optimized RPC file logger.info("Writing final optimized file: " + options.outputCalCamera) os.system("mkdir -p " + os.path.dirname(options.outputCalCamera)) with open(options.outputCalCamera, 'w') as f: for line in lines: f.write(line + "\n")
def fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, path, fileType): '''Retrieve the index file for a folder of data and create a parsed version of it that contains frame number / filename pairs.''' # Download the html file curlCmd = baseCurlCmd + ' ' + folderUrl + ' > ' + path logger.info(curlCmd) p = subprocess.Popen(curlCmd, shell=True) os.waitpid(p.pid, 0) # Find all the file names in the index file and # dump them to a new index file logger.info('Extracting file name list from index.html file...') with open(path, 'r') as f: indexText = f.read() # Must wipe this html file. We fetch it too often in different # contexts. If not wiped, the code fails to work in some # very rare but real situations. if os.path.exists(path): os.remove(path) # Extract just the file names fileList = [] # ensure initialization if fileType == 'jpeg': fileList = re.findall(">[0-9_]*.JPG", indexText, re.IGNORECASE) if fileType == 'ortho': fileList = re.findall(">DMS\w*.tif<", indexText, re.IGNORECASE) if fileType == 'fireball': # Fireball DEMs fileList = re.findall(">IODMS\w*DEM.tif", indexText, re.IGNORECASE) if fileType == 'lvis': fileList = re.findall(">ILVIS\w+.TXT", indexText, re.IGNORECASE) if fileType == 'atm1': fileList = re.findall(">ILATM1B[0-9_]*.ATM4\w+.qi", indexText, re.IGNORECASE) # >ILATM1B_20111018_145455.ATM4BT4.qi # or >ILATM1B_20091016_165112.atm4cT3.qi if fileType == 'atm2': # Match ILATM1B_20160713_195419.ATM5BT5.h5 fileList = re.findall(">ILATM1B[0-9_]*.ATM\w+.h5", indexText, re.IGNORECASE) # Get rid of '>' and '<' for fileIter in range(len(fileList)): fileList[fileIter] = fileList[fileIter].replace(">", "") fileList[fileIter] = fileList[fileIter].replace("<", "") # Some runs, eg, https://n5eil01u.ecs.nsidc.org/ICEBRIDGE/IODMS1B.001/2015.09.24 # have files for both GR and AN, with same frame number. Those need to be separated # by latitude. This is a problem only with orthoimages. badXmls = set() outputFolder = os.path.dirname(path) if separateByLat: allFilesToFetch = [] allUrlsToFetch = [] for filename in fileList: xmlFile = icebridge_common.xmlFile(filename) url = os.path.join(folderUrl, xmlFile) outputPath = os.path.join(outputFolder, xmlFile) allFilesToFetch.append(outputPath) allUrlsToFetch.append(url) dryRun = False icebridge_common.fetchFilesInBatches(baseCurlCmd, MAX_IN_ONE_CALL, dryRun, outputFolder, allFilesToFetch, allUrlsToFetch, logger) # Mark the bad ones for xmlFile in allFilesToFetch: latitude = icebridge_common.parseLatitude(xmlFile) isGood = hasGoodLat(latitude, isSouth) if not isGood: badXmls.add(xmlFile) elif (fileType == 'ortho' or fileType == 'fireball'): # Sometimes there is a large gap in the timestamp. That means orthoimages # from previous day are spilling over. If dayVal is 0, we must ignore # the spillover images. If dayVal is 1, we must keep the spillover images # and igore the others. list1 = [] list2 = [] isBigGap = False prevStamp = -1 for filename in fileList: [imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename) currStamp = float(imageTimeString) / 1000000.0 # hours if prevStamp < 0: list1.append(filename) prevStamp = currStamp continue # Note that once isBigGap becomes true, it stays true # even when the gap gets small again if currStamp - prevStamp >= 6: # six hour gap is a lot isBigGap = True if not isBigGap: list1.append(filename) else: list2.append(filename) prevStamp = currStamp # for next iteration if isBigGap: if dayVal == 0: fileList = list2[:] # current day else: fileList = list1[:] # spillover from prev day # For each entry that matched the regex, record: the frame number and the file name. frameDict = {} urlDict = {} badFiles = [] for filename in fileList: if len(badXmls) > 0: xmlFile = os.path.join(outputFolder, icebridge_common.xmlFile(filename)) if xmlFile in badXmls: continue frame = icebridge_common.getFrameNumberFromFilename(filename) if frame in frameDict.keys(): # The same frame must not occur twice. if fileType not in LIDAR_TYPES: logger.error("Error: Found two file names with same frame number: " + \ frameDict[frame] + " and " + filename) badFiles.append(filename) badFiles.append(frameDict[frame]) # note that folderUrl can vary among orthoimages, as sometimes # some of them are in a folder for the next day. frameDict[frame] = filename urlDict[frame] = folderUrl # Wipe them all, to be sorted later for badFile in badFiles: if os.path.exists(badFile): logger.info("Deleting: " + badFile) os.remove(badFile) xmlFile = icebridge_common.xmlFile(badFile) if os.path.exists(xmlFile): logger.info("Deleting: " + xmlFile) os.remove(xmlFile) if len(badFiles) > 0: raise Exception("Found files with same frame number") return (frameDict, urlDict)
logger.info('\nStarting processing...') # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs( imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) numFiles = len(imageCameraPairs) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') # Check that the files are properly aligned lastFrame = -1 availableFrames = [] for (image, camera) in imageCameraPairs: frameNumber = icebridge_common.getFrameNumberFromFilename(image) availableFrames.append(frameNumber) if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber): logger.error('Error: input files do not align!\n' + str((image, camera))) return -1 if frameNumber <= lastFrame: logger.error('Error: input frames not sorted properly!\n') return -1 lastFrame = frameNumber # Set the output resolution as the computed mean GSD # - Currently we process ten frames total but this should be increased for production! # TODO: This should be cashed, and recomputed only when the batch file changes! GSD_RESOLUTION_MULTIPLIER = 4.0
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, inputCalCamera, cameraLookupPath, noNav, navCameraFolder, yyyymmdd, site, refDemPath, cameraFolder, simpleCameras, startFrame, stopFrame, framesFile, numProcesses, numThreads, logger): '''Generate camera models from the ortho files. Returns false if any files were not generated.''' logger.info('Generating camera models from ortho images...') imageFiles = icebridge_common.getTifs(imageFolder) orthoFiles = icebridge_common.getTifs(orthoFolder) if navCameraFolder != "": estimateFiles = icebridge_common.getByExtension( navCameraFolder, '.tsai') else: estimateFiles = [] # See if to process frames from file filesSet = set() if framesFile != "": filesSet = icebridge_common.readLinesInSet(framesFile) # Make a dictionary of ortho files by frame # - The orthoFiles list contains _gray.tif as well as the original # images. Prefer the gray versions because it saves a bit of time # in the ortho2pinhole process. orthoFrames = {} for f in orthoFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue # Record this file if it is the first of this frame or # if it is the gray version of this frame. if (frame not in orthoFrames) or ('_gray.tif' in f): orthoFrames[frame] = f # Make a dictionary of estimated camera files by frame estimatedFrames = {} for f in estimateFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue estimatedFrames[frame] = f imageFiles.sort() logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] outputFiles = [] for imageFile in imageFiles: # Skip non-image files (including _sub images made by stereo_gui) # TODO: Use a function here from icebridge_common. Also replace # all similar locations. ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue if not frame in orthoFrames.keys(): continue # Find the estimated camera file to use with this ortho frame. orthoFile = orthoFrames[frame] try: estimatedCameraFile = estimatedFrames[frame] estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile) except: # For now treat this as an error, a missing nav file suggests # that something is going wrong with the flight! if not noNav: logger.error('Missing nav estimated camera for frame ' + str(frame)) continue else: estimatedCameraFile = None estimatedCameraPath = None #estimatedCameraFile = None #estimatedCameraPath = None # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join( cameraFolder, icebridge_common.getCameraFileName(imageFile)) outputFiles.append(outputCamFile) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs continue # Determine which input camera file will be used for this frame if inputCalCamera == "": inputCamFile = getCalibrationFileForFrame(cameraLookupPath, inputCalFolder, frame, yyyymmdd, site) else: # This logic will force to use a given camera rather than # looking it up. This is not the usual way of doing things. inputCamFile = inputCalCamera if not os.path.exists(inputCalCamera): raise Exception("Could not find: " + inputCalCamera) orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCameras, numThreads) if numProcesses > 1: # Add ortho2pinhole command to the task pool taskHandles.append( pool.apply_async(cameraFromOrthoWrapper, orthoArgs)) else: # Single process, more logging info this way cameraFromOrthoWrapper(*orthoArgs) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q') # All tasks should be finished icebridge_common.stopTaskPool(pool) logger.info('Finished ortho processing.') # Run a check to see if we got all the output files for f in outputFiles: if not os.path.exists(f): return False return True
def getImageSpacing(orthoFolder): '''Find a good image stereo spacing interval that gives us a good balance between coverage and baseline width. Also detect all frames where this is a large break after the current frame.''' # Do nothing if this option was not provided if not orthoFolder: return None logger.info('Computing optimal image stereo interval...') breaks = [] # Generate a list of valid, full path ortho files fileList = os.listdir(orthoFolder) orthoFiles = [] for orthoFile in fileList: # Skip non-image files (including junk from stereo_gui) and duplicate grayscale files ext = os.path.splitext(orthoFile)[1] if (ext != '.tif') or ('_sub' in orthoFile) or ('.tif_gray.tif' in orthoFile): continue orthoPath = os.path.join(orthoFolder, orthoFile) orthoFiles.append(orthoPath) orthoFiles.sort() numOrthos = len(orthoFiles) # Get the bounding box and frame number of each ortho image logger.info('Loading bounding boxes...') bboxes = [] frames = [] for i in range(0, numOrthos): imageGeoInfo = asp_geo_utils.getImageGeoInfo(orthoFiles[i], getStats=False) thisBox = imageGeoInfo['projection_bounds'] thisFrame = icebridge_common.getFrameNumberFromFilename(orthoFiles[i]) bboxes.append(thisBox) frames.append(thisFrame) # Since we are only comparing the image bounding boxes, not their exact corners, # these ratios are only estimates. MAX_RATIO = 0.8 # Increase skip until we get below this... MIN_RATIO = 0.4 # ... but don't go below this value! def getBboxArea(bbox): '''Return the area of a bounding box in form of (minX, maxX, minY, maxY)''' width = bbox[1] - bbox[0] height = bbox[3] - bbox[2] if (width < 0) or (height < 0): return 0 return width*height # Iterate over stereo image intervals to try to get our target numbers interval = 0 meanRatio = 1.0 while meanRatio > MAX_RATIO: meanRatio = 0 count = 0 interval = interval + 1 logger.info('Trying stereo image interval ' + str(interval)) if numOrthos <= interval: raise Exception('Error: There are too few images and they overlap too much. ' + \ 'Consider processing more images in the given batch.') for i in range(0, numOrthos-interval): # Compute intersection area between this and next image thisBox = bboxes[i ] lastBox = bboxes[i+interval] intersect = [max(lastBox[0], thisBox[0]), # Min X min(lastBox[1], thisBox[1]), # Max X max(lastBox[2], thisBox[2]), # Min Y min(lastBox[3], thisBox[3])] # Max Y thisArea = getBboxArea(thisBox) area = getBboxArea(intersect) ratio = area / thisArea # Don't include non-overlapping frames in the statistics if area > 0: meanRatio = meanRatio + ratio count = count + 1 # On the first pass (with interval 1) check for gaps in coverage. if (interval == 1) and (area <= 0): breaks.append(frames[i]) logger.info('Detected large break after frame ' + str(frames[i])) # Get the mean intersection ratio meanRatio = meanRatio / count logger.info(' --> meanRatio = ' + str(meanRatio)) # If we increased the interval too much, back it off by one step. if (meanRatio < MIN_RATIO) and (interval > 1): interval = interval - 1 logger.info('Computed automatic image stereo interval: ' + str(interval)) logger.info('Detected ' + str(interval) + ' breaks in image coverage.') return (interval, breaks)
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList(os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder = True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame]) except Exception as e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90 ' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90 ' cmd = ('%s %s -colorspace Gray %s%s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE|re.MULTILINE|re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) if badFiles: logger.error("Converstion of JPEGs failed. If any files were corrupted, " + "they were removed, and need to be re-fetched.") return (not badFiles)
def fetchAndParseIndexFileAux(isSouth, separateByLat, dayVal, baseCurlCmd, folderUrl, path, fileType): '''Retrieve the index file for a folder of data and create a parsed version of it that contains frame number / filename pairs.''' # Download the html file curlCmd = baseCurlCmd + ' ' + folderUrl + ' > ' + path logger.info(curlCmd) p = subprocess.Popen(curlCmd, shell=True, universal_newlines=True) os.waitpid(p.pid, 0) # Find all the file names in the index file and # dump them to a new index file logger.info('Extracting file name list from index.html file...') with open(path, 'r') as f: indexText = f.read() # Must wipe this html file. We fetch it too often in different # contexts. If not wiped, the code fails to work in some # very rare but real situations. if os.path.exists(path): os.remove(path) # Extract just the file names fileList = [] # ensure initialization if fileType == 'jpeg': fileList = re.findall(">[0-9_]*.JPG", indexText, re.IGNORECASE) if fileType == 'ortho': fileList = re.findall(">DMS\w*.tif<", indexText, re.IGNORECASE) if fileType == 'fireball': # Fireball DEMs fileList = re.findall(">IODMS\w*DEM.tif", indexText, re.IGNORECASE) if fileType == 'lvis': fileList = re.findall(">ILVIS\w+.TXT", indexText, re.IGNORECASE) if fileType == 'atm1': fileList = re.findall(">ILATM1B[0-9_]*.ATM4\w+.qi", indexText, re.IGNORECASE) # >ILATM1B_20111018_145455.ATM4BT4.qi # or >ILATM1B_20091016_165112.atm4cT3.qi if fileType == 'atm2': # Match ILATM1B_20160713_195419.ATM5BT5.h5 fileList = re.findall(">ILATM1B[0-9_]*.ATM\w+.h5", indexText, re.IGNORECASE) # Get rid of '>' and '<' for fileIter in range(len(fileList)): fileList[fileIter] = fileList[fileIter].replace(">", "") fileList[fileIter] = fileList[fileIter].replace("<", "") # Some runs, eg, https://n5eil01u.ecs.nsidc.org/ICEBRIDGE/IODMS1B.001/2015.09.24 # have files for both GR and AN, with same frame number. Those need to be separated # by latitude. This is a problem only with orthoimages. badXmls = set() outputFolder = os.path.dirname(path) if separateByLat: allFilesToFetch = [] allUrlsToFetch = [] for filename in fileList: xmlFile = icebridge_common.xmlFile(filename) url = os.path.join(folderUrl, xmlFile) outputPath = os.path.join(outputFolder, xmlFile) allFilesToFetch.append(outputPath) allUrlsToFetch.append(url) dryRun = False icebridge_common.fetchFilesInBatches(baseCurlCmd, MAX_IN_ONE_CALL, dryRun, outputFolder, allFilesToFetch, allUrlsToFetch, logger) # Mark the bad ones for xmlFile in allFilesToFetch: latitude = icebridge_common.parseLatitude(xmlFile) isGood = hasGoodLat(latitude, isSouth) if not isGood: badXmls.add(xmlFile) elif (fileType == 'ortho' or fileType == 'fireball'): # Sometimes there is a large gap in the timestamp. That means orthoimages # from previous day are spilling over. If dayVal is 0, we must ignore # the spillover images. If dayVal is 1, we must keep the spillover images # and igore the others. list1 = [] list2 = [] isBigGap = False prevStamp = -1 for filename in fileList: [imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename) currStamp = float(imageTimeString)/1000000.0 # hours if prevStamp < 0: list1.append(filename) prevStamp = currStamp continue # Note that once isBigGap becomes true, it stays true # even when the gap gets small again if currStamp - prevStamp >= 6: # six hour gap is a lot isBigGap = True if not isBigGap: list1.append(filename) else: list2.append(filename) prevStamp = currStamp # for next iteration if isBigGap: if dayVal == 0: fileList = list2[:] # current day else: fileList = list1[:] # spillover from prev day # For each entry that matched the regex, record: the frame number and the file name. frameDict = {} urlDict = {} badFiles = [] for filename in fileList: if len(badXmls) > 0: xmlFile = os.path.join(outputFolder, icebridge_common.xmlFile(filename)) if xmlFile in badXmls: continue frame = icebridge_common.getFrameNumberFromFilename(filename) if frame in frameDict.keys(): # The same frame must not occur twice. if fileType not in LIDAR_TYPES: logger.error("Error: Found two file names with same frame number: " + \ frameDict[frame] + " and " + filename) badFiles.append(filename) badFiles.append(frameDict[frame]) # note that folderUrl can vary among orthoimages, as sometimes # some of them are in a folder for the next day. frameDict[frame] = filename urlDict[frame] = folderUrl # Wipe them all, to be sorted later for badFile in badFiles: if os.path.exists(badFile): logger.info("Deleting: " + badFile) os.remove(badFile) xmlFile = icebridge_common.xmlFile(badFile) if os.path.exists(xmlFile): logger.info("Deleting: " + xmlFile) os.remove(xmlFile) if len(badFiles) > 0: raise Exception("Found files with same frame number") return (frameDict, urlDict)
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, cameraLookupPath, yyyymmdd, site, refDemPath, cameraFolder, numProcesses, numThreads): '''Generate camera models from the ortho files''' logger = logging.getLogger(__name__) logger.info('Generating camera models from ortho images...') imageFiles = os.listdir(imageFolder) orthoFiles = os.listdir(orthoFolder) # Make a dictionary of ortho files by frame orthoFrames = {} for f in orthoFiles: # Skip non-image files ext = os.path.splitext(f)[1] if (ext != '.tif') or ('_sub' in f): continue frame = icebridge_common.getFrameNumberFromFilename(f) orthoFrames[frame] = f logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] for imageFile in imageFiles: # Skip non-image files (including junk from stereo_gui) ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) orthoFile = orthoFrames[frame] # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join( cameraFolder, icebridge_common.getCameraFileName(imageFile)) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) continue # Determine which input camera file will be used for this frame inputCamFile = getCalibrationFileForFrame(cameraLookupPath, inputCalFolder, frame, yyyymmdd, site) # Add ortho2pinhole command to the task pool taskHandles.append( pool.apply_async(cameraFromOrthoWrapper, (inputPath, orthoPath, inputCamFile, outputCamFile, refDemPath, numThreads))) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False, quitKey='q') # All tasks should be finished, clean up the processing pool logger.info('Cleaning up the ortho processing pool...') icebridge_common.stopTaskPool(pool) logger.info('Finished cleaning up the ortho processing pool')
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, inputCalCamera, cameraLookupFile, noNav, navCameraFolder, yyyymmdd, site, refDemPath, cameraFolder, simpleCameras, startFrame, stopFrame, framesFile, numProcesses, numThreads, logger): '''Generate camera models from the ortho files. Returns false if any files were not generated.''' logger.info('Generating camera models from ortho images...') imageFiles = icebridge_common.getTifs(imageFolder) orthoFiles = icebridge_common.getTifs(orthoFolder) if navCameraFolder != "": estimateFiles = icebridge_common.getByExtension(navCameraFolder, '.tsai') else: estimateFiles = [] # See if to process frames from file filesSet = set() if framesFile != "": filesSet = icebridge_common.readLinesInSet(framesFile) # Make a dictionary of ortho files by frame # - The orthoFiles list contains _gray.tif as well as the original # images. Prefer the gray versions because it saves a bit of time # in the ortho2pinhole process. orthoFrames = {} for f in orthoFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue # Record this file if it is the first of this frame or # if it is the gray version of this frame. if (frame not in orthoFrames) or ('_gray.tif' in f): orthoFrames[frame] = f # Make a dictionary of estimated camera files by frame estimatedFrames = {} for f in estimateFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue estimatedFrames[frame] = f imageFiles.sort() logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] outputFiles = [] for imageFile in imageFiles: # Skip non-image files (including _sub images made by stereo_gui) # TODO: Use a function here from icebridge_common. Also replace # all similar locations. ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue if not frame in orthoFrames.keys(): continue # Find the estimated camera file to use with this ortho frame. orthoFile = orthoFrames[frame] try: estimatedCameraFile = estimatedFrames[frame] estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile) except: # For now treat this as an error, a missing nav file suggests # that something is going wrong with the flight! if not noNav: logger.error('Missing nav estimated camera for frame ' + str(frame)) continue else: estimatedCameraFile = None estimatedCameraPath = None #estimatedCameraFile = None #estimatedCameraPath = None # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join(cameraFolder, icebridge_common.getCameraFileName(imageFile)) outputFiles.append(outputCamFile) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs continue # Determine which input camera file will be used for this frame if inputCalCamera == "": inputCamFile = getCalibrationFileForFrame(cameraLookupFile, inputCalFolder, frame, yyyymmdd, site, logger) else: # This logic will force to use a given camera rather than # looking it up. This is not the usual way of doing things. inputCamFile = inputCalCamera if not os.path.exists(inputCalCamera): raise Exception("Could not find: " + inputCalCamera) orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCameras, numThreads) if numProcesses > 1: # Add ortho2pinhole command to the task pool taskHandles.append(pool.apply_async(cameraFromOrthoWrapper, orthoArgs)) else: # Single process, more logging info this way cameraFromOrthoWrapper(*orthoArgs) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q') # All tasks should be finished icebridge_common.stopTaskPool(pool) logger.info('Finished ortho processing.') # Run a check to see if we got all the output files for f in outputFiles: if not os.path.exists(f): return False return True
def main(argsIn): try: usage = '''generate_fake_camera_models.py <options>''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--work-folder", dest="workFolder", default=None, help="Temporary download folder.") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) # Fetch the jpeg files for missing camera files fetch_options = options fetch_options.type = 'jpeg' fetch_options.year = int(options.yyyymmdd[0:4]) fetch_options.month = int(options.yyyymmdd[4:6]) fetch_options.day = int(options.yyyymmdd[6:8]) fetch_options.skipValidate = True fetch_options.ignoreMissingLidar = True fetch_options.maxNumLidarToFetch = 0 fetch_options.refetchIndex = False fetch_options.refetchNav = False fetch_options.stopAfterIndexFetch = False fetch_options.dryRun = False fetch_options.allFrames = False fetch_options.frameSkip = 0 fetch_icebridge_data.doFetch(fetch_options, options.workFolder) if not os.path.exists(options.outputFolder): os.mkdir(options.outputFolder) # For each jpeg file, generate an empty file with the correct file name. inputFiles = os.listdir(options.workFolder) for f in inputFiles: if os.path.splitext(f)[1] != '.JPG': continue inputPath = os.path.join(options.workFolder, f) print inputPath # Get image info frame = icebridge_common.getFrameNumberFromFilename(inputPath) (datestr, timestr) = icebridge_common.getJpegDateTime(inputPath) # Pick output name outputName = icebridge_common.formFilePrefix(datestr, timestr, frame) + '.tsai' outputPath = os.path.join(options.outputFolder, outputName) cmd = 'touch ' + outputPath print cmd os.system(cmd)
def main(argsIn): # Command line parsing try: usage = "usage: camera_models_from_nav.py <image_folder> <ortho_folder> <cal_folder> <nav_folder> <output_folder> [options]" parser = optparse.OptionParser(usage=usage) parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=999999, type='int', help='The frame number to finish processing with.') parser.add_option("--input-calibration-camera", dest="inputCalCamera", default="", help="Use this input calibrated camera.") parser.add_option( '--camera-mounting', dest='cameraMounting', default=0, type='int', help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) (options, args) = parser.parse_args(argsIn) if len(args) < 5: print('Error: Missing arguments.') print(usage) return -1 imageFolder = os.path.abspath(args[0]) orthoFolder = os.path.abspath(args[1]) calFolder = os.path.abspath(args[2]) navFolder = os.path.abspath(args[3]) outputFolder = os.path.abspath(args[4]) except optparse.OptionError as msg: raise Usage(msg) runDir = os.path.dirname(orthoFolder) os.system("mkdir -p " + runDir) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(runDir, logLevel, 'camera_models_from_nav_log') if not os.path.exists(orthoFolder): logger.error('Ortho folder ' + orthoFolder + ' does not exist!') return -1 # Find the nav file # - There should only be one or two nav files per flight. fileList = os.listdir(navFolder) fileList = [x for x in fileList if '.out' in x] if len(fileList) == 0: logger.error('No nav files in: ' + navFolder) return -1 navPath = os.path.join(navFolder, fileList[0]) parsedNavPath = navPath.replace('.out', '.txt') if not asp_file_utils.fileIsNonZero(navPath): logger.error('Nav file ' + navPath + ' is invalid!') return -1 # Create the output file only if it is empty or does not exist isNonEmpty = asp_file_utils.fileIsNonZero(parsedNavPath) if not isNonEmpty: # Initialize the output file as being empty logger.info("Create empty file: " + parsedNavPath) open(parsedNavPath, 'w').close() # Append to the output parsed nav file for fileName in fileList: # Convert the nav file from binary to text navPath = os.path.join(navFolder, fileName) with open(navPath, 'r') as f: try: text = f.readline() if 'HTML' in text: # Sometimes the server is down, and instead of the binary nav file # we are given an html file with an error message. logger.info("Have invalid nav file: " + navPath) return -1 # Die in this case! except UnicodeDecodeError as e: # Got a binary file, that means likely we are good pass cmd = asp_system_utils.which( 'sbet2txt.pl') + ' -q ' + navPath + ' >> ' + parsedNavPath logger.info(cmd) if not isNonEmpty: os.system(cmd) cameraPath = options.inputCalCamera if cameraPath == "": # No input camera file provided, look one up. It does not matter much, # as later ortho2pinhole will insert the correct intrinsics. goodFile = False fileList = os.listdir(calFolder) fileList = [x for x in fileList if (('.tsai' in x) and ('~' not in x))] if not fileList: logger.error('Unable to find any camera files in ' + calFolder) return -1 for fileName in fileList: cameraPath = os.path.join(calFolder, fileName) # Check if this path is valid with open(cameraPath, 'r') as f: for line in f: if 'fu' in line: goodFile = True break if goodFile: break # Get the ortho list orthoFiles = icebridge_common.getTifs(orthoFolder) logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.') # Look up the frame numbers for each ortho file infoDict = {} for ortho in orthoFiles: if ('gray' in ortho) or ('sub' in ortho): continue frame = icebridge_common.getFrameNumberFromFilename(ortho) if frame < options.startFrame or frame > options.stopFrame: continue infoDict[frame] = [ortho, ''] # Get the image file list try: imageFiles = icebridge_common.getTifs(imageFolder) except Exception as e: raise Exception( "Cannot continue with nav generation, will resume later when images are created. This is not a fatal error. " + str(e)) logger.info('Found ' + str(len(imageFiles)) + ' image files.') # Update the second part of each dictionary object for image in imageFiles: if ('gray' in image) or ('sub' in image): continue frame = icebridge_common.getFrameNumberFromFilename(image) if frame < options.startFrame or frame > options.stopFrame: continue if frame not in infoDict: logger.info('Image missing ortho file: ' + image) # don't throw here, that will mess the whole batch, we will recover # the missing one later. continue infoDict[frame][1] = image os.system('mkdir -p ' + outputFolder) orthoListFile = os.path.join( outputFolder, 'ortho_file_list_' + str(options.startFrame) + "_" + str(options.stopFrame) + '.csv') # Open the output file for writing logger.info("Writing: " + orthoListFile) with open(orthoListFile, 'w') as outputFile: # Loop through frames in order for key in sorted(infoDict): # Write the ortho name and the output camera name to the file (ortho, image) = infoDict[key] if not image: #raise Exception('Ortho missing image file: ' +ortho) continue camera = image.replace('.tif', '.tsai') outputFile.write(ortho + ', ' + camera + '\n') # Check if we already have all of the output camera files. haveAllFiles = True with open(orthoListFile, 'r') as inputFile: for line in inputFile: parts = line.split(',') camPath = os.path.join(outputFolder, parts[1].strip()) if not asp_file_utils.fileIsNonZero(camPath): logger.info('Missing file -> ' + camPath) haveAllFiles = False break # Call the C++ tool to generate a camera model for each ortho file if not haveAllFiles: cmd = ( 'nav2cam --input-cam %s --nav-file %s --cam-list %s --output-folder %s --camera-mounting %d' % (cameraPath, parsedNavPath, orthoListFile, outputFolder, options.cameraMounting)) logger.info(cmd) os.system(cmd) else: logger.info("All nav files were already generated.") # Generate a kml file for the nav camera files kmlPath = os.path.join(outputFolder, 'nav_cameras.kml') # This is a hack. If we are invoked from a Pleiades node, do not # create this kml file, as nodes will just overwrite each other. # This job may happen anyway earlier or later when on the head node. if not 'PBS_NODEFILE' in os.environ: try: tempPath = os.path.join(outputFolder, 'list.txt') logger.info('Generating nav camera kml file: ' + kmlPath) os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath) orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole') cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath logger.info(cmd) asp_system_utils.executeCommand(cmd, kmlPath, suppressOutput=True, redo=False) os.remove(tempPath) except Exception as e: logger.info("Warning: " + str(e)) logger.info('Finished generating camera models from nav!') return 0
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) if not skipValidate: validFilesList = icebridge_common.validFilesList( os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder=True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ((frame >= startFrame) and (frame <= stopFrame)): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile( inputPath, orthoFrameDict[frame]) except Exception, e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90' cmd = ('%s %s -colorspace Gray %s %s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE | re.MULTILINE | re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw
cameraFiles = [os.path.join(cameraFolder,f) for f in cameraFiles] numFiles = len(imageFiles) if (len(cameraFiles) != numFiles): print imageFiles print cameraFiles logger.error('process_icebridge.py: counted ' + str(len(imageFiles)) + ' image files.\n'+ 'and ' + str(len(cameraFiles)) + ' camera files.\n'+ 'Error: Number of image files and number of camera files must match!') return -1 imageCameraPairs = zip(imageFiles, cameraFiles) # Check that the files are properly aligned for (image, camera) in imageCameraPairs: frameNumber = icebridge_common.getFrameNumberFromFilename(image) if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber): logger.error('Error: input files do not align!\n' + str((image, camera))) return -1 # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in imageCameraPairs: vizString += image +' ' + camera+' ' cmd = 'orbitviz --hide-labels -t nadirpinhole -r wgs84 -o '+ orbitvizBefore +' '+ vizString logger.info(cmd) asp_system_utils.executeCommand(cmd, orbitvizBefore, suppressOutput, redo) # Set up options for process_icebridge_batch extraOptions = ' --stereo-algorithm ' + str(options.stereoAlgo)