asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # COLORMAP colormapMin = -10 colormapMax = 10 colorOutput = outputPrefix + '-DEM_CMAP.tif' cmd = ('colormap --min %f --max %f %s -o %s' % (colormapMin, colormapMax, p2dOutput, colorOutput)) asp_system_utils.executeCommand(cmd, colorOutput, suppressOutput, redo) if options.lidarOverlay: LIDAR_DEM_RESOLUTION = 5 LIDAR_PROJ_BUFFER_METERS = 100 # Get buffered projection bounds of this image demGeoInfo = asp_geo_utils.getImageGeoInfo(p2dOutput, getStats=False) projBounds = demGeoInfo['projection_bounds'] minX = projBounds[ 0] - LIDAR_PROJ_BUFFER_METERS # Expand the bounds a bit minY = projBounds[2] - LIDAR_PROJ_BUFFER_METERS maxX = projBounds[1] + LIDAR_PROJ_BUFFER_METERS maxY = projBounds[3] + LIDAR_PROJ_BUFFER_METERS # Generate a DEM from the lidar point cloud in this region lidarDemPrefix = os.path.join(outputFolder, 'cropped_lidar') cmd = ( 'point2dem --t_projwin %f %f %f %f --tr %lf --t_srs %s %s %s --csv-format %s -o %s' % (minX, minY, maxX, maxY, LIDAR_DEM_RESOLUTION, projString, lidarFile, threadText, csvFormatString, lidarDemPrefix)) lidarDemOutput = lidarDemPrefix + '-DEM.tif' asp_system_utils.executeCommand(cmd, lidarDemOutput, suppressOutput,
def getImageSpacing(orthoFolder, availableFrames, startFrame, stopFrame, forceAllFramesInRange): '''Find a good image stereo spacing interval that gives us a good balance between coverage and baseline width. Also detect all frames where this is a large break after the current frame.''' logger.info('Computing optimal image stereo interval...') ## With very few cameras this is the only possible way to process them #if len(availableFrames) < 3 and not forceAllFramesInRange: # return ([], {}) # No skip, no breaks # Retrieve a list of the ortho files orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) # From the dictionary create a sorted list of ortho files in the frame range breaks = [] largeSkips = {} orthoFiles = [] for frame in sorted(orthoFrameDict.keys()): # Only process frames within the range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue orthoPath = orthoFrameDict[frame] frame = icebridge_common.getFrameNumberFromFilename(orthoPath) if not forceAllFramesInRange: if frame not in availableFrames: # Skip frames we did not compute a camera for continue orthoFiles.append(orthoPath) orthoFiles.sort() numOrthos = len(orthoFiles) # First load whatever boxes are there projectionIndexFile = icebridge_common.projectionBoundsFile(os.path.dirname(orthoFolder)) logger.info("Reading: " + projectionIndexFile) boundsDict = icebridge_common.readProjectionBounds(projectionIndexFile) # Get the bounding box and frame number of each ortho image logger.info('Loading bounding boxes...') frames = [] updatedBounds = False # will be true if some computation got done count = 0 for i in range(0, numOrthos): # This can be slow, so add a progress dialong count = count + 1 if (count - 1) % 1000 == 0: logger.info('Progress: ' + str(count) + '/' + str(numOrthos)) thisFrame = icebridge_common.getFrameNumberFromFilename(orthoFiles[i]) if thisFrame not in boundsDict: imageGeoInfo = asp_geo_utils.getImageGeoInfo(orthoFiles[i], getStats=False) thisBox = imageGeoInfo['projection_bounds'] boundsDict[thisFrame] = thisBox updatedBounds = True frames.append(thisFrame) # Read this file again, in case some other process modified it in the meantime. # This won't happen in production mode, but can during testing with partial sequences. boundsDictRecent = icebridge_common.readProjectionBounds(projectionIndexFile) for frame in sorted(boundsDictRecent.keys()): if not frame in boundsDict.keys(): boundsDict[frame] = boundsDictRecent[frame] updatedBounds = True # Save the bounds. There is always the danger that two processes will # do that at the same time, but this is rare, as hopefully we get here # only once from the manager. It is not a big loss if this file gets messed up. if updatedBounds: logger.info("Writing: " + projectionIndexFile) icebridge_common.writeProjectionBounds(projectionIndexFile, boundsDict) # Since we are only comparing the image bounding boxes, not their exact corners, # these ratios are only estimates. MAX_RATIO = 0.85 # Increase skip until we get below this... MIN_RATIO = 0.75 # ... but don't go below this value! NOTRY_RATIO = 0.0001 # Don't bother with overlap amounts less than this (small on purpose) def getBboxArea(bbox): '''Return the area of a bounding box in form of (minX, maxX, minY, maxY)''' width = bbox[1] - bbox[0] height = bbox[3] - bbox[2] if (width < 0) or (height < 0): return 0 return width*height # Iterate over the frames and find the best stereo frame for each for i in range(0, numOrthos-1): thisFrame = frames[i] thisBox = boundsDict[thisFrame] thisArea = getBboxArea(thisBox) interval = 1 while(True): # Compute intersection area between this and next image nextFrame = frames[i+interval] nextBox = boundsDict[nextFrame] intersect = [max(thisBox[0], nextBox[0]), # Min X min(thisBox[1], nextBox[1]), # Max X max(thisBox[2], nextBox[2]), # Min Y min(thisBox[3], nextBox[3])] # Max Y area = getBboxArea(intersect) ratio = 0 if area > 0: ratio = area / thisArea if interval == 1: # Cases for the smallest interval... if ratio < NOTRY_RATIO: breaks.append(thisFrame) # No match for this frame logger.info('Detected large break after frame ' + str(thisFrame)) break if ratio < MIN_RATIO: break # No reason to try increasing skip amounts for this frame else: # interval > 1 if ratio < MIN_RATIO: # Went too small, walk back the interval. interval = interval - 1 break if ratio > MAX_RATIO: # Too much overlap, increase interval interval = interval + 1 else: # Overlap is fine, keep this interval. break # Handle the case where we go past the end of frames looking for a match. if i+interval >= len(frames): interval = interval - 1 break if interval > 1: # Only record larger than normal intervals. largeSkips[thisFrame] = interval logger.info('Detected ' + str(len(breaks)) + ' breaks in image coverage.') logger.info('Detected ' + str(len(largeSkips)) + ' images with interval > 1.') return (breaks, largeSkips)
def runOrtho(frame, processFolder, imageFile, bundleLength, threadText, redo, suppressOutput): os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others # This will run as multiple processes. Hence have to catch all exceptions: projBounds = () try: alignCamFile, batchFolder = \ icebridge_common.frameToFile(frame, icebridge_common.alignedBundleStr() + '*' + str(frame) + '.tsai', processFolder, bundleLength) if alignCamFile == "": print("Could not find aligned camera for frame: " + str(frame)) return # To ensure we mapproject the image fully, mosaic the several DEMs # around it. Keep the closest 5. Try to grab more first to account # for skipped frames. frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4] dems = [] for offset in frameOffsets: demFile, batchFolder = icebridge_common.frameToFile( frame + offset, icebridge_common.blendFileName(), processFolder, bundleLength) # If the central DEM is missing, we are out of luck if offset == 0 and demFile == "": print("Could not find DEM for frame: " + str(frame + offset)) return if offset == 0: demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False) projBounds = demGeoInfo[ 'projection_bounds'] # minX maxX minY maxY if demFile == "": # Missing DEM continue if len(dems) >= 5: break # too many already dems.append(demFile) demList = " ".join(dems) # Call this one more time, to get the current batch folder currDemFile, batchFolder = icebridge_common.frameToFile( frame, icebridge_common.blendFileName(), processFolder, bundleLength) # The names for the final results finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName()) finalOrthoPreview = os.path.join( batchFolder, icebridge_common.orthoPreviewFileName()) if (not redo) and os.path.exists(finalOrtho): print("File exists: " + finalOrtho + ".") else: filesToWipe = [] # If the center dem spans say 1 km, there's no way the # ortho can span more than 5 km, unless something is # seriously out of whack, such as alignment failing for # some neighbours. In the best case, if the center dem is # 1 km by 1 km, the obtained ortho will likely be 1.4 km # by 1 km, as an image extends beyond its stereo dem with # a neighbor. factor = float(2.0) projWinStr = "" if len(projBounds) >= 4: # projBounds is in the format minX maxX minY maxY widX = float(projBounds[1]) - float(projBounds[0]) widY = float(projBounds[3]) - float(projBounds[2]) projBounds = ( float(projBounds[0]) - factor * widX, # minX float(projBounds[1]) + factor * widX, # maxX float(projBounds[2]) - factor * widY, # minY float(projBounds[3]) + factor * widY # maxY ) projWinStr = ("--t_projwin %f %f %f %f" % \ (projBounds[0], projBounds[2], projBounds[1], projBounds[3])) # See if we have a pre-existing DEM to use as footprint mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic') mosaicOutput = mosaicPrefix + '-tile-0.tif' cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' % (demList, threadText, projWinStr, mosaicPrefix)) filesToWipe.append(mosaicOutput) # no longer needed print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo) # Borow some pixels from the footprint DEM,just to grow a bit the real estate finalFootprintDEM = os.path.join( batchFolder, icebridge_common.footprintFileName()) if os.path.exists(finalFootprintDEM): mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2') mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif' cmd = ( 'dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s' % (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2)) print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo, noThrow=True) if os.path.exists(mosaicOutput2): cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput print(cmd) os.system(cmd) # TODO: Look at more aggressive hole-filling. But need a testcase. filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*') # First mapproject to create a tif image with 4 channels. # Then pull 3 channels and compress them as jpeg, while keeping the # image a geotiff. tempOrtho = os.path.join( batchFolder, icebridge_common.orthoFileName() + "_tmp.tif") # There is no need for this file to exist unless it is stray junk if os.path.exists(tempOrtho): os.remove(tempOrtho) # Run mapproject. The grid size is auto-determined. cmd = ( 'mapproject --no-geoheader-info %s %s %s %s %s' % (mosaicOutput, imageFile, alignCamFile, tempOrtho, threadText)) print(cmd) asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo) filesToWipe.append(tempOrtho) # This makes the images smaller than Rose's by a factor of about 4, # even though both types are jpeg compressed. Rose's images filtered # through this command also get compressed by a factor of 4. # I conclude that the jpeg compression used by Rose was not as # aggressive as the one used in gdal_translate, but there is no # apparent knob to control that. cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho print(cmd) asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo) # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) if (not redo) and os.path.exists(finalOrthoPreview): print("File exists: " + finalOrthoPreview + ".") else: cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \ ' ' + finalOrthoPreview print(cmd) asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo) except Exception as e: print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc())) os.system("rm -f core.*") # these keep on popping up # To ensure we print promptly what we did so far sys.stdout.flush()
def runOrtho(frame, processFolder, imageFile, bundleLength, cameraMounting, threadText, redo, suppressOutput): os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others # This will run as multiple processes. Hence have to catch all exceptions: projBounds = () try: # Retrieve the aligned camera file alignCamFile, batchFolder = \ icebridge_common.frameToFile(frame, icebridge_common.alignedBundleStr() + '*' + str(frame) + '.tsai', processFolder, bundleLength) if alignCamFile == "": print("Could not find aligned camera for frame: " + str(frame)) return # To ensure we mapproject the image fully, mosaic the several DEMs # around it. Keep the closest 5. Try to grab more first to account # for skipped frames. frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4] dems = [] for offset in frameOffsets: # Find the DEM file for the desired frame demFile, batchFolder = icebridge_common.frameToFile(frame + offset, icebridge_common.blendFileName(), processFolder, bundleLength) # If the central DEM is missing, we are out of luck if offset == 0 and demFile == "": print("Could not find blended DEM for frame: " + str(frame + offset)) return if offset == 0: demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False) projBounds = demGeoInfo['projection_bounds'] # minX maxX minY maxY if demFile == "": # Missing DEM continue if len(dems) >= 5: break # too many already dems.append(demFile) demList = " ".join(dems) # Call this one more time, to get the current batch folder currDemFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.blendFileName(), processFolder, bundleLength) # The names for the final results finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName()) finalOrthoPreview = os.path.join(batchFolder, icebridge_common.orthoPreviewFileName()) if (not redo) and os.path.exists(finalOrtho): print("File exists: " + finalOrtho + ".") else: filesToWipe = [] # If the center dem spans say 1 km, there's no way the # ortho can span more than 5 km, unless something is # seriously out of whack, such as alignment failing for # some neighbours. In the best case, if the center dem is # 1 km by 1 km, the obtained ortho will likely be 1.4 km # by 1 km, as an image extends beyond its stereo dem with # a neighbor. factor = float(2.0) projWinStr = "" if len(projBounds) >= 4: # projBounds is in the format minX maxX minY maxY widX = float(projBounds[1]) - float(projBounds[0]) widY = float(projBounds[3]) - float(projBounds[2]) projBounds = ( float(projBounds[0]) - factor*widX, # minX float(projBounds[1]) + factor*widX, # maxX float(projBounds[2]) - factor*widY, # minY float(projBounds[3]) + factor*widY # maxY ) projWinStr = ("--t_projwin %f %f %f %f" % \ (projBounds[0], projBounds[2], projBounds[1], projBounds[3])) # See if we have a pre-existing DEM to use as footprint mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic') mosaicOutput = mosaicPrefix + '-tile-0.tif' cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' % (demList, threadText, projWinStr, mosaicPrefix)) filesToWipe.append(mosaicOutput) # no longer needed # Generate the DEM mosaic print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo) # Borow some pixels from the footprint DEM,just to grow a bit the real estate finalFootprintDEM = os.path.join(batchFolder, icebridge_common.footprintFileName()) if os.path.exists(finalFootprintDEM): mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2') mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif' cmd = ('dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s' % (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2)) print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo, noThrow = True) if os.path.exists(mosaicOutput2): cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput print(cmd) os.system(cmd) # TODO: Look at more aggressive hole-filling. But need a testcase. filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*') # First mapproject to create a tif image with 4 channels. # Then pull 3 channels and compress them as jpeg, while keeping the # image a geotiff. tempOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName() + "_tmp.tif") # There is no need for this file to exist unless it is stray junk if os.path.exists(tempOrtho): os.remove(tempOrtho) # If needed, generate a temporary camera file to correct a mounting rotation. # - When the camera mount is rotated 90 degrees stereo is run on a corrected version # but ortho needs to work on the original uncorrected jpeg image. tempCamFile = alignCamFile + '_temp_rot.tsai' tempCamFile = createRotatedCameraFile(alignCamFile, tempCamFile, cameraMounting) # Run mapproject. The grid size is auto-determined. cmd = ('mapproject --no-geoheader-info %s %s %s %s %s' % (mosaicOutput, imageFile, tempCamFile, tempOrtho, threadText)) print(cmd) asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo) # Set temporary files to be cleaned up filesToWipe.append(tempOrtho) if tempCamFile != alignCamFile: filesToWipe.append(tempCamFile) # This makes the images smaller than Rose's by a factor of about 4, # even though both types are jpeg compressed. Rose's images filtered # through this command also get compressed by a factor of 4. # I conclude that the jpeg compression used by Rose was not as # aggressive as the one used in gdal_translate, but there is no # apparent knob to control that. cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho print(cmd) asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo) # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) if (not redo) and os.path.exists(finalOrthoPreview): print("File exists: " + finalOrthoPreview + ".") else: cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \ ' ' + finalOrthoPreview print(cmd) asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo) except Exception as e: print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc())) os.system("rm -f core.*") # these keep on popping up # To ensure we print promptly what we did so far sys.stdout.flush()
cmd = "hillshade " + p2dOutput + " -o " + hillOutput asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # COLORMAP colormapMin = -10 colormapMax = 10 colorOutput = outputPrefix + "-DEM_CMAP.tif" cmd = "colormap --min %f --max %f %s -o %s" % (colormapMin, colormapMax, p2dOutput, colorOutput) asp_system_utils.executeCommand(cmd, colorOutput, suppressOutput, redo) if options.lidarOverlay: LIDAR_DEM_RESOLUTION = 5 LIDAR_PROJ_BUFFER_METERS = 100 # Get buffered projection bounds of this image demGeoInfo = asp_geo_utils.getImageGeoInfo(p2dOutput, getStats=False) projBounds = demGeoInfo["projection_bounds"] minX = projBounds[0] - LIDAR_PROJ_BUFFER_METERS # Expand the bounds a bit minY = projBounds[2] - LIDAR_PROJ_BUFFER_METERS maxX = projBounds[1] + LIDAR_PROJ_BUFFER_METERS maxY = projBounds[3] + LIDAR_PROJ_BUFFER_METERS # Generate a DEM from the lidar point cloud in this region lidarDemPrefix = os.path.join(outputFolder, "cropped_lidar") cmd = "point2dem --t_projwin %f %f %f %f --tr %lf --t_srs %s %s %s --csv-format %s -o %s" % ( minX, minY, maxX, maxY, LIDAR_DEM_RESOLUTION, projString,
def getImageSpacing(orthoFolder): '''Find a good image stereo spacing interval that gives us a good balance between coverage and baseline width. Also detect all frames where this is a large break after the current frame.''' # Do nothing if this option was not provided if not orthoFolder: return None logger.info('Computing optimal image stereo interval...') breaks = [] # Generate a list of valid, full path ortho files fileList = os.listdir(orthoFolder) orthoFiles = [] for orthoFile in fileList: # Skip non-image files (including junk from stereo_gui) and duplicate grayscale files ext = os.path.splitext(orthoFile)[1] if (ext != '.tif') or ('_sub' in orthoFile) or ('.tif_gray.tif' in orthoFile): continue orthoPath = os.path.join(orthoFolder, orthoFile) orthoFiles.append(orthoPath) orthoFiles.sort() numOrthos = len(orthoFiles) # Get the bounding box and frame number of each ortho image logger.info('Loading bounding boxes...') bboxes = [] frames = [] for i in range(0, numOrthos): imageGeoInfo = asp_geo_utils.getImageGeoInfo(orthoFiles[i], getStats=False) thisBox = imageGeoInfo['projection_bounds'] thisFrame = icebridge_common.getFrameNumberFromFilename(orthoFiles[i]) bboxes.append(thisBox) frames.append(thisFrame) # Since we are only comparing the image bounding boxes, not their exact corners, # these ratios are only estimates. MAX_RATIO = 0.8 # Increase skip until we get below this... MIN_RATIO = 0.4 # ... but don't go below this value! def getBboxArea(bbox): '''Return the area of a bounding box in form of (minX, maxX, minY, maxY)''' width = bbox[1] - bbox[0] height = bbox[3] - bbox[2] if (width < 0) or (height < 0): return 0 return width*height # Iterate over stereo image intervals to try to get our target numbers interval = 0 meanRatio = 1.0 while meanRatio > MAX_RATIO: meanRatio = 0 count = 0 interval = interval + 1 logger.info('Trying stereo image interval ' + str(interval)) if numOrthos <= interval: raise Exception('Error: There are too few images and they overlap too much. ' + \ 'Consider processing more images in the given batch.') for i in range(0, numOrthos-interval): # Compute intersection area between this and next image thisBox = bboxes[i ] lastBox = bboxes[i+interval] intersect = [max(lastBox[0], thisBox[0]), # Min X min(lastBox[1], thisBox[1]), # Max X max(lastBox[2], thisBox[2]), # Min Y min(lastBox[3], thisBox[3])] # Max Y thisArea = getBboxArea(thisBox) area = getBboxArea(intersect) ratio = area / thisArea # Don't include non-overlapping frames in the statistics if area > 0: meanRatio = meanRatio + ratio count = count + 1 # On the first pass (with interval 1) check for gaps in coverage. if (interval == 1) and (area <= 0): breaks.append(frames[i]) logger.info('Detected large break after frame ' + str(frames[i])) # Get the mean intersection ratio meanRatio = meanRatio / count logger.info(' --> meanRatio = ' + str(meanRatio)) # If we increased the interval too much, back it off by one step. if (meanRatio < MIN_RATIO) and (interval > 1): interval = interval - 1 logger.info('Computed automatic image stereo interval: ' + str(interval)) logger.info('Detected ' + str(interval) + ' breaks in image coverage.') return (interval, breaks)