def packAndSendAlignedCameras(run, logger): '''Archive the pc_align-ed cameras for later use''' logger.info('Archiving aligned cameras for run ' + str(run)) cwd = os.getcwd() os.chdir(run.parentFolder) runFolder = str(run) fileName = run.getAlignedCameraTarName() lfePath = os.path.join(REMOTE_ALIGN_CAM_FOLDER, fileName) # First remove any existing tar file cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null" logger.info(cmd) os.system(cmd) # Create a new archive cmd = 'shiftc --wait -d -r --include=\'^.*?' + icebridge_common.alignedBundleStr() + \ '.*?\.tsai$\' --create-tar ' + runFolder + \ ' ' + lfePath robust_shiftc(cmd, logger) os.chdir(cwd) logger.info('Finished sending aligned cameras to lfe.')
def massRename(self, startFrame, stopFrame, logger): '''We changed how the timestamp for images and cameras is computed. Make all existing converted images, cameras, nav cameras, and aligned cameras conform.''' logger.info("Renaming files with timestamp. This is slow.") # Need to do a mass rename for incorrect timestamp for: # converted images, nav cameras, cameras, and bundle aligned cameras outputFolder = self.getFolder() cameraFolder = icebridge_common.getCameraFolder(outputFolder) imageFolder = icebridge_common.getImageFolder(outputFolder) jpegFolder = icebridge_common.getJpegFolder(outputFolder) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) processedFolder = icebridge_common.getProcessedFolder(outputFolder) navFolder = icebridge_common.getNavFolder(outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder( os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) logger.info('Renaming camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(cameraFolder, '*DMS*tsai'), logger) logger.info('Renaming nav camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(navCameraFolder, '*DMS*tsai'), logger) logger.info('Renaming converted images...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(imageFolder, '*DMS*tif'), logger) logger.info('Renaming aligned cameras...') self.massRenameByGlob( startFrame, stopFrame, orthoFrameDict, os.path.join(processedFolder, 'batch*', icebridge_common.alignedBundleStr() + '*DMS*tsai'), logger)
def massRename(self, startFrame, stopFrame, logger): '''We changed how the timestamp for images and cameras is computed. Make all existing converted images, cameras, nav cameras, and aligned cameras conform.''' logger.info("Renaming files with timestamp. This is slow.") # Need to do a mass rename for incorrect timestamp for: # converted images, nav cameras, cameras, and bundle aligned cameras outputFolder = self.getFolder() cameraFolder = icebridge_common.getCameraFolder(outputFolder) imageFolder = icebridge_common.getImageFolder(outputFolder) jpegFolder = icebridge_common.getJpegFolder(outputFolder) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) processedFolder = icebridge_common.getProcessedFolder(outputFolder) navFolder = icebridge_common.getNavFolder(outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) logger.info('Renaming camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(cameraFolder, '*DMS*tsai'), logger) logger.info('Renaming nav camera files...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(navCameraFolder, '*DMS*tsai'), logger) logger.info('Renaming converted images...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(imageFolder, '*DMS*tif'), logger) logger.info('Renaming aligned cameras...') self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict, os.path.join(processedFolder, 'batch*', icebridge_common.alignedBundleStr() + '*DMS*tsai'), logger)
def runOrtho(frame, processFolder, imageFile, bundleLength, threadText, redo, suppressOutput): os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others # This will run as multiple processes. Hence have to catch all exceptions: projBounds = () try: alignCamFile, batchFolder = \ icebridge_common.frameToFile(frame, icebridge_common.alignedBundleStr() + '*' + str(frame) + '.tsai', processFolder, bundleLength) if alignCamFile == "": print("Could not find aligned camera for frame: " + str(frame)) return # To ensure we mapproject the image fully, mosaic the several DEMs # around it. Keep the closest 5. Try to grab more first to account # for skipped frames. frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4] dems = [] for offset in frameOffsets: demFile, batchFolder = icebridge_common.frameToFile( frame + offset, icebridge_common.blendFileName(), processFolder, bundleLength) # If the central DEM is missing, we are out of luck if offset == 0 and demFile == "": print("Could not find DEM for frame: " + str(frame + offset)) return if offset == 0: demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False) projBounds = demGeoInfo[ 'projection_bounds'] # minX maxX minY maxY if demFile == "": # Missing DEM continue if len(dems) >= 5: break # too many already dems.append(demFile) demList = " ".join(dems) # Call this one more time, to get the current batch folder currDemFile, batchFolder = icebridge_common.frameToFile( frame, icebridge_common.blendFileName(), processFolder, bundleLength) # The names for the final results finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName()) finalOrthoPreview = os.path.join( batchFolder, icebridge_common.orthoPreviewFileName()) if (not redo) and os.path.exists(finalOrtho): print("File exists: " + finalOrtho + ".") else: filesToWipe = [] # If the center dem spans say 1 km, there's no way the # ortho can span more than 5 km, unless something is # seriously out of whack, such as alignment failing for # some neighbours. In the best case, if the center dem is # 1 km by 1 km, the obtained ortho will likely be 1.4 km # by 1 km, as an image extends beyond its stereo dem with # a neighbor. factor = float(2.0) projWinStr = "" if len(projBounds) >= 4: # projBounds is in the format minX maxX minY maxY widX = float(projBounds[1]) - float(projBounds[0]) widY = float(projBounds[3]) - float(projBounds[2]) projBounds = ( float(projBounds[0]) - factor * widX, # minX float(projBounds[1]) + factor * widX, # maxX float(projBounds[2]) - factor * widY, # minY float(projBounds[3]) + factor * widY # maxY ) projWinStr = ("--t_projwin %f %f %f %f" % \ (projBounds[0], projBounds[2], projBounds[1], projBounds[3])) # See if we have a pre-existing DEM to use as footprint mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic') mosaicOutput = mosaicPrefix + '-tile-0.tif' cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' % (demList, threadText, projWinStr, mosaicPrefix)) filesToWipe.append(mosaicOutput) # no longer needed print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo) # Borow some pixels from the footprint DEM,just to grow a bit the real estate finalFootprintDEM = os.path.join( batchFolder, icebridge_common.footprintFileName()) if os.path.exists(finalFootprintDEM): mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2') mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif' cmd = ( 'dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s' % (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2)) print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo, noThrow=True) if os.path.exists(mosaicOutput2): cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput print(cmd) os.system(cmd) # TODO: Look at more aggressive hole-filling. But need a testcase. filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*') # First mapproject to create a tif image with 4 channels. # Then pull 3 channels and compress them as jpeg, while keeping the # image a geotiff. tempOrtho = os.path.join( batchFolder, icebridge_common.orthoFileName() + "_tmp.tif") # There is no need for this file to exist unless it is stray junk if os.path.exists(tempOrtho): os.remove(tempOrtho) # Run mapproject. The grid size is auto-determined. cmd = ( 'mapproject --no-geoheader-info %s %s %s %s %s' % (mosaicOutput, imageFile, alignCamFile, tempOrtho, threadText)) print(cmd) asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo) filesToWipe.append(tempOrtho) # This makes the images smaller than Rose's by a factor of about 4, # even though both types are jpeg compressed. Rose's images filtered # through this command also get compressed by a factor of 4. # I conclude that the jpeg compression used by Rose was not as # aggressive as the one used in gdal_translate, but there is no # apparent knob to control that. cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho print(cmd) asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo) # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) if (not redo) and os.path.exists(finalOrthoPreview): print("File exists: " + finalOrthoPreview + ".") else: cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \ ' ' + finalOrthoPreview print(cmd) asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo) except Exception as e: print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc())) os.system("rm -f core.*") # these keep on popping up # To ensure we print promptly what we did so far sys.stdout.flush()
def runOrtho(frame, processFolder, imageFile, bundleLength, cameraMounting, threadText, redo, suppressOutput): os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others # This will run as multiple processes. Hence have to catch all exceptions: projBounds = () try: # Retrieve the aligned camera file alignCamFile, batchFolder = \ icebridge_common.frameToFile(frame, icebridge_common.alignedBundleStr() + '*' + str(frame) + '.tsai', processFolder, bundleLength) if alignCamFile == "": print("Could not find aligned camera for frame: " + str(frame)) return # To ensure we mapproject the image fully, mosaic the several DEMs # around it. Keep the closest 5. Try to grab more first to account # for skipped frames. frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4] dems = [] for offset in frameOffsets: # Find the DEM file for the desired frame demFile, batchFolder = icebridge_common.frameToFile(frame + offset, icebridge_common.blendFileName(), processFolder, bundleLength) # If the central DEM is missing, we are out of luck if offset == 0 and demFile == "": print("Could not find blended DEM for frame: " + str(frame + offset)) return if offset == 0: demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False) projBounds = demGeoInfo['projection_bounds'] # minX maxX minY maxY if demFile == "": # Missing DEM continue if len(dems) >= 5: break # too many already dems.append(demFile) demList = " ".join(dems) # Call this one more time, to get the current batch folder currDemFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.blendFileName(), processFolder, bundleLength) # The names for the final results finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName()) finalOrthoPreview = os.path.join(batchFolder, icebridge_common.orthoPreviewFileName()) if (not redo) and os.path.exists(finalOrtho): print("File exists: " + finalOrtho + ".") else: filesToWipe = [] # If the center dem spans say 1 km, there's no way the # ortho can span more than 5 km, unless something is # seriously out of whack, such as alignment failing for # some neighbours. In the best case, if the center dem is # 1 km by 1 km, the obtained ortho will likely be 1.4 km # by 1 km, as an image extends beyond its stereo dem with # a neighbor. factor = float(2.0) projWinStr = "" if len(projBounds) >= 4: # projBounds is in the format minX maxX minY maxY widX = float(projBounds[1]) - float(projBounds[0]) widY = float(projBounds[3]) - float(projBounds[2]) projBounds = ( float(projBounds[0]) - factor*widX, # minX float(projBounds[1]) + factor*widX, # maxX float(projBounds[2]) - factor*widY, # minY float(projBounds[3]) + factor*widY # maxY ) projWinStr = ("--t_projwin %f %f %f %f" % \ (projBounds[0], projBounds[2], projBounds[1], projBounds[3])) # See if we have a pre-existing DEM to use as footprint mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic') mosaicOutput = mosaicPrefix + '-tile-0.tif' cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' % (demList, threadText, projWinStr, mosaicPrefix)) filesToWipe.append(mosaicOutput) # no longer needed # Generate the DEM mosaic print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo) # Borow some pixels from the footprint DEM,just to grow a bit the real estate finalFootprintDEM = os.path.join(batchFolder, icebridge_common.footprintFileName()) if os.path.exists(finalFootprintDEM): mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2') mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif' cmd = ('dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s' % (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2)) print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo, noThrow = True) if os.path.exists(mosaicOutput2): cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput print(cmd) os.system(cmd) # TODO: Look at more aggressive hole-filling. But need a testcase. filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*') # First mapproject to create a tif image with 4 channels. # Then pull 3 channels and compress them as jpeg, while keeping the # image a geotiff. tempOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName() + "_tmp.tif") # There is no need for this file to exist unless it is stray junk if os.path.exists(tempOrtho): os.remove(tempOrtho) # If needed, generate a temporary camera file to correct a mounting rotation. # - When the camera mount is rotated 90 degrees stereo is run on a corrected version # but ortho needs to work on the original uncorrected jpeg image. tempCamFile = alignCamFile + '_temp_rot.tsai' tempCamFile = createRotatedCameraFile(alignCamFile, tempCamFile, cameraMounting) # Run mapproject. The grid size is auto-determined. cmd = ('mapproject --no-geoheader-info %s %s %s %s %s' % (mosaicOutput, imageFile, tempCamFile, tempOrtho, threadText)) print(cmd) asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo) # Set temporary files to be cleaned up filesToWipe.append(tempOrtho) if tempCamFile != alignCamFile: filesToWipe.append(tempCamFile) # This makes the images smaller than Rose's by a factor of about 4, # even though both types are jpeg compressed. Rose's images filtered # through this command also get compressed by a factor of 4. # I conclude that the jpeg compression used by Rose was not as # aggressive as the one used in gdal_translate, but there is no # apparent knob to control that. cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho print(cmd) asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo) # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) if (not redo) and os.path.exists(finalOrthoPreview): print("File exists: " + finalOrthoPreview + ".") else: cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \ ' ' + finalOrthoPreview print(cmd) asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo) except Exception as e: print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc())) os.system("rm -f core.*") # these keep on popping up # To ensure we print promptly what we did so far sys.stdout.flush()