def packAndSendAlignedCameras(run, logger):
'''Archive the pc_align-ed cameras for later use'''
logger.info('Archiving aligned cameras for run ' + str(run))
cwd = os.getcwd()
os.chdir(run.parentFolder)
runFolder = str(run)
fileName = run.getAlignedCameraTarName()
lfePath = os.path.join(REMOTE_ALIGN_CAM_FOLDER, fileName)
# First remove any existing tar file
cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
# Create a new archive
cmd = 'shiftc --wait -d -r --include=\'^.*?' + icebridge_common.alignedBundleStr() + \
'.*?\.tsai$\' --create-tar ' + runFolder + \
' ' + lfePath
robust_shiftc(cmd, logger)
os.chdir(cwd)
logger.info('Finished sending aligned cameras to lfe.')
def massRename(self, startFrame, stopFrame, logger):
'''We changed how the timestamp for images and cameras is computed.
Make all existing converted images, cameras, nav cameras, and aligned cameras conform.'''
logger.info("Renaming files with timestamp. This is slow.")
# Need to do a mass rename for incorrect timestamp for:
# converted images, nav cameras, cameras, and bundle aligned cameras
outputFolder = self.getFolder()
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
imageFolder = icebridge_common.getImageFolder(outputFolder)
jpegFolder = icebridge_common.getJpegFolder(outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
processedFolder = icebridge_common.getProcessedFolder(outputFolder)
navFolder = icebridge_common.getNavFolder(outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(
os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " +
orthoIndexPath + ".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder=True)
logger.info('Renaming camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(cameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming nav camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(navCameraFolder, '*DMS*tsai'),
logger)
logger.info('Renaming converted images...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(imageFolder, '*DMS*tif'), logger)
logger.info('Renaming aligned cameras...')
self.massRenameByGlob(
startFrame, stopFrame, orthoFrameDict,
os.path.join(processedFolder, 'batch*',
icebridge_common.alignedBundleStr() + '*DMS*tsai'),
logger)
def massRename(self, startFrame, stopFrame, logger):
'''We changed how the timestamp for images and cameras is computed.
Make all existing converted images, cameras, nav cameras, and aligned cameras conform.'''
logger.info("Renaming files with timestamp. This is slow.")
# Need to do a mass rename for incorrect timestamp for:
# converted images, nav cameras, cameras, and bundle aligned cameras
outputFolder = self.getFolder()
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
imageFolder = icebridge_common.getImageFolder(outputFolder)
jpegFolder = icebridge_common.getJpegFolder(outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
processedFolder = icebridge_common.getProcessedFolder(outputFolder)
navFolder = icebridge_common.getNavFolder(outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder = True)
logger.info('Renaming camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(cameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming nav camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(navCameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming converted images...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(imageFolder, '*DMS*tif'), logger)
logger.info('Renaming aligned cameras...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(processedFolder, 'batch*',
icebridge_common.alignedBundleStr() + '*DMS*tsai'),
logger)
def runOrtho(frame, processFolder, imageFile, bundleLength, threadText, redo,
suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
demFile, batchFolder = icebridge_common.frameToFile(
frame + offset, icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile,
getStats=False)
projBounds = demGeoInfo[
'projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.blendFileName(), processFolder,
bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder,
icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(
batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor * widX, # minX
float(projBounds[1]) + factor * widX, # maxX
float(projBounds[2]) - factor * widY, # minY
float(projBounds[3]) + factor * widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' %
(demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput,
localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(
batchFolder, icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = (
'dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr,
mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd,
mosaicOutput2,
suppressOutput,
localRedo,
noThrow=True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(
batchFolder,
icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# Run mapproject. The grid size is auto-determined.
cmd = (
'mapproject --no-geoheader-info %s %s %s %s %s' %
(mosaicOutput, imageFile, alignCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput,
redo)
filesToWipe.append(tempOrtho)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput,
redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview,
suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " +
str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()
def runOrtho(frame, processFolder, imageFile, bundleLength, cameraMounting,
threadText, redo, suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
# Retrieve the aligned camera file
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
# Find the DEM file for the desired frame
demFile, batchFolder = icebridge_common.frameToFile(frame + offset,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find blended DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False)
projBounds = demGeoInfo['projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(frame,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor*widX, # minX
float(projBounds[1]) + factor*widX, # maxX
float(projBounds[2]) - factor*widY, # minY
float(projBounds[3]) + factor*widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s'
% (demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
# Generate the DEM mosaic
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(batchFolder,
icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = ('dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo,
noThrow = True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# If needed, generate a temporary camera file to correct a mounting rotation.
# - When the camera mount is rotated 90 degrees stereo is run on a corrected version
# but ortho needs to work on the original uncorrected jpeg image.
tempCamFile = alignCamFile + '_temp_rot.tsai'
tempCamFile = createRotatedCameraFile(alignCamFile, tempCamFile, cameraMounting)
# Run mapproject. The grid size is auto-determined.
cmd = ('mapproject --no-geoheader-info %s %s %s %s %s'
% (mosaicOutput, imageFile, tempCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo)
# Set temporary files to be cleaned up
filesToWipe.append(tempOrtho)
if tempCamFile != alignCamFile:
filesToWipe.append(tempCamFile)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()
