def packAndSendCompletedRun(run, logger):
'''Assembles and compresses the deliverable parts of the run'''
logger.info('Getting ready to pack up run ' + str(run))
cwd = os.getcwd()
os.chdir(run.parentFolder)
runFolder = str(run)
# Use symlinks to assemble a fake file structure to tar up
assemblyFolder = run.getAssemblyFolder()
batchFolders = run.getBatchFolderList()
os.system('mkdir -p ' + assemblyFolder)
# Wipe any dead symlinks, as maybe this is not the first time the assembly is made
pattern = os.path.join(assemblyFolder, '*')
currFiles = glob.glob(pattern)
for filename in currFiles:
if not os.path.exists(os.path.realpath(filename)):
logger.info("Will wipe dead link: " + filename)
os.system("rm -f " + filename)
# For each batch folder, start adding links to files that we want in the tarball
for batch in batchFolders:
# Skip folders where we did not produce final output
finalDemFile = os.path.join(batch, icebridge_common.blendFileName())
if not os.path.exists(finalDemFile):
continue
# Need to change the name of these files when they go in the output folder
(startFrame,
stopFrame) = icebridge_common.getFrameRangeFromBatchFolder(batch)
# We respect the convention below in push_to_nsidc.py.
prefix = ('F_%05d_%05d' % (startFrame, stopFrame))
prefix = os.path.join(assemblyFolder, prefix)
target = prefix + '_DEM.tif'
try:
if os.path.exists(target):
os.system("rm -f " + target) # to wipe whatever was there
os.symlink(finalDemFile, target)
except Exception, e:
logger.info(
str(e) + " when doing: ln -s " + finalDemFile + " " + target)
def runOrtho(frame, processFolder, imageFile, bundleLength, threadText, redo,
suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
demFile, batchFolder = icebridge_common.frameToFile(
frame + offset, icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile,
getStats=False)
projBounds = demGeoInfo[
'projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.blendFileName(), processFolder,
bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder,
icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(
batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor * widX, # minX
float(projBounds[1]) + factor * widX, # maxX
float(projBounds[2]) - factor * widY, # minY
float(projBounds[3]) + factor * widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' %
(demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput,
localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(
batchFolder, icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = (
'dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr,
mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd,
mosaicOutput2,
suppressOutput,
localRedo,
noThrow=True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(
batchFolder,
icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# Run mapproject. The grid size is auto-determined.
cmd = (
'mapproject --no-geoheader-info %s %s %s %s %s' %
(mosaicOutput, imageFile, alignCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput,
redo)
filesToWipe.append(tempOrtho)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput,
redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview,
suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " +
str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()
def packAndSendCompletedRun(run, logger):
'''Assembles and compresses the deliverable parts of the run'''
logger.info('Getting ready to pack up run ' + str(run))
cwd = os.getcwd()
os.chdir(run.parentFolder)
runFolder = str(run)
# Use symlinks to assemble a fake file structure to tar up
assemblyFolder = run.getAssemblyFolder()
batchFolders = run.getBatchFolderList()
os.system('mkdir -p ' + assemblyFolder)
# Wipe any dead symlinks, as maybe this is not the first time the assembly is made
pattern = os.path.join(assemblyFolder, '*')
currFiles = glob.glob(pattern)
for filename in currFiles:
if not os.path.exists(os.path.realpath(filename)):
logger.info("Will wipe dead link: " + filename)
os.system("rm -f " + filename)
# For each batch folder, start adding links to files that we want in the tarball
for batch in batchFolders:
# Skip folders where we did not produce final output
finalDemFile = os.path.join(batch, icebridge_common.blendFileName())
if not os.path.exists(finalDemFile):
continue
# Need to change the name of these files when they go in the output folder
(startFrame, stopFrame) = icebridge_common.getFrameRangeFromBatchFolder(batch)
# We respect the convention below in push_to_nsidc.py.
prefix = ('F_%05d_%05d' % (startFrame, stopFrame))
prefix = os.path.join(assemblyFolder, prefix)
target = prefix + '_DEM.tif'
try:
if os.path.exists(target):
os.system("rm -f " + target) # to wipe whatever was there
os.symlink(finalDemFile, target)
except Exception as e:
logger.info(str(e) + " when doing: ln -s " + finalDemFile + " " + target)
# Tar up the assembled files and send them at the same time using the shiftc command
# - No need to use a compression algorithm here
fileName = run.getOutputTarName()
lfePath = os.path.join(REMOTE_OUTPUT_FOLDER, fileName)
logger.info('Sending run to lfe...')
cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
cmd = 'shiftc --wait -d -r --dereference --create-tar ' + \
os.path.join(runFolder, os.path.basename(assemblyFolder)) + ' ' + lfePath
# Command to transfer the files as they are in the batch dirs without symlink
#cmd = 'shiftc --wait -d -r --include=\'^.*?' + icebridge_common.blendFileName() + \
# '$\' --create-tar ' + runFolder + \
# ' ' + lfePath
try:
robust_shiftc(cmd, logger)
except Exception as e:
logger.info(str(e))
raise Exception('Failed to pack/send results for run ' + str(run) + \
'. Maybe not all sym links are valid.')
os.chdir(cwd)
logger.info('Finished sending run to lfe.')
# Collect per-batch information
batchInfoPath = os.path.join(options.outputFolder, 'batchInfoSummary.csv')
failedBatchPath = os.path.join(options.outputFolder, 'failedBatchList.csv')
print("Writing statistics to: " + batchInfoPath)
print("Writing failures to: " + failedBatchPath)
batchInfoLog = open(batchInfoPath, 'w')
failureLog = open(failedBatchPath, 'w')
# Write the header for the batch log file
batchInfoLog.write('# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' +
'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' +
'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + \
'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n')
failureLog.write('# startFrame, stopFrame, errorCode, errorText\n')
demList = run.getOutputFileList(icebridge_common.blendFileName())
for (dem, frames) in demList:
demFolder = os.path.dirname(dem)
# Handle frame range option
if (frames[0] < options.startFrame):
continue
if (frames[1] > options.stopFrame):
break
# Progress indication
if frames[0] % 100 == 0:
print("Frame: " + str(frames[0]))
batchInfoLog.flush() # for instant gratification
failureLog.flush()
def runOrtho(frame, processFolder, imageFile, bundleLength, cameraMounting,
threadText, redo, suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
# Retrieve the aligned camera file
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
# Find the DEM file for the desired frame
demFile, batchFolder = icebridge_common.frameToFile(frame + offset,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find blended DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False)
projBounds = demGeoInfo['projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(frame,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor*widX, # minX
float(projBounds[1]) + factor*widX, # maxX
float(projBounds[2]) - factor*widY, # minY
float(projBounds[3]) + factor*widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s'
% (demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
# Generate the DEM mosaic
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(batchFolder,
icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = ('dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo,
noThrow = True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# If needed, generate a temporary camera file to correct a mounting rotation.
# - When the camera mount is rotated 90 degrees stereo is run on a corrected version
# but ortho needs to work on the original uncorrected jpeg image.
tempCamFile = alignCamFile + '_temp_rot.tsai'
tempCamFile = createRotatedCameraFile(alignCamFile, tempCamFile, cameraMounting)
# Run mapproject. The grid size is auto-determined.
cmd = ('mapproject --no-geoheader-info %s %s %s %s %s'
% (mosaicOutput, imageFile, tempCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo)
# Set temporary files to be cleaned up
filesToWipe.append(tempOrtho)
if tempCamFile != alignCamFile:
filesToWipe.append(tempCamFile)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()
def generateFlightSummary(run, options):
'''Generate a folder containing handy debugging files including output thumbnails'''
# Copy logs to the output folder
print 'Copying log files...'
badImageFolder = os.path.join(options.outputFolder, 'badImages')
runFolder = run.getFolder()
procFolder = run.getProcessFolder()
navCameraFolder = run.getNavCameraFolder()
os.system('mkdir -p ' + options.outputFolder)
os.system('mkdir -p ' + badImageFolder)
packedErrorLog = os.path.join(runFolder, 'packedErrors.log')
if os.path.exists(packedErrorLog):
try:
shutil.copy(packedErrorLog, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
if not options.skipKml:
# Copy the input camera kml file
camerasInKmlPath = os.path.join(procFolder, 'cameras_in.kml')
try:
shutil.copy(camerasInKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Copy the input camera kml file
navCamerasKmlPath = os.path.join(navCameraFolder, 'nav_cameras.kml')
try:
shutil.copy(navCamerasKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Create a merged version of all the bundle adjusted camera files
# - The tool currently includes cameras more than once if they appear
# in multiple bundles.
print 'Merging output camera kml files...'
cmd = "find " + procFolder + " -name cameras_out.kml"
p = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
shell=False,
universal_newlines=True)
textOutput, err = p.communicate()
camKmlFiles = textOutput.replace('\n', ' ')
# Write the list of files to process to disk. Otherwise, if we just
# pass the full list on the command line, it may be too big
# and the call will fail.
kmlList = os.path.join(options.outputFolder, 'kml_list.txt')
print("Writing: " + kmlList)
with open(kmlList, 'w') as f:
for filename in sorted(camKmlFiles.split()):
filename = filename.strip()
if filename == "":
continue
f.write(filename + "\n")
outputKml = os.path.join(options.outputFolder, 'cameras_out.kml')
scriptPath = icebridge_common.fullPath('merge_orbitviz.py')
cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList
print(cmd)
os.system(cmd)
# Generate lidar kml files
print("Generating lidar kml files")
LIDAR_POINT_SKIP = 1527
lidarFiles = run.getLidarList(prependFolder=True)
lidarOutputFolder = os.path.join(options.outputFolder, 'lidar')
os.system('mkdir -p ' + lidarOutputFolder)
for f in lidarFiles:
inputPath = os.path.splitext(f)[0] + '.csv'
outputPath = os.path.join(lidarOutputFolder,
os.path.basename(f) + '.kml')
args = [
inputPath, outputPath, '--skip',
str(LIDAR_POINT_SKIP), '--color', 'red'
]
if not os.path.exists(outputPath): # Don't recreate these files
print("Generating: " + outputPath
) # This can be very slow, so print what is going on
try:
lvis2kml.main(args)
except Exception as e:
# Do not let this make our life miserable
print("Problem: " + str(e))
# Collect per-batch information
batchInfoPath = os.path.join(options.outputFolder, 'batchInfoSummary.csv')
failedBatchPath = os.path.join(options.outputFolder, 'failedBatchList.csv')
print("Writing statistics to: " + batchInfoPath)
print("Writing failures to: " + failedBatchPath)
batchInfoLog = open(batchInfoPath, 'w')
failureLog = open(failedBatchPath, 'w')
# Write the header for the batch log file
batchInfoLog.write('# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' +
'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' +
'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + \
'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n')
failureLog.write('# startFrame, stopFrame, errorCode, errorText\n')
demList = run.getOutputFileList(icebridge_common.blendFileName())
for (dem, frames) in demList:
demFolder = os.path.dirname(dem)
# Handle frame range option
if (frames[0] < options.startFrame):
continue
if (frames[1] > options.stopFrame):
break
# Progress indication
if frames[0] % 100 == 0:
print("Frame: " + str(frames[0]))
batchInfoLog.flush() # for instant gratification
failureLog.flush()
# Read in blend results which are not part of the consolidated stats file
blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv')
try:
blendDiffResults = icebridge_common.readGeodiffOutput(
blendDiffPath)
except:
blendDiffResults = {'Mean': -999}
# Read in blend results which are not part of the consolidated stats file
# for the blend done in the fireball footprint
fireballBlendDiffPath = os.path.join(
demFolder, 'out-blend-fb-footprint-diff.csv')
try:
fireballBlendDiffResults = icebridge_common.readGeodiffOutput(
fireballBlendDiffPath)
except:
fireballBlendDiffResults = {'Mean': -999}
runStatsFile = os.path.join(demFolder,
icebridge_common.getRunStatsFile())
statsLine = icebridge_common.readStats(runStatsFile)
# All of the other results should be in a consolidated stats file
consolidatedStatsPath = os.path.join(demFolder,
'out-consolidated_stats.txt')
if not os.path.exists(consolidatedStatsPath):
# Stats file not present, recreate it.
print 'Recreating missing stats file: ' + consolidatedStatsPath
# Get paths to the files of interest
# This logic must be sync-ed up with cleanBatch().
lidarDiffPath = os.path.join(demFolder, 'out-diff.csv')
interDiffPath = os.path.join(demFolder,
'out_inter_diff_summary.csv')
fireDiffPath = os.path.join(demFolder,
'out_fireball_diff_summary.csv')
fireLidarDiffPath = os.path.join(demFolder,
'out_fireLidar_diff_summary.csv')
fractionValidPath = os.path.join(demFolder,
'valid_pixel_fraction.txt')
process_icebridge_batch.consolidateStats(
lidarDiffPath, interDiffPath, fireDiffPath, fireLidarDiffPath,
dem, consolidatedStatsPath, fractionValidPath, None,
options.skipGeo)
# Now the consolidated file should always be present
with open(consolidatedStatsPath, 'r') as f:
statsText = f.read()
# Write info to summary file
try:
batchInfoLog.write(
'%d, %d, %s, %f, %f, %s\n' %
(frames[0], frames[1], statsText, blendDiffResults['Mean'],
fireballBlendDiffResults['Mean'], statsLine))
except:
# Bugfix for corrupted data
print("Problem parsing frame: " + str(frames[0]))
# Keep a list of batches that did not generate an output DEM
parts = statsText.split(',')
if len(parts) <= 2:
print("Cannot parse " + consolidatedStatsPath + ", skipping.")
else:
if (float(parts[0]) == 0) and (float(parts[1]) == 0) and (float(
parts[2]) == -999):
if os.path.exists(dem):
# Handle the case where the statistics are bad for some reason
errorCode = 0
errorText = 'Success but statistics are bad'
else: # A real failure, figure out the cause
batchFolder = os.path.dirname(dem)
(errorCode, errorText) = getFailureCause(batchFolder)
#print str((errorCode, errorText))
#if errorCode < 0: # Debug code for unknown errors
#print str((errorCode, errorText))
#print statsText
#print batchFolder
#raise Exception('DEBUG')
failureLog.write('%d, %d, %d, %s\n' %
(frames[0], frames[1], errorCode, errorText))
# Make a link to the DEM thumbnail file in our summary folder
hillshadePath = os.path.join(demFolder,
'out-blend-DEM_HILLSHADE_browse.tif')
if os.path.exists(hillshadePath):
thumbName = ('dem_%05d_%05d_browse.tif' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(hillshadePath,
thumbPath,
verbose=False)
else:
# If the DEM thumbnail does not exist, look for the input frame thumbnail.
inPath = os.path.join(demFolder, 'first_image_browse.tif')
thumbName = ('input_%05d_browse.tif' % (frames[0]))
thumbPath = os.path.join(badImageFolder, thumbName)
if os.path.exists(inPath):
icebridge_common.makeSymLink(inPath, thumbPath, verbose=False)
# Make a link to the ortho thumbnail file in our summary folder
orthoPath = os.path.join(demFolder,
icebridge_common.orthoPreviewFileName())
if os.path.exists(orthoPath):
thumbName = ('ortho_%05d_%05d_browse.jpg' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(orthoPath, thumbPath, verbose=False)
# End loop through expected DEMs and writing log files
batchInfoLog.close()
failureLog.close()
print 'Finished generating flight summary in folder: ' + options.outputFolder
print("Wrote: " + batchInfoPath)
print("Wrote: " + failedBatchPath)