blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv')
try:
blendDiffResults = icebridge_common.readGeodiffOutput(blendDiffPath)
except:
blendDiffResults = {'Mean':-999}
# Read in blend results which are not part of the consolidated stats file
# for the blend done in the fireball footprint
fireballBlendDiffPath = os.path.join(demFolder, 'out-blend-fb-footprint-diff.csv')
try:
fireballBlendDiffResults = icebridge_common.readGeodiffOutput(fireballBlendDiffPath)
except:
fireballBlendDiffResults = {'Mean':-999}
runStatsFile = os.path.join(demFolder, icebridge_common.getRunStatsFile())
statsLine = icebridge_common.readStats(runStatsFile)
# All of the other results should be in a consolidated stats file
consolidatedStatsPath = os.path.join(demFolder, 'out-consolidated_stats.txt')
if not os.path.exists(consolidatedStatsPath):
# Stats file not present, recreate it.
print 'Recreating missing stats file: ' + consolidatedStatsPath
# Get paths to the files of interest
# This logic must be sync-ed up with cleanBatch().
lidarDiffPath = os.path.join(demFolder, 'out-diff.csv')
interDiffPath = os.path.join(demFolder, 'out_inter_diff_summary.csv')
fireDiffPath = os.path.join(demFolder, 'out_fireball_diff_summary.csv')
except:
blendDiffResults = {'Mean': -999}
# Read in blend results which are not part of the consolidated stats file
# for the blend done in the fireball footprint
fireballBlendDiffPath = os.path.join(
demFolder, 'out-blend-fb-footprint-diff.csv')
try:
fireballBlendDiffResults = icebridge_common.readGeodiffOutput(
fireballBlendDiffPath)
except:
fireballBlendDiffResults = {'Mean': -999}
runStatsFile = os.path.join(demFolder,
icebridge_common.getRunStatsFile())
statsLine = icebridge_common.readStats(runStatsFile)
# All of the other results should be in a consolidated stats file
consolidatedStatsPath = os.path.join(demFolder,
'out-consolidated_stats.txt')
if not os.path.exists(consolidatedStatsPath):
# Stats file not present, recreate it.
print 'Recreating missing stats file: ' + consolidatedStatsPath
# Get paths to the files of interest
# This logic must be sync-ed up with cleanBatch().
lidarDiffPath = os.path.join(demFolder, 'out-diff.csv')
interDiffPath = os.path.join(demFolder,
def generateFlightSummary(run, options):
'''Generate a folder containing handy debugging files including output thumbnails'''
# Copy logs to the output folder
print 'Copying log files...'
badImageFolder = os.path.join(options.outputFolder, 'badImages')
runFolder = run.getFolder()
procFolder = run.getProcessFolder()
navCameraFolder = run.getNavCameraFolder()
os.system('mkdir -p ' + options.outputFolder)
os.system('mkdir -p ' + badImageFolder)
packedErrorLog = os.path.join(runFolder, 'packedErrors.log')
if os.path.exists(packedErrorLog):
try:
shutil.copy(packedErrorLog, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
if not options.skipKml:
# Copy the input camera kml file
camerasInKmlPath = os.path.join(procFolder, 'cameras_in.kml')
try:
shutil.copy(camerasInKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Copy the input camera kml file
navCamerasKmlPath = os.path.join(navCameraFolder, 'nav_cameras.kml')
try:
shutil.copy(navCamerasKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Create a merged version of all the bundle adjusted camera files
# - The tool currently includes cameras more than once if they appear
# in multiple bundles.
print 'Merging output camera kml files...'
cmd = "find " + procFolder + " -name cameras_out.kml"
p = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
shell=False,
universal_newlines=True)
textOutput, err = p.communicate()
camKmlFiles = textOutput.replace('\n', ' ')
# Write the list of files to process to disk. Otherwise, if we just
# pass the full list on the command line, it may be too big
# and the call will fail.
kmlList = os.path.join(options.outputFolder, 'kml_list.txt')
print("Writing: " + kmlList)
with open(kmlList, 'w') as f:
for filename in sorted(camKmlFiles.split()):
filename = filename.strip()
if filename == "":
continue
f.write(filename + "\n")
outputKml = os.path.join(options.outputFolder, 'cameras_out.kml')
scriptPath = icebridge_common.fullPath('merge_orbitviz.py')
cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList
print(cmd)
os.system(cmd)
# Generate lidar kml files
print("Generating lidar kml files")
LIDAR_POINT_SKIP = 1527
lidarFiles = run.getLidarList(prependFolder=True)
lidarOutputFolder = os.path.join(options.outputFolder, 'lidar')
os.system('mkdir -p ' + lidarOutputFolder)
for f in lidarFiles:
inputPath = os.path.splitext(f)[0] + '.csv'
outputPath = os.path.join(lidarOutputFolder,
os.path.basename(f) + '.kml')
args = [
inputPath, outputPath, '--skip',
str(LIDAR_POINT_SKIP), '--color', 'red'
]
if not os.path.exists(outputPath): # Don't recreate these files
print("Generating: " + outputPath
) # This can be very slow, so print what is going on
try:
lvis2kml.main(args)
except Exception as e:
# Do not let this make our life miserable
print("Problem: " + str(e))
# Collect per-batch information
batchInfoPath = os.path.join(options.outputFolder, 'batchInfoSummary.csv')
failedBatchPath = os.path.join(options.outputFolder, 'failedBatchList.csv')
print("Writing statistics to: " + batchInfoPath)
print("Writing failures to: " + failedBatchPath)
batchInfoLog = open(batchInfoPath, 'w')
failureLog = open(failedBatchPath, 'w')
# Write the header for the batch log file
batchInfoLog.write('# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' +
'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' +
'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + \
'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n')
failureLog.write('# startFrame, stopFrame, errorCode, errorText\n')
demList = run.getOutputFileList(icebridge_common.blendFileName())
for (dem, frames) in demList:
demFolder = os.path.dirname(dem)
# Handle frame range option
if (frames[0] < options.startFrame):
continue
if (frames[1] > options.stopFrame):
break
# Progress indication
if frames[0] % 100 == 0:
print("Frame: " + str(frames[0]))
batchInfoLog.flush() # for instant gratification
failureLog.flush()
# Read in blend results which are not part of the consolidated stats file
blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv')
try:
blendDiffResults = icebridge_common.readGeodiffOutput(
blendDiffPath)
except:
blendDiffResults = {'Mean': -999}
# Read in blend results which are not part of the consolidated stats file
# for the blend done in the fireball footprint
fireballBlendDiffPath = os.path.join(
demFolder, 'out-blend-fb-footprint-diff.csv')
try:
fireballBlendDiffResults = icebridge_common.readGeodiffOutput(
fireballBlendDiffPath)
except:
fireballBlendDiffResults = {'Mean': -999}
runStatsFile = os.path.join(demFolder,
icebridge_common.getRunStatsFile())
statsLine = icebridge_common.readStats(runStatsFile)
# All of the other results should be in a consolidated stats file
consolidatedStatsPath = os.path.join(demFolder,
'out-consolidated_stats.txt')
if not os.path.exists(consolidatedStatsPath):
# Stats file not present, recreate it.
print 'Recreating missing stats file: ' + consolidatedStatsPath
# Get paths to the files of interest
# This logic must be sync-ed up with cleanBatch().
lidarDiffPath = os.path.join(demFolder, 'out-diff.csv')
interDiffPath = os.path.join(demFolder,
'out_inter_diff_summary.csv')
fireDiffPath = os.path.join(demFolder,
'out_fireball_diff_summary.csv')
fireLidarDiffPath = os.path.join(demFolder,
'out_fireLidar_diff_summary.csv')
fractionValidPath = os.path.join(demFolder,
'valid_pixel_fraction.txt')
process_icebridge_batch.consolidateStats(
lidarDiffPath, interDiffPath, fireDiffPath, fireLidarDiffPath,
dem, consolidatedStatsPath, fractionValidPath, None,
options.skipGeo)
# Now the consolidated file should always be present
with open(consolidatedStatsPath, 'r') as f:
statsText = f.read()
# Write info to summary file
try:
batchInfoLog.write(
'%d, %d, %s, %f, %f, %s\n' %
(frames[0], frames[1], statsText, blendDiffResults['Mean'],
fireballBlendDiffResults['Mean'], statsLine))
except:
# Bugfix for corrupted data
print("Problem parsing frame: " + str(frames[0]))
# Keep a list of batches that did not generate an output DEM
parts = statsText.split(',')
if len(parts) <= 2:
print("Cannot parse " + consolidatedStatsPath + ", skipping.")
else:
if (float(parts[0]) == 0) and (float(parts[1]) == 0) and (float(
parts[2]) == -999):
if os.path.exists(dem):
# Handle the case where the statistics are bad for some reason
errorCode = 0
errorText = 'Success but statistics are bad'
else: # A real failure, figure out the cause
batchFolder = os.path.dirname(dem)
(errorCode, errorText) = getFailureCause(batchFolder)
#print str((errorCode, errorText))
#if errorCode < 0: # Debug code for unknown errors
#print str((errorCode, errorText))
#print statsText
#print batchFolder
#raise Exception('DEBUG')
failureLog.write('%d, %d, %d, %s\n' %
(frames[0], frames[1], errorCode, errorText))
# Make a link to the DEM thumbnail file in our summary folder
hillshadePath = os.path.join(demFolder,
'out-blend-DEM_HILLSHADE_browse.tif')
if os.path.exists(hillshadePath):
thumbName = ('dem_%05d_%05d_browse.tif' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(hillshadePath,
thumbPath,
verbose=False)
else:
# If the DEM thumbnail does not exist, look for the input frame thumbnail.
inPath = os.path.join(demFolder, 'first_image_browse.tif')
thumbName = ('input_%05d_browse.tif' % (frames[0]))
thumbPath = os.path.join(badImageFolder, thumbName)
if os.path.exists(inPath):
icebridge_common.makeSymLink(inPath, thumbPath, verbose=False)
# Make a link to the ortho thumbnail file in our summary folder
orthoPath = os.path.join(demFolder,
icebridge_common.orthoPreviewFileName())
if os.path.exists(orthoPath):
thumbName = ('ortho_%05d_%05d_browse.jpg' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(orthoPath, thumbPath, verbose=False)
# End loop through expected DEMs and writing log files
batchInfoLog.close()
failureLog.close()
print 'Finished generating flight summary in folder: ' + options.outputFolder
print("Wrote: " + batchInfoPath)
print("Wrote: " + failedBatchPath)