def getMeanDemDiff(dems, outputPrefix):
'''Get the mean distance between the input DEMs to the main DEM.'''
mainDem = dems[0]
meanDiff = 0.0
meanCount = 0.0
for i in range(1,len(dems)):
thisDem = dems[i]
if not thisDem:
continue
diffPrefix = outputPrefix + '_' + str(i)
diffPath = diffPrefix + '-diff.tif'
cmd = ('geodiff --absolute %s %s -o %s' % (mainDem, thisDem, diffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, True, False)
try:
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(diffPath)
print("Found inter-DEM diff " + str(i) + " = " + str(results['Mean']))
meanDiff = meanDiff + results['Mean']
meanCount = meanCount + 1.0
except:
print('No overlap with DEM ' + thisDem)
if meanCount < 1: # Handle degenerate cases
return 0
meanDiff = meanDiff / meanCount
print('Mean of DEM diffs = ' + str(meanDiff))
return meanDiff
def getMeanDemDiff(dems, outputPrefix):
'''Get the mean distance between the input DEMs to the main DEM.'''
mainDem = dems[0]
meanDiff = 0.0
meanCount = 0.0
for i in range(1, len(dems)):
thisDem = dems[i]
if not thisDem:
continue
diffPrefix = outputPrefix + '_' + str(i)
diffPath = diffPrefix + '-diff.tif'
cmd = ('geodiff --absolute %s %s -o %s' %
(mainDem, thisDem, diffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, True, False)
try:
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(diffPath)
print("Found inter-DEM diff " + str(i) + " = " +
str(results['Mean']))
meanDiff = meanDiff + results['Mean']
meanCount = meanCount + 1.0
except:
print('No overlap with DEM ' + thisDem)
if meanCount < 1: # Handle degenerate cases
return 0
meanDiff = meanDiff / meanCount
print('Mean of DEM diffs = ' + str(meanDiff))
return meanDiff
def runBlend(frame, processFolder, lidarFile, fireballDEM, options, threadText,
redo, suppressOutput):
WEIGHT_EXP = 1.3
# This will run as multiple processes. Hence have to catch all exceptions.
try:
demFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.alignFileName(), processFolder,
options.bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder,
'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
# This is turned off for now. Find the diff between neighboring
# aligned DEMs before blending.
prevDemFile, prevBatchFolder = \
icebridge_common.frameToFile(frame-1,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
prevDiffPrefix = os.path.join(batchFolder, 'out-prev')
prevDiffFile = prevDiffPrefix + '-diff.tif'
if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile):
os.system("rm -f " + prevDiffFile)
if os.path.exists(prevDemFile) and os.path.exists(demFile):
if os.path.exists(prevDiffFile):
print("File exists: " + prevDiffFile)
else:
cmd = ('geodiff --absolute %s %s -o %s' %
(prevDemFile, demFile, prevDiffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, prevDiffFile,
suppressOutput, redo)
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput)
and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " +
fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index + 1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
# Compute the mean distance between the DEMs
# TODO: Make sure this gets cleaned up!
meanWorkPrefix = os.path.join(batchFolder, 'bd')
meanDiff = getMeanDemDiff(dems, meanWorkPrefix)
# If the mean error between DEMs is creater than this,
# use a less aggressive blending method.
MEAN_DIFF_BLEND_THRESHOLD = 1.0
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder,
'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = (
'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, footprintDEM, demString, threadText,
outputPrefix))
else:
cmd = (
'dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s'
% (WEIGHT_EXP, demString, threadText, outputPrefix))
if meanDiff > MEAN_DIFF_BLEND_THRESHOLD:
cmd += ' --propagate-nodata --use-centerline-weights '
print(cmd)
# Execute the blend command.
# - Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
print(cmd)
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput,
localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
# Compute post-blending error to lidar
cmd = (
'geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput,
redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
except Exception, e:
pass
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) +
" when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix + '_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.system("rm -f " +
hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = (
'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, fireballDEM, demString, threadText,
fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput,
suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile,
fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath,
suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " +
str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput,
suppressOutput, redo)
except Exception, e:
pass
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
def runBlend(frame, processFolder, lidarFile, fireballDEM, options,
threadText, redo, suppressOutput):
WEIGHT_EXP = 1.3
# This will run as multiple processes. Hence have to catch all exceptions.
try:
demFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.alignFileName(),
processFolder, options.bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
# This is turned off for now. Find the diff between neighboring
# aligned DEMs before blending.
prevDemFile, prevBatchFolder = \
icebridge_common.frameToFile(frame-1,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
prevDiffPrefix = os.path.join(batchFolder, 'out-prev')
prevDiffFile = prevDiffPrefix + '-diff.tif'
if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile):
os.system("rm -f " + prevDiffFile)
if os.path.exists(prevDemFile) and os.path.exists(demFile):
if os.path.exists(prevDiffFile):
print("File exists: " + prevDiffFile)
else:
cmd = ('geodiff --absolute %s %s -o %s' %
(prevDemFile, demFile, prevDiffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, prevDiffFile, suppressOutput, redo)
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index+1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
# Compute the mean distance between the DEMs
# TODO: Make sure this gets cleaned up!
meanWorkPrefix = os.path.join(batchFolder, 'bd')
meanDiff = getMeanDemDiff(dems, meanWorkPrefix)
# If the mean error between DEMs is creater than this,
# use a less aggressive blending method.
MEAN_DIFF_BLEND_THRESHOLD = 1.0
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, footprintDEM, demString, threadText, outputPrefix))
else:
cmd = ('dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s'
% (WEIGHT_EXP, demString, threadText, outputPrefix))
if meanDiff > MEAN_DIFF_BLEND_THRESHOLD:
cmd += ' --propagate-nodata --use-centerline-weights '
print(cmd)
# Execute the blend command.
# - Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
print(cmd)
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
# Compute post-blending error to lidar
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
except Exception as e:
pass
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix+'_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend +' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, fireballDEM, demString, threadText, fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " + str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo)
except Exception as e:
pass
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Done with dealing with the fireball footprint
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.system("rm -f " + fileName)
# TODO: Handle this cleanup better!
os.system('rm -f ' + meanWorkPrefix + '*')
except Exception as e:
print('Blending failed!\n' + str(e) + ". " + str(traceback.print_exc()))
sys.stdout.flush()
# Handle frame range option
if (frames[0] < options.startFrame):
continue
if (frames[1] > options.stopFrame):
break
# Progress indication
if frames[0] % 100 == 0:
print("Frame: " + str(frames[0]))
batchInfoLog.flush() # for instant gratification
failureLog.flush()
# Read in blend results which are not part of the consolidated stats file
blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv')
try:
blendDiffResults = icebridge_common.readGeodiffOutput(
blendDiffPath)
except:
blendDiffResults = {'Mean': -999}
# Read in blend results which are not part of the consolidated stats file
# for the blend done in the fireball footprint
fireballBlendDiffPath = os.path.join(
demFolder, 'out-blend-fb-footprint-diff.csv')
try:
fireballBlendDiffResults = icebridge_common.readGeodiffOutput(
fireballBlendDiffPath)
except:
fireballBlendDiffResults = {'Mean': -999}
runStatsFile = os.path.join(demFolder,
def runBlend(frame, processFolder, lidarFile, fireballDEM, bundleLength,
threadText, redo, suppressOutput):
# This will run as multiple processes. Hence have to catch all exceptions:
try:
demFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.alignFileName(), processFolder,
bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder,
'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput)
and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " +
fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index + 1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder,
'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' %
(footprintDEM, demString, threadText, outputPrefix))
#filesToWipe.append(footprintDEM) # no longer needed
else:
cmd = ('dem_mosaic --first-dem-as-reference %s %s -o %s' %
(demString, threadText, outputPrefix))
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput,
localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
cmd = (
'geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput,
redo)
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) +
" when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix + '_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.remove(hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' %
(fireballDEM, demString, threadText, fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput,
suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile,
fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath,
suppressOutput, redo)
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " +
str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput,
suppressOutput, redo)
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.remove(hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Done with dealing with the fireball footprint
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
except Exception as e:
print('Blending failed!\n' + str(e) + ". " +
str(traceback.print_exc()))
sys.stdout.flush()
def generateFlightSummary(run, options):
'''Generate a folder containing handy debugging files including output thumbnails'''
# Copy logs to the output folder
print 'Copying log files...'
badImageFolder = os.path.join(options.outputFolder, 'badImages')
runFolder = run.getFolder()
procFolder = run.getProcessFolder()
navCameraFolder = run.getNavCameraFolder()
os.system('mkdir -p ' + options.outputFolder)
os.system('mkdir -p ' + badImageFolder)
packedErrorLog = os.path.join(runFolder, 'packedErrors.log')
if os.path.exists(packedErrorLog):
try:
shutil.copy(packedErrorLog, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
if not options.skipKml:
# Copy the input camera kml file
camerasInKmlPath = os.path.join(procFolder, 'cameras_in.kml')
try:
shutil.copy(camerasInKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Copy the input camera kml file
navCamerasKmlPath = os.path.join(navCameraFolder, 'nav_cameras.kml')
try:
shutil.copy(navCamerasKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Create a merged version of all the bundle adjusted camera files
# - The tool currently includes cameras more than once if they appear
# in multiple bundles.
print 'Merging output camera kml files...'
cmd = "find " + procFolder + " -name cameras_out.kml"
p = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
shell=False,
universal_newlines=True)
textOutput, err = p.communicate()
camKmlFiles = textOutput.replace('\n', ' ')
# Write the list of files to process to disk. Otherwise, if we just
# pass the full list on the command line, it may be too big
# and the call will fail.
kmlList = os.path.join(options.outputFolder, 'kml_list.txt')
print("Writing: " + kmlList)
with open(kmlList, 'w') as f:
for filename in sorted(camKmlFiles.split()):
filename = filename.strip()
if filename == "":
continue
f.write(filename + "\n")
outputKml = os.path.join(options.outputFolder, 'cameras_out.kml')
scriptPath = icebridge_common.fullPath('merge_orbitviz.py')
cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList
print(cmd)
os.system(cmd)
# Generate lidar kml files
print("Generating lidar kml files")
LIDAR_POINT_SKIP = 1527
lidarFiles = run.getLidarList(prependFolder=True)
lidarOutputFolder = os.path.join(options.outputFolder, 'lidar')
os.system('mkdir -p ' + lidarOutputFolder)
for f in lidarFiles:
inputPath = os.path.splitext(f)[0] + '.csv'
outputPath = os.path.join(lidarOutputFolder,
os.path.basename(f) + '.kml')
args = [
inputPath, outputPath, '--skip',
str(LIDAR_POINT_SKIP), '--color', 'red'
]
if not os.path.exists(outputPath): # Don't recreate these files
print("Generating: " + outputPath
) # This can be very slow, so print what is going on
try:
lvis2kml.main(args)
except Exception as e:
# Do not let this make our life miserable
print("Problem: " + str(e))
# Collect per-batch information
batchInfoPath = os.path.join(options.outputFolder, 'batchInfoSummary.csv')
failedBatchPath = os.path.join(options.outputFolder, 'failedBatchList.csv')
print("Writing statistics to: " + batchInfoPath)
print("Writing failures to: " + failedBatchPath)
batchInfoLog = open(batchInfoPath, 'w')
failureLog = open(failedBatchPath, 'w')
# Write the header for the batch log file
batchInfoLog.write('# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' +
'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' +
'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + \
'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n')
failureLog.write('# startFrame, stopFrame, errorCode, errorText\n')
demList = run.getOutputFileList(icebridge_common.blendFileName())
for (dem, frames) in demList:
demFolder = os.path.dirname(dem)
# Handle frame range option
if (frames[0] < options.startFrame):
continue
if (frames[1] > options.stopFrame):
break
# Progress indication
if frames[0] % 100 == 0:
print("Frame: " + str(frames[0]))
batchInfoLog.flush() # for instant gratification
failureLog.flush()
# Read in blend results which are not part of the consolidated stats file
blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv')
try:
blendDiffResults = icebridge_common.readGeodiffOutput(
blendDiffPath)
except:
blendDiffResults = {'Mean': -999}
# Read in blend results which are not part of the consolidated stats file
# for the blend done in the fireball footprint
fireballBlendDiffPath = os.path.join(
demFolder, 'out-blend-fb-footprint-diff.csv')
try:
fireballBlendDiffResults = icebridge_common.readGeodiffOutput(
fireballBlendDiffPath)
except:
fireballBlendDiffResults = {'Mean': -999}
runStatsFile = os.path.join(demFolder,
icebridge_common.getRunStatsFile())
statsLine = icebridge_common.readStats(runStatsFile)
# All of the other results should be in a consolidated stats file
consolidatedStatsPath = os.path.join(demFolder,
'out-consolidated_stats.txt')
if not os.path.exists(consolidatedStatsPath):
# Stats file not present, recreate it.
print 'Recreating missing stats file: ' + consolidatedStatsPath
# Get paths to the files of interest
# This logic must be sync-ed up with cleanBatch().
lidarDiffPath = os.path.join(demFolder, 'out-diff.csv')
interDiffPath = os.path.join(demFolder,
'out_inter_diff_summary.csv')
fireDiffPath = os.path.join(demFolder,
'out_fireball_diff_summary.csv')
fireLidarDiffPath = os.path.join(demFolder,
'out_fireLidar_diff_summary.csv')
fractionValidPath = os.path.join(demFolder,
'valid_pixel_fraction.txt')
process_icebridge_batch.consolidateStats(
lidarDiffPath, interDiffPath, fireDiffPath, fireLidarDiffPath,
dem, consolidatedStatsPath, fractionValidPath, None,
options.skipGeo)
# Now the consolidated file should always be present
with open(consolidatedStatsPath, 'r') as f:
statsText = f.read()
# Write info to summary file
try:
batchInfoLog.write(
'%d, %d, %s, %f, %f, %s\n' %
(frames[0], frames[1], statsText, blendDiffResults['Mean'],
fireballBlendDiffResults['Mean'], statsLine))
except:
# Bugfix for corrupted data
print("Problem parsing frame: " + str(frames[0]))
# Keep a list of batches that did not generate an output DEM
parts = statsText.split(',')
if len(parts) <= 2:
print("Cannot parse " + consolidatedStatsPath + ", skipping.")
else:
if (float(parts[0]) == 0) and (float(parts[1]) == 0) and (float(
parts[2]) == -999):
if os.path.exists(dem):
# Handle the case where the statistics are bad for some reason
errorCode = 0
errorText = 'Success but statistics are bad'
else: # A real failure, figure out the cause
batchFolder = os.path.dirname(dem)
(errorCode, errorText) = getFailureCause(batchFolder)
#print str((errorCode, errorText))
#if errorCode < 0: # Debug code for unknown errors
#print str((errorCode, errorText))
#print statsText
#print batchFolder
#raise Exception('DEBUG')
failureLog.write('%d, %d, %d, %s\n' %
(frames[0], frames[1], errorCode, errorText))
# Make a link to the DEM thumbnail file in our summary folder
hillshadePath = os.path.join(demFolder,
'out-blend-DEM_HILLSHADE_browse.tif')
if os.path.exists(hillshadePath):
thumbName = ('dem_%05d_%05d_browse.tif' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(hillshadePath,
thumbPath,
verbose=False)
else:
# If the DEM thumbnail does not exist, look for the input frame thumbnail.
inPath = os.path.join(demFolder, 'first_image_browse.tif')
thumbName = ('input_%05d_browse.tif' % (frames[0]))
thumbPath = os.path.join(badImageFolder, thumbName)
if os.path.exists(inPath):
icebridge_common.makeSymLink(inPath, thumbPath, verbose=False)
# Make a link to the ortho thumbnail file in our summary folder
orthoPath = os.path.join(demFolder,
icebridge_common.orthoPreviewFileName())
if os.path.exists(orthoPath):
thumbName = ('ortho_%05d_%05d_browse.jpg' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(orthoPath, thumbPath, verbose=False)
# End loop through expected DEMs and writing log files
batchInfoLog.close()
failureLog.close()
print 'Finished generating flight summary in folder: ' + options.outputFolder
print("Wrote: " + batchInfoPath)
print("Wrote: " + failedBatchPath)