def runOrtho(frame, processFolder, imageFile, bundleLength, threadText, redo,
suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
demFile, batchFolder = icebridge_common.frameToFile(
frame + offset, icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile,
getStats=False)
projBounds = demGeoInfo[
'projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.blendFileName(), processFolder,
bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder,
icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(
batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor * widX, # minX
float(projBounds[1]) + factor * widX, # maxX
float(projBounds[2]) - factor * widY, # minY
float(projBounds[3]) + factor * widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' %
(demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput,
localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(
batchFolder, icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = (
'dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr,
mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd,
mosaicOutput2,
suppressOutput,
localRedo,
noThrow=True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(
batchFolder,
icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# Run mapproject. The grid size is auto-determined.
cmd = (
'mapproject --no-geoheader-info %s %s %s %s %s' %
(mosaicOutput, imageFile, alignCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput,
redo)
filesToWipe.append(tempOrtho)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput,
redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview,
suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " +
str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()
def runBlend(frame, processFolder, lidarFile, fireballDEM, options, threadText,
redo, suppressOutput):
WEIGHT_EXP = 1.3
# This will run as multiple processes. Hence have to catch all exceptions.
try:
demFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.alignFileName(), processFolder,
options.bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder,
'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
# This is turned off for now. Find the diff between neighboring
# aligned DEMs before blending.
prevDemFile, prevBatchFolder = \
icebridge_common.frameToFile(frame-1,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
prevDiffPrefix = os.path.join(batchFolder, 'out-prev')
prevDiffFile = prevDiffPrefix + '-diff.tif'
if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile):
os.system("rm -f " + prevDiffFile)
if os.path.exists(prevDemFile) and os.path.exists(demFile):
if os.path.exists(prevDiffFile):
print("File exists: " + prevDiffFile)
else:
cmd = ('geodiff --absolute %s %s -o %s' %
(prevDemFile, demFile, prevDiffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, prevDiffFile,
suppressOutput, redo)
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput)
and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " +
fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index + 1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
# Compute the mean distance between the DEMs
# TODO: Make sure this gets cleaned up!
meanWorkPrefix = os.path.join(batchFolder, 'bd')
meanDiff = getMeanDemDiff(dems, meanWorkPrefix)
# If the mean error between DEMs is creater than this,
# use a less aggressive blending method.
MEAN_DIFF_BLEND_THRESHOLD = 1.0
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder,
'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = (
'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, footprintDEM, demString, threadText,
outputPrefix))
else:
cmd = (
'dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s'
% (WEIGHT_EXP, demString, threadText, outputPrefix))
if meanDiff > MEAN_DIFF_BLEND_THRESHOLD:
cmd += ' --propagate-nodata --use-centerline-weights '
print(cmd)
# Execute the blend command.
# - Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
print(cmd)
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput,
localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
# Compute post-blending error to lidar
cmd = (
'geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput,
redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
except Exception, e:
pass
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) +
" when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix + '_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.system("rm -f " +
hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = (
'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, fireballDEM, demString, threadText,
fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput,
suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile,
fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath,
suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " +
str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput,
suppressOutput, redo)
except Exception, e:
pass
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
def runBlend(frame, processFolder, lidarFile, fireballDEM, bundleLength,
threadText, redo, suppressOutput):
# This will run as multiple processes. Hence have to catch all exceptions:
try:
demFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.alignFileName(), processFolder,
bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder,
'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput)
and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " +
fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index + 1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder,
'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' %
(footprintDEM, demString, threadText, outputPrefix))
#filesToWipe.append(footprintDEM) # no longer needed
else:
cmd = ('dem_mosaic --first-dem-as-reference %s %s -o %s' %
(demString, threadText, outputPrefix))
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput,
localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
cmd = (
'geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput,
redo)
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) +
" when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix + '_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.remove(hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' %
(fireballDEM, demString, threadText, fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput,
suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile,
fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath,
suppressOutput, redo)
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " +
str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput,
suppressOutput, redo)
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.remove(hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Done with dealing with the fireball footprint
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
except Exception as e:
print('Blending failed!\n' + str(e) + ". " +
str(traceback.print_exc()))
sys.stdout.flush()
def runBlend(frame, processFolder, lidarFile, fireballDEM, options,
threadText, redo, suppressOutput):
WEIGHT_EXP = 1.3
# This will run as multiple processes. Hence have to catch all exceptions.
try:
demFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.alignFileName(),
processFolder, options.bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
# This is turned off for now. Find the diff between neighboring
# aligned DEMs before blending.
prevDemFile, prevBatchFolder = \
icebridge_common.frameToFile(frame-1,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
prevDiffPrefix = os.path.join(batchFolder, 'out-prev')
prevDiffFile = prevDiffPrefix + '-diff.tif'
if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile):
os.system("rm -f " + prevDiffFile)
if os.path.exists(prevDemFile) and os.path.exists(demFile):
if os.path.exists(prevDiffFile):
print("File exists: " + prevDiffFile)
else:
cmd = ('geodiff --absolute %s %s -o %s' %
(prevDemFile, demFile, prevDiffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, prevDiffFile, suppressOutput, redo)
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index+1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
# Compute the mean distance between the DEMs
# TODO: Make sure this gets cleaned up!
meanWorkPrefix = os.path.join(batchFolder, 'bd')
meanDiff = getMeanDemDiff(dems, meanWorkPrefix)
# If the mean error between DEMs is creater than this,
# use a less aggressive blending method.
MEAN_DIFF_BLEND_THRESHOLD = 1.0
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, footprintDEM, demString, threadText, outputPrefix))
else:
cmd = ('dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s'
% (WEIGHT_EXP, demString, threadText, outputPrefix))
if meanDiff > MEAN_DIFF_BLEND_THRESHOLD:
cmd += ' --propagate-nodata --use-centerline-weights '
print(cmd)
# Execute the blend command.
# - Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
print(cmd)
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
# Compute post-blending error to lidar
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
except Exception as e:
pass
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix+'_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend +' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, fireballDEM, demString, threadText, fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " + str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo)
except Exception as e:
pass
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Done with dealing with the fireball footprint
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.system("rm -f " + fileName)
# TODO: Handle this cleanup better!
os.system('rm -f ' + meanWorkPrefix + '*')
except Exception as e:
print('Blending failed!\n' + str(e) + ". " + str(traceback.print_exc()))
sys.stdout.flush()
def runOrtho(frame, processFolder, imageFile, bundleLength, cameraMounting,
threadText, redo, suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
# Retrieve the aligned camera file
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
# Find the DEM file for the desired frame
demFile, batchFolder = icebridge_common.frameToFile(frame + offset,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find blended DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False)
projBounds = demGeoInfo['projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(frame,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor*widX, # minX
float(projBounds[1]) + factor*widX, # maxX
float(projBounds[2]) - factor*widY, # minY
float(projBounds[3]) + factor*widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s'
% (demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
# Generate the DEM mosaic
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(batchFolder,
icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = ('dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo,
noThrow = True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# If needed, generate a temporary camera file to correct a mounting rotation.
# - When the camera mount is rotated 90 degrees stereo is run on a corrected version
# but ortho needs to work on the original uncorrected jpeg image.
tempCamFile = alignCamFile + '_temp_rot.tsai'
tempCamFile = createRotatedCameraFile(alignCamFile, tempCamFile, cameraMounting)
# Run mapproject. The grid size is auto-determined.
cmd = ('mapproject --no-geoheader-info %s %s %s %s %s'
% (mosaicOutput, imageFile, tempCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo)
# Set temporary files to be cleaned up
filesToWipe.append(tempOrtho)
if tempCamFile != alignCamFile:
filesToWipe.append(tempCamFile)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()