def getMeanDemDiff(dems, outputPrefix): '''Get the mean distance between the input DEMs to the main DEM.''' mainDem = dems[0] meanDiff = 0.0 meanCount = 0.0 for i in range(1,len(dems)): thisDem = dems[i] if not thisDem: continue diffPrefix = outputPrefix + '_' + str(i) diffPath = diffPrefix + '-diff.tif' cmd = ('geodiff --absolute %s %s -o %s' % (mainDem, thisDem, diffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, True, False) try: # Read in and examine the results results = icebridge_common.readGeodiffOutput(diffPath) print("Found inter-DEM diff " + str(i) + " = " + str(results['Mean'])) meanDiff = meanDiff + results['Mean'] meanCount = meanCount + 1.0 except: print('No overlap with DEM ' + thisDem) if meanCount < 1: # Handle degenerate cases return 0 meanDiff = meanDiff / meanCount print('Mean of DEM diffs = ' + str(meanDiff)) return meanDiff
def getMeanDemDiff(dems, outputPrefix): '''Get the mean distance between the input DEMs to the main DEM.''' mainDem = dems[0] meanDiff = 0.0 meanCount = 0.0 for i in range(1, len(dems)): thisDem = dems[i] if not thisDem: continue diffPrefix = outputPrefix + '_' + str(i) diffPath = diffPrefix + '-diff.tif' cmd = ('geodiff --absolute %s %s -o %s' % (mainDem, thisDem, diffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, True, False) try: # Read in and examine the results results = icebridge_common.readGeodiffOutput(diffPath) print("Found inter-DEM diff " + str(i) + " = " + str(results['Mean'])) meanDiff = meanDiff + results['Mean'] meanCount = meanCount + 1.0 except: print('No overlap with DEM ' + thisDem) if meanCount < 1: # Handle degenerate cases return 0 meanDiff = meanDiff / meanCount print('Mean of DEM diffs = ' + str(meanDiff)) return meanDiff
def runBlend(frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput): WEIGHT_EXP = 1.3 # This will run as multiple processes. Hence have to catch all exceptions. try: demFile, batchFolder = icebridge_common.frameToFile( frame, icebridge_common.alignFileName(), processFolder, options.bundleLength) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) if demFile == "": print("Could not find DEM for frame: " + str(frame)) return # The names for the final results finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM') finalBlend = finalOutputPrefix + '.tif' finalDiff = finalOutputPrefix + "-diff.csv" fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint') fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif' finalFireballOutput = fireballOutputPrefix + '-DEM.tif' fireballDiffPath = fireballOutputPrefix + "-diff.csv" # This is turned off for now. Find the diff between neighboring # aligned DEMs before blending. prevDemFile, prevBatchFolder = \ icebridge_common.frameToFile(frame-1, icebridge_common.alignFileName(), processFolder, options.bundleLength) prevDiffPrefix = os.path.join(batchFolder, 'out-prev') prevDiffFile = prevDiffPrefix + '-diff.tif' if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile): os.system("rm -f " + prevDiffFile) if os.path.exists(prevDemFile) and os.path.exists(demFile): if os.path.exists(prevDiffFile): print("File exists: " + prevDiffFile) else: cmd = ('geodiff --absolute %s %s -o %s' % (prevDemFile, demFile, prevDiffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, prevDiffFile, suppressOutput, redo) if not redo: set1Exists = False if (os.path.exists(finalBlend) and os.path.exists(finalDiff)): print("Files exist: " + finalBlend + " " + finalDiff + ".") set1Exists = True set2Exists = True if fireballDEM != "": if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)): print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".") set2Exists = True else: set2Exists = False if set1Exists and set2Exists: return # We will blend the dems with frame offsets within frameOffsets[0:index] filesToWipe = [] bestMean = 1.0e+100 bestBlend = '' bestVals = '' bestDiff = '' # Look at frames with these offsets when blending frameOffsets = [0, 1, -1, 2, -2] for index in range(len(frameOffsets)): # Find all the DEMs up to the current index dems = [] currDemFile = "" for val in range(0, index + 1): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) if currDemFile == "": # The last DEM was not present. Hence this iteration will add nothing new. continue # Compute the mean distance between the DEMs # TODO: Make sure this gets cleaned up! meanWorkPrefix = os.path.join(batchFolder, 'bd') meanDiff = getMeanDemDiff(dems, meanWorkPrefix) # If the mean error between DEMs is creater than this, # use a less aggressive blending method. MEAN_DIFF_BLEND_THRESHOLD = 1.0 demString = " ".join(dems) outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index)) # See if we have a pre-existing DEM to use as footprint footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif') blendOutput = outputPrefix + '-tile-0.tif' if os.path.exists(footprintDEM): cmd = ( 'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, footprintDEM, demString, threadText, outputPrefix)) else: cmd = ( 'dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s' % (WEIGHT_EXP, demString, threadText, outputPrefix)) if meanDiff > MEAN_DIFF_BLEND_THRESHOLD: cmd += ' --propagate-nodata --use-centerline-weights ' print(cmd) # Execute the blend command. # - Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True print(cmd) asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo) filesToWipe.append(blendOutput) diffPath = outputPrefix + "-diff.csv" filesToWipe.append(diffPath) # Compute post-blending error to lidar cmd = ( 'geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, blendOutput, lidarFile, outputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(diffPath) print("Current mean error to lidar is " + str(results['Mean'])) if bestMean > float(results['Mean']): bestMean = float(results['Mean']) bestBlend = blendOutput bestVals = demString bestDiff = diffPath except Exception, e: pass logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Update the filenames of the output files print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals) cmd = "mv " + bestBlend + " " + finalBlend print(cmd) asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo) cmd = "mv " + bestDiff + " " + finalDiff print(cmd) asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo) # Generate a thumbnail of the final DEM hillOutput = finalOutputPrefix + '_HILLSHADE.tif' cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # Generate a low resolution compressed thumbnail of the hillshade for debugging thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif' cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) os.system("rm -f " + hillOutput) # Remove this file to keep down the file count # Do another blend, to this DEM's footprint, but not using it if fireballDEM != "": # Find all the DEMs dems = [] for val in range(0, len(frameOffsets)): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) demString = " ".join(dems) cmd = ( 'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, fireballDEM, demString, threadText, fireballOutputPrefix)) #filesToWipe.append(fireballBlendOutput) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo) #filesToWipe.append(fireballDiffPath) cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(fireballDiffPath) print("Mean error to lidar in fireball footprint is " + str(results['Mean'])) cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput print(cmd) asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo) except Exception, e: pass # Generate a thumbnail of the final DEM #hillOutput = fireballOutputPrefix+'_HILLSHADE.tif' #cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput #print(cmd) #asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) ## Generate a low resolution compressed thumbnail of the hillshade for debugging #thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif' #cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' #asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) #os.system("rm -f " + hillOutput) # Remove this file to keep down the file count logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles
def runBlend(frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput): WEIGHT_EXP = 1.3 # This will run as multiple processes. Hence have to catch all exceptions. try: demFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.alignFileName(), processFolder, options.bundleLength) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) if demFile == "": print("Could not find DEM for frame: " + str(frame)) return # The names for the final results finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM') finalBlend = finalOutputPrefix + '.tif' finalDiff = finalOutputPrefix + "-diff.csv" fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint') fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif' finalFireballOutput = fireballOutputPrefix + '-DEM.tif' fireballDiffPath = fireballOutputPrefix + "-diff.csv" # This is turned off for now. Find the diff between neighboring # aligned DEMs before blending. prevDemFile, prevBatchFolder = \ icebridge_common.frameToFile(frame-1, icebridge_common.alignFileName(), processFolder, options.bundleLength) prevDiffPrefix = os.path.join(batchFolder, 'out-prev') prevDiffFile = prevDiffPrefix + '-diff.tif' if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile): os.system("rm -f " + prevDiffFile) if os.path.exists(prevDemFile) and os.path.exists(demFile): if os.path.exists(prevDiffFile): print("File exists: " + prevDiffFile) else: cmd = ('geodiff --absolute %s %s -o %s' % (prevDemFile, demFile, prevDiffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, prevDiffFile, suppressOutput, redo) if not redo: set1Exists = False if (os.path.exists(finalBlend) and os.path.exists(finalDiff)): print("Files exist: " + finalBlend + " " + finalDiff + ".") set1Exists = True set2Exists = True if fireballDEM != "": if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)): print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".") set2Exists = True else: set2Exists = False if set1Exists and set2Exists: return # We will blend the dems with frame offsets within frameOffsets[0:index] filesToWipe = [] bestMean = 1.0e+100 bestBlend = '' bestVals = '' bestDiff = '' # Look at frames with these offsets when blending frameOffsets = [0, 1, -1, 2, -2] for index in range(len(frameOffsets)): # Find all the DEMs up to the current index dems = [] currDemFile = "" for val in range(0, index+1): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) if currDemFile == "": # The last DEM was not present. Hence this iteration will add nothing new. continue # Compute the mean distance between the DEMs # TODO: Make sure this gets cleaned up! meanWorkPrefix = os.path.join(batchFolder, 'bd') meanDiff = getMeanDemDiff(dems, meanWorkPrefix) # If the mean error between DEMs is creater than this, # use a less aggressive blending method. MEAN_DIFF_BLEND_THRESHOLD = 1.0 demString = " ".join(dems) outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index)) # See if we have a pre-existing DEM to use as footprint footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif') blendOutput = outputPrefix + '-tile-0.tif' if os.path.exists(footprintDEM): cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, footprintDEM, demString, threadText, outputPrefix)) else: cmd = ('dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s' % (WEIGHT_EXP, demString, threadText, outputPrefix)) if meanDiff > MEAN_DIFF_BLEND_THRESHOLD: cmd += ' --propagate-nodata --use-centerline-weights ' print(cmd) # Execute the blend command. # - Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True print(cmd) asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo) filesToWipe.append(blendOutput) diffPath = outputPrefix + "-diff.csv" filesToWipe.append(diffPath) # Compute post-blending error to lidar cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, blendOutput, lidarFile, outputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(diffPath) print("Current mean error to lidar is " + str(results['Mean'])) if bestMean > float(results['Mean']): bestMean = float(results['Mean']) bestBlend = blendOutput bestVals = demString bestDiff = diffPath except Exception as e: pass logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Update the filenames of the output files print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals) cmd = "mv " + bestBlend + " " + finalBlend print(cmd) asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo) cmd = "mv " + bestDiff + " " + finalDiff print(cmd) asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo) # Generate a thumbnail of the final DEM hillOutput = finalOutputPrefix+'_HILLSHADE.tif' cmd = 'hillshade ' + finalBlend +' -o ' + hillOutput asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # Generate a low resolution compressed thumbnail of the hillshade for debugging thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif' cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) os.system("rm -f " + hillOutput) # Remove this file to keep down the file count # Do another blend, to this DEM's footprint, but not using it if fireballDEM != "": # Find all the DEMs dems = [] for val in range(0, len(frameOffsets)): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) demString = " ".join(dems) cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, fireballDEM, demString, threadText, fireballOutputPrefix)) #filesToWipe.append(fireballBlendOutput) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo) #filesToWipe.append(fireballDiffPath) cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(fireballDiffPath) print("Mean error to lidar in fireball footprint is " + str(results['Mean'])) cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput print(cmd) asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo) except Exception as e: pass # Generate a thumbnail of the final DEM #hillOutput = fireballOutputPrefix+'_HILLSHADE.tif' #cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput #print(cmd) #asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) ## Generate a low resolution compressed thumbnail of the hillshade for debugging #thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif' #cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' #asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) #os.system("rm -f " + hillOutput) # Remove this file to keep down the file count logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Done with dealing with the fireball footprint # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.system("rm -f " + fileName) # TODO: Handle this cleanup better! os.system('rm -f ' + meanWorkPrefix + '*') except Exception as e: print('Blending failed!\n' + str(e) + ". " + str(traceback.print_exc())) sys.stdout.flush()
# Handle frame range option if (frames[0] < options.startFrame): continue if (frames[1] > options.stopFrame): break # Progress indication if frames[0] % 100 == 0: print("Frame: " + str(frames[0])) batchInfoLog.flush() # for instant gratification failureLog.flush() # Read in blend results which are not part of the consolidated stats file blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv') try: blendDiffResults = icebridge_common.readGeodiffOutput( blendDiffPath) except: blendDiffResults = {'Mean': -999} # Read in blend results which are not part of the consolidated stats file # for the blend done in the fireball footprint fireballBlendDiffPath = os.path.join( demFolder, 'out-blend-fb-footprint-diff.csv') try: fireballBlendDiffResults = icebridge_common.readGeodiffOutput( fireballBlendDiffPath) except: fireballBlendDiffResults = {'Mean': -999} runStatsFile = os.path.join(demFolder,
def runBlend(frame, processFolder, lidarFile, fireballDEM, bundleLength, threadText, redo, suppressOutput): # This will run as multiple processes. Hence have to catch all exceptions: try: demFile, batchFolder = icebridge_common.frameToFile( frame, icebridge_common.alignFileName(), processFolder, bundleLength) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) if demFile == "": print("Could not find DEM for frame: " + str(frame)) return # The names for the final results finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM') finalBlend = finalOutputPrefix + '.tif' finalDiff = finalOutputPrefix + "-diff.csv" fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint') fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif' finalFireballOutput = fireballOutputPrefix + '-DEM.tif' fireballDiffPath = fireballOutputPrefix + "-diff.csv" if not redo: set1Exists = False if (os.path.exists(finalBlend) and os.path.exists(finalDiff)): print("Files exist: " + finalBlend + " " + finalDiff + ".") set1Exists = True set2Exists = True if fireballDEM != "": if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)): print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".") set2Exists = True else: set2Exists = False if set1Exists and set2Exists: return # We will blend the dems with frame offsets within frameOffsets[0:index] filesToWipe = [] bestMean = 1.0e+100 bestBlend = '' bestVals = '' bestDiff = '' # Look at frames with these offsets when blending frameOffsets = [0, 1, -1, 2, -2] for index in range(len(frameOffsets)): # Find all the DEMs up to the current index dems = [] currDemFile = "" for val in range(0, index + 1): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, bundleLength) if currDemFile == "": continue dems.append(currDemFile) if currDemFile == "": # The last DEM was not present. Hence this iteration will add nothing new. continue demString = " ".join(dems) outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index)) # See if we have a pre-existing DEM to use as footprint footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif') blendOutput = outputPrefix + '-tile-0.tif' if os.path.exists(footprintDEM): cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' % (footprintDEM, demString, threadText, outputPrefix)) #filesToWipe.append(footprintDEM) # no longer needed else: cmd = ('dem_mosaic --first-dem-as-reference %s %s -o %s' % (demString, threadText, outputPrefix)) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo) filesToWipe.append(blendOutput) diffPath = outputPrefix + "-diff.csv" filesToWipe.append(diffPath) cmd = ( 'geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, blendOutput, lidarFile, outputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo) # Read in and examine the results results = icebridge_common.readGeodiffOutput(diffPath) print("Current mean error to lidar is " + str(results['Mean'])) if bestMean > float(results['Mean']): bestMean = float(results['Mean']) bestBlend = blendOutput bestVals = demString bestDiff = diffPath logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Update the filenames of the output files print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals) cmd = "mv " + bestBlend + " " + finalBlend print(cmd) asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo) cmd = "mv " + bestDiff + " " + finalDiff print(cmd) asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo) # Generate a thumbnail of the final DEM hillOutput = finalOutputPrefix + '_HILLSHADE.tif' cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # Generate a low resolution compressed thumbnail of the hillshade for debugging thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif' cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) os.remove(hillOutput) # Remove this file to keep down the file count # Do another blend, to this DEM's footprint, but not using it if fireballDEM != "": # Find all the DEMs dems = [] for val in range(0, len(frameOffsets)): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, bundleLength) if currDemFile == "": continue dems.append(currDemFile) demString = " ".join(dems) cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' % (fireballDEM, demString, threadText, fireballOutputPrefix)) #filesToWipe.append(fireballBlendOutput) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo) #filesToWipe.append(fireballDiffPath) cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo) # Read in and examine the results results = icebridge_common.readGeodiffOutput(fireballDiffPath) print("Mean error to lidar in fireball footprint is " + str(results['Mean'])) cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput print(cmd) asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo) # Generate a thumbnail of the final DEM #hillOutput = fireballOutputPrefix+'_HILLSHADE.tif' #cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput #print(cmd) #asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) ## Generate a low resolution compressed thumbnail of the hillshade for debugging #thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif' #cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' #asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) #os.remove(hillOutput) # Remove this file to keep down the file count logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Done with dealing with the fireball footprint # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) except Exception as e: print('Blending failed!\n' + str(e) + ". " + str(traceback.print_exc())) sys.stdout.flush()
def generateFlightSummary(run, options): '''Generate a folder containing handy debugging files including output thumbnails''' # Copy logs to the output folder print 'Copying log files...' badImageFolder = os.path.join(options.outputFolder, 'badImages') runFolder = run.getFolder() procFolder = run.getProcessFolder() navCameraFolder = run.getNavCameraFolder() os.system('mkdir -p ' + options.outputFolder) os.system('mkdir -p ' + badImageFolder) packedErrorLog = os.path.join(runFolder, 'packedErrors.log') if os.path.exists(packedErrorLog): try: shutil.copy(packedErrorLog, options.outputFolder) except Exception as e: # In case it complains about copying a file onto itself print("Warning: " + str(e)) if not options.skipKml: # Copy the input camera kml file camerasInKmlPath = os.path.join(procFolder, 'cameras_in.kml') try: shutil.copy(camerasInKmlPath, options.outputFolder) except Exception as e: # In case it complains about copying a file onto itself print("Warning: " + str(e)) # Copy the input camera kml file navCamerasKmlPath = os.path.join(navCameraFolder, 'nav_cameras.kml') try: shutil.copy(navCamerasKmlPath, options.outputFolder) except Exception as e: # In case it complains about copying a file onto itself print("Warning: " + str(e)) # Create a merged version of all the bundle adjusted camera files # - The tool currently includes cameras more than once if they appear # in multiple bundles. print 'Merging output camera kml files...' cmd = "find " + procFolder + " -name cameras_out.kml" p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, shell=False, universal_newlines=True) textOutput, err = p.communicate() camKmlFiles = textOutput.replace('\n', ' ') # Write the list of files to process to disk. Otherwise, if we just # pass the full list on the command line, it may be too big # and the call will fail. kmlList = os.path.join(options.outputFolder, 'kml_list.txt') print("Writing: " + kmlList) with open(kmlList, 'w') as f: for filename in sorted(camKmlFiles.split()): filename = filename.strip() if filename == "": continue f.write(filename + "\n") outputKml = os.path.join(options.outputFolder, 'cameras_out.kml') scriptPath = icebridge_common.fullPath('merge_orbitviz.py') cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList print(cmd) os.system(cmd) # Generate lidar kml files print("Generating lidar kml files") LIDAR_POINT_SKIP = 1527 lidarFiles = run.getLidarList(prependFolder=True) lidarOutputFolder = os.path.join(options.outputFolder, 'lidar') os.system('mkdir -p ' + lidarOutputFolder) for f in lidarFiles: inputPath = os.path.splitext(f)[0] + '.csv' outputPath = os.path.join(lidarOutputFolder, os.path.basename(f) + '.kml') args = [ inputPath, outputPath, '--skip', str(LIDAR_POINT_SKIP), '--color', 'red' ] if not os.path.exists(outputPath): # Don't recreate these files print("Generating: " + outputPath ) # This can be very slow, so print what is going on try: lvis2kml.main(args) except Exception as e: # Do not let this make our life miserable print("Problem: " + str(e)) # Collect per-batch information batchInfoPath = os.path.join(options.outputFolder, 'batchInfoSummary.csv') failedBatchPath = os.path.join(options.outputFolder, 'failedBatchList.csv') print("Writing statistics to: " + batchInfoPath) print("Writing failures to: " + failedBatchPath) batchInfoLog = open(batchInfoPath, 'w') failureLog = open(failedBatchPath, 'w') # Write the header for the batch log file batchInfoLog.write('# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' + 'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' + 'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + \ 'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n') failureLog.write('# startFrame, stopFrame, errorCode, errorText\n') demList = run.getOutputFileList(icebridge_common.blendFileName()) for (dem, frames) in demList: demFolder = os.path.dirname(dem) # Handle frame range option if (frames[0] < options.startFrame): continue if (frames[1] > options.stopFrame): break # Progress indication if frames[0] % 100 == 0: print("Frame: " + str(frames[0])) batchInfoLog.flush() # for instant gratification failureLog.flush() # Read in blend results which are not part of the consolidated stats file blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv') try: blendDiffResults = icebridge_common.readGeodiffOutput( blendDiffPath) except: blendDiffResults = {'Mean': -999} # Read in blend results which are not part of the consolidated stats file # for the blend done in the fireball footprint fireballBlendDiffPath = os.path.join( demFolder, 'out-blend-fb-footprint-diff.csv') try: fireballBlendDiffResults = icebridge_common.readGeodiffOutput( fireballBlendDiffPath) except: fireballBlendDiffResults = {'Mean': -999} runStatsFile = os.path.join(demFolder, icebridge_common.getRunStatsFile()) statsLine = icebridge_common.readStats(runStatsFile) # All of the other results should be in a consolidated stats file consolidatedStatsPath = os.path.join(demFolder, 'out-consolidated_stats.txt') if not os.path.exists(consolidatedStatsPath): # Stats file not present, recreate it. print 'Recreating missing stats file: ' + consolidatedStatsPath # Get paths to the files of interest # This logic must be sync-ed up with cleanBatch(). lidarDiffPath = os.path.join(demFolder, 'out-diff.csv') interDiffPath = os.path.join(demFolder, 'out_inter_diff_summary.csv') fireDiffPath = os.path.join(demFolder, 'out_fireball_diff_summary.csv') fireLidarDiffPath = os.path.join(demFolder, 'out_fireLidar_diff_summary.csv') fractionValidPath = os.path.join(demFolder, 'valid_pixel_fraction.txt') process_icebridge_batch.consolidateStats( lidarDiffPath, interDiffPath, fireDiffPath, fireLidarDiffPath, dem, consolidatedStatsPath, fractionValidPath, None, options.skipGeo) # Now the consolidated file should always be present with open(consolidatedStatsPath, 'r') as f: statsText = f.read() # Write info to summary file try: batchInfoLog.write( '%d, %d, %s, %f, %f, %s\n' % (frames[0], frames[1], statsText, blendDiffResults['Mean'], fireballBlendDiffResults['Mean'], statsLine)) except: # Bugfix for corrupted data print("Problem parsing frame: " + str(frames[0])) # Keep a list of batches that did not generate an output DEM parts = statsText.split(',') if len(parts) <= 2: print("Cannot parse " + consolidatedStatsPath + ", skipping.") else: if (float(parts[0]) == 0) and (float(parts[1]) == 0) and (float( parts[2]) == -999): if os.path.exists(dem): # Handle the case where the statistics are bad for some reason errorCode = 0 errorText = 'Success but statistics are bad' else: # A real failure, figure out the cause batchFolder = os.path.dirname(dem) (errorCode, errorText) = getFailureCause(batchFolder) #print str((errorCode, errorText)) #if errorCode < 0: # Debug code for unknown errors #print str((errorCode, errorText)) #print statsText #print batchFolder #raise Exception('DEBUG') failureLog.write('%d, %d, %d, %s\n' % (frames[0], frames[1], errorCode, errorText)) # Make a link to the DEM thumbnail file in our summary folder hillshadePath = os.path.join(demFolder, 'out-blend-DEM_HILLSHADE_browse.tif') if os.path.exists(hillshadePath): thumbName = ('dem_%05d_%05d_browse.tif' % (frames[0], frames[1])) thumbPath = os.path.join(options.outputFolder, thumbName) icebridge_common.makeSymLink(hillshadePath, thumbPath, verbose=False) else: # If the DEM thumbnail does not exist, look for the input frame thumbnail. inPath = os.path.join(demFolder, 'first_image_browse.tif') thumbName = ('input_%05d_browse.tif' % (frames[0])) thumbPath = os.path.join(badImageFolder, thumbName) if os.path.exists(inPath): icebridge_common.makeSymLink(inPath, thumbPath, verbose=False) # Make a link to the ortho thumbnail file in our summary folder orthoPath = os.path.join(demFolder, icebridge_common.orthoPreviewFileName()) if os.path.exists(orthoPath): thumbName = ('ortho_%05d_%05d_browse.jpg' % (frames[0], frames[1])) thumbPath = os.path.join(options.outputFolder, thumbName) icebridge_common.makeSymLink(orthoPath, thumbPath, verbose=False) # End loop through expected DEMs and writing log files batchInfoLog.close() failureLog.close() print 'Finished generating flight summary in folder: ' + options.outputFolder print("Wrote: " + batchInfoPath) print("Wrote: " + failedBatchPath)