def processBatch(imageCameraPairs, lidarFolder, referenceDem, skipInterval, outputFolder, extraOptions,
outputResolution, stereoArgs, batchNum, batchLogPath=''):
'''Processes a batch of images at once'''
suppressOutput = False
redo = False
argString = ''
for pair in imageCameraPairs:
argString += pair[0] + ' '
for pair in imageCameraPairs:
argString += pair[1] + ' '
# Just set the options and call the pair python tool.
# We can try out bundle adjustment for intrinsic parameters here.
cmd = ('--lidar-folder %s --reference-dem %s --stereo-image-interval %d --dem-resolution %f' \
' --output-folder %s %s %s --stereo-arguments '
% (lidarFolder, referenceDem, skipInterval, outputResolution, outputFolder, argString, extraOptions))
if batchLogPath:
# With this option we just log the commands to a text file
# - Setting this option limits to one process so there will be only one
# simultaneous file writer.
with open(batchLogPath, 'a') as f:
f.write('python ' + icebridge_common.fullPath('process_icebridge_batch.py') + ' ' + \
cmd +'"'+ stereoArgs +'"\n')
return
try:
args = cmd.split()
args += (stereoArgs.strip(),) # Make sure this is properly passed
process_icebridge_batch.main(args)
except Exception as e:
logger.error('Batch processing failed!\n' + str(e) +
traceback.print_exc())
def fetchIndices(options, logger):
'''Fetch the csv indices of available files.'''
logger.info("Fetch indices from NSIDC.")
pythonPath = asp_system_utils.which('python')
cmd = ( (pythonPath + ' ' + icebridge_common.fullPath('full_processing_script.py') + \
' --camera-calibration-folder %s --reference-dem-folder %s --site %s ' + \
'--yyyymmdd %s --stop-after-index-fetch --no-nav ' ) % \
(options.inputCalFolder, options.refDemFolder, options.site, options.yyyymmdd))
logger.info(cmd)
start_time()
os.system(cmd)
stop_time("fetch index", logger)
def fetchIndices(options, logger):
'''Fetch the csv indices of available files.'''
logger.info("Fetch indices from NSIDC.")
pythonPath = asp_system_utils.which('python')
cmd = ( (pythonPath + ' ' + icebridge_common.fullPath('full_processing_script.py') + \
' --camera-calibration-folder %s --reference-dem-folder %s --site %s ' + \
'--yyyymmdd %s --stop-after-index-fetch --no-nav ' ) % \
(options.inputCalFolder, options.refDemFolder, options.site, options.yyyymmdd))
logger.info(cmd)
start_time()
os.system(cmd)
stop_time("fetch index", logger)
def submitBatchJobs(commandFileList, options, pbsLogFolder, run, logger):
'''Read all the batch jobs required for a run and distribute them across job submissions.
Returns the common string in the job names.'''
# Retrieve parallel processing parameters
(numProcesses, numThreads, tasksPerJob,
maxHours) = getParallelParams(options.nodeType)
numBatches = len(commandFileList)
logger.info(
("Num batches: %d, tasks per job: %d" % (numBatches, tasksPerJob)))
baseName = run.shortName(
) # SITE + YYMMDD = 8 chars, leaves seven for frame digits.
# Call the tool which just executes commands from a file
scriptPath = icebridge_common.fullPath('multi_process_command_runner.py')
index = 0
jobIDs = []
for commandFile in commandFileList:
jobName = ('%s%05d%s' % ('RS', index, baseName))
# Specify the range of lines in the file we want this node to execute
args = ('--command-file-path %s --start-frame -1 --stop-frame -1 --num-processes %d' % \
(commandFile, numProcesses))
logPrefix = os.path.join(pbsLogFolder, 'batch_' + jobName)
logger.info('Submitting summary regen job: ' + scriptPath + ' ' + args)
BATCH_PBS_QUEUE = 'normal'
jobID = pbs_functions.submitJob(jobName, BATCH_PBS_QUEUE, maxHours,
logger, options.minutesInDevelQueue,
GROUP_ID, options.nodeType,
'~/repo/python_env/bin/python',
scriptPath + ' ' + args, logPrefix)
jobIDs.append(jobID)
index += 1
# Waiting on these jobs happens outside this function
return (baseName, jobIDs)
def submitBatchJobs(commandFileList, options, pbsLogFolder, run, logger):
'''Read all the batch jobs required for a run and distribute them across job submissions.
Returns the common string in the job names.'''
# Retrieve parallel processing parameters
(numProcesses, numThreads, tasksPerJob, maxHours) = getParallelParams(options.nodeType)
numBatches = len(commandFileList)
logger.info( ("Num batches: %d, tasks per job: %d" % (numBatches, tasksPerJob) ) )
baseName = run.shortName() # SITE + YYMMDD = 8 chars, leaves seven for frame digits.
# Call the tool which just executes commands from a file
scriptPath = icebridge_common.fullPath('multi_process_command_runner.py')
index = 0
jobIDs = []
for commandFile in commandFileList:
jobName = ('%s%05d%s' % ('RS', index, baseName) )
# Specify the range of lines in the file we want this node to execute
args = ('--command-file-path %s --start-frame -1 --stop-frame -1 --num-processes %d' % \
(commandFile, numProcesses))
logPrefix = os.path.join(pbsLogFolder, 'batch_' + jobName)
logger.info('Submitting summary regen job: ' + scriptPath + ' ' + args)
BATCH_PBS_QUEUE = 'normal'
jobID = pbs_functions.submitJob(jobName, BATCH_PBS_QUEUE, maxHours, logger,
options.minutesInDevelQueue,
GROUP_ID,
options.nodeType, '~/repo/python_env/bin/python',
scriptPath + ' ' + args, logPrefix)
jobIDs.append(jobID)
index += 1
# Waiting on these jobs happens outside this function
return (baseName, jobIDs)
def processBatch(imageCameraPairs, lidarFolder, referenceDem, skipInterval,
outputFolder, extraOptions, outputResolution, stereoArgs,
batchNum, batchLogPath=''):
'''Processes a batch of images at once'''
suppressOutput = False
redo = False
argString = ''
for pair in imageCameraPairs:
argString += pair[0] + ' '
for pair in imageCameraPairs:
argString += pair[1] + ' '
# Just set the options and call the pair python tool.
# We can try out bundle adjustment for intrinsic parameters here.
cmd = ('--lidar-folder %s --reference-dem %s --stereo-image-interval %d --dem-resolution %f' \
' --output-folder %s %s %s --stereo-arguments '
% (lidarFolder, referenceDem, skipInterval, outputResolution,
outputFolder, argString, extraOptions))
if batchLogPath:
# With this option we just log the commands to a text file
# - Setting this option limits to one process so there will be only one
# simultaneous file writer.
with open(batchLogPath, 'a') as f:
f.write('python ' + icebridge_common.fullPath('process_icebridge_batch.py') + ' ' + \
cmd +'"'+ stereoArgs +'"\n')
return
try:
args = cmd.split()
args += (stereoArgs.strip(),) # Make sure this is properly passed
process_icebridge_batch.main(args)
except Exception as e:
logger.error('Batch processing failed!\n' + str(e) +
traceback.print_exc())
camKmlFiles = textOutput.replace('\n', ' ')
# Write the list of files to process to disk. Otherwise, if we just
# pass the full list on the command line, it may be too big
# and the call will fail.
kmlList = os.path.join(options.outputFolder, 'kml_list.txt')
print("Writing: " + kmlList)
with open(kmlList, 'w') as f:
for filename in sorted(camKmlFiles.split()):
filename = filename.strip()
if filename == "":
continue
f.write(filename + "\n")
outputKml = os.path.join(options.outputFolder, 'cameras_out.kml')
scriptPath = icebridge_common.fullPath('merge_orbitviz.py')
cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList
print(cmd)
os.system(cmd)
# Generate lidar kml files
print("Generating lidar kml files")
LIDAR_POINT_SKIP = 1527
lidarFiles = run.getLidarList(prependFolder=True)
lidarOutputFolder = os.path.join(options.outputFolder, 'lidar')
os.system('mkdir -p ' + lidarOutputFolder)
for f in lidarFiles:
inputPath = os.path.splitext(f)[0] + '.csv'
outputPath = os.path.join(lidarOutputFolder,
os.path.basename(f) + '.kml')
args = [
def generateFlightSummary(run, options):
'''Generate a folder containing handy debugging files including output thumbnails'''
# Copy logs to the output folder
print 'Copying log files...'
badImageFolder = os.path.join(options.outputFolder, 'badImages')
runFolder = run.getFolder()
procFolder = run.getProcessFolder()
navCameraFolder = run.getNavCameraFolder()
os.system('mkdir -p ' + options.outputFolder)
os.system('mkdir -p ' + badImageFolder)
packedErrorLog = os.path.join(runFolder, 'packedErrors.log')
if os.path.exists(packedErrorLog):
try:
shutil.copy(packedErrorLog, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
if not options.skipKml:
# Copy the input camera kml file
camerasInKmlPath = os.path.join(procFolder, 'cameras_in.kml')
try:
shutil.copy(camerasInKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Copy the input camera kml file
navCamerasKmlPath = os.path.join(navCameraFolder, 'nav_cameras.kml')
try:
shutil.copy(navCamerasKmlPath, options.outputFolder)
except Exception as e:
# In case it complains about copying a file onto itself
print("Warning: " + str(e))
# Create a merged version of all the bundle adjusted camera files
# - The tool currently includes cameras more than once if they appear
# in multiple bundles.
print 'Merging output camera kml files...'
cmd = "find " + procFolder + " -name cameras_out.kml"
p = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
shell=False,
universal_newlines=True)
textOutput, err = p.communicate()
camKmlFiles = textOutput.replace('\n', ' ')
# Write the list of files to process to disk. Otherwise, if we just
# pass the full list on the command line, it may be too big
# and the call will fail.
kmlList = os.path.join(options.outputFolder, 'kml_list.txt')
print("Writing: " + kmlList)
with open(kmlList, 'w') as f:
for filename in sorted(camKmlFiles.split()):
filename = filename.strip()
if filename == "":
continue
f.write(filename + "\n")
outputKml = os.path.join(options.outputFolder, 'cameras_out.kml')
scriptPath = icebridge_common.fullPath('merge_orbitviz.py')
cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList
print(cmd)
os.system(cmd)
# Generate lidar kml files
print("Generating lidar kml files")
LIDAR_POINT_SKIP = 1527
lidarFiles = run.getLidarList(prependFolder=True)
lidarOutputFolder = os.path.join(options.outputFolder, 'lidar')
os.system('mkdir -p ' + lidarOutputFolder)
for f in lidarFiles:
inputPath = os.path.splitext(f)[0] + '.csv'
outputPath = os.path.join(lidarOutputFolder,
os.path.basename(f) + '.kml')
args = [
inputPath, outputPath, '--skip',
str(LIDAR_POINT_SKIP), '--color', 'red'
]
if not os.path.exists(outputPath): # Don't recreate these files
print("Generating: " + outputPath
) # This can be very slow, so print what is going on
try:
lvis2kml.main(args)
except Exception as e:
# Do not let this make our life miserable
print("Problem: " + str(e))
# Collect per-batch information
batchInfoPath = os.path.join(options.outputFolder, 'batchInfoSummary.csv')
failedBatchPath = os.path.join(options.outputFolder, 'failedBatchList.csv')
print("Writing statistics to: " + batchInfoPath)
print("Writing failures to: " + failedBatchPath)
batchInfoLog = open(batchInfoPath, 'w')
failureLog = open(failedBatchPath, 'w')
# Write the header for the batch log file
batchInfoLog.write('# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' +
'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' +
'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + \
'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n')
failureLog.write('# startFrame, stopFrame, errorCode, errorText\n')
demList = run.getOutputFileList(icebridge_common.blendFileName())
for (dem, frames) in demList:
demFolder = os.path.dirname(dem)
# Handle frame range option
if (frames[0] < options.startFrame):
continue
if (frames[1] > options.stopFrame):
break
# Progress indication
if frames[0] % 100 == 0:
print("Frame: " + str(frames[0]))
batchInfoLog.flush() # for instant gratification
failureLog.flush()
# Read in blend results which are not part of the consolidated stats file
blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv')
try:
blendDiffResults = icebridge_common.readGeodiffOutput(
blendDiffPath)
except:
blendDiffResults = {'Mean': -999}
# Read in blend results which are not part of the consolidated stats file
# for the blend done in the fireball footprint
fireballBlendDiffPath = os.path.join(
demFolder, 'out-blend-fb-footprint-diff.csv')
try:
fireballBlendDiffResults = icebridge_common.readGeodiffOutput(
fireballBlendDiffPath)
except:
fireballBlendDiffResults = {'Mean': -999}
runStatsFile = os.path.join(demFolder,
icebridge_common.getRunStatsFile())
statsLine = icebridge_common.readStats(runStatsFile)
# All of the other results should be in a consolidated stats file
consolidatedStatsPath = os.path.join(demFolder,
'out-consolidated_stats.txt')
if not os.path.exists(consolidatedStatsPath):
# Stats file not present, recreate it.
print 'Recreating missing stats file: ' + consolidatedStatsPath
# Get paths to the files of interest
# This logic must be sync-ed up with cleanBatch().
lidarDiffPath = os.path.join(demFolder, 'out-diff.csv')
interDiffPath = os.path.join(demFolder,
'out_inter_diff_summary.csv')
fireDiffPath = os.path.join(demFolder,
'out_fireball_diff_summary.csv')
fireLidarDiffPath = os.path.join(demFolder,
'out_fireLidar_diff_summary.csv')
fractionValidPath = os.path.join(demFolder,
'valid_pixel_fraction.txt')
process_icebridge_batch.consolidateStats(
lidarDiffPath, interDiffPath, fireDiffPath, fireLidarDiffPath,
dem, consolidatedStatsPath, fractionValidPath, None,
options.skipGeo)
# Now the consolidated file should always be present
with open(consolidatedStatsPath, 'r') as f:
statsText = f.read()
# Write info to summary file
try:
batchInfoLog.write(
'%d, %d, %s, %f, %f, %s\n' %
(frames[0], frames[1], statsText, blendDiffResults['Mean'],
fireballBlendDiffResults['Mean'], statsLine))
except:
# Bugfix for corrupted data
print("Problem parsing frame: " + str(frames[0]))
# Keep a list of batches that did not generate an output DEM
parts = statsText.split(',')
if len(parts) <= 2:
print("Cannot parse " + consolidatedStatsPath + ", skipping.")
else:
if (float(parts[0]) == 0) and (float(parts[1]) == 0) and (float(
parts[2]) == -999):
if os.path.exists(dem):
# Handle the case where the statistics are bad for some reason
errorCode = 0
errorText = 'Success but statistics are bad'
else: # A real failure, figure out the cause
batchFolder = os.path.dirname(dem)
(errorCode, errorText) = getFailureCause(batchFolder)
#print str((errorCode, errorText))
#if errorCode < 0: # Debug code for unknown errors
#print str((errorCode, errorText))
#print statsText
#print batchFolder
#raise Exception('DEBUG')
failureLog.write('%d, %d, %d, %s\n' %
(frames[0], frames[1], errorCode, errorText))
# Make a link to the DEM thumbnail file in our summary folder
hillshadePath = os.path.join(demFolder,
'out-blend-DEM_HILLSHADE_browse.tif')
if os.path.exists(hillshadePath):
thumbName = ('dem_%05d_%05d_browse.tif' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(hillshadePath,
thumbPath,
verbose=False)
else:
# If the DEM thumbnail does not exist, look for the input frame thumbnail.
inPath = os.path.join(demFolder, 'first_image_browse.tif')
thumbName = ('input_%05d_browse.tif' % (frames[0]))
thumbPath = os.path.join(badImageFolder, thumbName)
if os.path.exists(inPath):
icebridge_common.makeSymLink(inPath, thumbPath, verbose=False)
# Make a link to the ortho thumbnail file in our summary folder
orthoPath = os.path.join(demFolder,
icebridge_common.orthoPreviewFileName())
if os.path.exists(orthoPath):
thumbName = ('ortho_%05d_%05d_browse.jpg' % (frames[0], frames[1]))
thumbPath = os.path.join(options.outputFolder, thumbName)
icebridge_common.makeSymLink(orthoPath, thumbPath, verbose=False)
# End loop through expected DEMs and writing log files
batchInfoLog.close()
failureLog.close()
print 'Finished generating flight summary in folder: ' + options.outputFolder
print("Wrote: " + batchInfoPath)
print("Wrote: " + failedBatchPath)