def getMeanDemDiff(dems, outputPrefix):
'''Get the mean distance between the input DEMs to the main DEM.'''
mainDem = dems[0]
meanDiff = 0.0
meanCount = 0.0
for i in range(1,len(dems)):
thisDem = dems[i]
if not thisDem:
continue
diffPrefix = outputPrefix + '_' + str(i)
diffPath = diffPrefix + '-diff.tif'
cmd = ('geodiff --absolute %s %s -o %s' % (mainDem, thisDem, diffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, True, False)
try:
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(diffPath)
print("Found inter-DEM diff " + str(i) + " = " + str(results['Mean']))
meanDiff = meanDiff + results['Mean']
meanCount = meanCount + 1.0
except:
print('No overlap with DEM ' + thisDem)
if meanCount < 1: # Handle degenerate cases
return 0
meanDiff = meanDiff / meanCount
print('Mean of DEM diffs = ' + str(meanDiff))
return meanDiff
def getMeanDemDiff(dems, outputPrefix):
'''Get the mean distance between the input DEMs to the main DEM.'''
mainDem = dems[0]
meanDiff = 0.0
meanCount = 0.0
for i in range(1, len(dems)):
thisDem = dems[i]
if not thisDem:
continue
diffPrefix = outputPrefix + '_' + str(i)
diffPath = diffPrefix + '-diff.tif'
cmd = ('geodiff --absolute %s %s -o %s' %
(mainDem, thisDem, diffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, True, False)
try:
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(diffPath)
print("Found inter-DEM diff " + str(i) + " = " +
str(results['Mean']))
meanDiff = meanDiff + results['Mean']
meanCount = meanCount + 1.0
except:
print('No overlap with DEM ' + thisDem)
if meanCount < 1: # Handle degenerate cases
return 0
meanDiff = meanDiff / meanCount
print('Mean of DEM diffs = ' + str(meanDiff))
return meanDiff
def processPair(imageA, imageB, cameraA, cameraB, lidarFolder,
outputFolder, options):
'''Processes a single image pair'''
suppressOutput = False
redo = False
# Just set the options and call the pair python tool.
# We can try out bundle adjustment for intrinsic parameters here.
cmd = ('python process_icebridge_pair.py --lidar-overlay --align-max-displacement 100 %s %s %s %s %s %s %s'
% (imageA, imageB, cameraA, cameraB, lidarFolder, outputFolder, options))
asp_system_utils.executeCommand(cmd, None, suppressOutput, redo)
def processPair(imageA, imageB, cameraA, cameraB, lidarFolder, outputFolder,
options):
'''Processes a single image pair'''
suppressOutput = False
redo = False
# Just set the options and call the pair python tool.
# We can try out bundle adjustment for intrinsic parameters here.
cmd = (
'python process_icebridge_pair.py --lidar-overlay --align-max-displacement 100 %s %s %s %s %s %s %s'
%
(imageA, imageB, cameraA, cameraB, lidarFolder, outputFolder, options))
asp_system_utils.executeCommand(cmd, None, suppressOutput, redo)
def fetchCameraFolder(run, logger):
'''Fetch a camera folder from the archive if it exists. Returns
True if we got the file. If more than one, return the latest by
modification time.'''
logger.info('Fetching camera folder for ' + str(run))
# Tar up the camera files and send them at the same time using the shiftc command
cameraFolder = run.getCameraFolder()
fileName = run.getCameraTarName()
# There could be multiple camera folders, fetch the latest by modification time.
strippedName = fileName
m = re.match('^(.*?)\.tar', strippedName)
if m:
strippedName = m.group(1)
m = re.match('^(.*?)' + run.suffix, strippedName)
if m:
strippedName = m.group(1)
# Look for the tarred camera files in each remote folder
# - TODO: This may not get the latest if cameras are in each folder!
found_rcf = ''
for rcf in [REMOTE_CAMERA_FOLDER_OLEG, REMOTE_CAMERA_FOLDER_SCOTT]:
cmd = 'ssh lfe "ls -dt ' + stripHost(
rcf) + '/' + strippedName + '*.tar"'
logger.info(cmd)
(out, err,
status) = asp_system_utils.executeCommand(cmd,
outputPath=None,
suppressOutput=True,
redo=True,
noThrow=True)
out = out.strip()
vals = out.split()
if len(vals) >= 1:
# Pick the first one, which is the newest
out = vals[0]
if out != '':
found_rcf = rcf
break
if found_rcf == '':
logger.info('Did not find camera file for run.')
return False
fileName = os.path.basename(out)
lfePath = os.path.join(found_rcf, fileName)
cmd = 'shiftc --wait -d -r --extract-tar ' + lfePath + ' .'
# Don't try too hard below, if failed first time that means there are no cameras
# to fetch
logger.info(cmd)
status = os.system(cmd)
if status != 0:
logger.info('Did not find camera file for run.')
return False
else:
logger.info('Finished retrieving cameras from lfe.')
return True
def packAndSendSummaryFolder(run, folder, logger):
'''Archive the summary folder in case we want to look at it later'''
logger.info('Archiving summary folder for run ' + str(run))
# Create a local tar file
# - Some fiddling to make the packed folders convenient
fileName = run.getSummaryTarName()
cmd = 'tar -chf '+ fileName +' -C '+ folder +'/.. ' + os.path.basename(folder)
logger.info(cmd)
(out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None,
suppressOutput = True, redo = True,
noThrow = True)
# This tends to print a very verbose message
ans = out + '\n' + err
vals = ans.split('\n')
if len(vals) < 10:
logger.info(ans)
else:
vals = vals[0:10]
logger.info("\n".join(vals))
logger.info("Above output truncated.")
# Delete any existing copy of the file on lfe
lfePath = os.path.join(REMOTE_SUMMARY_FOLDER, fileName)
cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
# Send the file to lfe using shiftc
cmd = 'shiftc --wait -d -r ' + fileName + ' ' + lfePath
logger.info(cmd)
status = os.system(cmd)
if status != 0:
raise Exception('Failed to pack/send summary folder for run ' + str(run))
logger.info('Finished sending summary to lfe.')
if icebridge_common.getUser() != 'oalexan1':
# Wipe the copy on lunokhod
l2Path = os.path.join(L_SUMMARY_FOLDER, fileName)
cmd = "ssh " + LUNOKHOD + " 'rm -f "+ stripHost(l2Path) +"' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
# Make target directory on lunokhod
cmd = "ssh " + LUNOKHOD + " 'mkdir -p " + os.path.dirname(stripHost(l2Path)) + \
"' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
# Send a copy of the file to Lunokhod for convenience
cmd = 'scp ' + fileName + ' ' + l2Path + ' 2>/dev/null'
logger.info(cmd)
os.system(cmd)
# Clean up the local tar file
os.system('rm -f ' + fileName)
def fetchCameraFolder(run, logger):
'''Fetch a camera folder from the archive if it exists. Returns
True if we got the file. If more than one, return the latest by
modification time.'''
logger.info('Fetching camera folder for ' + str(run))
# Tar up the camera files and send them at the same time using the shiftc command
cameraFolder = run.getCameraFolder()
fileName = run.getCameraTarName()
# There could be multiple camera folders, fetch the latest by modification time.
strippedName = fileName
m = re.match('^(.*?)\.tar', strippedName)
if m:
strippedName = m.group(1)
m = re.match('^(.*?)' + run.suffix, strippedName)
if m:
strippedName = m.group(1)
# Look for the tarred camera files in each remote folder
# - TODO: This may not get the latest if cameras are in each folder!
found_rcf = ''
for rcf in [REMOTE_CAMERA_FOLDER_OLEG, REMOTE_CAMERA_FOLDER_SCOTT]:
cmd = 'ssh lfe "ls -dt ' + stripHost(rcf) + '/' + strippedName + '*.tar"'
logger.info(cmd)
(out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None,
suppressOutput = True, redo = True,
noThrow = True)
out = out.strip()
vals = out.split()
if len(vals) >= 1:
# Pick the first one, which is the newest
out = vals[0]
if out != '':
found_rcf = rcf
break
if found_rcf == '':
logger.info('Did not find camera file for run.')
return False
fileName = os.path.basename(out)
lfePath = os.path.join(found_rcf, fileName)
cmd = 'shiftc --wait -d -r --extract-tar ' + lfePath + ' .'
# Don't try too hard below, if failed first time that means there are no cameras
# to fetch
logger.info(cmd)
status = os.system(cmd)
if status != 0:
logger.info('Did not find camera file for run.')
return False
else:
logger.info('Finished retrieving cameras from lfe.')
return True
def convertToTsai(path, demPath):
'''Given the json metadata, generate a .tsai camera file'''
# Get the other file paths
tifPath = path.replace('metadata.json', 'basic_panchromatic_dn.tif')
outputPath = tifPath.replace('.tif', '.tsai')
# Get the image size (for the center coordinate)
width, height = asp_image_utils.getImageSize(tifPath)
# Set up the required coordinate string
with open(path, 'r') as f:
contents = f.read()
parts = json.loads(contents)
coords = parts['geometry']['coordinates'][0]
coordString = (('%f %f %f %f %f %f %f %f') % (coords[0][0], coords[0][1],
coords[3][0], coords[3][1],
coords[2][0], coords[2][1],
coords[1][0], coords[1][1]))
cmd = (('cam_gen %s --reference-dem %s --focal-length 553846.153846 --optical-center %f %f --pixel-pitch 1 --refine-camera --lon-lat-values "%s" -o %s') % (tifPath, demPath, width/2.0, height/2.0, coordString, outputPath))
asp_system_utils.executeCommand(cmd, outputPath=outputPath)
def convertToTsai(path, demPath):
'''Given the json metadata, generate a .tsai camera file'''
# Get the other file paths
tifPath = path.replace('metadata.json', 'basic_panchromatic_dn.tif')
outputPath = tifPath.replace('.tif', '.tsai')
# Get the image size (for the center coordinate)
width, height = asp_image_utils.getImageSize(tifPath)
# Set up the required coordinate string
with open(path, 'r') as f:
contents = f.read()
parts = json.loads(contents)
coords = parts['geometry']['coordinates'][0]
coordString = (('%f %f %f %f %f %f %f %f') %
(coords[0][0], coords[0][1], coords[3][0], coords[3][1],
coords[2][0], coords[2][1], coords[1][0], coords[1][1]))
cmd = ((
'cam_gen %s --reference-dem %s --focal-length 553846.153846 --optical-center %f %f --pixel-pitch 1 --refine-camera --lon-lat-values "%s" -o %s'
) % (tifPath, demPath, width / 2.0, height / 2.0, coordString, outputPath))
asp_system_utils.executeCommand(cmd, outputPath=outputPath)
def getActiveJobs(user):
'''Returns a list of the currently active jobs and their status'''
# Run qstat command to get a list of all active jobs by this user
cmd = ['qstat', '-u', user]
(textOutput, err, status) = \
asp_system_utils.executeCommand(cmd,
outputPath=None,
suppressOutput = True,
redo=True,
noThrow=True,
numAttempts = 1000,
sleepTime = 60
)
lines = textOutput.split('\n')
# Strip header lines
NUM_HEADER_LINES = 3
JOBID_INDEX = 0
NAME_INDEX = 3
STATUS_INDEX = 7
if len(lines) <= NUM_HEADER_LINES:
return []
lines = lines[NUM_HEADER_LINES:]
# Pick out all the job names and put them in a list
jobs = []
for line in lines:
parts = line.split()
if len(parts) < STATUS_INDEX:
continue
jobID = cleanJobID(parts[JOBID_INDEX])
jobName = parts[NAME_INDEX]
status = parts[STATUS_INDEX]
jobs.append((jobID, jobName, status))
return jobs
def getActiveJobs(user):
'''Returns a list of the currently active jobs and their status'''
# Run qstat command to get a list of all active jobs by this user
cmd = ['qstat', '-u', user]
(textOutput, err, status) = \
asp_system_utils.executeCommand(cmd,
outputPath=None,
suppressOutput = True,
redo=True,
noThrow=True,
numAttempts = 1000,
sleepTime = 60
)
lines = textOutput.split('\n')
# Strip header lines
NUM_HEADER_LINES = 3
JOBID_INDEX = 0
NAME_INDEX = 3
STATUS_INDEX = 7
if len(lines) <= NUM_HEADER_LINES:
return []
lines = lines[NUM_HEADER_LINES:]
# Pick out all the job names and put them in a list
jobs = []
for line in lines:
parts = line.split()
if len(parts) < STATUS_INDEX:
continue
jobID = cleanJobID(parts[JOBID_INDEX])
jobName = parts[NAME_INDEX]
status = parts[STATUS_INDEX]
jobs.append((jobID, jobName, status))
return jobs
def pushByType(run, options, logger, dataType):
# Fetch the ortho index from NSIDC if missing
outputFolder = run.getFolder()
logger.info("Output folder is " + outputFolder)
os.system("mkdir -p " + outputFolder)
# Current directory. It is important to go from /u to the real dir which is /nobackup...
unpackDir = os.path.realpath(os.getcwd())
logger.info("Unpack directory is " + unpackDir)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
fetchIndices(options, logger)
logger.info("Reading ortho index: " + orthoIndexPath)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
# Fetch unarchived folder if missing
if dataType == 'DEM':
unarchivedFolder = run.getAssemblyFolder()
elif dataType == 'ORTHO':
unarchivedFolder = run.getProcessFolder()
else:
raise Exception("Unknown data type: " + dataType)
logger.info("Unarchived data folder is " + unarchivedFolder)
# Especially for ortho, force-fetch each time, as there is no good way
# of checking if we fetched well before.
start_time()
if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType):
return
stop_time("fetching archived data by type: " + dataType, logger)
# Make the output directory at NSIDC
m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd)
if m:
outDir = options.site + "_" + m.group(1) + "." + m.group(2) + "." + m.group(3)
else:
raise Exception("Could not parse: " + options.yyyymmdd)
# Keep the output directory locally here
localDirPath = os.path.join(outputFolder, dataType, outDir)
os.system("mkdir -p " + localDirPath)
logger.info("Storing the renamed " + dataType + " files in " + localDirPath)
logger.info("Directory name at NSIDC: " + outDir)
# Read the DEMs and orthos, and copy them to outDir according to the final convention
if dataType == 'DEM':
dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True)
else:
dataFiles = glob.glob(os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif'))
for dataFile in dataFiles:
# Here we use the convention from archive_functions.py for DEMs and from how we store orthos.
if dataType == 'DEM':
m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \
"\.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
else:
m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \
"out-ortho.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
if frameNumber < options.startFrame or frameNumber > options.stopFrame:
continue
# For each data file, copy from the ortho its meta info
if not frameNumber in orthoFrameDict.keys():
# Bugfix: Ortho fetching failed, try again
fetchIndices(options, logger)
logger.info("Re-reading ortho index: " + orthoIndexPath)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if not frameNumber in orthoFrameDict.keys():
# This time there is nothing we can do
raise Exception("Cannot find ortho for frame: " + str(frameNumber))
orthoFile = orthoFrameDict[frameNumber]
[dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile)
# It is always possible that the ortho file date will be the next day
# after the current flight date, if the flight goes after midnight.
# So it is not unreasonable that options.yyyymmdd != dateString.
if dataType == 'DEM':
outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber))
else:
# TODO: Need to think more of the naming convention.
outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber))
cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(localDirPath, outFile)
logger.info(cmd)
os.system(cmd)
# Push the directory to NSIDC
remoteDirPath = os.path.join(os.path.basename(os.path.dirname(localDirPath)),
os.path.basename(localDirPath))
remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath)
logger.info("Storing at NSIDC in: " + remoteDirPath)
cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \
' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo
logger.info(cmd)
start_time()
(output, err, status) = asp_system_utils.executeCommand(cmd,
suppressOutput = True)
#status = os.system(cmd)
logger.info("LFTP output and error: " + output + ' ' + err)
logger.info("LFTP status: " + str(status))
#if status != 0:
# raise Exception("Problem pushing")
stop_time("push to NSIDC", logger)
def fetchProcessedByType(run, unpackFolder, logger, dataType):
'''Fetch from lfe the latest archive of processed DEMs or orthos by modification time.
For now we ignore the timestamp.'''
logger.info('Fetching processed ' + dataType + ' data for ' + str(run))
# Fetch the latest file by modification time as returned by 'ls'.
# A given file can be either in Scott's or Oleg's archive.
# It can be in both, and it can be multiple instances.
# Need to list all and fetch the latest.
if dataType == 'DEM':
fileName = run.getOutputTarName(useWildCard = True)
lfePaths = os.path.join(stripHost(get_remote_output_folder('oalexan1')), fileName) + " " + \
os.path.join(stripHost(get_remote_output_folder('smcmich1')), fileName)
elif dataType == 'ORTHO':
fileName = run.getOrthoTarName(useWildCard = True)
lfePaths = os.path.join(stripHost(get_remote_ortho_folder('oalexan1')), fileName) + " " + \
os.path.join(stripHost(get_remote_ortho_folder('smcmich1')), fileName)
else:
raise Exception("Unknown data type: " + dataType)
# Get the list
cmd = 'ssh lfe "ls -altd ' + lfePaths + '"'
logger.info(cmd)
(out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None,
suppressOutput = True, redo = True,
noThrow = True)
# Parse the answer and keep the latest.
out = out.strip()
vals = out.split('\n')
for val in vals:
logger.info("Found: " + val)
lfePath = ""
for val in vals:
if len(val) >= 1 and val[0] == 'l':
continue # this is a symlink
arr = val.split()
lfePath = arr[-1]
break # found what we needed
if lfePath == "":
logger.info("Could not locate: " + fileName)
return False
cmd = 'shiftc --wait -d -r --verify --extract-tar ' + 'lfe:' + lfePath + ' ' + unpackFolder
logger.info(cmd)
status = os.system(cmd)
# Try tar if shift failed
if status != 0:
logger.info('Did not sucessfully fetch archived processed data of type: ' + dataType)
logger.info('Will try with tar.''')
tar_cmd = 'ssh lfe "cd ' + os.path.dirname(lfePath) + "; tar xfv " + \
os.path.basename(lfePath) + " -C " + os.path.realpath(unpackFolder) + '"'
logger.info(tar_cmd)
status = os.system(tar_cmd)
if status != 0:
logger.info('Failed using tar as well.')
return False
return True
# Generate a kml file for the nav camera files
kmlPath = os.path.join(outputFolder, 'nav_cameras.kml')
# This is a hack. If we are invoked from a Pleiades node, do not
# create this kml file, as nodes will just overwrite each other.
# This job may happen anyway earlier or later when on the head node.
if not 'PBS_NODEFILE' in os.environ:
try:
tempPath = os.path.join(outputFolder, 'list.txt')
logger.info('Generating nav camera kml file: ' + kmlPath)
os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath)
orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole')
cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath
logger.info(cmd)
asp_system_utils.executeCommand(cmd, kmlPath, suppressOutput=True, redo=False)
os.remove(tempPath)
except Exception, e:
logger.info("Warning: " + str(e))
logger.info('Finished generating camera models from nav!')
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
def submitJob(jobName, queueName, maxHours, logger, minutesInDevelQueue,
groupId, nodeType, commandPath, args, logPrefix, priority=None,
pythonPath='/u/oalexan1/.local/lib/python2.7/site-packages'):
'''Submits a job to the PBS system.
Any such job must invoke icebridge_common.switchWorkDir().'''
if len(queueName) > MAX_PBS_NAME_LENGTH:
raise Exception('Job name "'+queueName+'" exceeds the maximum length of ' + str(MAX_PBS_NAME_LENGTH))
numCpus = getNumCores(nodeType) # Cores or CPUs?
hourString = '"'+str(maxHours)+':00:00"'
# We must create and execute a shell script, to be able to explicitely
# direct the output and error to files, without counting
# on PBS to manage that, as that one overfloes.
# All our outputs and errors will go to the "verbose" files,
# while PBS will write its summaries, etc, to the other files.
shellScriptPath = logPrefix + '_script.sh'
verboseOutputPath = logPrefix + '_verbose_output.log'
verboseErrorsPath = logPrefix + '_verbose_errors.log'
outputPath = logPrefix + '_output.log'
errorsPath = logPrefix + '_errors.log'
workDir = os.getcwd()
numAttempts = 1000
# For debugging
if minutesInDevelQueue > 0:
queueName = 'devel'
hourString = '00:' + str(minutesInDevelQueue).zfill(2) + ':00'
#numAttempts = 1 # For some reason qsub errors out in the devel queue
# The "-m eb" option sends the user an email when the process begins and when it ends.
# The -r n ensures the job does not restart if it runs out of memory.
# Debug the environment
#for v in os.environ.keys():
# logger.info("env is " + v + '=' + str(os.environ[v]))
# We empty PYTHONSTARTUP and LD_LIBRARY_PATH so that python can function
# properly on the nodes.
priorityString = ''
if priority:
priorityString = ' -p ' + str(priority) + ' '
# Generate the shell command
shellCommand = ( "%s %s > %s 2> %s\n" % (commandPath, args,
verboseOutputPath, verboseErrorsPath) )
with open(shellScriptPath, 'w') as f:
f.write("#!/bin/bash\n")
f.write(shellCommand)
# Make it executable
os.system("chmod a+rx " + shellScriptPath)
# Run it
pbsCommand = ('qsub -r y -q %s -N %s %s -l walltime=%s -W group_list=%s -j oe -e %s -o %s -S /bin/bash -V -C %s -l select=1:ncpus=%d:model=%s -- /usr/bin/env OIB_WORK_DIR=%s PYTHONPATH=%s PYTHONSTARTUP="" LD_LIBRARY_PATH="" %s' %
(queueName, jobName, priorityString, hourString, groupId, errorsPath, outputPath, workDir, numCpus, nodeType, workDir, pythonPath, shellScriptPath))
#~/repo/python_env/lib/python2.7/site-packages/
#/u/oalexan1/.local/lib/python2.7/site-packages
logger.info(pbsCommand)
outputPath = None
suppressOutput = True
redo = True
noThrow = True
(out, err, status) = \
asp_system_utils.executeCommand(pbsCommand, outputPath,
suppressOutput, redo,
noThrow, numAttempts, SLEEP_TIME)
if status != 0:
logger.info(out)
logger.info(err)
logger.info("Status is: " + str(status))
jobID = ''
else:
jobID = cleanJobID(out)
logger.info("Submitted job named " + str(jobName) + " with id " + str(jobID))
return jobID
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 3.
usage = '''gen_ortho.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument('--camera-mounting', dest='cameraMounting', default=0, type=int,
help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.')
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
# Performance options
parser.add_argument('--num-processes', dest='numProcesses', default=1,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=8,
type=int, help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput = True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder, options.processingSubfolder)
logger.info('Reading from processing subfolder: ' + options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".")
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder = True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(jpegFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No jpeg files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in jpegFrameDict:
logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".")
continue
# Find the right image
currImage = jpegFrameDict[frame]
args = (frame, processFolder, currImage, options.bundleLength,
options.cameraMounting, threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runOrtho, args))
else:
runOrtho(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False,
quitKey='q', sleepTime=20)
icebridge_common.stopTaskPool(pool)
def runBlend(frame, processFolder, lidarFile, fireballDEM, options, threadText,
redo, suppressOutput):
WEIGHT_EXP = 1.3
# This will run as multiple processes. Hence have to catch all exceptions.
try:
demFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.alignFileName(), processFolder,
options.bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder,
'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
# This is turned off for now. Find the diff between neighboring
# aligned DEMs before blending.
prevDemFile, prevBatchFolder = \
icebridge_common.frameToFile(frame-1,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
prevDiffPrefix = os.path.join(batchFolder, 'out-prev')
prevDiffFile = prevDiffPrefix + '-diff.tif'
if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile):
os.system("rm -f " + prevDiffFile)
if os.path.exists(prevDemFile) and os.path.exists(demFile):
if os.path.exists(prevDiffFile):
print("File exists: " + prevDiffFile)
else:
cmd = ('geodiff --absolute %s %s -o %s' %
(prevDemFile, demFile, prevDiffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, prevDiffFile,
suppressOutput, redo)
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput)
and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " +
fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index + 1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
# Compute the mean distance between the DEMs
# TODO: Make sure this gets cleaned up!
meanWorkPrefix = os.path.join(batchFolder, 'bd')
meanDiff = getMeanDemDiff(dems, meanWorkPrefix)
# If the mean error between DEMs is creater than this,
# use a less aggressive blending method.
MEAN_DIFF_BLEND_THRESHOLD = 1.0
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder,
'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = (
'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, footprintDEM, demString, threadText,
outputPrefix))
else:
cmd = (
'dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s'
% (WEIGHT_EXP, demString, threadText, outputPrefix))
if meanDiff > MEAN_DIFF_BLEND_THRESHOLD:
cmd += ' --propagate-nodata --use-centerline-weights '
print(cmd)
# Execute the blend command.
# - Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
print(cmd)
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput,
localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
# Compute post-blending error to lidar
cmd = (
'geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput,
redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
except Exception, e:
pass
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) +
" when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix + '_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.system("rm -f " +
hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = (
'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, fireballDEM, demString, threadText,
fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput,
suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile,
fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath,
suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " +
str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput,
suppressOutput, redo)
except Exception, e:
pass
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 10
usage = '''blend_dems.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
parser.add_argument("--compute-diff-to-prev-dem", action="store_true",
dest="computeDiffToPrev", default=False,
help="Compute the absolute difference between the current DEM " +
"and the one before it.")
parser.add_argument("--blend-to-fireball-footprint", action="store_true",
dest="blendToFireball", default=False,
help="Create additional blended DEMs having the same " + \
"footprint as Fireball DEMs.")
# Performance options
parser.add_argument('--num-processes', dest='numProcesses', default=1,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=8,
type=int, help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_blend_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput = True)
logger.info("Running on machine: " + out)
logger.info(str(argsIn))
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder, options.processingSubfolder)
logger.info('Reading from processing subfolder: ' + options.processingSubfolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if options.blendToFireball:
fireballFrameDict = icebridge_common.getCorrectedFireballDems(options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(orthoFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No ortho files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in orthoFrameDict:
logger.info("Error: Missing ortho file for frame: " + str(frame) + ".")
continue
orthoFile = orthoFrameDict[frame]
try:
lidarFile = icebridge_common.findMatchingLidarFile(orthoFile, lidarFolder)
except: # Skip if no lidar file matches this frame
continue
fireballDEM = ""
if options.blendToFireball:
if frame in fireballFrameDict:
fireballDEM = fireballFrameDict[frame]
else:
logger.info("No fireball DEM for frame: " + str(frame))
args = (frame, processFolder, lidarFile, fireballDEM, options, threadText,
redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runBlend, args))
else:
runBlend(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False,
quitKey='q', sleepTime=20)
icebridge_common.stopTaskPool(pool)
def pushByType(run, options, logger, dataType):
# Fetch the ortho index from NSIDC if missing
outputFolder = run.getFolder()
logger.info("Output folder is " + outputFolder)
os.system("mkdir -p " + outputFolder)
# Current directory. It is important to go from /u to the real dir which is /nobackup...
unpackDir = os.path.realpath(os.getcwd())
logger.info("Unpack directory is " + unpackDir)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
fetchIndices(options, logger)
logger.info("Reading ortho index: " + orthoIndexPath)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
# Fetch unarchived folder if missing
if dataType == 'DEM':
unarchivedFolder = run.getAssemblyFolder()
elif dataType == 'ORTHO':
unarchivedFolder = run.getProcessFolder()
else:
raise Exception("Unknown data type: " + dataType)
logger.info("Unarchived data folder is " + unarchivedFolder)
# Especially for ortho, force-fetch each time, as there is no good way
# of checking if we fetched well before.
start_time()
if not archive_functions.fetchProcessedByType(run, unpackDir, logger,
dataType):
return
stop_time("fetching archived data by type: " + dataType, logger)
# Make the output directory at NSIDC
m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd)
if m:
outDir = options.site + "_" + m.group(1) + "." + m.group(
2) + "." + m.group(3)
else:
raise Exception("Could not parse: " + options.yyyymmdd)
# Keep the output directory locally here
localDirPath = os.path.join(outputFolder, dataType, outDir)
os.system("mkdir -p " + localDirPath)
logger.info("Storing the renamed " + dataType + " files in " +
localDirPath)
logger.info("Directory name at NSIDC: " + outDir)
# Read the DEMs and orthos, and copy them to outDir according to the final convention
if dataType == 'DEM':
dataFiles = icebridge_common.getTifs(unarchivedFolder,
prependFolder=True)
else:
dataFiles = glob.glob(
os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif'))
for dataFile in dataFiles:
# Here we use the convention from archive_functions.py for DEMs and from how we store orthos.
if dataType == 'DEM':
m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \
"\.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
else:
m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \
"out-ortho.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
if frameNumber < options.startFrame or frameNumber > options.stopFrame:
continue
# For each data file, copy from the ortho its meta info
if not frameNumber in orthoFrameDict.keys():
# Bugfix: Ortho fetching failed, try again
fetchIndices(options, logger)
logger.info("Re-reading ortho index: " + orthoIndexPath)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if not frameNumber in orthoFrameDict.keys():
# This time there is nothing we can do
raise Exception("Cannot find ortho for frame: " +
str(frameNumber))
orthoFile = orthoFrameDict[frameNumber]
[dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile)
# It is always possible that the ortho file date will be the next day
# after the current flight date, if the flight goes after midnight.
# So it is not unreasonable that options.yyyymmdd != dateString.
if dataType == 'DEM':
outFile = ('IODEM3_%s_%s_%05d_DEM.tif' %
(dateString, timeString, frameNumber))
else:
# TODO: Need to think more of the naming convention.
outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' %
(dateString, timeString, frameNumber))
cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(
localDirPath, outFile)
logger.info(cmd)
os.system(cmd)
# Push the directory to NSIDC
remoteDirPath = os.path.join(
os.path.basename(os.path.dirname(localDirPath)),
os.path.basename(localDirPath))
remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath)
logger.info("Storing at NSIDC in: " + remoteDirPath)
cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \
' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo
logger.info(cmd)
start_time()
(output, err,
status) = asp_system_utils.executeCommand(cmd, suppressOutput=True)
#status = os.system(cmd)
logger.info("LFTP output and error: " + output + ' ' + err)
logger.info("LFTP status: " + str(status))
#if status != 0:
# raise Exception("Problem pushing")
stop_time("push to NSIDC", logger)
def main(argsIn):
try:
usage = '''usage: regenerate_summary_images.py <options> '''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument("--data-folder", dest="dataFolder",
help="Where all the inputs and outputs are stored.")
parser.add_argument("--work-folder", dest="workFolder",
help="Where working files are stored.")
parser.add_argument("--site", dest="site", help="Site code.")
parser.add_argument("--yyyymmdd", dest="yyyymmdd", help="Date.")
#parser.add_argument("--dem-tarball", dest="demTarball", default=os.getcwd(),
# help="Where all the inputs and outputs are stored.")
#parser.add_argument("--ortho-tarball", dest="orthoTarball", default=None,
# help="Where to unpack the data.")
#parser.add_argument("--summary-tarball", dest="summaryTarball", default=None,
# help="Where to unpack the data.")
#parser.add_argument("--unpack-dir", dest="unpackDir", default=None,
# help="Where to unpack the data.")
parser.add_argument("--node-type", dest="nodeType", default='san',
help="Node type to use (wes[mfe], san, ivy, has, bro)")
#parser.add_argument("--skip-archive-summary", action="store_true",
# dest="skipArchiveSummary", default=False,
# help="Skip archiving the summary.")
# Debug option
parser.add_argument('--minutes-in-devel-queue', dest='minutesInDevelQueue', type=int,
default=0,
help="If positive, submit to the devel queue for this many minutes.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
# Check if we are on the right machine
(host, err, status) = asp_system_utils.executeCommand(['uname', '-n'],
suppressOutput = True)
host = host.strip()
if 'pfe' in host and options.nodeType not in PFE_NODES:
raise Exception("From machine " + host + " can only launch on: " + " ".join(PFE_NODES))
if 'mfe' in host and options.nodeType != 'wes':
raise Exception("From machine " + host + " can only launch on: wes")
# Make sure our paths will work when called from PBS
options.dataFolder= os.path.abspath(options.dataFolder)
#os.system('mkdir -p ' + options.unpackDir)
# TODO: Check folders!
run = run_helper.RunHelper(options.site, options.yyyymmdd, options.workFolder)
runFolder = os.path.join(options.workFolder, str(run))
os.system('mkdir -p ' + runFolder)
logFolder = os.path.join(runFolder, 'logs')
# Set up logging in the run directory
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(logFolder, logLevel,
icebridge_common.manager_log_prefix())
logger.info("Logging in: " + logFolder)
checkRequiredTools() # Make sure all the needed tools can be found before we start
logger.info("Disabling core dumps.") # these just take a lot of room
os.system("ulimit -c 0")
os.system("umask 022") # enforce files be readable by others
# See how many hours we used so far. I think this counter gets updated once a day.
(out, err, status) = asp_system_utils.executeCommand("acct_ytd", outputPath = None,
suppressOutput = True, redo = True,
noThrow = True)
logger.info("Hours used so far:\n" + out + '\n' + err)
try:
# Fetch and extract the tarball files from Lou
localDemFolder = os.path.join(options.dataFolder, run.name()+'_dems')
localOrthoFolder = os.path.join(options.dataFolder, run.name()+'_orthos')
demSummaryFolder = os.path.join(options.dataFolder, run.name()+'_dem_summaries')
orthoSummaryFolder = os.path.join(options.dataFolder, run.name()+'_ortho_summaries')
missingDemFiles = []
missingOrthoFiles = []
for f in os.listdir(localDemFolder):
if 'temp' in f:
raise Exception('Bad file: ' + f)
if ('IODEM3' in f) and (f[-4:] == '.tif'):
inputPath = os.path.join(localDemFolder, f)
outputPath = os.path.join(demSummaryFolder, f.replace('DEM.tif', 'DEM_browse.tif'))
if not os.path.exists(outputPath):
missingDemFiles.append((inputPath, outputPath))
for f in os.listdir(localOrthoFolder):
if 'temp' in f:
raise Exception('Bad file: ' + f)
if ('IODIM3' in f) and (f[-4:] == '.tif'):
inputPath = os.path.join(localOrthoFolder, f)
outputPath = os.path.join(orthoSummaryFolder, f.replace('ORTHO.tif', 'ORTHO.jpg'))
if not os.path.exists(outputPath):
missingOrthoFiles.append((inputPath, outputPath))
#print 'Fetching and unpacking tarballs...'
#fetchTarball(options.demTarball, localDemFolder)
#fetchTarball(options.orthoTarball, localOrthoFolder)
#fetchTarball(options.summaryTarball, localSummaryFolder)
# If the summary tarball unpacked to []/summary/summary,
# work with the lower level folder from now on.
#localSummaryFolder = descendIfNeeded(localSummaryFolder)
# Make a list of all input files that are missing their summary file, and
# the desired output path for that file.
#missingDemFiles = getMissingSummaryFiles(localDemFolder, localSummaryFolder, isOrtho=False)
#missingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, localSummaryFolder, isOrtho=True )
# Divide this list into chunks and for each chunk generate a file containing all of
# the gdal_translate commands that need to be executed.
print 'Writing command files...'
commandFileLength = getParallelParams(options.nodeType)[2]
commandFilePrefix = os.path.join(runFolder, 'convert_commands_')
print 'Clearing existing command files.'
os.system('rm ' + commandFilePrefix + '*')
commandFileList = writeCommandFiles(missingDemFiles, missingOrthoFiles,
commandFilePrefix, commandFileLength)
#raise Exception('DEBUG')
# Get the location to store the logs
pbsLogFolder = run.getPbsLogFolder()
logger.info("Storing logs in: " + pbsLogFolder)
os.system('mkdir -p ' + pbsLogFolder)
# Call multi_process_command_runner.py through PBS for each chunk.
start_time()
(baseName, jobIDs) = submitBatchJobs(commandFileList, options, pbsLogFolder, run, logger)
# Wait for everything to finish.
pbs_functions.waitForJobCompletion(jobIDs, logger, baseName)
stop_time("pbs_jobs", logger)
# Check that we now have all of the summary files.
# - Both of these should now be empty.
#newMissingDemFiles = getMissingSummaryFiles(localDemFolder, demSummaryFolder, isOrtho=False)
#newMissingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, orthoSummaryFolder, isOrtho=True )
numDemsMissing = 0
numOrthosMissing = 0
for pair in missingDemFiles:
if not os.path.exists(pair[1]):
numDemsMissing += 1
for pair in missingOrthoFiles:
if not os.path.exists(pair[1]):
numOrthosMissing += 1
resultText = ('After regeneration, missing %d DEM summaries and %d ORTHO summaries'
% (numDemsMissing, numOrthosMissing))
logger.info(resultText)
runWasSuccess = ((numDemsMissing == 0) and (numOrthosMissing == 0))
# If successful, create a new tarball and send it to Lou.
#if runWasSuccess and (not options.skipArchiveSummary):
# start_time()
# archive_functions.packAndSendSummaryFolder(run, localSummaryFolder, logger)
# stop_time("archive summary", logger)
except Exception as e:
resultText = 'Caught exception: ' + str(e) + '\n' + traceback.format_exc()
runWasSuccess = False
# Send a summary email.
emailAddress = getEmailAddress(icebridge_common.getUser())
logger.info("Sending email to: " + emailAddress)
if runWasSuccess:
sendEmail(emailAddress, 'OIB summary regen passed', resultText)
else:
sendEmail(emailAddress, '"OIB summary regen failed', resultText)
# TODO: Add automated delete command!
#if options.wipeProcessed:
# processedFolder = run.getProcessFolder()
# logger.info("Will delete: " + processedFolder)
# os.system("rm -rf " + processedFolder)
logger.info('==== regenerate_summary_images script has finished for run: ' + str(run) + ' ====')
def submitJob(jobName,
queueName,
maxHours,
logger,
minutesInDevelQueue,
groupId,
nodeType,
commandPath,
args,
logPrefix,
priority=None,
pythonPath='/u/oalexan1/.local/lib/python2.7/site-packages'):
'''Submits a job to the PBS system.
Any such job must invoke icebridge_common.switchWorkDir().'''
if len(queueName) > MAX_PBS_NAME_LENGTH:
raise Exception('Job name "' + queueName +
'" exceeds the maximum length of ' +
str(MAX_PBS_NAME_LENGTH))
numCpus = getNumCores(nodeType) # Cores or CPUs?
hourString = '"' + str(maxHours) + ':00:00"'
# We must create and execute a shell script, to be able to explicitely
# direct the output and error to files, without counting
# on PBS to manage that, as that one overfloes.
# All our outputs and errors will go to the "verbose" files,
# while PBS will write its summaries, etc, to the other files.
shellScriptPath = logPrefix + '_script.sh'
verboseOutputPath = logPrefix + '_verbose_output.log'
verboseErrorsPath = logPrefix + '_verbose_errors.log'
outputPath = logPrefix + '_output.log'
errorsPath = logPrefix + '_errors.log'
workDir = os.getcwd()
numAttempts = 1000
# For debugging
if minutesInDevelQueue > 0:
queueName = 'devel'
hourString = '00:' + str(minutesInDevelQueue).zfill(2) + ':00'
#numAttempts = 1 # For some reason qsub errors out in the devel queue
# The "-m eb" option sends the user an email when the process begins and when it ends.
# The -r n ensures the job does not restart if it runs out of memory.
# Debug the environment
#for v in os.environ.keys():
# logger.info("env is " + v + '=' + str(os.environ[v]))
# We empty PYTHONSTARTUP and LD_LIBRARY_PATH so that python can function
# properly on the nodes.
priorityString = ''
if priority:
priorityString = ' -p ' + str(priority) + ' '
# Generate the shell command
shellCommand = ("%s %s > %s 2> %s\n" %
(commandPath, args, verboseOutputPath, verboseErrorsPath))
with open(shellScriptPath, 'w') as f:
f.write("#!/bin/bash\n")
f.write(shellCommand)
# Make it executable
os.system("chmod a+rx " + shellScriptPath)
# Run it
pbsCommand = (
'qsub -r y -q %s -N %s %s -l walltime=%s -W group_list=%s -j oe -e %s -o %s -S /bin/bash -V -C %s -l select=1:ncpus=%d:model=%s -- /usr/bin/env OIB_WORK_DIR=%s PYTHONPATH=%s PYTHONSTARTUP="" LD_LIBRARY_PATH="" %s'
% (queueName, jobName, priorityString, hourString, groupId, errorsPath,
outputPath, workDir, numCpus, nodeType, workDir, pythonPath,
shellScriptPath))
#~/repo/python_env/lib/python2.7/site-packages/
#/u/oalexan1/.local/lib/python2.7/site-packages
logger.info(pbsCommand)
outputPath = None
suppressOutput = True
redo = True
noThrow = True
(out, err, status) = \
asp_system_utils.executeCommand(pbsCommand, outputPath,
suppressOutput, redo,
noThrow, numAttempts, SLEEP_TIME)
if status != 0:
logger.info(out)
logger.info(err)
logger.info("Status is: " + str(status))
jobID = ''
else:
jobID = cleanJobID(out)
logger.info("Submitted job named " + str(jobName) + " with id " +
str(jobID))
return jobID
def runBlend(frame, processFolder, lidarFile, fireballDEM, bundleLength,
threadText, redo, suppressOutput):
# This will run as multiple processes. Hence have to catch all exceptions:
try:
demFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.alignFileName(), processFolder,
bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder,
'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput)
and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " +
fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index + 1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder,
'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' %
(footprintDEM, demString, threadText, outputPrefix))
#filesToWipe.append(footprintDEM) # no longer needed
else:
cmd = ('dem_mosaic --first-dem-as-reference %s %s -o %s' %
(demString, threadText, outputPrefix))
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput,
localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
cmd = (
'geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput,
redo)
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) +
" when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix + '_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.remove(hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' %
(fireballDEM, demString, threadText, fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput,
suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile,
fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath,
suppressOutput, redo)
# Read in and examine the results
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " +
str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput,
suppressOutput, redo)
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.remove(hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Done with dealing with the fireball footprint
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
except Exception as e:
print('Blending failed!\n' + str(e) + ". " +
str(traceback.print_exc()))
sys.stdout.flush()
def main(argsIn):
try:
usage = '''usage: push_to_nsidc.py <options> '''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
default="",
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
default="",
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument(
"--site_yyyymmdd",
dest="site_yyyymmdd",
default="",
help=
"A value like GR_20150330, which will be split into site and yyyymmdd by this script."
)
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
parser.add_argument("--camera-calibration-folder",
dest="inputCalFolder",
default=None,
help="The folder containing camera calibration.")
parser.add_argument(
"--reference-dem-folder",
dest="refDemFolder",
default=None,
help="The folder containing DEMs that created orthoimages.")
parser.add_argument("--login-info",
dest="loginInfo",
default=None,
help="user,password destination.nsidc.org.")
parser.add_argument("--done-file",
dest="doneFile",
default=None,
help="List of runs that were done by now.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
# parse --site_yyyymmdd. Sometimes it is easier to pass this than
# to pass separately --site and --yyyymmdd.
m = re.match('^(\w+)_(\w+)', options.site_yyyymmdd)
if m:
options.site = m.group(1)
options.yyyymmdd = m.group(2)
else:
options.site_yyyymmdd = options.site + "_" + options.yyyymmdd
# Read the done file and exit if the current flight is done
done = set()
if options.doneFile != "":
with open(options.doneFile, 'r') as f:
for val in f:
val = val.strip()
done.add(val)
if options.site_yyyymmdd in done:
print("Skipping done flight: " + options.site_yyyymmdd)
return 0
run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd())
# Set up logging in the run directory. Log outside of the run dir,
# as that one we will wipe
logFolder = os.path.abspath(
os.path.join(run.getFolder(), '..', 'push_logs'))
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(logFolder, logLevel, "push")
logger.info("Logging in: " + logFolder)
# Check the lftp version. On some machines it is too old.
(out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'],
suppressOutput=True)
m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out)
if not m:
raise Exception('Expecting LFTP version 4.5.')
else:
logger.info("Found an acceptable version of LFTP.")
pushByType(run, options, logger, 'DEM')
#pushByType(run, options, logger, 'ORTHO') # need to wait for format decision
# Wipe at the end
cmd = "rm -rf " + run.getFolder()
logger.info(cmd)
os.system(cmd)
def main(argsIn):
# Command line parsing
try:
usage = "usage: camera_models_from_nav.py <image_folder> <ortho_folder> <cal_folder> <nav_folder> <output_folder> [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option('--start-frame',
dest='startFrame',
default=-1,
type='int',
help='The frame number to start processing with.')
parser.add_option('--stop-frame',
dest='stopFrame',
default=999999,
type='int',
help='The frame number to finish processing with.')
parser.add_option("--input-calibration-camera",
dest="inputCalCamera",
default="",
help="Use this input calibrated camera.")
parser.add_option(
'--camera-mounting',
dest='cameraMounting',
default=0,
type='int',
help=
'0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.'
)
(options, args) = parser.parse_args(argsIn)
if len(args) < 5:
print('Error: Missing arguments.')
print(usage)
return -1
imageFolder = os.path.abspath(args[0])
orthoFolder = os.path.abspath(args[1])
calFolder = os.path.abspath(args[2])
navFolder = os.path.abspath(args[3])
outputFolder = os.path.abspath(args[4])
except optparse.OptionError as msg:
raise Usage(msg)
runDir = os.path.dirname(orthoFolder)
os.system("mkdir -p " + runDir)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(runDir, logLevel,
'camera_models_from_nav_log')
if not os.path.exists(orthoFolder):
logger.error('Ortho folder ' + orthoFolder + ' does not exist!')
return -1
# Find the nav file
# - There should only be one or two nav files per flight.
fileList = os.listdir(navFolder)
fileList = [x for x in fileList if '.out' in x]
if len(fileList) == 0:
logger.error('No nav files in: ' + navFolder)
return -1
navPath = os.path.join(navFolder, fileList[0])
parsedNavPath = navPath.replace('.out', '.txt')
if not asp_file_utils.fileIsNonZero(navPath):
logger.error('Nav file ' + navPath + ' is invalid!')
return -1
# Create the output file only if it is empty or does not exist
isNonEmpty = asp_file_utils.fileIsNonZero(parsedNavPath)
if not isNonEmpty:
# Initialize the output file as being empty
logger.info("Create empty file: " + parsedNavPath)
open(parsedNavPath, 'w').close()
# Append to the output parsed nav file
for fileName in fileList:
# Convert the nav file from binary to text
navPath = os.path.join(navFolder, fileName)
with open(navPath, 'r') as f:
try:
text = f.readline()
if 'HTML' in text:
# Sometimes the server is down, and instead of the binary nav file
# we are given an html file with an error message.
logger.info("Have invalid nav file: " + navPath)
return -1 # Die in this case!
except UnicodeDecodeError as e:
# Got a binary file, that means likely we are good
pass
cmd = asp_system_utils.which(
'sbet2txt.pl') + ' -q ' + navPath + ' >> ' + parsedNavPath
logger.info(cmd)
if not isNonEmpty:
os.system(cmd)
cameraPath = options.inputCalCamera
if cameraPath == "":
# No input camera file provided, look one up. It does not matter much,
# as later ortho2pinhole will insert the correct intrinsics.
goodFile = False
fileList = os.listdir(calFolder)
fileList = [x for x in fileList if (('.tsai' in x) and ('~' not in x))]
if not fileList:
logger.error('Unable to find any camera files in ' + calFolder)
return -1
for fileName in fileList:
cameraPath = os.path.join(calFolder, fileName)
# Check if this path is valid
with open(cameraPath, 'r') as f:
for line in f:
if 'fu' in line:
goodFile = True
break
if goodFile:
break
# Get the ortho list
orthoFiles = icebridge_common.getTifs(orthoFolder)
logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.')
# Look up the frame numbers for each ortho file
infoDict = {}
for ortho in orthoFiles:
if ('gray' in ortho) or ('sub' in ortho):
continue
frame = icebridge_common.getFrameNumberFromFilename(ortho)
if frame < options.startFrame or frame > options.stopFrame:
continue
infoDict[frame] = [ortho, '']
# Get the image file list
try:
imageFiles = icebridge_common.getTifs(imageFolder)
except Exception as e:
raise Exception(
"Cannot continue with nav generation, will resume later when images are created. This is not a fatal error. "
+ str(e))
logger.info('Found ' + str(len(imageFiles)) + ' image files.')
# Update the second part of each dictionary object
for image in imageFiles:
if ('gray' in image) or ('sub' in image):
continue
frame = icebridge_common.getFrameNumberFromFilename(image)
if frame < options.startFrame or frame > options.stopFrame:
continue
if frame not in infoDict:
logger.info('Image missing ortho file: ' + image)
# don't throw here, that will mess the whole batch, we will recover
# the missing one later.
continue
infoDict[frame][1] = image
os.system('mkdir -p ' + outputFolder)
orthoListFile = os.path.join(
outputFolder, 'ortho_file_list_' + str(options.startFrame) + "_" +
str(options.stopFrame) + '.csv')
# Open the output file for writing
logger.info("Writing: " + orthoListFile)
with open(orthoListFile, 'w') as outputFile:
# Loop through frames in order
for key in sorted(infoDict):
# Write the ortho name and the output camera name to the file
(ortho, image) = infoDict[key]
if not image:
#raise Exception('Ortho missing image file: ' +ortho)
continue
camera = image.replace('.tif', '.tsai')
outputFile.write(ortho + ', ' + camera + '\n')
# Check if we already have all of the output camera files.
haveAllFiles = True
with open(orthoListFile, 'r') as inputFile:
for line in inputFile:
parts = line.split(',')
camPath = os.path.join(outputFolder, parts[1].strip())
if not asp_file_utils.fileIsNonZero(camPath):
logger.info('Missing file -> ' + camPath)
haveAllFiles = False
break
# Call the C++ tool to generate a camera model for each ortho file
if not haveAllFiles:
cmd = (
'nav2cam --input-cam %s --nav-file %s --cam-list %s --output-folder %s --camera-mounting %d'
% (cameraPath, parsedNavPath, orthoListFile, outputFolder,
options.cameraMounting))
logger.info(cmd)
os.system(cmd)
else:
logger.info("All nav files were already generated.")
# Generate a kml file for the nav camera files
kmlPath = os.path.join(outputFolder, 'nav_cameras.kml')
# This is a hack. If we are invoked from a Pleiades node, do not
# create this kml file, as nodes will just overwrite each other.
# This job may happen anyway earlier or later when on the head node.
if not 'PBS_NODEFILE' in os.environ:
try:
tempPath = os.path.join(outputFolder, 'list.txt')
logger.info('Generating nav camera kml file: ' + kmlPath)
os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath)
orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole')
cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath
logger.info(cmd)
asp_system_utils.executeCommand(cmd,
kmlPath,
suppressOutput=True,
redo=False)
os.remove(tempPath)
except Exception as e:
logger.info("Warning: " + str(e))
logger.info('Finished generating camera models from nav!')
return 0
def packAndSendSummaryFolder(run, folder, logger, backupToL2=False):
'''Archive the summary folder in case we want to look at it later'''
logger.info('Archiving summary folder for run ' + str(run))
cwd = os.getcwd()
os.chdir(run.parentFolder)
fileName = run.getSummaryTarName()
# Create a local tar file.
# Turn this approach off, new approach below.
# - Some fiddling to make the packed folders convenient
cmd = 'tar -chf '+ fileName +' -C '+ folder +'/.. ' + os.path.basename(folder)
logger.info(cmd)
(out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None,
suppressOutput = True, redo = True,
noThrow = True)
# This tends to print a very verbose message
ans = out + '\n' + err
vals = ans.split('\n')
if len(vals) < 10:
logger.info(ans)
else:
vals = vals[0:10]
logger.info("\n".join(vals))
logger.info("Above output truncated.")
# Use shiftc to create a local copy, and we want to include log files too
#runFolder = str(run)
#sumName = os.path.basename(run.getSummaryFolder())
#cmd = 'shiftc --wait -d -r --dereference --include=\'^.*?(' \
# + icebridge_common.logFilePrefix() + '|' \
# + runFolder + '/' + sumName + '|' \
# + icebridge_common.manager_log_prefix() \
# + ')\' --create-tar ' + runFolder \
# + ' ' + fileName
#logger.info(cmd)
#os.system(cmd)
# Delete any existing copy of the file on lfe
lfePath = os.path.join(REMOTE_SUMMARY_FOLDER, fileName)
cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
# Send the file to lfe using shiftc
cmd = 'shiftc --wait -d -r ' + fileName + ' ' + lfePath
robust_shiftc(cmd, logger)
logger.info('Finished sending summary to lfe.')
if backupToL2:
# Wipe the copy on lunokhod
l2Path = os.path.join(L_SUMMARY_FOLDER, fileName)
cmd = "ssh " + LUNOKHOD + " 'rm -f "+ stripHost(l2Path) +"' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
# Make target directory on lunokhod
cmd = "ssh " + LUNOKHOD + " 'mkdir -p " + os.path.dirname(stripHost(l2Path)) + \
"' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
# Send a copy of the file to Lunokhod for convenience
cmd = 'scp ' + fileName + ' ' + l2Path + ' 2>/dev/null'
logger.info(cmd)
os.system(cmd)
# Clean up the local tar file
cmd = 'rm -f ' + fileName
logger.info(cmd)
os.system(cmd)
os.chdir(cwd)
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 3.
usage = '''gen_ortho.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument(
'--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument(
'--camera-mounting',
dest='cameraMounting',
default=0,
type=int,
help=
'0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.'
)
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
# Performance options
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=1,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=8,
type=int,
help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(jpegFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No jpeg files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in jpegFrameDict:
logger.info("Error: Missing jpeg file for frame: " + str(frame) +
".")
continue
# Find the right image
currImage = jpegFrameDict[frame]
args = (frame, processFolder, currImage, options.bundleLength,
options.cameraMounting, threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runOrtho, args))
else:
runOrtho(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
logger,
interactive=False,
quitKey='q',
sleepTime=20)
icebridge_common.stopTaskPool(pool)
print("Skipping done flight: " + options.site_yyyymmdd)
return 0
run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd())
# Set up logging in the run directory. Log outside of the run dir,
# as that one we will wipe
logFolder = os.path.abspath(
os.path.join(run.getFolder(), '..', 'push_logs'))
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(logFolder, logLevel, "push")
logger.info("Logging in: " + logFolder)
# Check the lftp version. On some machines it is too old.
(out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'],
suppressOutput=True)
m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out)
if not m:
raise Exception('Expecting LFTP version 4.5.')
else:
logger.info("Found an acceptable version of LFTP.")
pushByType(run, options, logger, 'DEM')
#pushByType(run, options, logger, 'ORTHO') # need to wait for format decision
# Wipe at the end
cmd = "rm -rf " + run.getFolder()
logger.info(cmd)
os.system(cmd)
def main(argsIn):
try:
usage = '''usage: regenerate_summary_images.py <options> '''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"--data-folder",
dest="dataFolder",
help="Where all the inputs and outputs are stored.")
parser.add_argument("--work-folder",
dest="workFolder",
help="Where working files are stored.")
parser.add_argument("--site", dest="site", help="Site code.")
parser.add_argument("--yyyymmdd", dest="yyyymmdd", help="Date.")
#parser.add_argument("--dem-tarball", dest="demTarball", default=os.getcwd(),
# help="Where all the inputs and outputs are stored.")
#parser.add_argument("--ortho-tarball", dest="orthoTarball", default=None,
# help="Where to unpack the data.")
#parser.add_argument("--summary-tarball", dest="summaryTarball", default=None,
# help="Where to unpack the data.")
#parser.add_argument("--unpack-dir", dest="unpackDir", default=None,
# help="Where to unpack the data.")
parser.add_argument(
"--node-type",
dest="nodeType",
default='san',
help="Node type to use (wes[mfe], san, ivy, has, bro)")
#parser.add_argument("--skip-archive-summary", action="store_true",
# dest="skipArchiveSummary", default=False,
# help="Skip archiving the summary.")
# Debug option
parser.add_argument(
'--minutes-in-devel-queue',
dest='minutesInDevelQueue',
type=int,
default=0,
help="If positive, submit to the devel queue for this many minutes."
)
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
# Check if we are on the right machine
(host, err, status) = asp_system_utils.executeCommand(['uname', '-n'],
suppressOutput=True)
host = host.strip()
if 'pfe' in host and options.nodeType not in PFE_NODES:
raise Exception("From machine " + host + " can only launch on: " +
" ".join(PFE_NODES))
if 'mfe' in host and options.nodeType != 'wes':
raise Exception("From machine " + host + " can only launch on: wes")
# Make sure our paths will work when called from PBS
options.dataFolder = os.path.abspath(options.dataFolder)
#os.system('mkdir -p ' + options.unpackDir)
# TODO: Check folders!
run = run_helper.RunHelper(options.site, options.yyyymmdd,
options.workFolder)
runFolder = os.path.join(options.workFolder, str(run))
os.system('mkdir -p ' + runFolder)
logFolder = os.path.join(runFolder, 'logs')
# Set up logging in the run directory
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(
logFolder, logLevel, icebridge_common.manager_log_prefix())
logger.info("Logging in: " + logFolder)
checkRequiredTools(
) # Make sure all the needed tools can be found before we start
logger.info("Disabling core dumps.") # these just take a lot of room
os.system("ulimit -c 0")
os.system("umask 022") # enforce files be readable by others
# See how many hours we used so far. I think this counter gets updated once a day.
(out, err, status) = asp_system_utils.executeCommand("acct_ytd",
outputPath=None,
suppressOutput=True,
redo=True,
noThrow=True)
logger.info("Hours used so far:\n" + out + '\n' + err)
try:
# Fetch and extract the tarball files from Lou
localDemFolder = os.path.join(options.dataFolder, run.name() + '_dems')
localOrthoFolder = os.path.join(options.dataFolder,
run.name() + '_orthos')
demSummaryFolder = os.path.join(options.dataFolder,
run.name() + '_dem_summaries')
orthoSummaryFolder = os.path.join(options.dataFolder,
run.name() + '_ortho_summaries')
missingDemFiles = []
missingOrthoFiles = []
for f in os.listdir(localDemFolder):
if 'temp' in f:
raise Exception('Bad file: ' + f)
if ('IODEM3' in f) and (f[-4:] == '.tif'):
inputPath = os.path.join(localDemFolder, f)
outputPath = os.path.join(
demSummaryFolder, f.replace('DEM.tif', 'DEM_browse.tif'))
if not os.path.exists(outputPath):
missingDemFiles.append((inputPath, outputPath))
for f in os.listdir(localOrthoFolder):
if 'temp' in f:
raise Exception('Bad file: ' + f)
if ('IODIM3' in f) and (f[-4:] == '.tif'):
inputPath = os.path.join(localOrthoFolder, f)
outputPath = os.path.join(orthoSummaryFolder,
f.replace('ORTHO.tif', 'ORTHO.jpg'))
if not os.path.exists(outputPath):
missingOrthoFiles.append((inputPath, outputPath))
#print 'Fetching and unpacking tarballs...'
#fetchTarball(options.demTarball, localDemFolder)
#fetchTarball(options.orthoTarball, localOrthoFolder)
#fetchTarball(options.summaryTarball, localSummaryFolder)
# If the summary tarball unpacked to []/summary/summary,
# work with the lower level folder from now on.
#localSummaryFolder = descendIfNeeded(localSummaryFolder)
# Make a list of all input files that are missing their summary file, and
# the desired output path for that file.
#missingDemFiles = getMissingSummaryFiles(localDemFolder, localSummaryFolder, isOrtho=False)
#missingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, localSummaryFolder, isOrtho=True )
# Divide this list into chunks and for each chunk generate a file containing all of
# the gdal_translate commands that need to be executed.
print 'Writing command files...'
commandFileLength = getParallelParams(options.nodeType)[2]
commandFilePrefix = os.path.join(runFolder, 'convert_commands_')
print 'Clearing existing command files.'
os.system('rm ' + commandFilePrefix + '*')
commandFileList = writeCommandFiles(missingDemFiles, missingOrthoFiles,
commandFilePrefix,
commandFileLength)
#raise Exception('DEBUG')
# Get the location to store the logs
pbsLogFolder = run.getPbsLogFolder()
logger.info("Storing logs in: " + pbsLogFolder)
os.system('mkdir -p ' + pbsLogFolder)
# Call multi_process_command_runner.py through PBS for each chunk.
start_time()
(baseName, jobIDs) = submitBatchJobs(commandFileList, options,
pbsLogFolder, run, logger)
# Wait for everything to finish.
pbs_functions.waitForJobCompletion(jobIDs, logger, baseName)
stop_time("pbs_jobs", logger)
# Check that we now have all of the summary files.
# - Both of these should now be empty.
#newMissingDemFiles = getMissingSummaryFiles(localDemFolder, demSummaryFolder, isOrtho=False)
#newMissingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, orthoSummaryFolder, isOrtho=True )
numDemsMissing = 0
numOrthosMissing = 0
for pair in missingDemFiles:
if not os.path.exists(pair[1]):
numDemsMissing += 1
for pair in missingOrthoFiles:
if not os.path.exists(pair[1]):
numOrthosMissing += 1
resultText = (
'After regeneration, missing %d DEM summaries and %d ORTHO summaries'
% (numDemsMissing, numOrthosMissing))
logger.info(resultText)
runWasSuccess = ((numDemsMissing == 0) and (numOrthosMissing == 0))
# If successful, create a new tarball and send it to Lou.
#if runWasSuccess and (not options.skipArchiveSummary):
# start_time()
# archive_functions.packAndSendSummaryFolder(run, localSummaryFolder, logger)
# stop_time("archive summary", logger)
except Exception as e:
resultText = 'Caught exception: ' + str(
e) + '\n' + traceback.format_exc()
runWasSuccess = False
# Send a summary email.
emailAddress = getEmailAddress(icebridge_common.getUser())
logger.info("Sending email to: " + emailAddress)
if runWasSuccess:
sendEmail(emailAddress, 'OIB summary regen passed', resultText)
else:
sendEmail(emailAddress, '"OIB summary regen failed', resultText)
# TODO: Add automated delete command!
#if options.wipeProcessed:
# processedFolder = run.getProcessFolder()
# logger.info("Will delete: " + processedFolder)
# os.system("rm -rf " + processedFolder)
logger.info(
'==== regenerate_summary_images script has finished for run: ' +
str(run) + ' ====')
def runBlend(frame, processFolder, lidarFile, fireballDEM, options,
threadText, redo, suppressOutput):
WEIGHT_EXP = 1.3
# This will run as multiple processes. Hence have to catch all exceptions.
try:
demFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.alignFileName(),
processFolder, options.bundleLength)
lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile)
if demFile == "":
print("Could not find DEM for frame: " + str(frame))
return
# The names for the final results
finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM')
finalBlend = finalOutputPrefix + '.tif'
finalDiff = finalOutputPrefix + "-diff.csv"
fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint')
fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif'
finalFireballOutput = fireballOutputPrefix + '-DEM.tif'
fireballDiffPath = fireballOutputPrefix + "-diff.csv"
# This is turned off for now. Find the diff between neighboring
# aligned DEMs before blending.
prevDemFile, prevBatchFolder = \
icebridge_common.frameToFile(frame-1,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
prevDiffPrefix = os.path.join(batchFolder, 'out-prev')
prevDiffFile = prevDiffPrefix + '-diff.tif'
if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile):
os.system("rm -f " + prevDiffFile)
if os.path.exists(prevDemFile) and os.path.exists(demFile):
if os.path.exists(prevDiffFile):
print("File exists: " + prevDiffFile)
else:
cmd = ('geodiff --absolute %s %s -o %s' %
(prevDemFile, demFile, prevDiffPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, prevDiffFile, suppressOutput, redo)
if not redo:
set1Exists = False
if (os.path.exists(finalBlend) and os.path.exists(finalDiff)):
print("Files exist: " + finalBlend + " " + finalDiff + ".")
set1Exists = True
set2Exists = True
if fireballDEM != "":
if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)):
print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".")
set2Exists = True
else:
set2Exists = False
if set1Exists and set2Exists:
return
# We will blend the dems with frame offsets within frameOffsets[0:index]
filesToWipe = []
bestMean = 1.0e+100
bestBlend = ''
bestVals = ''
bestDiff = ''
# Look at frames with these offsets when blending
frameOffsets = [0, 1, -1, 2, -2]
for index in range(len(frameOffsets)):
# Find all the DEMs up to the current index
dems = []
currDemFile = ""
for val in range(0, index+1):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
if currDemFile == "":
# The last DEM was not present. Hence this iteration will add nothing new.
continue
# Compute the mean distance between the DEMs
# TODO: Make sure this gets cleaned up!
meanWorkPrefix = os.path.join(batchFolder, 'bd')
meanDiff = getMeanDemDiff(dems, meanWorkPrefix)
# If the mean error between DEMs is creater than this,
# use a less aggressive blending method.
MEAN_DIFF_BLEND_THRESHOLD = 1.0
demString = " ".join(dems)
outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index))
# See if we have a pre-existing DEM to use as footprint
footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif')
blendOutput = outputPrefix + '-tile-0.tif'
if os.path.exists(footprintDEM):
cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, footprintDEM, demString, threadText, outputPrefix))
else:
cmd = ('dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s'
% (WEIGHT_EXP, demString, threadText, outputPrefix))
if meanDiff > MEAN_DIFF_BLEND_THRESHOLD:
cmd += ' --propagate-nodata --use-centerline-weights '
print(cmd)
# Execute the blend command.
# - Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
print(cmd)
asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo)
filesToWipe.append(blendOutput)
diffPath = outputPrefix + "-diff.csv"
filesToWipe.append(diffPath)
# Compute post-blending error to lidar
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, blendOutput, lidarFile, outputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(diffPath)
print("Current mean error to lidar is " + str(results['Mean']))
if bestMean > float(results['Mean']):
bestMean = float(results['Mean'])
bestBlend = blendOutput
bestVals = demString
bestDiff = diffPath
except Exception as e:
pass
logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Update the filenames of the output files
print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals)
cmd = "mv " + bestBlend + " " + finalBlend
print(cmd)
asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo)
cmd = "mv " + bestDiff + " " + finalDiff
print(cmd)
asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo)
# Generate a thumbnail of the final DEM
hillOutput = finalOutputPrefix+'_HILLSHADE.tif'
cmd = 'hillshade ' + finalBlend +' -o ' + hillOutput
asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
# Generate a low resolution compressed thumbnail of the hillshade for debugging
thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif'
cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
# Do another blend, to this DEM's footprint, but not using it
if fireballDEM != "":
# Find all the DEMs
dems = []
for val in range(0, len(frameOffsets)):
offset = frameOffsets[val]
currDemFile, currBatchFolder = \
icebridge_common.frameToFile(frame + offset,
icebridge_common.alignFileName(),
processFolder, options.bundleLength)
if currDemFile == "":
continue
dems.append(currDemFile)
demString = " ".join(dems)
cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s'
% (WEIGHT_EXP, fireballDEM, demString, threadText, fireballOutputPrefix))
#filesToWipe.append(fireballBlendOutput)
print(cmd)
# Sometimes there is junk left from a previous interrupted run. So if we
# got so far, recreate all files.
localRedo = True
asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo)
#filesToWipe.append(fireballDiffPath)
cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' %
(lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix))
print(cmd)
asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo)
# Read in and examine the results
try:
results = icebridge_common.readGeodiffOutput(fireballDiffPath)
print("Mean error to lidar in fireball footprint is " + str(results['Mean']))
cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput
print(cmd)
asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo)
except Exception as e:
pass
# Generate a thumbnail of the final DEM
#hillOutput = fireballOutputPrefix+'_HILLSHADE.tif'
#cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput
#print(cmd)
#asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo)
## Generate a low resolution compressed thumbnail of the hillshade for debugging
#thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif'
#cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"'
#asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo)
#os.system("rm -f " + hillOutput) # Remove this file to keep down the file count
logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*")
filesToWipe += logFiles
# Done with dealing with the fireball footprint
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.system("rm -f " + fileName)
# TODO: Handle this cleanup better!
os.system('rm -f ' + meanWorkPrefix + '*')
except Exception as e:
print('Blending failed!\n' + str(e) + ". " + str(traceback.print_exc()))
sys.stdout.flush()
def runOrtho(frame, processFolder, imageFile, bundleLength, cameraMounting,
threadText, redo, suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
# Retrieve the aligned camera file
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
# Find the DEM file for the desired frame
demFile, batchFolder = icebridge_common.frameToFile(frame + offset,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find blended DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False)
projBounds = demGeoInfo['projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(frame,
icebridge_common.blendFileName(),
processFolder, bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor*widX, # minX
float(projBounds[1]) + factor*widX, # maxX
float(projBounds[2]) - factor*widY, # minY
float(projBounds[3]) + factor*widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s'
% (demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
# Generate the DEM mosaic
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(batchFolder,
icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = ('dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo,
noThrow = True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# If needed, generate a temporary camera file to correct a mounting rotation.
# - When the camera mount is rotated 90 degrees stereo is run on a corrected version
# but ortho needs to work on the original uncorrected jpeg image.
tempCamFile = alignCamFile + '_temp_rot.tsai'
tempCamFile = createRotatedCameraFile(alignCamFile, tempCamFile, cameraMounting)
# Run mapproject. The grid size is auto-determined.
cmd = ('mapproject --no-geoheader-info %s %s %s %s %s'
% (mosaicOutput, imageFile, tempCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo)
# Set temporary files to be cleaned up
filesToWipe.append(tempOrtho)
if tempCamFile != alignCamFile:
filesToWipe.append(tempCamFile)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()
#medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth,
# gsdFrameSkip)
#outputResolution = icebridge_common.gsdToDemRes(medianGsd)
#logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution))
outputResolution = 0.4
# Generate a map of initial camera positions
orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml')
vizString = ''
for (image, camera) in imageCameraPairs:
vizString += camera+' '
cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString
logger.info('Running orbitviz on input files...')
# Suppress (potentially long) output
asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo)
# Set up options for process_icebridge_batch
extraOptions = ''
if options.numThreads:
extraOptions += ' --num-threads ' + str(options.numThreads)
if options.solve_intr:
extraOptions += ' --solve-intrinsics '
if options.isSouth:
extraOptions += ' --south '
if options.maxDisplacement:
extraOptions += ' --max-displacement ' + str(options.maxDisplacement)
if options.fireballFolder:
extraOptions += ' --fireball-folder ' + str(options.fireballFolder)
if options.cleanup:
extraOptions += ' --cleanup '
for (image, camera) in zip(imageFiles, cameraFiles):
frameNumber = getFrameNumberFromFilename(image)
if (getFrameNumberFromFilename(camera) != frameNumber):
print 'Error: input files do not align!'
print (image, camera)
return -1
imageString += image +' ' # Build strings for the bundle_adjust step
cameraString += camera+' '
# Generate a map of initial camera positions
orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml')
vizString = ''
for (image, camera) in zip(imageFiles, cameraFiles):
vizString += image +' ' + camera+' '
cmd = 'orbitviz --hide-labels -t nadirpinhole -r wgs84 -o '+ orbitvizBefore +' '+ vizString
asp_system_utils.executeCommand(cmd, orbitvizBefore, suppressOutput, redo)
if options.globalBundleAdjust:
# TODO: Intrinsics???
# Bundle adjust all of the cameras
# - Could use an overlap of 4 but there is very little overlap at that point.
# - If we start dealing with crossover paths we can use the KML overlap method.
print 'Setting up bundle adjustment...'
baFolder = os.path.join(outputFolder, 'group_bundle')
baPrefix = os.path.join(baFolder, 'out')
cmd = ('bundle_adjust '+ imageString + cameraString
+ ' --overlap-limit 3 --local-pinhole --solve-intrinsics -o '+ baPrefix)
suppressOutput = False
redo = False
baOutFile = baPrefix +'-'+ os.path.basename(cameraFiles[-1])
asp_system_utils.executeCommand(cmd, baOutFile, suppressOutput, redo)
gsdFrameSkip = 1
medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem,
options.isSouth, gsdFrameSkip)
outputResolution = medianGsd * GSD_RESOLUTION_MULTIPLIER
logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution))
#return 0
# Generate a map of initial camera positions
orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml')
vizString = ''
for (image, camera) in imageCameraPairs:
vizString += camera + ' '
cmd = 'orbitviz_pinhole --hide-labels -o ' + orbitvizBefore + ' ' + vizString
logger.info('Running orbitviz on input files...')
asp_system_utils.executeCommand(cmd, orbitvizBefore, True,
redo) # Suppress (potentially long) output
#raise Exception('DEBUG')
# Set up options for process_icebridge_batch
extraOptions = ''
if options.numThreads:
extraOptions += ' --num-threads ' + str(options.numThreads)
if options.numProcessesPerBatch:
extraOptions += ' --num-processes-per-batch ' + str(
options.numProcessesPerBatch)
if options.solve_intr:
extraOptions += ' --solve-intrinsics '
if options.isSouth:
extraOptions += ' --south '
if options.maxDisplacement:
extraOptions += ' --max-displacement ' + str(options.maxDisplacement)
def runOrtho(frame, processFolder, imageFile, bundleLength, threadText, redo,
suppressOutput):
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
# This will run as multiple processes. Hence have to catch all exceptions:
projBounds = ()
try:
alignCamFile, batchFolder = \
icebridge_common.frameToFile(frame,
icebridge_common.alignedBundleStr() +
'*' + str(frame) + '.tsai',
processFolder, bundleLength)
if alignCamFile == "":
print("Could not find aligned camera for frame: " + str(frame))
return
# To ensure we mapproject the image fully, mosaic the several DEMs
# around it. Keep the closest 5. Try to grab more first to account
# for skipped frames.
frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4]
dems = []
for offset in frameOffsets:
demFile, batchFolder = icebridge_common.frameToFile(
frame + offset, icebridge_common.blendFileName(),
processFolder, bundleLength)
# If the central DEM is missing, we are out of luck
if offset == 0 and demFile == "":
print("Could not find DEM for frame: " + str(frame + offset))
return
if offset == 0:
demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile,
getStats=False)
projBounds = demGeoInfo[
'projection_bounds'] # minX maxX minY maxY
if demFile == "":
# Missing DEM
continue
if len(dems) >= 5:
break # too many already
dems.append(demFile)
demList = " ".join(dems)
# Call this one more time, to get the current batch folder
currDemFile, batchFolder = icebridge_common.frameToFile(
frame, icebridge_common.blendFileName(), processFolder,
bundleLength)
# The names for the final results
finalOrtho = os.path.join(batchFolder,
icebridge_common.orthoFileName())
finalOrthoPreview = os.path.join(
batchFolder, icebridge_common.orthoPreviewFileName())
if (not redo) and os.path.exists(finalOrtho):
print("File exists: " + finalOrtho + ".")
else:
filesToWipe = []
# If the center dem spans say 1 km, there's no way the
# ortho can span more than 5 km, unless something is
# seriously out of whack, such as alignment failing for
# some neighbours. In the best case, if the center dem is
# 1 km by 1 km, the obtained ortho will likely be 1.4 km
# by 1 km, as an image extends beyond its stereo dem with
# a neighbor.
factor = float(2.0)
projWinStr = ""
if len(projBounds) >= 4:
# projBounds is in the format minX maxX minY maxY
widX = float(projBounds[1]) - float(projBounds[0])
widY = float(projBounds[3]) - float(projBounds[2])
projBounds = (
float(projBounds[0]) - factor * widX, # minX
float(projBounds[1]) + factor * widX, # maxX
float(projBounds[2]) - factor * widY, # minY
float(projBounds[3]) + factor * widY # maxY
)
projWinStr = ("--t_projwin %f %f %f %f" % \
(projBounds[0], projBounds[2], projBounds[1], projBounds[3]))
# See if we have a pre-existing DEM to use as footprint
mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic')
mosaicOutput = mosaicPrefix + '-tile-0.tif'
cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' %
(demList, threadText, projWinStr, mosaicPrefix))
filesToWipe.append(mosaicOutput) # no longer needed
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput,
localRedo)
# Borow some pixels from the footprint DEM,just to grow a bit the real estate
finalFootprintDEM = os.path.join(
batchFolder, icebridge_common.footprintFileName())
if os.path.exists(finalFootprintDEM):
mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2')
mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif'
cmd = (
'dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s'
% (mosaicOutput, finalFootprintDEM, threadText, projWinStr,
mosaicPrefix2))
print(cmd)
localRedo = True # The file below should not exist unless there was a crash
asp_system_utils.executeCommand(cmd,
mosaicOutput2,
suppressOutput,
localRedo,
noThrow=True)
if os.path.exists(mosaicOutput2):
cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput
print(cmd)
os.system(cmd)
# TODO: Look at more aggressive hole-filling. But need a testcase.
filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*')
# First mapproject to create a tif image with 4 channels.
# Then pull 3 channels and compress them as jpeg, while keeping the
# image a geotiff.
tempOrtho = os.path.join(
batchFolder,
icebridge_common.orthoFileName() + "_tmp.tif")
# There is no need for this file to exist unless it is stray junk
if os.path.exists(tempOrtho):
os.remove(tempOrtho)
# Run mapproject. The grid size is auto-determined.
cmd = (
'mapproject --no-geoheader-info %s %s %s %s %s' %
(mosaicOutput, imageFile, alignCamFile, tempOrtho, threadText))
print(cmd)
asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput,
redo)
filesToWipe.append(tempOrtho)
# This makes the images smaller than Rose's by a factor of about 4,
# even though both types are jpeg compressed. Rose's images filtered
# through this command also get compressed by a factor of 4.
# I conclude that the jpeg compression used by Rose was not as
# aggressive as the one used in gdal_translate, but there is no
# apparent knob to control that.
cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput,
redo)
# Clean up extra files
for fileName in filesToWipe:
if os.path.exists(fileName):
print("Removing: " + fileName)
os.remove(fileName)
if (not redo) and os.path.exists(finalOrthoPreview):
print("File exists: " + finalOrthoPreview + ".")
else:
cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \
' ' + finalOrthoPreview
print(cmd)
asp_system_utils.executeCommand(cmd, finalOrthoPreview,
suppressOutput, redo)
except Exception as e:
print('Ortho creation failed!\n' + str(e) + ". " +
str(traceback.print_exc()))
os.system("rm -f core.*") # these keep on popping up
# To ensure we print promptly what we did so far
sys.stdout.flush()
def main(argsIn):
try:
# Sample usage:
# python full_processing_script.py \
# --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \
# --start-frame 350 --stop-frame 353 --skip-validate \
# --camera-calibration-folder camera_calib \
# --reference-dem-folder ref_dem_folder
# An output folder will be crated automatically (with a name like
# AN_20091016), or its name can be specified via the --output-folder
# option.
usage = '''full_processing_script.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None,
help="The file to use to find which camera was used for which " + \
"flight. By default it is in the same directory as this script " + \
"and named camera_lookup.txt.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
# TODO: Compute this automatically??
parser.add_argument('--overlap-limit', dest='overlapLimit', default=2,
type=int, help="The number of images to treat as overlapping for " + \
"bundle adjustment.")
parser.add_argument('--stereo-arguments', dest='stereoArgs',
# set --min-xcorr-level 0 to do the left-to-right
# and right-to-left consistency check at the lowest level.
default='--stereo-algorithm 2 --min-xcorr-level 0',
help='Extra arguments to pass to stereo.')
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
parser.add_argument('--frames-file', dest='framesFile', default="",
help='Specific frames to run ortho2pinhole on within this frame range.')
parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None,
type=int, help="The maximum number of lidar files to fetch. " + \
"This is used in debugging.")
parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None,
help="The folder containing camera calibration.")
parser.add_argument("--input-calibration-camera", dest="inputCalCamera", default="",
help="Instead of looking up the calibrated camera in the calibration folder, use this one.")
parser.add_argument("--output-calibration-camera", dest="outputCalCamera", default="",
help="If specified, float the intrinsics and write the optimized model here.")
parser.add_argument("--output-model-type", dest="outputModelType", default="RPC",
help="Generate a distortion model of type RPC, RPC5, or RPC6.")
parser.add_argument("--reference-dem-folder", dest="refDemFolder", default=None,
help="The folder containing DEMs that created orthoimages.")
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. " + \
"fault is no additional folder")
parser.add_argument("--simple-cameras", action="store_true", dest="simpleCameras", default=False,
help="Don't use orthoimages to refine the camera models.")
# This option is only needed when generating camera models from the nav files.
parser.add_argument('--camera-mounting', default=0, dest='cameraMounting', type=int,
help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.')
# Performance options
parser.add_argument('--num-processes', dest='numProcesses', default=1,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-ortho-processes', dest='numOrthoProcesses', default=-1,
type=int, help='The number of simultaneous ortho processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=8,
type=int, help='The number of threads per process.')
# Action control
parser.add_argument("--skip-fetch", action="store_true", dest="noFetch", default=False,
help="Skip data fetching.")
parser.add_argument("--skip-convert", action="store_true", dest="noConvert", default=False,
help="Skip data conversion.")
parser.add_argument("--stop-after-fetch", action="store_true", dest="stopAfterFetch",
default=False,
help="Stop program after data fetching.")
parser.add_argument("--stop-after-convert", action="store_true", dest="stopAfterConvert",
default=False,
help="Stop program after data conversion.")
parser.add_argument("--skip-validate", action="store_true", dest="skipValidate",
default=False,
help="Skip input data validation.")
parser.add_argument("--ignore-missing-lidar", action="store_true", dest="ignoreMissingLidar",
default=False,
help="Keep going if the lidar is missing.")
parser.add_argument("--log-batches", action="store_true", dest="logBatches", default=False,
help="Log the required batch commands without running them.")
parser.add_argument('--cleanup', action='store_true', default=False, dest='cleanup',
help='If the final result is produced delete intermediate files.')
parser.add_argument('--many-ip', action='store_true', default=False, dest='manyip',
help='If to use a lot of IP in bundle adjustment from the beginning.')
parser.add_argument("--dry-run", action="store_true", dest="dryRun", default=False,
help="Set up the input directories but do not fetch/process any imagery.")
parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False,
help="Try fetching again if some files turned out invalid " + \
"during conversions.")
parser.add_argument("--refetch-index", action="store_true", dest="refetchIndex",
default=False,
help="Force refetch of the index file.")
parser.add_argument("--refetch-nav", action="store_true", dest="refetchNav",
default=False,
help="Force refetch of the nav file.")
parser.add_argument("--stop-after-index-fetch", action="store_true",
dest="stopAfterIndexFetch", default=False,
help="Stop after fetching the indices.")
parser.add_argument("--no-nav", action="store_true", dest="noNavFetch",
default=False, help="Don't fetch or convert the nav data.")
parser.add_argument("--no-lidar-convert", action="store_true", dest="noLidarConvert",
default=False,
help="Skip lidar files in the conversion step.")
parser.add_argument("--no-ortho-convert", action="store_true", dest="noOrthoConvert",
default=False,
help="Skip generating camera models in the conversion step.")
parser.add_argument("--skip-fast-conversions", action="store_true", dest="skipFastConvert",
default=False,
help="Skips all non-ortho conversions.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if options.numOrthoProcesses < 0:
options.numOrthoProcesses = options.numProcesses
isSouth = icebridge_common.checkSite(options.site)
# Turned off elevation limits here since they are being set from LIDAR data.
## Add the site based elevation limits to the stereoArgs option
#altLimits = icebridge_common.getElevationLimits(options.site)
#options.stereoArgs = (' %s --elevation-limit %f %f '
# % (options.stereoArgs, altLimits[0], altLimits[1]))
options.stereoArgs = (' %s ' % (options.stereoArgs))
if options.cameraLookupFile is None:
options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt')
if not os.path.isfile(options.cameraLookupFile):
raise Exception("Can't find camera file: " + options.cameraLookupFile)
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
if options.stopAfterIndexFetch:
options.stopAfterFetch = True
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Record everything
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_processing_log_frames_' + \
str(options.startFrame) + "_" + str(options.stopFrame))
# Make sure we later know what we were doing
logger.info("full_processing_script.py " + " ".join(argsIn))
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput = True)
logger.info("Running on machine: " + out)
logger.info("Work dir is " + os.getcwd())
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
# Perform some input checks and initializations
# These are not needed unless cameras are initialized
if options.inputCalFolder is None or not os.path.exists(options.inputCalFolder):
raise Exception("Missing camera calibration folder.")
if options.refDemFolder is None or not os.path.exists(options.refDemFolder):
raise Exception("Missing reference DEM folder.")
refDemName = icebridge_common.getReferenceDemName(options.site)
refDemPath = os.path.join(options.refDemFolder, refDemName)
if not os.path.exists(refDemPath):
raise Exception("Missing reference DEM: " + refDemPath)
# TODO: CLEAN UP!!!
# Set up the output folders
cameraFolder = icebridge_common.getCameraFolder(options.outputFolder)
imageFolder = icebridge_common.getImageFolder(options.outputFolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
fireballFolder = icebridge_common.getFireballFolder(options.outputFolder)
corrFireballFolder = icebridge_common.getCorrFireballFolder(options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
navFolder = icebridge_common.getNavFolder(options.outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder)
processedFolder = icebridge_common.getProcessedFolder(options.outputFolder)
if options.outputCalCamera != "":
# Prepare to solve for intrinsics. Note that this modifies some things along the way.
(options, cameraFolder, navCameraFolder, processedFolder) = \
solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder,
processedFolder, logger)
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processedFolder = os.path.join(processedFolder, options.processingSubfolder)
logger.info('Will write to processing subfolder: ' + options.processingSubfolder)
# If something failed in the first attempt either in fetch or in
# convert, we will wipe bad files, and try to refetch/re-convert.
numAttempts = 1
if options.reFetch and (not options.noFetch):
numAttempts = 2
for attempt in range(numAttempts):
if numAttempts > 1:
logger.info("Fetch/convert attempt: " + str(attempt+1))
ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder, jpegFolder, orthoFolder,
fireballFolder, corrFireballFolder, lidarFolder, processedFolder,
navFolder, navCameraFolder, refDemPath, logger)
if ans == 0:
break
if options.stopAfterFetch or options.dryRun or options.stopAfterConvert:
logger.info('Fetch/convert finished!')
return 0
# Call the processing routine
processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder,
corrFireballFolder, processedFolder,
isSouth, refDemPath)
if options.outputCalCamera != "":
# Finish solving for intrinscs.
solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder,
processedFolder, isSouth, logger)
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 10
usage = '''blend_dems.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument(
'--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
parser.add_argument(
"--compute-diff-to-prev-dem",
action="store_true",
dest="computeDiffToPrev",
default=False,
help="Compute the absolute difference between the current DEM " +
"and the one before it.")
parser.add_argument("--blend-to-fireball-footprint", action="store_true",
dest="blendToFireball", default=False,
help="Create additional blended DEMs having the same " + \
"footprint as Fireball DEMs.")
# Performance options
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=1,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=8,
type=int,
help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_blend_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
logger.info(str(argsIn))
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath +
".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if options.blendToFireball:
fireballFrameDict = icebridge_common.getCorrectedFireballDems(
options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(orthoFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No ortho files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in orthoFrameDict:
logger.info("Error: Missing ortho file for frame: " + str(frame) +
".")
continue
orthoFile = orthoFrameDict[frame]
try:
lidarFile = icebridge_common.findMatchingLidarFile(
orthoFile, lidarFolder)
except: # Skip if no lidar file matches this frame
continue
fireballDEM = ""
if options.blendToFireball:
if frame in fireballFrameDict:
fireballDEM = fireballFrameDict[frame]
else:
logger.info("No fireball DEM for frame: " + str(frame))
args = (frame, processFolder, lidarFile, fireballDEM, options,
threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runBlend, args))
else:
runBlend(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
logger,
interactive=False,
quitKey='q',
sleepTime=20)
icebridge_common.stopTaskPool(pool)
for (image, camera) in zip(imageFiles, cameraFiles):
frameNumber = getFrameNumberFromFilename(image)
if (getFrameNumberFromFilename(camera) != frameNumber):
print 'Error: input files do not align!'
print(image, camera)
return -1
imageString += image + ' ' # Build strings for the bundle_adjust step
cameraString += camera + ' '
# Generate a map of initial camera positions
orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml')
vizString = ''
for (image, camera) in zip(imageFiles, cameraFiles):
vizString += image + ' ' + camera + ' '
cmd = 'orbitviz --hide-labels -t nadirpinhole -r wgs84 -o ' + orbitvizBefore + ' ' + vizString
asp_system_utils.executeCommand(cmd, orbitvizBefore, suppressOutput, redo)
if options.globalBundleAdjust:
# TODO: Intrinsics???
# Bundle adjust all of the cameras
# - Could use an overlap of 4 but there is very little overlap at that point.
# - If we start dealing with crossover paths we can use the KML overlap method.
print 'Setting up bundle adjustment...'
baFolder = os.path.join(outputFolder, 'group_bundle')
baPrefix = os.path.join(baFolder, 'out')
cmd = ('bundle_adjust ' + imageString + cameraString +
' --overlap-limit 3 --local-pinhole --solve-intrinsics -o ' +
baPrefix)
suppressOutput = False
redo = False
baOutFile = baPrefix + '-' + os.path.basename(cameraFiles[-1])
def main(argsIn):
try:
usage = '''usage: push_to_nsidc.py <options> '''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument("--yyyymmdd", dest="yyyymmdd", default="",
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", default = "",
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--site_yyyymmdd", dest="site_yyyymmdd", default = "",
help="A value like GR_20150330, which will be split into site and yyyymmdd by this script.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None,
help="The folder containing camera calibration.")
parser.add_argument("--reference-dem-folder", dest="refDemFolder", default=None,
help="The folder containing DEMs that created orthoimages.")
parser.add_argument("--login-info", dest="loginInfo", default=None,
help="user,password destination.nsidc.org.")
parser.add_argument("--done-file", dest="doneFile", default=None,
help="List of runs that were done by now.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
# parse --site_yyyymmdd. Sometimes it is easier to pass this than
# to pass separately --site and --yyyymmdd.
m = re.match('^(\w+)_(\w+)', options.site_yyyymmdd)
if m:
options.site = m.group(1)
options.yyyymmdd = m.group(2)
else:
options.site_yyyymmdd = options.site + "_" + options.yyyymmdd
# Read the done file and exit if the current flight is done
done = set()
if options.doneFile != "":
with open(options.doneFile, 'r') as f:
for val in f:
val = val.strip()
done.add(val)
if options.site_yyyymmdd in done:
print("Skipping done flight: " + options.site_yyyymmdd)
return 0
run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd())
# Set up logging in the run directory. Log outside of the run dir,
# as that one we will wipe
logFolder = os.path.abspath(os.path.join(run.getFolder(), '..', 'push_logs'))
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(logFolder, logLevel, "push")
logger.info("Logging in: " + logFolder)
# Check the lftp version. On some machines it is too old.
(out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'],
suppressOutput = True)
m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out)
if not m:
raise Exception('Expecting LFTP version 4.5.')
else:
logger.info("Found an acceptable version of LFTP.")
pushByType(run, options, logger, 'DEM')
#pushByType(run, options, logger, 'ORTHO') # need to wait for format decision
# Wipe at the end
cmd = "rm -rf " + run.getFolder()
logger.info(cmd)
os.system(cmd)
icebridge_common.switchWorkDir()
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
if options.bundleAdjust:
# TODO: Solve for intrinsics?
bundlePrefix = os.path.join(outputFolder, "bundle/out")
cmd = "bundle_adjust %s %s %s %s -o %s %s -t nadirpinhole --local-pinhole" % (
imageA,
imageB,
cameraA,
cameraB,
bundlePrefix,
threadText,
)
print (cmd)
# Point to the new camera models
cameraA = bundlePrefix + "-" + os.path.basename(cameraA)
cameraB = bundlePrefix + "-" + os.path.basename(cameraB)
asp_system_utils.executeCommand(cmd, cameraA, suppressOutput, redo)
# Update the baseArgString
baseArgString = "%s %s %s %s %s -t nadirpinhole --alignment-method epipolar" % (
imageA,
imageB,
cameraA,
cameraB,
outputPrefix,
)
if options.use_sgm:
# PPRC
cmd = "stereo_pprc %s %s" % (baseArgString, threadText)
pprcOutput = outputPrefix + "-L.tif"
asp_system_utils.executeCommand(cmd, pprcOutput, suppressOutput, redo)
def main(argsIn):
try:
usage = '''usage: process_icebridge_run.py <image_folder> <camera_folder>
<lidar_folder> <output_folder>'''
parser = optparse.OptionParser(usage=usage)
# Data selection optios
parser.add_option('--start-frame', dest='startFrame', default=-1,
type='int', help='The frame number to start processing with.')
parser.add_option('--stop-frame', dest='stopFrame', default=-1,
type='int', help='The frame number to finish processing with.')
parser.add_option('--south', action='store_true', default=False, dest='isSouth',
help='MUST be set if the images are in the southern hemisphere.')
# Processing options
parser.add_option('--stereo-arguments', dest='stereoArgs', default='',
help='Additional argument string to be passed to the stereo command.')
parser.add_option('--bundle-length', dest='bundleLength', default=2,
type='int', help='Number of images to bundle adjust and process at once.')
parser.add_option('--image-stereo-interval', dest='imageStereoInterval', default=None,
type='int', help='Advance this many frames to get the stereo pair. Default is auto-calculate')
parser.add_option('--solve-intrinsics', action='store_true', default=False,
dest='solve_intr',
help='If to float the intrinsics params.')
#parser.add_option('--dem-resolution', dest='demResolution', default=0.4,
# type='float', help='Generate output DEMs at this resolution.')
parser.add_option('--max-displacement', dest='maxDisplacement', default=20,
type='float', help='Max displacement value passed to pc_align.')
# Performance options
parser.add_option('--num-processes', dest='numProcesses', default=1,
type='int', help='The number of simultaneous processes to run.')
parser.add_option('--num-threads', dest='numThreads', default=None,
type='int', help='The number threads to use per process.')
# Action options
parser.add_option('--interactive', action='store_true', default=False, dest='interactive',
help='If to wait on user input to terminate the jobs.')
parser.add_option('--log-batches', action='store_true', default=False, dest='logBatches',
help="Just log the batch commands to a file.")
parser.add_option('--cleanup', action='store_true', default=False, dest='cleanup',
help='If the final result is produced delete intermediate files.')
parser.add_option('--many-ip', action='store_true', default=False, dest='manyip',
help='If to use a lot of IP in bundle adjustment from the beginning.')
parser.add_option('--dry-run', action='store_true', default=False, dest='dryRun',
help="Print but don't launch the processing jobs.")
parser.add_option('--ortho-folder', dest='orthoFolder', default=None,
help='Use ortho files to adjust processing to the image spacing.')
parser.add_option('--fireball-folder', dest='fireballFolder', default=None,
help='Location of fireball DEMs for comparison.')
parser.add_option('--reference-dem', dest='referenceDem', default=None,
help='Reference DEM used to calculate the expected GSD.')
(options, args) = parser.parse_args(argsIn)
if len(args) < 4:
print(usage)
return 0
imageFolder = args[0]
cameraFolder = args[1]
lidarFolder = args[2]
outputFolder = args[3]
except optparse.OptionError as msg:
raise Usage(msg)
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
# Check the inputs
for f in [imageFolder, cameraFolder, lidarFolder]:
if not os.path.exists(f):
logger.error('Input folder '+ f +' does not exist!')
return 0
asp_system_utils.mkdir_p(outputFolder)
suppressOutput = False
redo = False
logger.info('\nStarting processing...')
# Get a list of all the input files
imageCameraPairs = icebridge_common.getImageCameraPairs(imageFolder, cameraFolder,
options.startFrame, options.stopFrame,
logger)
numFiles = len(list(imageCameraPairs))
if numFiles < 2:
raise Exception('Failed to find any image camera pairs!')
# Check that the files are properly aligned
lastFrame = -1
availableFrames = []
for (image, camera) in imageCameraPairs:
frameNumber = icebridge_common.getFrameNumberFromFilename(image)
availableFrames.append(frameNumber)
if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber):
logger.error('Error: input files do not align!\n' + str((image, camera)))
return -1
if frameNumber <= lastFrame:
logger.error('Error: input frames not sorted properly!\n')
return -1
lastFrame = frameNumber
# Do not compute output resolution. Will be overwritten anyway per frame.
## Set the output resolution as the computed mean GSD
## TODO: This should be cashed, and recomputed only when the batch file changes!
#NUM_GSD_FRAMES = 20
#logger.info('Computing GSD with ' + str(NUM_GSD_FRAMES) + ' frames.')
#gsdFrameSkip = len(imageCameraPairs) / NUM_GSD_FRAMES
#if gsdFrameSkip < 1:
# gsdFrameSkip = 1
#medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth,
# gsdFrameSkip)
#outputResolution = icebridge_common.gsdToDemRes(medianGsd)
#logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution))
outputResolution = 0.4
# Generate a map of initial camera positions
orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml')
vizString = ''
for (image, camera) in imageCameraPairs:
vizString += camera+' '
cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString
logger.info('Running orbitviz on input files...')
# Suppress (potentially long) output
asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo)
# Set up options for process_icebridge_batch
extraOptions = ''
if options.numThreads:
extraOptions += ' --num-threads ' + str(options.numThreads)
if options.solve_intr:
extraOptions += ' --solve-intrinsics '
if options.isSouth:
extraOptions += ' --south '
if options.maxDisplacement:
extraOptions += ' --max-displacement ' + str(options.maxDisplacement)
if options.fireballFolder:
extraOptions += ' --fireball-folder ' + str(options.fireballFolder)
if options.cleanup:
extraOptions += ' --cleanup '
if options.manyip:
extraOptions += ' --many-ip '
# We ran this before, as part of fetching, so hopefully all the data is cached
forceAllFramesInRange = False
(breaks, largeSkips) = getImageSpacing(options.orthoFolder, availableFrames,
options.startFrame, options.stopFrame,
forceAllFramesInRange)
if options.imageStereoInterval:
logger.info('Using manually specified image stereo interval: ' +
str(options.imageStereoInterval))
largeSkips = [] # Always use a manually specified skip interval
else:
options.imageStereoInterval = 1
sleepTime = 20
# If all we are doing is logging commands then one process is sufficient.
# - Wipe the output file while we are at it.
batchLogPath = ''
batchNum = 0
if options.logBatches:
options.numProcesses = 1
sleepTime = 1
batchLogPath = os.path.join(outputFolder, BATCH_COMMAND_LOG_FILE)
os.system('rm -f ' + batchLogPath)
logger.info('Just generating batch log file '+batchLogPath+', no processing will occur.')
logger.info('Starting processing pool with ' + str(options.numProcesses) +' processes.')
pool = NonDaemonPool(options.numProcesses)
# Call process_icebridge_batch on each batch of images.
# - Batch size should be the largest number of images which can be effectively bundle-adjusted.
taskHandles = []
batchImageCameraPairs = []
frameNumbers = []
i = 0 # The frame index that starts the current batch
while True: # Loop for adding batches
# Bugfix: arrived at the last feasible frame (maybe there are more but
# they lack cameras).
if i >= len(list(imageCameraPairs)):
break
firstBundleFrame = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[i][0])
# Determine the frame skip amount for this batch (set by the first frame)
thisSkipInterval = options.imageStereoInterval
if firstBundleFrame in largeSkips:
#print(" ", firstBundleFrame, " in largeskips!" )
thisSkipInterval = largeSkips[firstBundleFrame]
thisBatchSize = options.bundleLength + thisSkipInterval - 1
# Keep adding frames until we have enough or run out of frames
j = i # The frame index that ends the batch
while True:
if j >= len(imageCameraPairs):
# Bugfix: arrived at the last feasible frame.
break
frameNumber = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[j][0])
# Useful debugging code
#print("Framenumber is ", frameNumber)
#if int(frameNumber) > 8531:
# # pass
# for t in range(8340, 8360):
# print("i frame is ", t, imageCameraPairs[t][0])
# print("breaks are ", breaks)
# sys.exit(1)
# Update conditions
hitBreakFrame = (frameNumber in breaks)
lastFrame = (frameNumber > options.stopFrame) or (j >= numFiles)
endBatch = ( len(frameNumbers) >= thisBatchSize )
if lastFrame or endBatch:
break # The new frame is too much, don't add it to the batch
# Add frame to the list for the current batch
batchImageCameraPairs.append(imageCameraPairs[j])
frameNumbers.append(frameNumber)
if hitBreakFrame:
logger.info("Hit a break, won't start a batch with frame: " + str(frameNumber))
break # Break after this frame, it is the last one added to the batch.
j = j + 1
# Done adding frames to this batch
if len(frameNumbers) <= thisSkipInterval:
logger.info('Batch from frame: ' + str(firstBundleFrame) +
' is too small to run. Skipping.')
else:
# Submit the batch
if not options.logBatches:
logger.info('Processing frame number: ' + str(firstBundleFrame))
# The output folder is named after the first and last frame in the batch.
# We count on this convention in blend_dems.py.
batchFolderName = icebridge_common.batchFolderName(frameNumbers[0], frameNumbers[-1],
options.bundleLength)
thisOutputFolder = os.path.join(outputFolder, batchFolderName)
if not options.logBatches:
logger.info('Running processing batch in output folder: ' + \
thisOutputFolder + '\n' + 'with options: ' + \
extraOptions + ' --stereo-arguments ' + \
options.stereoArgs)
if not options.dryRun:
# Generate the command call
taskHandles.append(pool.apply_async(processBatch,
(batchImageCameraPairs, lidarFolder, options.referenceDem, thisSkipInterval,
thisOutputFolder, extraOptions,
outputResolution, options.stereoArgs, batchNum, batchLogPath)))
batchNum += 1
# Reset these lists
batchImageCameraPairs = []
frameNumbers = []
# Advance to the frame that starts the next batch
if hitBreakFrame:
# When we hit a break in the frames we need to start the
# next batch after the break frame
#print("Hit break frame for i, j, frameNumber", i, j, frameNumber)
i = j + 1
else:
# Start in the next frame that was not used as a "left" stereo image.
i = i + options.bundleLength - 1
if lastFrame:
break # Quit the main loop if we hit the end of the frame list.
# End of loop through input file pairs
logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.')
# Wait for all the tasks to complete
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, options.interactive,
quitKey='q', sleepTime=sleepTime)
# Either all the tasks are finished or the user requested a cancel.
# Clean up the processing pool
icebridge_common.stopTaskPool(pool)
logger.info('Finished process_icebridge_run.') # to avoid ending a log with 'waiting ...'
def main(argsIn):
try:
# Sample usage:
# python full_processing_script.py \
# --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \
# --start-frame 350 --stop-frame 353 --skip-validate \
# --camera-calibration-folder camera_calib \
# --reference-dem-folder ref_dem_folder
# An output folder will be crated automatically (with a name like
# AN_20091016), or its name can be specified via the --output-folder
# option.
usage = '''full_processing_script.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None,
help="The file to use to find which camera was used for which " + \
"flight. By default it is in the same directory as this script " + \
"and named camera_lookup.txt.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
# TODO: Compute this automatically??
parser.add_argument('--overlap-limit', dest='overlapLimit', default=2,
type=int, help="The number of images to treat as overlapping for " + \
"bundle adjustment.")
parser.add_argument(
'--max-overlap-ratio',
dest='maxOverlapRatio',
default=0.85,
type=float,
help=
'The maximum ratio of overlap between images to be accepted as part of a stereo pair. When floating intrinsics, this will be set to 1, to not upset some bookkeeping.'
)
parser.add_argument(
'--stereo-arguments',
dest='stereoArgs',
# set --min-xcorr-level 0 to do the left-to-right
# and right-to-left consistency check at the lowest level.
default='--stereo-algorithm 2 --min-xcorr-level 0',
help='Extra arguments to pass to stereo.')
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
parser.add_argument(
'--frames-file',
dest='framesFile',
default="",
help=
'Specific frames to run ortho2pinhole on within this frame range.')
parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None,
type=int, help="The maximum number of lidar files to fetch. " + \
"This is used in debugging.")
parser.add_argument("--camera-calibration-folder",
dest="inputCalFolder",
default=None,
help="The folder containing camera calibration.")
parser.add_argument(
"--input-calibration-camera",
dest="inputCalCamera",
default="",
help=
"Instead of looking up the calibrated camera in the calibration folder, use this one."
)
parser.add_argument(
"--output-calibration-camera",
dest="outputCalCamera",
default="",
help=
"If specified, float the intrinsics and write the optimized model here."
)
parser.add_argument(
"--output-model-type",
dest="outputModelType",
default="RPC",
help="Generate a distortion model of type RPC, RPC5, or RPC6.")
parser.add_argument(
"--reference-dem-folder",
dest="refDemFolder",
default=None,
help="The folder containing DEMs that created orthoimages.")
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. " + \
"fault is no additional folder")
parser.add_argument(
"--simple-cameras",
action="store_true",
dest="simpleCameras",
default=False,
help="Don't use orthoimages to refine the camera models.")
# This option is only needed when generating camera models from the nav files.
parser.add_argument(
'--camera-mounting',
default=0,
dest='cameraMounting',
type=int,
help=
'0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.'
)
# Performance options
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=1,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument(
'--num-ortho-processes',
dest='numOrthoProcesses',
default=-1,
type=int,
help='The number of simultaneous ortho processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=8,
type=int,
help='The number of threads per process.')
# Action control
parser.add_argument("--skip-fetch",
action="store_true",
dest="noFetch",
default=False,
help="Skip data fetching.")
parser.add_argument("--skip-convert",
action="store_true",
dest="noConvert",
default=False,
help="Skip data conversion.")
parser.add_argument("--stop-after-fetch",
action="store_true",
dest="stopAfterFetch",
default=False,
help="Stop program after data fetching.")
parser.add_argument("--stop-after-convert",
action="store_true",
dest="stopAfterConvert",
default=False,
help="Stop program after data conversion.")
parser.add_argument("--skip-validate",
action="store_true",
dest="skipValidate",
default=False,
help="Skip input data validation.")
parser.add_argument("--ignore-missing-lidar",
action="store_true",
dest="ignoreMissingLidar",
default=False,
help="Keep going if the lidar is missing.")
parser.add_argument(
"--log-batches",
action="store_true",
dest="logBatches",
default=False,
help="Log the required batch commands without running them.")
parser.add_argument(
'--cleanup',
action='store_true',
default=False,
dest='cleanup',
help='If the final result is produced delete intermediate files.')
parser.add_argument(
'--many-ip',
action='store_true',
default=False,
dest='manyip',
help=
'If to use a lot of IP in bundle adjustment from the beginning.')
parser.add_argument(
"--dry-run",
action="store_true",
dest="dryRun",
default=False,
help=
"Set up the input directories but do not fetch/process any imagery."
)
parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False,
help="Try fetching again if some files turned out invalid " + \
"during conversions.")
parser.add_argument("--refetch-index",
action="store_true",
dest="refetchIndex",
default=False,
help="Force refetch of the index file.")
parser.add_argument("--refetch-nav",
action="store_true",
dest="refetchNav",
default=False,
help="Force refetch of the nav file.")
parser.add_argument("--stop-after-index-fetch",
action="store_true",
dest="stopAfterIndexFetch",
default=False,
help="Stop after fetching the indices.")
parser.add_argument("--no-nav",
action="store_true",
dest="noNavFetch",
default=False,
help="Don't fetch or convert the nav data.")
parser.add_argument("--no-lidar-convert",
action="store_true",
dest="noLidarConvert",
default=False,
help="Skip lidar files in the conversion step.")
parser.add_argument(
"--no-ortho-convert",
action="store_true",
dest="noOrthoConvert",
default=False,
help="Skip generating camera models in the conversion step.")
parser.add_argument("--skip-fast-conversions",
action="store_true",
dest="skipFastConvert",
default=False,
help="Skips all non-ortho conversions.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if options.numOrthoProcesses < 0:
options.numOrthoProcesses = options.numProcesses
isSouth = icebridge_common.checkSite(options.site)
# Turned off elevation limits here since they are being set from LIDAR data.
## Add the site based elevation limits to the stereoArgs option
#altLimits = icebridge_common.getElevationLimits(options.site)
#options.stereoArgs = (' %s --elevation-limit %f %f '
# % (options.stereoArgs, altLimits[0], altLimits[1]))
options.stereoArgs = (' %s ' % (options.stereoArgs))
if options.cameraLookupFile is None:
options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt')
if not os.path.isfile(options.cameraLookupFile):
raise Exception("Can't find camera file: " + options.cameraLookupFile)
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
if options.stopAfterIndexFetch:
options.stopAfterFetch = True
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Record everything
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_processing_log_frames_' + \
str(options.startFrame) + "_" + str(options.stopFrame))
# Make sure we later know what we were doing
logger.info("full_processing_script.py " + " ".join(argsIn))
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
logger.info("Work dir is " + os.getcwd())
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
# Perform some input checks and initializations
# These are not needed unless cameras are initialized
if options.inputCalFolder is None or not os.path.exists(
options.inputCalFolder):
raise Exception("Missing camera calibration folder.")
if options.refDemFolder is None or not os.path.exists(
options.refDemFolder):
raise Exception("Missing reference DEM folder.")
refDemName = icebridge_common.getReferenceDemName(options.site)
refDemPath = os.path.join(options.refDemFolder, refDemName)
if not os.path.exists(refDemPath):
raise Exception("Missing reference DEM: " + refDemPath)
# TODO: CLEAN UP!!!
# Set up the output folders
cameraFolder = icebridge_common.getCameraFolder(options.outputFolder)
imageFolder = icebridge_common.getImageFolder(options.outputFolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
fireballFolder = icebridge_common.getFireballFolder(options.outputFolder)
corrFireballFolder = icebridge_common.getCorrFireballFolder(
options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
navFolder = icebridge_common.getNavFolder(options.outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder)
processedFolder = icebridge_common.getProcessedFolder(options.outputFolder)
if options.outputCalCamera != "":
if options.maxOverlapRatio < 1:
raise Exception ("For optimizing intrinsics, must set --max-overlap-ratio to 1, " + \
"to always use consecutive frames.")
# Prepare to solve for intrinsics. Note that this modifies some things along the way.
(options, cameraFolder, navCameraFolder, processedFolder) = \
solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder,
processedFolder, logger)
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processedFolder = os.path.join(processedFolder,
options.processingSubfolder)
logger.info('Will write to processing subfolder: ' +
options.processingSubfolder)
# If something failed in the first attempt either in fetch or in
# convert, we will wipe bad files, and try to refetch/re-convert.
numAttempts = 1
if options.reFetch and (not options.noFetch):
numAttempts = 2
for attempt in range(numAttempts):
if numAttempts > 1:
logger.info("Fetch/convert attempt: " + str(attempt + 1))
ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder,
jpegFolder, orthoFolder, fireballFolder,
corrFireballFolder, lidarFolder, processedFolder,
navFolder, navCameraFolder, refDemPath, logger)
if ans == 0:
break
if options.stopAfterFetch or options.dryRun or options.stopAfterConvert:
logger.info('Fetch/convert finished!')
return 0
# Call the processing routine
processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder,
corrFireballFolder, processedFolder, isSouth, refDemPath)
if options.outputCalCamera != "":
# Finish solving for intrinscs.
solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder,
orthoFolder, processedFolder, isSouth, logger)
threadText = ''
if options.numThreads:
threadText = ' --threads ' + str(options.numThreads) + ' '
# BUNDLE_ADJUST
if options.bundleAdjust:
# TODO: Solve for intrinsics?
bundlePrefix = os.path.join(outputFolder, 'bundle/out')
cmd = (
'bundle_adjust %s %s %s %s -o %s %s -t nadirpinhole --local-pinhole'
% (imageA, imageB, cameraA, cameraB, bundlePrefix, threadText))
print(cmd)
# Point to the new camera models
cameraA = bundlePrefix + '-' + os.path.basename(cameraA)
cameraB = bundlePrefix + '-' + os.path.basename(cameraB)
asp_system_utils.executeCommand(cmd, cameraA, suppressOutput, redo)
# Update the baseArgString
baseArgString = (
'%s %s %s %s %s -t nadirpinhole --alignment-method epipolar' %
(imageA, imageB, cameraA, cameraB, outputPrefix))
if options.use_sgm:
# PPRC
cmd = ('stereo_pprc %s %s' % (baseArgString, threadText))
pprcOutput = outputPrefix + '-L.tif'
asp_system_utils.executeCommand(cmd, pprcOutput, suppressOutput, redo)
# CORR
# - This should be single threaded to use the SGM processing.
# - TODO: Use blob filtering to reduce outliers?
if baOverlapLimit < MIN_BA_OVERLAP:
baOverlapLimit = MIN_BA_OVERLAP
cmd = (
'bundle_adjust %s -o %s %s --camera-weight %d -t nadirpinhole --local-pinhole --overlap-limit %d'
% (imageCameraString, bundlePrefix, threadText, CAMERA_WEIGHT,
baOverlapLimit))
if options.solve_intr:
cmd += ' --solve-intrinsics'
# Point to the new camera models
for pair in inputPairs:
newCamera = bundlePrefix + '-' + os.path.basename(pair[1])
pair[1] = newCamera
imageCameraString.replace(pair[1], newCamera)
# Run the BA command
asp_system_utils.executeCommand(cmd, newCamera, suppressOutput, redo)
# Generate a map of initial camera positions
orbitvizAfter = os.path.join(outputFolder, 'cameras_out.kml')
vizString = ''
for (image, camera) in inputPairs:
vizString += image + ' ' + camera + ' '
cmd = 'orbitviz --hide-labels -t nadirpinhole -r wgs84 -o ' + \
orbitvizAfter +' '+ vizString
asp_system_utils.executeCommand(cmd, orbitvizAfter, suppressOutput, redo)
# STEREO
# Call stereo seperately on each pair of cameras
demFiles = []