def main(argsIn):
try:
usage = '''label_image.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument("--training", dest="trainingPath", required=True,
help="Path to the training file.")
parser.add_argument('--num-processes', dest='numProcesses', default=8,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=1,
type=int, help='IGNORED.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
if not os.path.exists(options.trainingPath):
print 'Error: Input training file ' + options.trainingPath + ' does not exist!'
return -1
# TODO: Everything should use the RunHelper class for this!
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
# Input is raw jpeg files.
inputFolder = icebridge_common.getJpegFolder(options.outputFolder)
# Write all tool output to this folder.
outputFolder = icebridge_common.getLabelFolder(options.outputFolder)
# Do the work
label_images(inputFolder, outputFolder, options.startFrame, options.stopFrame,
options.trainingPath, options.numProcesses)
def getFrameRange(self):
'''Return the min and max frame currently stored for the run'''
jpegFolder = icebridge_common.getJpegFolder(self.getFolder())
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".")
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
frames = sorted(jpegFrameDict.keys())
if len(frames) == 0:
raise Exception("Empty folder: " + jpegFolder)
return (frames[0], frames[-1])
def massRename(self, startFrame, stopFrame, logger):
'''We changed how the timestamp for images and cameras is computed.
Make all existing converted images, cameras, nav cameras, and aligned cameras conform.'''
logger.info("Renaming files with timestamp. This is slow.")
# Need to do a mass rename for incorrect timestamp for:
# converted images, nav cameras, cameras, and bundle aligned cameras
outputFolder = self.getFolder()
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
imageFolder = icebridge_common.getImageFolder(outputFolder)
jpegFolder = icebridge_common.getJpegFolder(outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
processedFolder = icebridge_common.getProcessedFolder(outputFolder)
navFolder = icebridge_common.getNavFolder(outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(
os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " +
orthoIndexPath + ".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder=True)
logger.info('Renaming camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(cameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming nav camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(navCameraFolder, '*DMS*tsai'),
logger)
logger.info('Renaming converted images...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(imageFolder, '*DMS*tif'), logger)
logger.info('Renaming aligned cameras...')
self.massRenameByGlob(
startFrame, stopFrame, orthoFrameDict,
os.path.join(processedFolder, 'batch*',
icebridge_common.alignedBundleStr() + '*DMS*tsai'),
logger)
def getFrameRange(self):
'''Return the min and max frame currently stored for the run'''
jpegFolder = icebridge_common.getJpegFolder(self.getFolder())
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " +
jpegIndexPath + ".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
frames = sorted(jpegFrameDict.keys())
if len(frames) == 0:
raise Exception("Empty folder: " + jpegFolder)
return (frames[0], frames[-1])
def massRename(self, startFrame, stopFrame, logger):
'''We changed how the timestamp for images and cameras is computed.
Make all existing converted images, cameras, nav cameras, and aligned cameras conform.'''
logger.info("Renaming files with timestamp. This is slow.")
# Need to do a mass rename for incorrect timestamp for:
# converted images, nav cameras, cameras, and bundle aligned cameras
outputFolder = self.getFolder()
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
imageFolder = icebridge_common.getImageFolder(outputFolder)
jpegFolder = icebridge_common.getJpegFolder(outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
processedFolder = icebridge_common.getProcessedFolder(outputFolder)
navFolder = icebridge_common.getNavFolder(outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder = True)
logger.info('Renaming camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(cameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming nav camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(navCameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming converted images...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(imageFolder, '*DMS*tif'), logger)
logger.info('Renaming aligned cameras...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(processedFolder, 'batch*',
icebridge_common.alignedBundleStr() + '*DMS*tsai'),
logger)
def label_images(outputFolder, frameNum, trainingPath, site):
'''Apply the labeling algorithm to a single image, then map project the result.'''
# Get required paths
inputFolder = icebridge_common.getJpegFolder (options.outputFolder)
cameraFolder = icebridge_common.getCameraFolder (options.outputFolder)
labelFolder = icebridge_common.getLabelFolder (options.outputFolder)
orthoFolder = icebridge_common.getLabelOrthoFolder(options.outputFolder)
# Run the label tool
toolPath = 'python ~/repo/OSSP/ossp_process.py'
NO_SPLITTING = 1 # Plenty of RAM to load these images
ONE_PROCESS = 1
cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' %
(toolPath, inputFolder, labelFolder, frameNum, trainingPath, NO_SPLITTING, ONE_PROCESS))
print cmd
os.system(cmd)
# Also generate the map projected version of the image
# Figure out the camera and output path
(labelPath, cameraPath) = get_label_and_camera(labelFolder, cameraFolder, frameNum)
fname = os.path.basename(labelPath).replace('classified', 'classified_ortho')
mapProjPath = os.path.join(orthoFolder, fname)
# Set map projection parameters
toolPath = 'mapproject' # TODO: Locate
isSouth = (site == 'AN')
srs = projString = icebridge_common.getEpsgCode(isSouth, asString=True)
demPath = 'WGS84' # Map project on to a flat surface
# Mapproject
cmd = ('%s %s %s %s %s -t nadirpinhole -t_srs %s' %
(toolPath, demPath, labelPath, cameraPath, mapProjPath srs))
print cmd
os.system(cmd)
def label_images(outputFolder, frameNum, trainingPath, site, yyyymmdd,
numThreads):
'''Apply the labeling algorithm to a single image, then map project the result.'''
print 'Running label for frame: ' + str(frameNum)
# Get required paths
inputFolder = icebridge_common.getJpegFolder(outputFolder)
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
labelFolder = icebridge_common.getLabelFolder(outputFolder)
orthoFolder = icebridge_common.getLabelOrthoFolder(outputFolder)
# Hardcoded paths!!!
toolPath = 'python ~/repo/OSSP/ossp_process.py'
refDemFolder = '/nobackup/smcmich1/icebridge/reference_dems/'
# Run the label tool
NO_SPLITTING = 1 # Plenty of RAM to load these images
ONE_PROCESS = 1
# Figure out the label path
run = run_helper.RunHelper(site, yyyymmdd)
labelName = icebridge_common.makeLabelFileName(run, frameNum)
labelPath = os.path.join(labelFolder, labelName)
#cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' %
# (toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath, NO_SPLITTING, ONE_PROCESS))
cmd = (
'time %s %s --output_dir %s --min_frame %d --max_frame %d srgb %s' %
(toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath))
print cmd
if icebridge_common.isValidImage(labelPath):
print 'Skipping completed file: ' + labelPath
else:
os.system(cmd)
# Also generate the map projected version of the image
# Figure out the camera and output path
cameraPath = get_camera(cameraFolder, frameNum)
fname = os.path.basename(labelPath).replace('classified',
'classified_ortho')
mapProjPath = os.path.join(orthoFolder, fname)
if not icebridge_common.isValidImage(labelPath):
print 'ERROR: Failed to generate label file: ' + labelPath
return
# Set map projection parameters
toolPath = 'mapproject'
isSouth = (site == 'AN')
srs = projString = icebridge_common.getEpsgCode(isSouth, asString=True)
demPath = 'WGS84' # Map project on to a flat surface, elevation zero.
#demPath = os.path.join(refDemFolder, icebridge_common.getReferenceDemName(site)) # The NSIDC ortho DEM
# Mapproject
cmd = (
'time %s %s %s %s %s -t nadirpinhole --t_srs %s --threads %d --num-processes 1 --ot Byte --nearest-neighbor'
% (toolPath, demPath, labelPath, cameraPath, mapProjPath, srs,
numThreads))
print cmd
if icebridge_common.isValidImage(mapProjPath):
print 'Skipping existing file: ' + mapProjPath
else:
os.system(cmd)
if not os.path.exists(mapProjPath):
print 'ERROR: Failed to generate map projected label file: ' + mapProjPath
return
def label_images(outputFolder, frameNum, trainingPath, site, yyyymmdd, numThreads):
'''Apply the labeling algorithm to a single image, then map project the result.'''
print 'Running label for frame: ' + str(frameNum)
# Get required paths
inputFolder = icebridge_common.getJpegFolder (outputFolder)
cameraFolder = icebridge_common.getCameraFolder (outputFolder)
labelFolder = icebridge_common.getLabelFolder (outputFolder)
orthoFolder = icebridge_common.getLabelOrthoFolder(outputFolder)
# Hardcoded paths!!!
toolPath = 'python ~/repo/OSSP/ossp_process.py'
refDemFolder = '/nobackup/smcmich1/icebridge/reference_dems/'
# Run the label tool
NO_SPLITTING = 1 # Plenty of RAM to load these images
ONE_PROCESS = 1
# Figure out the label path
run = run_helper.RunHelper(site, yyyymmdd)
labelName = icebridge_common.makeLabelFileName(run, frameNum)
labelPath = os.path.join(labelFolder, labelName)
#cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' %
# (toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath, NO_SPLITTING, ONE_PROCESS))
cmd = ('time %s %s --output_dir %s --min_frame %d --max_frame %d srgb %s' %
(toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath))
print cmd
if icebridge_common.isValidImage(labelPath):
print 'Skipping completed file: ' + labelPath
else:
os.system(cmd)
# Also generate the map projected version of the image
# Figure out the camera and output path
cameraPath = get_camera(cameraFolder, frameNum)
fname = os.path.basename(labelPath).replace('classified', 'classified_ortho')
mapProjPath = os.path.join(orthoFolder, fname)
if not icebridge_common.isValidImage(labelPath):
print 'ERROR: Failed to generate label file: ' + labelPath
return
# Set map projection parameters
toolPath = 'mapproject'
isSouth = (site == 'AN')
srs = projString = icebridge_common.getEpsgCode(isSouth, asString=True)
demPath = 'WGS84' # Map project on to a flat surface, elevation zero.
#demPath = os.path.join(refDemFolder, icebridge_common.getReferenceDemName(site)) # The NSIDC ortho DEM
# Mapproject
cmd = ('time %s %s %s %s %s -t nadirpinhole --t_srs %s --threads %d --num-processes 1 --ot Byte --nearest-neighbor' %
(toolPath, demPath, labelPath, cameraPath, mapProjPath, srs, numThreads))
print cmd
if icebridge_common.isValidImage(mapProjPath):
print 'Skipping existing file: ' + mapProjPath
else:
os.system(cmd)
if not os.path.exists(mapProjPath):
print 'ERROR: Failed to generate map projected label file: ' + mapProjPath
return
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
def main(argsIn):
try:
usage = '''label_image.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument(
'--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument("--training",
dest="trainingPath",
required=True,
help="Path to the training file.")
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=8,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=1,
type=int,
help='IGNORED.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
if not os.path.exists(options.trainingPath):
print 'Error: Input training file ' + options.trainingPath + ' does not exist!'
return -1
# TODO: Everything should use the RunHelper class for this!
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
# Input is raw jpeg files.
inputFolder = icebridge_common.getJpegFolder(options.outputFolder)
# Write all tool output to this folder.
outputFolder = icebridge_common.getLabelFolder(options.outputFolder)
# Do the work
label_images(inputFolder, outputFolder, options.startFrame,
options.stopFrame, options.trainingPath, options.numProcesses)
def main(argsIn):
try:
# Sample usage:
# python full_processing_script.py \
# --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \
# --start-frame 350 --stop-frame 353 --skip-validate \
# --camera-calibration-folder camera_calib \
# --reference-dem-folder ref_dem_folder
# An output folder will be crated automatically (with a name like
# AN_20091016), or its name can be specified via the --output-folder
# option.
usage = '''full_processing_script.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None,
help="The file to use to find which camera was used for which " + \
"flight. By default it is in the same directory as this script " + \
"and named camera_lookup.txt.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
# TODO: Compute this automatically??
parser.add_argument('--overlap-limit', dest='overlapLimit', default=2,
type=int, help="The number of images to treat as overlapping for " + \
"bundle adjustment.")
parser.add_argument('--stereo-arguments', dest='stereoArgs',
# set --min-xcorr-level 0 to do the left-to-right
# and right-to-left consistency check at the lowest level.
default='--stereo-algorithm 2 --min-xcorr-level 0',
help='Extra arguments to pass to stereo.')
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
parser.add_argument('--frames-file', dest='framesFile', default="",
help='Specific frames to run ortho2pinhole on within this frame range.')
parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None,
type=int, help="The maximum number of lidar files to fetch. " + \
"This is used in debugging.")
parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None,
help="The folder containing camera calibration.")
parser.add_argument("--input-calibration-camera", dest="inputCalCamera", default="",
help="Instead of looking up the calibrated camera in the calibration folder, use this one.")
parser.add_argument("--output-calibration-camera", dest="outputCalCamera", default="",
help="If specified, float the intrinsics and write the optimized model here.")
parser.add_argument("--output-model-type", dest="outputModelType", default="RPC",
help="Generate a distortion model of type RPC, RPC5, or RPC6.")
parser.add_argument("--reference-dem-folder", dest="refDemFolder", default=None,
help="The folder containing DEMs that created orthoimages.")
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. " + \
"fault is no additional folder")
parser.add_argument("--simple-cameras", action="store_true", dest="simpleCameras", default=False,
help="Don't use orthoimages to refine the camera models.")
# This option is only needed when generating camera models from the nav files.
parser.add_argument('--camera-mounting', default=0, dest='cameraMounting', type=int,
help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.')
# Performance options
parser.add_argument('--num-processes', dest='numProcesses', default=1,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-ortho-processes', dest='numOrthoProcesses', default=-1,
type=int, help='The number of simultaneous ortho processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=8,
type=int, help='The number of threads per process.')
# Action control
parser.add_argument("--skip-fetch", action="store_true", dest="noFetch", default=False,
help="Skip data fetching.")
parser.add_argument("--skip-convert", action="store_true", dest="noConvert", default=False,
help="Skip data conversion.")
parser.add_argument("--stop-after-fetch", action="store_true", dest="stopAfterFetch",
default=False,
help="Stop program after data fetching.")
parser.add_argument("--stop-after-convert", action="store_true", dest="stopAfterConvert",
default=False,
help="Stop program after data conversion.")
parser.add_argument("--skip-validate", action="store_true", dest="skipValidate",
default=False,
help="Skip input data validation.")
parser.add_argument("--ignore-missing-lidar", action="store_true", dest="ignoreMissingLidar",
default=False,
help="Keep going if the lidar is missing.")
parser.add_argument("--log-batches", action="store_true", dest="logBatches", default=False,
help="Log the required batch commands without running them.")
parser.add_argument('--cleanup', action='store_true', default=False, dest='cleanup',
help='If the final result is produced delete intermediate files.')
parser.add_argument('--many-ip', action='store_true', default=False, dest='manyip',
help='If to use a lot of IP in bundle adjustment from the beginning.')
parser.add_argument("--dry-run", action="store_true", dest="dryRun", default=False,
help="Set up the input directories but do not fetch/process any imagery.")
parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False,
help="Try fetching again if some files turned out invalid " + \
"during conversions.")
parser.add_argument("--refetch-index", action="store_true", dest="refetchIndex",
default=False,
help="Force refetch of the index file.")
parser.add_argument("--refetch-nav", action="store_true", dest="refetchNav",
default=False,
help="Force refetch of the nav file.")
parser.add_argument("--stop-after-index-fetch", action="store_true",
dest="stopAfterIndexFetch", default=False,
help="Stop after fetching the indices.")
parser.add_argument("--no-nav", action="store_true", dest="noNavFetch",
default=False, help="Don't fetch or convert the nav data.")
parser.add_argument("--no-lidar-convert", action="store_true", dest="noLidarConvert",
default=False,
help="Skip lidar files in the conversion step.")
parser.add_argument("--no-ortho-convert", action="store_true", dest="noOrthoConvert",
default=False,
help="Skip generating camera models in the conversion step.")
parser.add_argument("--skip-fast-conversions", action="store_true", dest="skipFastConvert",
default=False,
help="Skips all non-ortho conversions.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if options.numOrthoProcesses < 0:
options.numOrthoProcesses = options.numProcesses
isSouth = icebridge_common.checkSite(options.site)
# Turned off elevation limits here since they are being set from LIDAR data.
## Add the site based elevation limits to the stereoArgs option
#altLimits = icebridge_common.getElevationLimits(options.site)
#options.stereoArgs = (' %s --elevation-limit %f %f '
# % (options.stereoArgs, altLimits[0], altLimits[1]))
options.stereoArgs = (' %s ' % (options.stereoArgs))
if options.cameraLookupFile is None:
options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt')
if not os.path.isfile(options.cameraLookupFile):
raise Exception("Can't find camera file: " + options.cameraLookupFile)
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
if options.stopAfterIndexFetch:
options.stopAfterFetch = True
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Record everything
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_processing_log_frames_' + \
str(options.startFrame) + "_" + str(options.stopFrame))
# Make sure we later know what we were doing
logger.info("full_processing_script.py " + " ".join(argsIn))
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput = True)
logger.info("Running on machine: " + out)
logger.info("Work dir is " + os.getcwd())
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
# Perform some input checks and initializations
# These are not needed unless cameras are initialized
if options.inputCalFolder is None or not os.path.exists(options.inputCalFolder):
raise Exception("Missing camera calibration folder.")
if options.refDemFolder is None or not os.path.exists(options.refDemFolder):
raise Exception("Missing reference DEM folder.")
refDemName = icebridge_common.getReferenceDemName(options.site)
refDemPath = os.path.join(options.refDemFolder, refDemName)
if not os.path.exists(refDemPath):
raise Exception("Missing reference DEM: " + refDemPath)
# TODO: CLEAN UP!!!
# Set up the output folders
cameraFolder = icebridge_common.getCameraFolder(options.outputFolder)
imageFolder = icebridge_common.getImageFolder(options.outputFolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
fireballFolder = icebridge_common.getFireballFolder(options.outputFolder)
corrFireballFolder = icebridge_common.getCorrFireballFolder(options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
navFolder = icebridge_common.getNavFolder(options.outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder)
processedFolder = icebridge_common.getProcessedFolder(options.outputFolder)
if options.outputCalCamera != "":
# Prepare to solve for intrinsics. Note that this modifies some things along the way.
(options, cameraFolder, navCameraFolder, processedFolder) = \
solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder,
processedFolder, logger)
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processedFolder = os.path.join(processedFolder, options.processingSubfolder)
logger.info('Will write to processing subfolder: ' + options.processingSubfolder)
# If something failed in the first attempt either in fetch or in
# convert, we will wipe bad files, and try to refetch/re-convert.
numAttempts = 1
if options.reFetch and (not options.noFetch):
numAttempts = 2
for attempt in range(numAttempts):
if numAttempts > 1:
logger.info("Fetch/convert attempt: " + str(attempt+1))
ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder, jpegFolder, orthoFolder,
fireballFolder, corrFireballFolder, lidarFolder, processedFolder,
navFolder, navCameraFolder, refDemPath, logger)
if ans == 0:
break
if options.stopAfterFetch or options.dryRun or options.stopAfterConvert:
logger.info('Fetch/convert finished!')
return 0
# Call the processing routine
processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder,
corrFireballFolder, processedFolder,
isSouth, refDemPath)
if options.outputCalCamera != "":
# Finish solving for intrinscs.
solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder,
processedFolder, isSouth, logger)
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder):
'''Create a list of all files that must be fetched unless done already.'''
# For AN 20091112, etc, some of the ortho images are stored at the
# beginning of the next day's flight. Need to sort this out, and
# it is tricky. More comments within the code.
fetchNextDay = True
separateByLat = (options.type == 'ortho'
and isInSeparateByLatTable(options.yyyymmdd))
if separateByLat:
# Here we won't fetch the next day, we will just separate by latitude within
# a given day
fetchNextDay = False
orthoOrFireball = ((options.type == 'ortho')
or (options.type == 'fireball'))
if fetchNextDay:
# Normally we fetch for next day only for ortho or fireball. However,
# for one single special flight, we do it for jpeg too, as then
# the jpegs are also split.
if orthoOrFireball or \
((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)):
fetchNextDay = True
else:
fetchNextDay = False
# If we need to parse the next flight day as well, as expected in some runs,
# we will fetch two html files, but create a single index out of them.
dayVals = [0]
if fetchNextDay:
dayVals.append(1)
indexPath = icebridge_common.htmlIndexFile(outputFolder)
currIndexPath = indexPath
parsedIndexPath = icebridge_common.csvIndexFile(outputFolder)
if options.refetchIndex:
os.system('rm -f ' + indexPath)
os.system('rm -f ' + parsedIndexPath)
if icebridge_common.fileNonEmpty(parsedIndexPath):
logger.info('Already have the index file ' + parsedIndexPath +
', keeping it.')
return parsedIndexPath
frameDict = {}
urlDict = {}
# We need the list of jpeg frames. Sometimes when fetching ortho images,
# and we have to fetch from the next day, don't fetch unless
# in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
jpegFolder = icebridge_common.getJpegFolder(
os.path.dirname(outputFolder))
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
orthoStamp = {}
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
orthoFolder = icebridge_common.getOrthoFolder(
os.path.dirname(outputFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
for frame in sorted(orthoFrameDict.keys()):
filename = orthoFrameDict[frame]
[imageDateString,
imageTimeString] = icebridge_common.parseTimeStamps(filename)
orthoStamp[frame] = imageTimeString
for dayVal in dayVals:
if len(dayVals) > 1:
currIndexPath = indexPath + '.day' + str(dayVal)
if options.refetchIndex:
os.system('rm -f ' + currIndexPath)
# Find folderUrl which contains all of the files
if options.type in LIDAR_TYPES:
options.allFrames = True # For lidar, always get all the frames!
# For lidar, the data can come from one of three sources.
# Unfortunately sometimes there is more than one source, and then
# we need to pick by latitude.
folderUrls = []
lidar_types = []
for lidar in LIDAR_TYPES:
folderUrl = getFolderUrl(
options.yyyymmdd,
options.year,
options.month,
options.day,
dayVal, # note here the dayVal
options.site,
lidar)
logger.info('Checking lidar URL: ' + folderUrl)
if checkIfUrlExists(folderUrl):
logger.info('Found match with lidar type: ' + lidar)
folderUrls.append(folderUrl)
lidar_types.append(lidar)
if len(folderUrls) == 0:
logger.info(
'WARNING: Could not find any lidar data for the given date!'
)
elif len(folderUrls) == 1:
# Unique solution
folderUrl = folderUrls[0]
options.type = lidar_types[0]
elif len(folderUrls) >= 2:
# Multiple solutions. Pick the good one by latitude.
logger.info("Multiples URLs to search: " +
" ".join(folderUrls))
count = -1
isGood = False
for folderUrl in folderUrls:
count += 1
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath,
lidar_types[count])
for frame in sorted(localFrameDict.keys()):
filename = localFrameDict[frame]
xmlFile = icebridge_common.xmlFile(filename)
url = os.path.join(folderUrl, xmlFile)
# Download the file
curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile
logger.info(curlCmd)
p = subprocess.Popen(curlCmd, shell=True)
os.waitpid(p.pid, 0)
latitude = icebridge_common.parseLatitude(xmlFile)
if os.path.exists(xmlFile): os.remove(xmlFile)
if hasGoodLat(latitude, isSouth):
isGood = True
options.type = lidar_types[count]
logger.info("Good latitude " + str(latitude) +
", will use " + folderUrl +
" of type " + lidar_types[count])
else:
logger.info("Bad latitude " + str(latitude) +
", will not use " + folderUrl +
" of type " + lidar_types[count])
# Stop at first file no matter what
break
if isGood:
break
if not isGood:
if options.type in LIDAR_TYPES and options.ignoreMissingLidar:
logger.info("No lidar. None of these URLs are good: " +
" ".join(folderUrls))
else:
raise Exception("None of these URLs are good: " +
" ".join(folderUrls))
else: # Other cases are simpler
folderUrl = getFolderUrl(
options.yyyymmdd,
options.year,
options.month,
options.day,
dayVal, # note here the dayVal
options.site,
options.type)
logger.info('Fetching from URL: ' + folderUrl)
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath, options.type)
# Append to the main index
for frame in sorted(localFrameDict.keys()):
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
# Otherwise we may accidentally getting one from next day.
[imageDateString, imageTimeString] = \
icebridge_common.parseTimeStamps(localFrameDict[frame])
if frame not in orthoStamp:
#logger.info("Missing ortho for fireball: " + localFrameDict[frame])
continue
if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000:
# Apparently a tolerance is needed. Use 10 seconds, so the number 1000.
#logger.info("Will not use fireball DEM whose timestamp differs from ortho.")
#logger.info("Fireball is: " + localFrameDict[frame])
#logger.info("Ortho is: " + orthoFrameDict[frame])
continue
# Fetch from next day, unless already have a value. And don't fetch
# frames not in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
if not frame in jpegFrameDict.keys(): continue
if frame in frameDict.keys(): continue
frameDict[frame] = localFrameDict[frame]
urlDict[frame] = localUrlDict[frame]
# Write the combined index file
icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict)
return parsedIndexPath
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 3.
usage = '''gen_ortho.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument(
'--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument(
'--camera-mounting',
dest='cameraMounting',
default=0,
type=int,
help=
'0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.'
)
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
# Performance options
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=1,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=8,
type=int,
help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(jpegFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No jpeg files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in jpegFrameDict:
logger.info("Error: Missing jpeg file for frame: " + str(frame) +
".")
continue
# Find the right image
currImage = jpegFrameDict[frame]
args = (frame, processFolder, currImage, options.bundleLength,
options.cameraMounting, threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runOrtho, args))
else:
runOrtho(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
logger,
interactive=False,
quitKey='q',
sleepTime=20)
icebridge_common.stopTaskPool(pool)
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder):
'''Create a list of all files that must be fetched unless done already.'''
# For AN 20091112, etc, some of the ortho images are stored at the
# beginning of the next day's flight. Need to sort this out, and
# it is tricky. More comments within the code.
fetchNextDay = True
separateByLat = (options.type == 'ortho' and isInSeparateByLatTable(options.yyyymmdd))
if separateByLat:
# Here we won't fetch the next day, we will just separate by latitude within
# a given day
fetchNextDay = False
orthoOrFireball = ( (options.type == 'ortho') or (options.type == 'fireball') )
if fetchNextDay:
# Normally we fetch for next day only for ortho or fireball. However,
# for one single special flight, we do it for jpeg too, as then
# the jpegs are also split.
if orthoOrFireball or \
((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)):
fetchNextDay = True
else:
fetchNextDay = False
# If we need to parse the next flight day as well, as expected in some runs,
# we will fetch two html files, but create a single index out of them.
dayVals = [0]
if fetchNextDay:
dayVals.append(1)
indexPath = icebridge_common.htmlIndexFile(outputFolder)
currIndexPath = indexPath
parsedIndexPath = icebridge_common.csvIndexFile(outputFolder)
if options.refetchIndex:
os.system('rm -f ' + indexPath)
os.system('rm -f ' + parsedIndexPath)
if icebridge_common.fileNonEmpty(parsedIndexPath):
logger.info('Already have the index file ' + parsedIndexPath + ', keeping it.')
return parsedIndexPath
frameDict = {}
urlDict = {}
# We need the list of jpeg frames. Sometimes when fetching ortho images,
# and we have to fetch from the next day, don't fetch unless
# in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
jpegFolder = icebridge_common.getJpegFolder(os.path.dirname(outputFolder))
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
orthoStamp = {}
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(outputFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
for frame in sorted(orthoFrameDict.keys()):
filename = orthoFrameDict[frame]
[imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename)
orthoStamp[frame] = imageTimeString
for dayVal in dayVals:
if len(dayVals) > 1:
currIndexPath = indexPath + '.day' + str(dayVal)
if options.refetchIndex:
os.system('rm -f ' + currIndexPath)
# Find folderUrl which contains all of the files
if options.type in LIDAR_TYPES:
options.allFrames = True # For lidar, always get all the frames!
# For lidar, the data can come from one of three sources.
# Unfortunately sometimes there is more than one source, and then
# we need to pick by latitude.
folderUrls = []
lidar_types = []
for lidar in LIDAR_TYPES:
folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month,
options.day, dayVal, # note here the dayVal
options.site, lidar)
logger.info('Checking lidar URL: ' + folderUrl)
if checkIfUrlExists(folderUrl, baseCurlCmd):
logger.info('Found match with lidar type: ' + lidar)
folderUrls.append(folderUrl)
lidar_types.append(lidar)
if len(folderUrls) == 0:
logger.info('WARNING: Could not find any lidar data for the given date!')
elif len(folderUrls) == 1:
# Unique solution
folderUrl = folderUrls[0]
options.type = lidar_types[0]
elif len(folderUrls) >= 2:
# Multiple solutions. Pick the good one by latitude.
logger.info("Multiples URLs to search: " + " ".join(folderUrls))
count = -1
isGood = False
for folderUrl in folderUrls:
count += 1
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath,
lidar_types[count])
for frame in sorted(localFrameDict.keys()):
filename = localFrameDict[frame]
xmlFile = icebridge_common.xmlFile(filename)
url = os.path.join(folderUrl, xmlFile)
# Download the file
curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile
logger.info(curlCmd)
p = subprocess.Popen(curlCmd, shell=True, universal_newlines=True)
os.waitpid(p.pid, 0)
latitude = icebridge_common.parseLatitude(xmlFile)
if os.path.exists(xmlFile): os.remove(xmlFile)
if hasGoodLat(latitude, isSouth):
isGood = True
options.type = lidar_types[count]
logger.info("Good latitude " + str(latitude) + ", will use " +
folderUrl + " of type " + lidar_types[count])
else:
logger.info("Bad latitude " + str(latitude) + ", will not use " +
folderUrl + " of type " + lidar_types[count])
# Stop at first file no matter what
break
if isGood:
break
if not isGood:
if options.type in LIDAR_TYPES and options.ignoreMissingLidar:
logger.info("No lidar. None of these URLs are good: " +
" ".join(folderUrls))
else:
raise Exception("None of these URLs are good: " +
" ".join(folderUrls))
else: # Other cases are simpler
folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month,
options.day, dayVal, # note here the dayVal
options.site, options.type)
logger.info('Fetching from URL: ' + folderUrl)
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath, options.type)
# Append to the main index
for frame in sorted(localFrameDict.keys()):
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
# Otherwise we may accidentally getting one from next day.
[imageDateString, imageTimeString] = \
icebridge_common.parseTimeStamps(localFrameDict[frame])
if frame not in orthoStamp:
#logger.info("Missing ortho for fireball: " + localFrameDict[frame])
continue
if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000:
# Apparently a tolerance is needed. Use 10 seconds, so the number 1000.
#logger.info("Will not use fireball DEM whose timestamp differs from ortho.")
#logger.info("Fireball is: " + localFrameDict[frame])
#logger.info("Ortho is: " + orthoFrameDict[frame])
continue
# Fetch from next day, unless already have a value. And don't fetch
# frames not in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
if not frame in jpegFrameDict.keys(): continue
if frame in frameDict.keys(): continue
frameDict[frame] = localFrameDict[frame]
urlDict[frame] = localUrlDict[frame]
# Write the combined index file
icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict)
return parsedIndexPath
def main(argsIn):
try:
# Sample usage:
# python full_processing_script.py \
# --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \
# --start-frame 350 --stop-frame 353 --skip-validate \
# --camera-calibration-folder camera_calib \
# --reference-dem-folder ref_dem_folder
# An output folder will be crated automatically (with a name like
# AN_20091016), or its name can be specified via the --output-folder
# option.
usage = '''full_processing_script.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None,
help="The file to use to find which camera was used for which " + \
"flight. By default it is in the same directory as this script " + \
"and named camera_lookup.txt.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
# TODO: Compute this automatically??
parser.add_argument('--overlap-limit', dest='overlapLimit', default=2,
type=int, help="The number of images to treat as overlapping for " + \
"bundle adjustment.")
parser.add_argument(
'--max-overlap-ratio',
dest='maxOverlapRatio',
default=0.85,
type=float,
help=
'The maximum ratio of overlap between images to be accepted as part of a stereo pair. When floating intrinsics, this will be set to 1, to not upset some bookkeeping.'
)
parser.add_argument(
'--stereo-arguments',
dest='stereoArgs',
# set --min-xcorr-level 0 to do the left-to-right
# and right-to-left consistency check at the lowest level.
default='--stereo-algorithm 2 --min-xcorr-level 0',
help='Extra arguments to pass to stereo.')
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
parser.add_argument(
'--frames-file',
dest='framesFile',
default="",
help=
'Specific frames to run ortho2pinhole on within this frame range.')
parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None,
type=int, help="The maximum number of lidar files to fetch. " + \
"This is used in debugging.")
parser.add_argument("--camera-calibration-folder",
dest="inputCalFolder",
default=None,
help="The folder containing camera calibration.")
parser.add_argument(
"--input-calibration-camera",
dest="inputCalCamera",
default="",
help=
"Instead of looking up the calibrated camera in the calibration folder, use this one."
)
parser.add_argument(
"--output-calibration-camera",
dest="outputCalCamera",
default="",
help=
"If specified, float the intrinsics and write the optimized model here."
)
parser.add_argument(
"--output-model-type",
dest="outputModelType",
default="RPC",
help="Generate a distortion model of type RPC, RPC5, or RPC6.")
parser.add_argument(
"--reference-dem-folder",
dest="refDemFolder",
default=None,
help="The folder containing DEMs that created orthoimages.")
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. " + \
"fault is no additional folder")
parser.add_argument(
"--simple-cameras",
action="store_true",
dest="simpleCameras",
default=False,
help="Don't use orthoimages to refine the camera models.")
# This option is only needed when generating camera models from the nav files.
parser.add_argument(
'--camera-mounting',
default=0,
dest='cameraMounting',
type=int,
help=
'0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.'
)
# Performance options
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=1,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument(
'--num-ortho-processes',
dest='numOrthoProcesses',
default=-1,
type=int,
help='The number of simultaneous ortho processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=8,
type=int,
help='The number of threads per process.')
# Action control
parser.add_argument("--skip-fetch",
action="store_true",
dest="noFetch",
default=False,
help="Skip data fetching.")
parser.add_argument("--skip-convert",
action="store_true",
dest="noConvert",
default=False,
help="Skip data conversion.")
parser.add_argument("--stop-after-fetch",
action="store_true",
dest="stopAfterFetch",
default=False,
help="Stop program after data fetching.")
parser.add_argument("--stop-after-convert",
action="store_true",
dest="stopAfterConvert",
default=False,
help="Stop program after data conversion.")
parser.add_argument("--skip-validate",
action="store_true",
dest="skipValidate",
default=False,
help="Skip input data validation.")
parser.add_argument("--ignore-missing-lidar",
action="store_true",
dest="ignoreMissingLidar",
default=False,
help="Keep going if the lidar is missing.")
parser.add_argument(
"--log-batches",
action="store_true",
dest="logBatches",
default=False,
help="Log the required batch commands without running them.")
parser.add_argument(
'--cleanup',
action='store_true',
default=False,
dest='cleanup',
help='If the final result is produced delete intermediate files.')
parser.add_argument(
'--many-ip',
action='store_true',
default=False,
dest='manyip',
help=
'If to use a lot of IP in bundle adjustment from the beginning.')
parser.add_argument(
"--dry-run",
action="store_true",
dest="dryRun",
default=False,
help=
"Set up the input directories but do not fetch/process any imagery."
)
parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False,
help="Try fetching again if some files turned out invalid " + \
"during conversions.")
parser.add_argument("--refetch-index",
action="store_true",
dest="refetchIndex",
default=False,
help="Force refetch of the index file.")
parser.add_argument("--refetch-nav",
action="store_true",
dest="refetchNav",
default=False,
help="Force refetch of the nav file.")
parser.add_argument("--stop-after-index-fetch",
action="store_true",
dest="stopAfterIndexFetch",
default=False,
help="Stop after fetching the indices.")
parser.add_argument("--no-nav",
action="store_true",
dest="noNavFetch",
default=False,
help="Don't fetch or convert the nav data.")
parser.add_argument("--no-lidar-convert",
action="store_true",
dest="noLidarConvert",
default=False,
help="Skip lidar files in the conversion step.")
parser.add_argument(
"--no-ortho-convert",
action="store_true",
dest="noOrthoConvert",
default=False,
help="Skip generating camera models in the conversion step.")
parser.add_argument("--skip-fast-conversions",
action="store_true",
dest="skipFastConvert",
default=False,
help="Skips all non-ortho conversions.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if options.numOrthoProcesses < 0:
options.numOrthoProcesses = options.numProcesses
isSouth = icebridge_common.checkSite(options.site)
# Turned off elevation limits here since they are being set from LIDAR data.
## Add the site based elevation limits to the stereoArgs option
#altLimits = icebridge_common.getElevationLimits(options.site)
#options.stereoArgs = (' %s --elevation-limit %f %f '
# % (options.stereoArgs, altLimits[0], altLimits[1]))
options.stereoArgs = (' %s ' % (options.stereoArgs))
if options.cameraLookupFile is None:
options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt')
if not os.path.isfile(options.cameraLookupFile):
raise Exception("Can't find camera file: " + options.cameraLookupFile)
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
if options.stopAfterIndexFetch:
options.stopAfterFetch = True
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Record everything
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_processing_log_frames_' + \
str(options.startFrame) + "_" + str(options.stopFrame))
# Make sure we later know what we were doing
logger.info("full_processing_script.py " + " ".join(argsIn))
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
logger.info("Work dir is " + os.getcwd())
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
# Perform some input checks and initializations
# These are not needed unless cameras are initialized
if options.inputCalFolder is None or not os.path.exists(
options.inputCalFolder):
raise Exception("Missing camera calibration folder.")
if options.refDemFolder is None or not os.path.exists(
options.refDemFolder):
raise Exception("Missing reference DEM folder.")
refDemName = icebridge_common.getReferenceDemName(options.site)
refDemPath = os.path.join(options.refDemFolder, refDemName)
if not os.path.exists(refDemPath):
raise Exception("Missing reference DEM: " + refDemPath)
# TODO: CLEAN UP!!!
# Set up the output folders
cameraFolder = icebridge_common.getCameraFolder(options.outputFolder)
imageFolder = icebridge_common.getImageFolder(options.outputFolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
fireballFolder = icebridge_common.getFireballFolder(options.outputFolder)
corrFireballFolder = icebridge_common.getCorrFireballFolder(
options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
navFolder = icebridge_common.getNavFolder(options.outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder)
processedFolder = icebridge_common.getProcessedFolder(options.outputFolder)
if options.outputCalCamera != "":
if options.maxOverlapRatio < 1:
raise Exception ("For optimizing intrinsics, must set --max-overlap-ratio to 1, " + \
"to always use consecutive frames.")
# Prepare to solve for intrinsics. Note that this modifies some things along the way.
(options, cameraFolder, navCameraFolder, processedFolder) = \
solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder,
processedFolder, logger)
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processedFolder = os.path.join(processedFolder,
options.processingSubfolder)
logger.info('Will write to processing subfolder: ' +
options.processingSubfolder)
# If something failed in the first attempt either in fetch or in
# convert, we will wipe bad files, and try to refetch/re-convert.
numAttempts = 1
if options.reFetch and (not options.noFetch):
numAttempts = 2
for attempt in range(numAttempts):
if numAttempts > 1:
logger.info("Fetch/convert attempt: " + str(attempt + 1))
ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder,
jpegFolder, orthoFolder, fireballFolder,
corrFireballFolder, lidarFolder, processedFolder,
navFolder, navCameraFolder, refDemPath, logger)
if ans == 0:
break
if options.stopAfterFetch or options.dryRun or options.stopAfterConvert:
logger.info('Fetch/convert finished!')
return 0
# Call the processing routine
processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder,
corrFireballFolder, processedFolder, isSouth, refDemPath)
if options.outputCalCamera != "":
# Finish solving for intrinscs.
solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder,
orthoFolder, processedFolder, isSouth, logger)
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 3.
usage = '''gen_ortho.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument('--camera-mounting', dest='cameraMounting', default=0, type=int,
help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.')
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
# Performance options
parser.add_argument('--num-processes', dest='numProcesses', default=1,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=8,
type=int, help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput = True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder, options.processingSubfolder)
logger.info('Reading from processing subfolder: ' + options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".")
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder = True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(jpegFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No jpeg files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in jpegFrameDict:
logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".")
continue
# Find the right image
currImage = jpegFrameDict[frame]
args = (frame, processFolder, currImage, options.bundleLength,
options.cameraMounting, threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runOrtho, args))
else:
runOrtho(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False,
quitKey='q', sleepTime=20)
icebridge_common.stopTaskPool(pool)
