def lidarFilesInRange(lidarDict, lidarFolder, startFrame, stopFrame):
'''Fetch only lidar files for the given frame range. Do that as follows.
For each ortho frame in [startFrame, stopFrame], find the lidar
file with the closest timestamp. Collect them all.
Add the two neighboring ones, to help with finding lidar pairs later.'''
lidarList = []
for frame in sorted(lidarDict.keys()):
lidarList.append(lidarDict[frame])
# If we requested all frames, also get all the lidar files.
if ((startFrame == icebridge_common.getSmallestFrame())
and (stopFrame == icebridge_common.getLargestFrame())):
minLidarIndex = 0
maxLidarIndex = len(lidarList) - 1
else:
minLidarIndex = len(lidarList)
maxLidarIndex = 0
# Build up a list of lidar files that match the requested input frames
orthoFolder = icebridge_common.getOrthoFolder(
os.path.dirname(lidarFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
for frame in sorted(orthoFrameDict.keys()):
if ((frame < startFrame) or (frame > stopFrame)): continue
orthoFrame = orthoFrameDict[frame]
try:
matchingLidar = icebridge_common.findMatchingLidarFileFromList(
orthoFrame, lidarList)
except:
# Some image files don't have a matching lidar file, just keep going.
continue
for index in range(len(lidarList)):
if lidarList[index] == matchingLidar:
if minLidarIndex > index:
minLidarIndex = index
if maxLidarIndex < index:
maxLidarIndex = index
# We will fetch neighboring lidar files as well
if minLidarIndex > 0:
minLidarIndex = minLidarIndex - 1
if maxLidarIndex + 1 < len(lidarList):
maxLidarIndex = maxLidarIndex + 1
lidarsToFetch = set()
if lidarList:
for index in range(minLidarIndex, maxLidarIndex +
1): # Fetch only the requested lidar files.
lidarsToFetch.add(lidarList[index])
return lidarsToFetch
def lidarFilesInRange(lidarDict, lidarFolder, startFrame, stopFrame):
'''Fetch only lidar files for the given frame range. Do that as follows.
For each ortho frame in [startFrame, stopFrame], find the lidar
file with the closest timestamp. Collect them all.
Add the two neighboring ones, to help with finding lidar pairs later.'''
lidarList = []
for frame in sorted(lidarDict.keys()):
lidarList.append(lidarDict[frame])
# If we requested all frames, also get all the lidar files.
if ((startFrame == icebridge_common.getSmallestFrame()) and
(stopFrame == icebridge_common.getLargestFrame() ) ):
minLidarIndex = 0
maxLidarIndex = len(lidarList)-1
else:
minLidarIndex = len(lidarList)
maxLidarIndex = 0
# Build up a list of lidar files that match the requested input frames
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(lidarFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
for frame in sorted(orthoFrameDict.keys()):
if ((frame < startFrame) or (frame > stopFrame) ): continue
orthoFrame = orthoFrameDict[frame]
try:
matchingLidar = icebridge_common.findMatchingLidarFileFromList(orthoFrame, lidarList)
except:
# Some image files don't have a matching lidar file, just keep going.
continue
for index in range(len(lidarList)):
if lidarList[index] == matchingLidar:
if minLidarIndex > index:
minLidarIndex = index
if maxLidarIndex < index:
maxLidarIndex = index
# We will fetch neighboring lidar files as well
if minLidarIndex > 0:
minLidarIndex = minLidarIndex -1
if maxLidarIndex + 1 < len(lidarList):
maxLidarIndex = maxLidarIndex + 1
lidarsToFetch = set()
if lidarList:
for index in range(minLidarIndex, maxLidarIndex+1): # Fetch only the requested lidar files.
lidarsToFetch.add(lidarList[index])
return lidarsToFetch
def allSourceDataFetched(self, noNav, verbose=False):
'''Return true if all the required source data has been downloaded'''
logger = logging.getLogger(__name__)
# Old file collections use a different naming scheme
altIndexName = 'image_index.html.csv'
# Verify these input folders
subFolders = [
self.getJpegFolder(),
self.getLidarFolder(),
self.getOrthoFolder()
]
for folder in subFolders:
# Make sure all the files specified in the parsed index file are present
indexFile = icebridge_common.csvIndexFile(folder)
if not os.path.exists(indexFile):
# See if the older naming scheme was used
altPath = os.path.join(folder, altIndexName)
if os.path.exists(altPath):
indexFile = altPath
(fileDict, urlDict) = icebridge_common.readIndexFile(indexFile)
logger.info("Checking all files listed in: " + indexFile)
num = len(fileDict.keys())
count = 0
for f in fileDict.itervalues():
# Add the progress, as this operation can be terribly slow
# when the filesystem is not doing too well, especially on mfe.
count = count + 1
if (count - 1) % 1000 == 0:
logger.info('Progress: ' + str(count) + '/' + str(num))
path = os.path.join(folder, f)
if not os.path.exists(path):
logger.error('Missing file ' + path)
return False
if not noNav:
# Simple nav file check
navFiles = os.listdir(self.getNavFolder())
navFiles = [x for x in navFiles if '.out' in navFiles]
if (len(navFiles) == 0) or (len(navFiles) > 2):
logger.error('Wrong number of nav files detected!')
return False
return True # Success!
def getFrameRange(self):
'''Return the min and max frame currently stored for the run'''
jpegFolder = icebridge_common.getJpegFolder(self.getFolder())
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".")
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
frames = sorted(jpegFrameDict.keys())
if len(frames) == 0:
raise Exception("Empty folder: " + jpegFolder)
return (frames[0], frames[-1])
def allSourceDataFetched(self, noNav, verbose=False):
'''Return true if all the required source data has been downloaded'''
logger = logging.getLogger(__name__)
# Old file collections use a different naming scheme
altIndexName = 'image_index.html.csv'
# Verify these input folders
subFolders = [self.getJpegFolder(), self.getLidarFolder(), self.getOrthoFolder()]
for folder in subFolders:
# Make sure all the files specified in the parsed index file are present
indexFile = icebridge_common.csvIndexFile(folder)
if not os.path.exists(indexFile):
# See if the older naming scheme was used
altPath = os.path.join(folder, altIndexName)
if os.path.exists(altPath):
indexFile = altPath
(fileDict, urlDict) = icebridge_common.readIndexFile(indexFile)
logger.info("Checking all files listed in: " + indexFile)
num = len(fileDict.keys())
count = 0
for f in fileDict.itervalues():
# Add the progress, as this operation can be terribly slow
# when the filesystem is not doing too well, especially on mfe.
count = count + 1
if (count - 1) % 1000 == 0:
logger.info('Progress: ' + str(count) + '/' + str(num))
path = os.path.join(folder, f)
if not os.path.exists(path):
logger.error('Missing file ' + path)
return False
if not noNav:
# Simple nav file check
navFiles = os.listdir(self.getNavFolder())
navFiles = [x for x in navFiles if '.out' in navFiles]
if (len(navFiles) == 0) or (len(navFiles) > 2):
logger.error('Wrong number of nav files detected!')
return False
return True # Success!
def massRename(self, startFrame, stopFrame, logger):
'''We changed how the timestamp for images and cameras is computed.
Make all existing converted images, cameras, nav cameras, and aligned cameras conform.'''
logger.info("Renaming files with timestamp. This is slow.")
# Need to do a mass rename for incorrect timestamp for:
# converted images, nav cameras, cameras, and bundle aligned cameras
outputFolder = self.getFolder()
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
imageFolder = icebridge_common.getImageFolder(outputFolder)
jpegFolder = icebridge_common.getJpegFolder(outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
processedFolder = icebridge_common.getProcessedFolder(outputFolder)
navFolder = icebridge_common.getNavFolder(outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(
os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " +
orthoIndexPath + ".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder=True)
logger.info('Renaming camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(cameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming nav camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(navCameraFolder, '*DMS*tsai'),
logger)
logger.info('Renaming converted images...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(imageFolder, '*DMS*tif'), logger)
logger.info('Renaming aligned cameras...')
self.massRenameByGlob(
startFrame, stopFrame, orthoFrameDict,
os.path.join(processedFolder, 'batch*',
icebridge_common.alignedBundleStr() + '*DMS*tsai'),
logger)
def getFrameRange(self):
'''Return the min and max frame currently stored for the run'''
jpegFolder = icebridge_common.getJpegFolder(self.getFolder())
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " +
jpegIndexPath + ".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
frames = sorted(jpegFrameDict.keys())
if len(frames) == 0:
raise Exception("Empty folder: " + jpegFolder)
return (frames[0], frames[-1])
def massRename(self, startFrame, stopFrame, logger):
'''We changed how the timestamp for images and cameras is computed.
Make all existing converted images, cameras, nav cameras, and aligned cameras conform.'''
logger.info("Renaming files with timestamp. This is slow.")
# Need to do a mass rename for incorrect timestamp for:
# converted images, nav cameras, cameras, and bundle aligned cameras
outputFolder = self.getFolder()
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
imageFolder = icebridge_common.getImageFolder(outputFolder)
jpegFolder = icebridge_common.getJpegFolder(outputFolder)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
processedFolder = icebridge_common.getProcessedFolder(outputFolder)
navFolder = icebridge_common.getNavFolder(outputFolder)
navCameraFolder = icebridge_common.getNavCameraFolder(outputFolder)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder = True)
logger.info('Renaming camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(cameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming nav camera files...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(navCameraFolder, '*DMS*tsai'), logger)
logger.info('Renaming converted images...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(imageFolder, '*DMS*tif'), logger)
logger.info('Renaming aligned cameras...')
self.massRenameByGlob(startFrame, stopFrame, orthoFrameDict,
os.path.join(processedFolder, 'batch*',
icebridge_common.alignedBundleStr() + '*DMS*tsai'),
logger)
def lidarFilesInRange(lidarDict, lidarFolder, startFrame, stopFrame):
'''Fetch only lidar files for the given frame range. Do that as follows. '''
'''For each ortho frame in [startFrame, stopFrame], find the lidar '''
'''file with the closest timestamp. Collect them all. '''
'''Add the two neighboring ones, to help with finding lidar pairs later. '''
lidarList = []
for frame in sorted(lidarDict.keys()):
lidarList.append(lidarDict[frame])
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(lidarFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
minLidarIndex = len(lidarList)
maxLidarIndex = 0
for frame in sorted(orthoFrameDict.keys()):
if ((frame < startFrame) or (frame > stopFrame)): continue
orthoFrame = orthoFrameDict[frame]
matchingLidar = icebridge_common.findMatchingLidarFileFromList(
orthoFrame, lidarList)
for index in range(len(lidarList)):
if lidarList[index] == matchingLidar:
if minLidarIndex > index:
minLidarIndex = index
if maxLidarIndex < index:
maxLidarIndex = index
# We will fetch neighboring lidar files as well
if minLidarIndex > 0:
minLidarIndex = minLidarIndex - 1
if maxLidarIndex + 1 < len(lidarList):
maxLidarIndex = maxLidarIndex + 1
lidarsToFetch = set()
for index in range(minLidarIndex, maxLidarIndex + 1):
lidarsToFetch.add(lidarList[index])
return lidarsToFetch
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate,
cameraMounting, logger):
'''Convert jpeg images from RGB to single channel.
Returns false if any files failed.'''
badFiles = False
logger.info('Converting input images to grayscale...')
os.system('mkdir -p ' + imageFolder)
# Loop through all the input images
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".")
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder = True)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder = True)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(os.path.dirname(jpegFolder),
startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
# Fast check for missing images. This is fragile, as maybe it gets
# the wrong file with a similar name, but an honest check is very slow.
imageFiles = icebridge_common.getTifs(imageFolder, prependFolder = True)
imageFrameDict = {}
for imageFile in imageFiles:
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if frame < startFrame or frame > stopFrame: continue
imageFrameDict[frame] = imageFile
for frame in sorted(jpegFrameDict.keys()):
inputPath = jpegFrameDict[frame]
# Only deal with frames in range
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if frame in imageFrameDict.keys() and skipValidate:
# Fast, hackish check
continue
if frame not in orthoFrameDict:
logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame))
# Don't want to throw here. Just ignore the missing ortho
continue
# Make sure the timestamp and frame number are in the output file name
try:
outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame])
except Exception as e:
logger.info(str(e))
logger.info("Removing bad file: " + inputPath)
os.system('rm -f ' + inputPath) # will not throw
badFiles = True
continue
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # very verbose
validFilesSet.add(inputPath) # Must have this
continue
if icebridge_common.isValidImage(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath) # verbose
if not skipValidate:
# Mark both the input and the output as validated
validFilesSet.add(inputPath)
validFilesSet.add(outputPath)
continue
# Use ImageMagick tool to convert from RGB to grayscale
# - Some image orientations are rotated to make stereo processing easier.
rotateString = ''
if cameraMounting == 2: # Flight direction towards top of image
rotateString = '-rotate 90 '
if cameraMounting == 3: # Flight direction towards bottom of image
rotateString = '-rotate -90 '
cmd = ('%s %s -colorspace Gray %s%s') % \
(asp_system_utils.which('convert'), inputPath, rotateString, outputPath)
logger.info(cmd)
# Run command and fetch its output
p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
output, error = p.communicate()
if p.returncode != 0:
badFiles = True
logger.error("Command failed.")
logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n'
+ output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
if not os.path.exists(outputPath):
badFiles = True
logger.error('Failed to convert jpeg file: ' + inputPath)
logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n'
+ output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
# Check for corrupted files
if error is not None:
output += error
m = re.match("^.*?premature\s+end", output, re.IGNORECASE|re.MULTILINE|re.DOTALL)
if m:
badFiles = True
logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n'
+ output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
if not skipValidate:
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList,
validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
if badFiles:
logger.error("Converstion of JPEGs failed. If any files were corrupted, " +
"they were removed, and need to be re-fetched.")
return (not badFiles)
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 10
usage = '''blend_dems.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
parser.add_argument("--compute-diff-to-prev-dem", action="store_true",
dest="computeDiffToPrev", default=False,
help="Compute the absolute difference between the current DEM " +
"and the one before it.")
parser.add_argument("--blend-to-fireball-footprint", action="store_true",
dest="blendToFireball", default=False,
help="Create additional blended DEMs having the same " + \
"footprint as Fireball DEMs.")
# Performance options
parser.add_argument('--num-processes', dest='numProcesses', default=1,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=8,
type=int, help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_blend_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput = True)
logger.info("Running on machine: " + out)
logger.info(str(argsIn))
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder, options.processingSubfolder)
logger.info('Reading from processing subfolder: ' + options.processingSubfolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if options.blendToFireball:
fireballFrameDict = icebridge_common.getCorrectedFireballDems(options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(orthoFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No ortho files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in orthoFrameDict:
logger.info("Error: Missing ortho file for frame: " + str(frame) + ".")
continue
orthoFile = orthoFrameDict[frame]
try:
lidarFile = icebridge_common.findMatchingLidarFile(orthoFile, lidarFolder)
except: # Skip if no lidar file matches this frame
continue
fireballDEM = ""
if options.blendToFireball:
if frame in fireballFrameDict:
fireballDEM = fireballFrameDict[frame]
else:
logger.info("No fireball DEM for frame: " + str(frame))
args = (frame, processFolder, lidarFile, fireballDEM, options, threadText,
redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runBlend, args))
else:
runBlend(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False,
quitKey='q', sleepTime=20)
icebridge_common.stopTaskPool(pool)
def pushByType(run, options, logger, dataType):
# Fetch the ortho index from NSIDC if missing
outputFolder = run.getFolder()
logger.info("Output folder is " + outputFolder)
os.system("mkdir -p " + outputFolder)
# Current directory. It is important to go from /u to the real dir which is /nobackup...
unpackDir = os.path.realpath(os.getcwd())
logger.info("Unpack directory is " + unpackDir)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
fetchIndices(options, logger)
logger.info("Reading ortho index: " + orthoIndexPath)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
# Fetch unarchived folder if missing
if dataType == 'DEM':
unarchivedFolder = run.getAssemblyFolder()
elif dataType == 'ORTHO':
unarchivedFolder = run.getProcessFolder()
else:
raise Exception("Unknown data type: " + dataType)
logger.info("Unarchived data folder is " + unarchivedFolder)
# Especially for ortho, force-fetch each time, as there is no good way
# of checking if we fetched well before.
start_time()
if not archive_functions.fetchProcessedByType(run, unpackDir, logger,
dataType):
return
stop_time("fetching archived data by type: " + dataType, logger)
# Make the output directory at NSIDC
m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd)
if m:
outDir = options.site + "_" + m.group(1) + "." + m.group(
2) + "." + m.group(3)
else:
raise Exception("Could not parse: " + options.yyyymmdd)
# Keep the output directory locally here
localDirPath = os.path.join(outputFolder, dataType, outDir)
os.system("mkdir -p " + localDirPath)
logger.info("Storing the renamed " + dataType + " files in " +
localDirPath)
logger.info("Directory name at NSIDC: " + outDir)
# Read the DEMs and orthos, and copy them to outDir according to the final convention
if dataType == 'DEM':
dataFiles = icebridge_common.getTifs(unarchivedFolder,
prependFolder=True)
else:
dataFiles = glob.glob(
os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif'))
for dataFile in dataFiles:
# Here we use the convention from archive_functions.py for DEMs and from how we store orthos.
if dataType == 'DEM':
m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \
"\.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
else:
m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \
"out-ortho.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
if frameNumber < options.startFrame or frameNumber > options.stopFrame:
continue
# For each data file, copy from the ortho its meta info
if not frameNumber in orthoFrameDict.keys():
# Bugfix: Ortho fetching failed, try again
fetchIndices(options, logger)
logger.info("Re-reading ortho index: " + orthoIndexPath)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if not frameNumber in orthoFrameDict.keys():
# This time there is nothing we can do
raise Exception("Cannot find ortho for frame: " +
str(frameNumber))
orthoFile = orthoFrameDict[frameNumber]
[dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile)
# It is always possible that the ortho file date will be the next day
# after the current flight date, if the flight goes after midnight.
# So it is not unreasonable that options.yyyymmdd != dateString.
if dataType == 'DEM':
outFile = ('IODEM3_%s_%s_%05d_DEM.tif' %
(dateString, timeString, frameNumber))
else:
# TODO: Need to think more of the naming convention.
outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' %
(dateString, timeString, frameNumber))
cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(
localDirPath, outFile)
logger.info(cmd)
os.system(cmd)
# Push the directory to NSIDC
remoteDirPath = os.path.join(
os.path.basename(os.path.dirname(localDirPath)),
os.path.basename(localDirPath))
remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath)
logger.info("Storing at NSIDC in: " + remoteDirPath)
cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \
' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo
logger.info(cmd)
start_time()
(output, err,
status) = asp_system_utils.executeCommand(cmd, suppressOutput=True)
#status = os.system(cmd)
logger.info("LFTP output and error: " + output + ' ' + err)
logger.info("LFTP status: " + str(status))
#if status != 0:
# raise Exception("Problem pushing")
stop_time("push to NSIDC", logger)
def convertLidarDataToCsv(lidarFolder, startFrame, stopFrame, skipValidate,
logger):
'''Make sure all lidar data is available in a readable text format.
Returns false if any files failed to convert.'''
logger.info('Converting LIDAR files...')
lidarIndexPath = icebridge_common.csvIndexFile(lidarFolder)
(frameDict, urlDict) = icebridge_common.readIndexFile(lidarIndexPath)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(
os.path.dirname(lidarFolder), startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(
validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
convDict = {}
# Loop through all files in the folder
badFiles = False
for frame in sorted(frameDict.keys()):
f = frameDict[frame]
extension = icebridge_common.fileExtension(f)
# Only interested in a few file types
if (extension != '.qi') and (extension != '.hdf5') and (extension !=
'.h5'):
convDict[frame] = f # these are already in plain text
continue
convDict[frame] = os.path.splitext(f)[0] + '.csv'
outputPath = os.path.join(lidarFolder, convDict[frame])
# Handle paths
fullPath = os.path.join(lidarFolder, f)
if not os.path.exists(fullPath):
logger.info("Cannot convert missing file: " + fullPath)
continue
# If the input is invalid, wipe both it, its xml, and the output
# Hopefully there will be a subsquent fetch step where it will get
# refetched.
if not icebridge_common.hasValidChkSum(fullPath, logger):
logger.info("Will wipe invalid file: " + fullPath)
xmlFile = icebridge_common.xmlFile(fullPath)
os.system('rm -f ' + fullPath) # will not throw
os.system('rm -f ' + xmlFile) # will not throw
os.system('rm -f ' + outputPath) # will not throw
badFiles = True
continue
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # verbose
continue
if icebridge_common.isValidLidarCSV(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath)
continue
# Call the conversion
logger.info("Process " + fullPath)
extract_icebridge_ATM_points.main([fullPath])
# Check the result
if not icebridge_common.isValidLidarCSV(outputPath):
logger.error('Failed to parse LIDAR file, will wipe: ' +
outputPath)
os.system('rm -f ' + outputPath) # will not throw
badFiles = True
else:
if not skipValidate:
validFilesSet.add(outputPath) # mark it as validated
convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder)
willWriteConvFile = False
if not os.path.exists(convLidarFile):
willWriteConvFile = True
else:
# Bugfix: Sometimes the written converted file has the wrong size, maybe
# something got interrupted earlier.
(lidarDictIn,
dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile)
if lidarDictIn != convDict:
willWriteConvFile = True
if willWriteConvFile:
logger.info("Writing: " + convLidarFile)
icebridge_common.writeIndexFile(convLidarFile, convDict, {})
if not skipValidate:
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = icebridge_common.updateValidFilesListFromDisk(
validFilesList, validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
return (not badFiles)
def correctFireballDems(fireballFolder, corrFireballFolder, startFrame, stopFrame, isNorth,
skipValidate, logger):
'''Fix the header problem in Fireball DEMs'''
logger.info('Correcting Fireball DEMs ...')
# Read the existing DEMs
fireballIndexPath = icebridge_common.csvIndexFile(fireballFolder)
if not os.path.exists(fireballIndexPath):
raise Exception("Error: Missing fireball index file: " + fireballIndexPath + ".")
(fireballFrameDict, fireballUrlDict) = \
icebridge_common.readIndexFile(fireballIndexPath, prependFolder = True)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(os.path.dirname(fireballFolder),
startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
# Loop through all the input images
os.system('mkdir -p ' + corrFireballFolder)
badFiles = False
for frame in sorted(fireballFrameDict.keys()):
# Skip if outside the frame range
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
inputPath = fireballFrameDict[frame]
if not icebridge_common.isDEM(inputPath):
continue
outputPath = os.path.join(corrFireballFolder, os.path.basename(inputPath))
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # very vebose
continue
if icebridge_common.isValidImage(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath)
validFilesSet.add(outputPath) # mark it as validated
continue
# Run the correction script
execPath = asp_system_utils.which('correct_icebridge_l3_dem')
cmd = (('%s %s %s %d') %
(execPath, inputPath, outputPath, isNorth))
logger.info(cmd)
# TODO: Run this as a subprocess and check the return code
os.system(cmd)
# Check if the output file is good
if not icebridge_common.isValidImage(outputPath):
logger.error('Failed to convert dem file, wiping: ' + inputPath + ' ' + outputPath)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
badFiles = True
else:
if not skipValidate:
validFilesSet.add(outputPath) # mark it as validated
if not skipValidate:
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList,
validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
return (not badFiles)
def validateOrthosAndFireball(options, fileType, logger):
'''Validate ortho and fireball files within the current frame range. This
is expected to be in called in parallel for smaller chunks. Lidar files
will be validated serially. Jpegs get validated when converted to tif.
Return True if all is good.'''
badFiles = False
logger.info("Validating files of type: " + fileType)
if fileType == 'ortho':
dataFolder = icebridge_common.getOrthoFolder(options.outputFolder)
elif fileType == 'fireball':
dataFolder = icebridge_common.getFireballFolder(options.outputFolder)
else:
raise Exception("Unknown file type: " + fileType)
indexPath = icebridge_common.csvIndexFile(dataFolder)
if not os.path.exists(indexPath):
# The issue of what to do when the index does not exist should
# have been settled by now.
return (not badFiles)
# Fetch from disk the set of already validated files, if any
validFilesList = icebridge_common.validFilesList(options.outputFolder,
options.startFrame, options.stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
(frameDict, urlDict) = icebridge_common.readIndexFile(indexPath, prependFolder = True)
for frame in frameDict.keys():
if frame < options.startFrame or frame > options.stopFrame:
continue
outputPath = frameDict[frame]
xmlFile = icebridge_common.xmlFile(outputPath)
if outputPath in validFilesSet and os.path.exists(outputPath) and \
xmlFile in validFilesSet and os.path.exists(xmlFile):
#logger.info('Previously validated: ' + outputPath + ' ' + xmlFile)
continue
else:
isGood = icebridge_common.hasValidChkSum(outputPath, logger)
if not isGood:
logger.info('Found invalid data. Will wipe: ' + outputPath + ' ' + xmlFile)
os.system('rm -f ' + outputPath) # will not throw
os.system('rm -f ' + xmlFile) # will not throw
badFiles = True
else:
logger.info('Valid file: ' + outputPath)
validFilesSet.add(outputPath)
validFilesSet.add(xmlFile)
if fileType != 'fireball':
continue
# Also validate tfw
tfwFile = icebridge_common.tfwFile(outputPath)
xmlFile = icebridge_common.xmlFile(tfwFile)
if tfwFile in validFilesSet and os.path.exists(tfwFile) and \
xmlFile in validFilesSet and os.path.exists(xmlFile):
#logger.info('Previously validated: ' + tfwFile + ' ' + xmlFile)
continue
else:
isGood = icebridge_common.isValidTfw(tfwFile, logger)
if not isGood:
logger.info('Found invalid tfw. Will wipe: ' + tfwFile + ' ' + xmlFile)
os.system('rm -f ' + tfwFile) # will not throw
os.system('rm -f ' + xmlFile) # will not throw
badFiles = True
else:
logger.info('Valid tfw file: ' + tfwFile)
validFilesSet.add(tfwFile)
validFilesSet.add(xmlFile)
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = \
icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
return (not badFiles)
def convertLidarDataToCsv(lidarFolder, startFrame, stopFrame,
skipValidate, logger):
'''Make sure all lidar data is available in a readable text format.
Returns false if any files failed to convert.'''
logger.info('Converting LIDAR files...')
lidarIndexPath = icebridge_common.csvIndexFile(lidarFolder)
(frameDict, urlDict) = icebridge_common.readIndexFile(lidarIndexPath)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(os.path.dirname(lidarFolder),
startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
convDict = {}
# Loop through all files in the folder
badFiles = False
for frame in sorted(frameDict.keys()):
f = frameDict[frame]
extension = icebridge_common.fileExtension(f)
# Only interested in a few file types
if (extension != '.qi') and (extension != '.hdf5') and (extension != '.h5'):
convDict[frame] = f # these are already in plain text
continue
convDict[frame] = os.path.splitext(f)[0] + '.csv'
outputPath = os.path.join(lidarFolder, convDict[frame])
# Handle paths
fullPath = os.path.join(lidarFolder, f)
if not os.path.exists(fullPath):
logger.info("Cannot convert missing file: " + fullPath)
continue
# If the input is invalid, wipe both it, its xml, and the output
# Hopefully there will be a subsquent fetch step where it will get
# refetched.
if not icebridge_common.hasValidChkSum(fullPath, logger):
logger.info("Will wipe invalid file: " + fullPath)
xmlFile = icebridge_common.xmlFile(fullPath)
os.system('rm -f ' + fullPath) # will not throw
os.system('rm -f ' + xmlFile) # will not throw
os.system('rm -f ' + outputPath) # will not throw
badFiles = True
continue
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # verbose
continue
if icebridge_common.isValidLidarCSV(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath)
continue
# Call the conversion
logger.info("Process " + fullPath)
extract_icebridge_ATM_points.main([fullPath])
# Check the result
if not icebridge_common.isValidLidarCSV(outputPath):
logger.error('Failed to parse LIDAR file, will wipe: ' + outputPath)
os.system('rm -f ' + outputPath) # will not throw
badFiles = True
else:
if not skipValidate:
validFilesSet.add(outputPath) # mark it as validated
convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder)
willWriteConvFile = False
if not os.path.exists(convLidarFile):
willWriteConvFile = True
else:
# Bugfix: Sometimes the written converted file has the wrong size, maybe
# something got interrupted earlier.
(lidarDictIn, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile)
if lidarDictIn != convDict:
willWriteConvFile = True
if willWriteConvFile:
logger.info("Writing: " + convLidarFile)
icebridge_common.writeIndexFile(convLidarFile, convDict, {})
if not skipValidate:
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList,
validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
return (not badFiles)
def solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder,
processedFolder, logger):
'''Some preliminary work before solving for intrinsics. Here we
look up the default calibration file, and generate an RPC
approximation of its distortion model with polynomials of degree
4. We will then create cameras and stereo DEMs using this initial
camera file with RPC distortion.'''
# Sanity checks
if options.startFrame == icebridge_common.getSmallestFrame() or \
options.stopFrame == icebridge_common.getLargestFrame():
raise Exception(
"When solving for intrinsics, must specify a frame range.")
if options.bundleLength != 2:
raise Exception(
"When solving for intrinsics, we assume bundle length of 2.")
if (options.stopFrame - options.startFrame) % 2 == 0:
raise Exception(
"When solving for intrinsics, must have an even number of frames, "
+ " so stopFrame - startFrame must be odd.")
if options.processingSubfolder:
raise Exception(
"Processing subfolder not supported when solving for intrinsics.")
# Generate extra data we will use later to float intrinsics
options.stereoArgs += " --num-matches-from-disp-triplets 10000 --unalign-disparity " # --enable-fill-holes "
# Create separate directories for cameras and processed data,
# as these will be distinct than what we will finally be
# using to do the full run.
suff = "_camgen"
cameraFolder += suff
navCameraFolder += suff
processedFolder += suff
# Get the input calibration file
defaultCalibFile = ""
for frame in range(options.startFrame, options.stopFrame + 1):
currCalibFile = input_conversions.getCalibrationFileForFrame(
options.cameraLookupFile, options.inputCalFolder, frame,
options.yyyymmdd, options.site)
if defaultCalibFile == "":
defaultCalibFile = currCalibFile
if defaultCalibFile != currCalibFile:
# This is important, the calibration file must be unique
raise Exception("Found two distinct calibration files: " + defaultCalibFile + \
" and " + currCalibFile)
logger.info("Default calibration file: " + defaultCalibFile)
if options.inputCalCamera != "":
defaultCalibFile = options.inputCalCamera
logger.info("Using instead the user-provided: " + defaultCalibFile)
# Find the first image in the range
jpegIndex = icebridge_common.csvIndexFile(jpegFolder)
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex,
prependFolder=True)
if options.startFrame not in jpegFrameDict.keys():
raise Exception("Could not find jpeg image for frame: " +
options.startFrame)
firstImage = jpegFrameDict[options.startFrame]
# Create the RPC file before optimization
rpcCalibFile = os.path.join(processedFolder,
os.path.basename(defaultCalibFile))
rpcCalibFile = rpcCalibFile.replace(".tsai", "_INIT_RPC.tsai")
logger.info("Will approximate camera model " + defaultCalibFile + " with " + \
options.outputModelType + " model " + rpcCalibFile)
os.system("mkdir -p " + os.path.dirname(rpcCalibFile))
cmd = "convert_pinhole_model --input-file " + firstImage + ' --camera-file ' + \
defaultCalibFile + ' --output-type ' + options.outputModelType + \
' --sample-spacing 50 -o ' + rpcCalibFile
logger.info(cmd)
os.system(cmd)
# Use this one from now on
options.inputCalCamera = rpcCalibFile
# Return the modified values
return (options, cameraFolder, navCameraFolder, processedFolder)
def pushByType(run, options, logger, dataType):
# Fetch the ortho index from NSIDC if missing
outputFolder = run.getFolder()
logger.info("Output folder is " + outputFolder)
os.system("mkdir -p " + outputFolder)
# Current directory. It is important to go from /u to the real dir which is /nobackup...
unpackDir = os.path.realpath(os.getcwd())
logger.info("Unpack directory is " + unpackDir)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
fetchIndices(options, logger)
logger.info("Reading ortho index: " + orthoIndexPath)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
# Fetch unarchived folder if missing
if dataType == 'DEM':
unarchivedFolder = run.getAssemblyFolder()
elif dataType == 'ORTHO':
unarchivedFolder = run.getProcessFolder()
else:
raise Exception("Unknown data type: " + dataType)
logger.info("Unarchived data folder is " + unarchivedFolder)
# Especially for ortho, force-fetch each time, as there is no good way
# of checking if we fetched well before.
start_time()
if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType):
return
stop_time("fetching archived data by type: " + dataType, logger)
# Make the output directory at NSIDC
m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd)
if m:
outDir = options.site + "_" + m.group(1) + "." + m.group(2) + "." + m.group(3)
else:
raise Exception("Could not parse: " + options.yyyymmdd)
# Keep the output directory locally here
localDirPath = os.path.join(outputFolder, dataType, outDir)
os.system("mkdir -p " + localDirPath)
logger.info("Storing the renamed " + dataType + " files in " + localDirPath)
logger.info("Directory name at NSIDC: " + outDir)
# Read the DEMs and orthos, and copy them to outDir according to the final convention
if dataType == 'DEM':
dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True)
else:
dataFiles = glob.glob(os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif'))
for dataFile in dataFiles:
# Here we use the convention from archive_functions.py for DEMs and from how we store orthos.
if dataType == 'DEM':
m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \
"\.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
else:
m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \
"out-ortho.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
if frameNumber < options.startFrame or frameNumber > options.stopFrame:
continue
# For each data file, copy from the ortho its meta info
if not frameNumber in orthoFrameDict.keys():
# Bugfix: Ortho fetching failed, try again
fetchIndices(options, logger)
logger.info("Re-reading ortho index: " + orthoIndexPath)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if not frameNumber in orthoFrameDict.keys():
# This time there is nothing we can do
raise Exception("Cannot find ortho for frame: " + str(frameNumber))
orthoFile = orthoFrameDict[frameNumber]
[dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile)
# It is always possible that the ortho file date will be the next day
# after the current flight date, if the flight goes after midnight.
# So it is not unreasonable that options.yyyymmdd != dateString.
if dataType == 'DEM':
outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber))
else:
# TODO: Need to think more of the naming convention.
outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber))
cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(localDirPath, outFile)
logger.info(cmd)
os.system(cmd)
# Push the directory to NSIDC
remoteDirPath = os.path.join(os.path.basename(os.path.dirname(localDirPath)),
os.path.basename(localDirPath))
remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath)
logger.info("Storing at NSIDC in: " + remoteDirPath)
cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \
' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo
logger.info(cmd)
start_time()
(output, err, status) = asp_system_utils.executeCommand(cmd,
suppressOutput = True)
#status = os.system(cmd)
logger.info("LFTP output and error: " + output + ' ' + err)
logger.info("LFTP status: " + str(status))
#if status != 0:
# raise Exception("Problem pushing")
stop_time("push to NSIDC", logger)
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 3.
usage = '''gen_ortho.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument("--site", dest="site", required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument('--camera-mounting', dest='cameraMounting', default=0, type=int,
help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.')
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
# Performance options
parser.add_argument('--num-processes', dest='numProcesses', default=1,
type=int, help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads', dest='numThreads', default=8,
type=int, help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput = True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder, options.processingSubfolder)
logger.info('Reading from processing subfolder: ' + options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".")
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder = True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(jpegFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No jpeg files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in jpegFrameDict:
logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".")
continue
# Find the right image
currImage = jpegFrameDict[frame]
args = (frame, processFolder, currImage, options.bundleLength,
options.cameraMounting, threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runOrtho, args))
else:
runOrtho(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False,
quitKey='q', sleepTime=20)
icebridge_common.stopTaskPool(pool)
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder):
'''Create a list of all files that must be fetched unless done already.'''
# For AN 20091112, etc, some of the ortho images are stored at the
# beginning of the next day's flight. Need to sort this out, and
# it is tricky. More comments within the code.
fetchNextDay = True
separateByLat = (options.type == 'ortho' and isInSeparateByLatTable(options.yyyymmdd))
if separateByLat:
# Here we won't fetch the next day, we will just separate by latitude within
# a given day
fetchNextDay = False
orthoOrFireball = ( (options.type == 'ortho') or (options.type == 'fireball') )
if fetchNextDay:
# Normally we fetch for next day only for ortho or fireball. However,
# for one single special flight, we do it for jpeg too, as then
# the jpegs are also split.
if orthoOrFireball or \
((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)):
fetchNextDay = True
else:
fetchNextDay = False
# If we need to parse the next flight day as well, as expected in some runs,
# we will fetch two html files, but create a single index out of them.
dayVals = [0]
if fetchNextDay:
dayVals.append(1)
indexPath = icebridge_common.htmlIndexFile(outputFolder)
currIndexPath = indexPath
parsedIndexPath = icebridge_common.csvIndexFile(outputFolder)
if options.refetchIndex:
os.system('rm -f ' + indexPath)
os.system('rm -f ' + parsedIndexPath)
if icebridge_common.fileNonEmpty(parsedIndexPath):
logger.info('Already have the index file ' + parsedIndexPath + ', keeping it.')
return parsedIndexPath
frameDict = {}
urlDict = {}
# We need the list of jpeg frames. Sometimes when fetching ortho images,
# and we have to fetch from the next day, don't fetch unless
# in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
jpegFolder = icebridge_common.getJpegFolder(os.path.dirname(outputFolder))
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
orthoStamp = {}
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(outputFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
for frame in sorted(orthoFrameDict.keys()):
filename = orthoFrameDict[frame]
[imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename)
orthoStamp[frame] = imageTimeString
for dayVal in dayVals:
if len(dayVals) > 1:
currIndexPath = indexPath + '.day' + str(dayVal)
if options.refetchIndex:
os.system('rm -f ' + currIndexPath)
# Find folderUrl which contains all of the files
if options.type in LIDAR_TYPES:
options.allFrames = True # For lidar, always get all the frames!
# For lidar, the data can come from one of three sources.
# Unfortunately sometimes there is more than one source, and then
# we need to pick by latitude.
folderUrls = []
lidar_types = []
for lidar in LIDAR_TYPES:
folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month,
options.day, dayVal, # note here the dayVal
options.site, lidar)
logger.info('Checking lidar URL: ' + folderUrl)
if checkIfUrlExists(folderUrl, baseCurlCmd):
logger.info('Found match with lidar type: ' + lidar)
folderUrls.append(folderUrl)
lidar_types.append(lidar)
if len(folderUrls) == 0:
logger.info('WARNING: Could not find any lidar data for the given date!')
elif len(folderUrls) == 1:
# Unique solution
folderUrl = folderUrls[0]
options.type = lidar_types[0]
elif len(folderUrls) >= 2:
# Multiple solutions. Pick the good one by latitude.
logger.info("Multiples URLs to search: " + " ".join(folderUrls))
count = -1
isGood = False
for folderUrl in folderUrls:
count += 1
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath,
lidar_types[count])
for frame in sorted(localFrameDict.keys()):
filename = localFrameDict[frame]
xmlFile = icebridge_common.xmlFile(filename)
url = os.path.join(folderUrl, xmlFile)
# Download the file
curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile
logger.info(curlCmd)
p = subprocess.Popen(curlCmd, shell=True, universal_newlines=True)
os.waitpid(p.pid, 0)
latitude = icebridge_common.parseLatitude(xmlFile)
if os.path.exists(xmlFile): os.remove(xmlFile)
if hasGoodLat(latitude, isSouth):
isGood = True
options.type = lidar_types[count]
logger.info("Good latitude " + str(latitude) + ", will use " +
folderUrl + " of type " + lidar_types[count])
else:
logger.info("Bad latitude " + str(latitude) + ", will not use " +
folderUrl + " of type " + lidar_types[count])
# Stop at first file no matter what
break
if isGood:
break
if not isGood:
if options.type in LIDAR_TYPES and options.ignoreMissingLidar:
logger.info("No lidar. None of these URLs are good: " +
" ".join(folderUrls))
else:
raise Exception("None of these URLs are good: " +
" ".join(folderUrls))
else: # Other cases are simpler
folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month,
options.day, dayVal, # note here the dayVal
options.site, options.type)
logger.info('Fetching from URL: ' + folderUrl)
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath, options.type)
# Append to the main index
for frame in sorted(localFrameDict.keys()):
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
# Otherwise we may accidentally getting one from next day.
[imageDateString, imageTimeString] = \
icebridge_common.parseTimeStamps(localFrameDict[frame])
if frame not in orthoStamp:
#logger.info("Missing ortho for fireball: " + localFrameDict[frame])
continue
if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000:
# Apparently a tolerance is needed. Use 10 seconds, so the number 1000.
#logger.info("Will not use fireball DEM whose timestamp differs from ortho.")
#logger.info("Fireball is: " + localFrameDict[frame])
#logger.info("Ortho is: " + orthoFrameDict[frame])
continue
# Fetch from next day, unless already have a value. And don't fetch
# frames not in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
if not frame in jpegFrameDict.keys(): continue
if frame in frameDict.keys(): continue
frameDict[frame] = localFrameDict[frame]
urlDict[frame] = localUrlDict[frame]
# Write the combined index file
icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict)
return parsedIndexPath
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 3.
usage = '''gen_ortho.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument(
'--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument(
'--camera-mounting',
dest='cameraMounting',
default=0,
type=int,
help=
'0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.'
)
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
# Performance options
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=1,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=8,
type=int,
help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_ortho_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(jpegFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No jpeg files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in jpegFrameDict:
logger.info("Error: Missing jpeg file for frame: " + str(frame) +
".")
continue
# Find the right image
currImage = jpegFrameDict[frame]
args = (frame, processFolder, currImage, options.bundleLength,
options.cameraMounting, threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runOrtho, args))
else:
runOrtho(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
logger,
interactive=False,
quitKey='q',
sleepTime=20)
icebridge_common.stopTaskPool(pool)
def fetchAllRunData(options, startFrame, stopFrame,
jpegFolder, orthoFolder, fireballFolder, lidarFolder, navFolder):
'''Download all data needed to process a run'''
logger = logging.getLogger(__name__)
logger.info('Downloading data for the run...')
baseCommand = (('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d')
% (options.yyyymmdd, options.site, startFrame, stopFrame))
if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0:
baseCommand += ' --max-num-lidar-to-fetch ' + str(options.maxNumLidarToFetch)
if options.refetchIndex:
baseCommand += ' --refetch-index' # this was not right in older fetched runs
if options.refetchNav:
baseCommand += ' --refetch-nav' # sometimes this was corrupted
if options.stopAfterIndexFetch:
baseCommand += ' --stop-after-index-fetch'
if options.skipValidate:
baseCommand += ' --skip-validate'
if options.ignoreMissingLidar:
baseCommand += ' --ignore-missing-lidar'
if options.dryRun:
baseCommand += ' --dry-run'
jpegCommand = baseCommand + ' ' + jpegFolder
orthoCommand = baseCommand + ' ' + orthoFolder
fireballCommand = baseCommand + ' ' + fireballFolder
lidarCommand = baseCommand + ' ' + lidarFolder
navCommand = baseCommand + ' ' + navFolder
# Try to do all the downloads one after another
# - On a failure the error message should already be printed.
# - The fetching tool will not redownload existing data.
if fetch_icebridge_data.main(jpegCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(orthoCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(fireballCommand.split()) < 0:
logger.info('Fireball DEM data is optional, continuing run.')
if not options.noNavFetch:
if fetch_icebridge_data.main(navCommand.split()) < 0:
return -1
# Skip the lidar fetch if the user requested no lidar files
if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch > 0):
if fetch_icebridge_data.main(lidarCommand.split()) < 0:
return -1
# jpeg and ortho indices must be consistent
if not options.skipValidate:
logger.info("Check for consistency between raw and ortho images.")
jpegIndex = icebridge_common.csvIndexFile(jpegFolder)
orthoIndex = icebridge_common.csvIndexFile(orthoFolder)
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex)
for jpegFrame in jpegFrameDict.keys():
if jpegFrame < startFrame or jpegFrame > stopFrame:
continue
if jpegFrame not in orthoFrameDict.keys():
logger.info("Found jpeg frame missing from ortho: " + str(jpegFrame))
#raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame))
for orthoFrame in orthoFrameDict.keys():
if orthoFrame < startFrame or orthoFrame > stopFrame:
continue
if orthoFrame not in jpegFrameDict.keys():
# This can happen, don't die because of it
logger.info("Found ortho frame missing from jpeg: " + str(orthoFrame))
#raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame))
# TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it.
return 0
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder):
'''Create a list of all files that must be fetched unless done already.'''
# For AN 20091112, etc, some of the ortho images are stored at the
# beginning of the next day's flight. Need to sort this out, and
# it is tricky. More comments within the code.
fetchNextDay = True
separateByLat = (options.type == 'ortho'
and isInSeparateByLatTable(options.yyyymmdd))
if separateByLat:
# Here we won't fetch the next day, we will just separate by latitude within
# a given day
fetchNextDay = False
orthoOrFireball = ((options.type == 'ortho')
or (options.type == 'fireball'))
if fetchNextDay:
# Normally we fetch for next day only for ortho or fireball. However,
# for one single special flight, we do it for jpeg too, as then
# the jpegs are also split.
if orthoOrFireball or \
((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)):
fetchNextDay = True
else:
fetchNextDay = False
# If we need to parse the next flight day as well, as expected in some runs,
# we will fetch two html files, but create a single index out of them.
dayVals = [0]
if fetchNextDay:
dayVals.append(1)
indexPath = icebridge_common.htmlIndexFile(outputFolder)
currIndexPath = indexPath
parsedIndexPath = icebridge_common.csvIndexFile(outputFolder)
if options.refetchIndex:
os.system('rm -f ' + indexPath)
os.system('rm -f ' + parsedIndexPath)
if icebridge_common.fileNonEmpty(parsedIndexPath):
logger.info('Already have the index file ' + parsedIndexPath +
', keeping it.')
return parsedIndexPath
frameDict = {}
urlDict = {}
# We need the list of jpeg frames. Sometimes when fetching ortho images,
# and we have to fetch from the next day, don't fetch unless
# in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
jpegFolder = icebridge_common.getJpegFolder(
os.path.dirname(outputFolder))
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)
orthoStamp = {}
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
orthoFolder = icebridge_common.getOrthoFolder(
os.path.dirname(outputFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
for frame in sorted(orthoFrameDict.keys()):
filename = orthoFrameDict[frame]
[imageDateString,
imageTimeString] = icebridge_common.parseTimeStamps(filename)
orthoStamp[frame] = imageTimeString
for dayVal in dayVals:
if len(dayVals) > 1:
currIndexPath = indexPath + '.day' + str(dayVal)
if options.refetchIndex:
os.system('rm -f ' + currIndexPath)
# Find folderUrl which contains all of the files
if options.type in LIDAR_TYPES:
options.allFrames = True # For lidar, always get all the frames!
# For lidar, the data can come from one of three sources.
# Unfortunately sometimes there is more than one source, and then
# we need to pick by latitude.
folderUrls = []
lidar_types = []
for lidar in LIDAR_TYPES:
folderUrl = getFolderUrl(
options.yyyymmdd,
options.year,
options.month,
options.day,
dayVal, # note here the dayVal
options.site,
lidar)
logger.info('Checking lidar URL: ' + folderUrl)
if checkIfUrlExists(folderUrl):
logger.info('Found match with lidar type: ' + lidar)
folderUrls.append(folderUrl)
lidar_types.append(lidar)
if len(folderUrls) == 0:
logger.info(
'WARNING: Could not find any lidar data for the given date!'
)
elif len(folderUrls) == 1:
# Unique solution
folderUrl = folderUrls[0]
options.type = lidar_types[0]
elif len(folderUrls) >= 2:
# Multiple solutions. Pick the good one by latitude.
logger.info("Multiples URLs to search: " +
" ".join(folderUrls))
count = -1
isGood = False
for folderUrl in folderUrls:
count += 1
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath,
lidar_types[count])
for frame in sorted(localFrameDict.keys()):
filename = localFrameDict[frame]
xmlFile = icebridge_common.xmlFile(filename)
url = os.path.join(folderUrl, xmlFile)
# Download the file
curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile
logger.info(curlCmd)
p = subprocess.Popen(curlCmd, shell=True)
os.waitpid(p.pid, 0)
latitude = icebridge_common.parseLatitude(xmlFile)
if os.path.exists(xmlFile): os.remove(xmlFile)
if hasGoodLat(latitude, isSouth):
isGood = True
options.type = lidar_types[count]
logger.info("Good latitude " + str(latitude) +
", will use " + folderUrl +
" of type " + lidar_types[count])
else:
logger.info("Bad latitude " + str(latitude) +
", will not use " + folderUrl +
" of type " + lidar_types[count])
# Stop at first file no matter what
break
if isGood:
break
if not isGood:
if options.type in LIDAR_TYPES and options.ignoreMissingLidar:
logger.info("No lidar. None of these URLs are good: " +
" ".join(folderUrls))
else:
raise Exception("None of these URLs are good: " +
" ".join(folderUrls))
else: # Other cases are simpler
folderUrl = getFolderUrl(
options.yyyymmdd,
options.year,
options.month,
options.day,
dayVal, # note here the dayVal
options.site,
options.type)
logger.info('Fetching from URL: ' + folderUrl)
(localFrameDict, localUrlDict) = \
fetchAndParseIndexFileAux(isSouth,
separateByLat, dayVal,
baseCurlCmd, folderUrl,
currIndexPath, options.type)
# Append to the main index
for frame in sorted(localFrameDict.keys()):
if options.type == 'fireball':
# This is a bugfix. Ensure that the fireball DEM has not just
# the same frame number, but also same timestamp as the ortho.
# Otherwise we may accidentally getting one from next day.
[imageDateString, imageTimeString] = \
icebridge_common.parseTimeStamps(localFrameDict[frame])
if frame not in orthoStamp:
#logger.info("Missing ortho for fireball: " + localFrameDict[frame])
continue
if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000:
# Apparently a tolerance is needed. Use 10 seconds, so the number 1000.
#logger.info("Will not use fireball DEM whose timestamp differs from ortho.")
#logger.info("Fireball is: " + localFrameDict[frame])
#logger.info("Ortho is: " + orthoFrameDict[frame])
continue
# Fetch from next day, unless already have a value. And don't fetch
# frames not in the jpeg index.
if len(dayVals) > 1 and options.type != 'jpeg':
if not frame in jpegFrameDict.keys(): continue
if frame in frameDict.keys(): continue
frameDict[frame] = localFrameDict[frame]
urlDict[frame] = localUrlDict[frame]
# Write the combined index file
icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict)
return parsedIndexPath
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
jpegFolder = icebridge_common.getJpegFolder(options.outputFolder)
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
def validateOrthosAndFireball(options, fileType, logger):
'''Validate ortho and fireball files within the current frame range. This
is expected to be in called in parallel for smaller chunks. Lidar files
will be validated serially. Jpegs get validated when converted to tif.
Return True if all is good.'''
badFiles = False
logger.info("Validating files of type: " + fileType)
if fileType == 'ortho':
dataFolder = icebridge_common.getOrthoFolder(options.outputFolder)
elif fileType == 'fireball':
dataFolder = icebridge_common.getFireballFolder(options.outputFolder)
else:
raise Exception("Unknown file type: " + fileType)
indexPath = icebridge_common.csvIndexFile(dataFolder)
if not os.path.exists(indexPath):
# The issue of what to do when the index does not exist should
# have been settled by now.
return (not badFiles)
# Fetch from disk the set of already validated files, if any
validFilesList = icebridge_common.validFilesList(options.outputFolder,
options.startFrame,
options.stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(
validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
(frameDict, urlDict) = icebridge_common.readIndexFile(indexPath,
prependFolder=True)
for frame in frameDict.keys():
if frame < options.startFrame or frame > options.stopFrame:
continue
outputPath = frameDict[frame]
xmlFile = icebridge_common.xmlFile(outputPath)
if outputPath in validFilesSet and os.path.exists(outputPath) and \
xmlFile in validFilesSet and os.path.exists(xmlFile):
#logger.info('Previously validated: ' + outputPath + ' ' + xmlFile)
continue
else:
isGood = icebridge_common.hasValidChkSum(outputPath, logger)
if not isGood:
logger.info('Found invalid data. Will wipe: ' + outputPath +
' ' + xmlFile)
os.system('rm -f ' + outputPath) # will not throw
os.system('rm -f ' + xmlFile) # will not throw
badFiles = True
else:
logger.info('Valid file: ' + outputPath)
validFilesSet.add(outputPath)
validFilesSet.add(xmlFile)
if fileType != 'fireball':
continue
# Also validate tfw
tfwFile = icebridge_common.tfwFile(outputPath)
xmlFile = icebridge_common.xmlFile(tfwFile)
if tfwFile in validFilesSet and os.path.exists(tfwFile) and \
xmlFile in validFilesSet and os.path.exists(xmlFile):
#logger.info('Previously validated: ' + tfwFile + ' ' + xmlFile)
continue
else:
isGood = icebridge_common.isValidTfw(tfwFile, logger)
if not isGood:
logger.info('Found invalid tfw. Will wipe: ' + tfwFile + ' ' +
xmlFile)
os.system('rm -f ' + tfwFile) # will not throw
os.system('rm -f ' + xmlFile) # will not throw
badFiles = True
else:
logger.info('Valid tfw file: ' + tfwFile)
validFilesSet.add(tfwFile)
validFilesSet.add(xmlFile)
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = \
icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
return (not badFiles)
def checkForImages(self, startFrame, stopFrame, logger):
'''Return true if all the images have been converted from jpeg.'''
logger.info("Checking if all jpegs have been converted.")
jpegFolder = self.getJpegFolder()
imageFolder = self.getImageFolder()
orthoFolder = self.getOrthoFolder()
if not os.path.exists(jpegFolder):
logger.info("Missing: " + jpegFolder)
return False
if not os.path.exists(imageFolder):
logger.info("Missing: " + imageFolder)
return False
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
logger.info("Missing: " + jpegIndexPath)
return False
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder = True)
# Need the orthos to get the timestamp
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder = True)
# Thorough check for missing images. It is very slow.
num = len(jpegFrameDict.keys())
allGood = True
count = 0
for frame in sorted(jpegFrameDict.keys()):
if frame < startFrame or frame > stopFrame: continue
# Add the progress, as this operation can be terribly slow
# when the filesystem is not doing too well, especially on mfe.
count = count + 1
if (count - 1) % 1000 == 0:
logger.info('Progress: ' + str(count) + '/' + str(num))
inputPath = jpegFrameDict[frame]
if not frame in orthoFrameDict:
logger.info("Missing ortho for frame: " + frame)
continue
# Make sure the timestamp and frame number are in the output file name
try:
outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame])
except Exception as e:
logger.info(str(e))
logger.info("Removing bad file: " + inputPath)
if os.path.exists(inputPath): os.remove(inputPath)
allGood = False
continue
if not os.path.exists(outputPath):
logger.info("Missing image file: " + outputPath)
allGood = False
return allGood
def fetchAllRunData(options, startFrame, stopFrame, jpegFolder, orthoFolder,
fireballFolder, lidarFolder, navFolder):
'''Download all data needed to process a run'''
logger = logging.getLogger(__name__)
logger.info('Downloading data for the run...')
baseCommand = (
('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') %
(options.yyyymmdd, options.site, startFrame, stopFrame))
if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0:
baseCommand += ' --max-num-lidar-to-fetch ' + str(
options.maxNumLidarToFetch)
if options.refetchIndex:
baseCommand += ' --refetch-index' # this was not right in older fetched runs
if options.refetchNav:
baseCommand += ' --refetch-nav' # sometimes this was corrupted
if options.stopAfterIndexFetch:
baseCommand += ' --stop-after-index-fetch'
if options.skipValidate:
baseCommand += ' --skip-validate'
if options.ignoreMissingLidar:
baseCommand += ' --ignore-missing-lidar'
if options.dryRun:
baseCommand += ' --dry-run'
jpegCommand = baseCommand + ' ' + jpegFolder
orthoCommand = baseCommand + ' ' + orthoFolder
fireballCommand = baseCommand + ' ' + fireballFolder
lidarCommand = baseCommand + ' ' + lidarFolder
navCommand = baseCommand + ' ' + navFolder
# Try to do all the downloads one after another
# - On a failure the error message should already be printed.
# - The fetching tool will not redownload existing data.
if fetch_icebridge_data.main(jpegCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(orthoCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(fireballCommand.split()) < 0:
logger.info('Fireball DEM data is optional, continuing run.')
if not options.noNavFetch:
if fetch_icebridge_data.main(navCommand.split()) < 0:
return -1
# Skip the lidar fetch if the user requested no lidar files
if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch >
0):
if fetch_icebridge_data.main(lidarCommand.split()) < 0:
return -1
# jpeg and ortho indices must be consistent
if not options.skipValidate:
logger.info("Check for consistency between raw and ortho images.")
jpegIndex = icebridge_common.csvIndexFile(jpegFolder)
orthoIndex = icebridge_common.csvIndexFile(orthoFolder)
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex)
for jpegFrame in jpegFrameDict.keys():
if jpegFrame < startFrame or jpegFrame > stopFrame:
continue
if jpegFrame not in orthoFrameDict.keys():
logger.info("Found jpeg frame missing from ortho: " +
str(jpegFrame))
#raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame))
for orthoFrame in orthoFrameDict.keys():
if orthoFrame < startFrame or orthoFrame > stopFrame:
continue
if orthoFrame not in jpegFrameDict.keys():
# This can happen, don't die because of it
logger.info("Found ortho frame missing from jpeg: " +
str(orthoFrame))
#raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame))
# TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it.
return (jpegFolder, orthoFolder, fireballFolder, lidarFolder)
def checkForImages(self, startFrame, stopFrame, logger):
'''Return true if all the images have been converted from jpeg.'''
logger.info("Checking if all jpegs have been converted.")
jpegFolder = self.getJpegFolder()
imageFolder = self.getImageFolder()
orthoFolder = self.getOrthoFolder()
if not os.path.exists(jpegFolder):
logger.info("Missing: " + jpegFolder)
return False
if not os.path.exists(imageFolder):
logger.info("Missing: " + imageFolder)
return False
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
logger.info("Missing: " + jpegIndexPath)
return False
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
# Need the orthos to get the timestamp
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " +
orthoIndexPath + ".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder=True)
# Thorough check for missing images. It is very slow.
num = len(jpegFrameDict.keys())
allGood = True
count = 0
for frame in sorted(jpegFrameDict.keys()):
if frame < startFrame or frame > stopFrame: continue
# Add the progress, as this operation can be terribly slow
# when the filesystem is not doing too well, especially on mfe.
count = count + 1
if (count - 1) % 1000 == 0:
logger.info('Progress: ' + str(count) + '/' + str(num))
inputPath = jpegFrameDict[frame]
if not frame in orthoFrameDict:
logger.info("Missing ortho for frame: " + frame)
continue
# Make sure the timestamp and frame number are in the output file name
try:
outputPath = icebridge_common.jpegToImageFile(
inputPath, orthoFrameDict[frame])
except Exception as e:
logger.info(str(e))
logger.info("Removing bad file: " + inputPath)
if os.path.exists(inputPath): os.remove(inputPath)
allGood = False
continue
if not os.path.exists(outputPath):
logger.info("Missing image file: " + outputPath)
allGood = False
return allGood
def solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder, processedFolder,
logger):
'''Some preliminary work before solving for intrinsics. Here we
look up the default calibration file, and generate an RPC
approximation of its distortion model with polynomials of degree
4. We will then create cameras and stereo DEMs using this initial
camera file with RPC distortion.'''
# Sanity checks
if options.startFrame == icebridge_common.getSmallestFrame() or \
options.stopFrame == icebridge_common.getLargestFrame():
raise Exception("When solving for intrinsics, must specify a frame range.")
if options.bundleLength != 2:
raise Exception("When solving for intrinsics, we assume bundle length of 2.")
if (options.stopFrame - options.startFrame) % 2 == 0:
raise Exception("When solving for intrinsics, must have an even number of frames, " +
" so stopFrame - startFrame must be odd.")
if options.processingSubfolder:
raise Exception("Processing subfolder not supported when solving for intrinsics.")
# Generate extra data we will use later to float intrinsics
options.stereoArgs += " --num-matches-from-disp-triplets 10000 --unalign-disparity " # --enable-fill-holes "
# Create separate directories for cameras and processed data,
# as these will be distinct than what we will finally be
# using to do the full run.
suff = "_camgen"
cameraFolder += suff
navCameraFolder += suff
processedFolder += suff
# Get the input calibration file
defaultCalibFile = ""
for frame in range(options.startFrame, options.stopFrame+1):
currCalibFile = input_conversions.getCalibrationFileForFrame(options.cameraLookupFile,
options.inputCalFolder,
frame, options.yyyymmdd,
options.site, logger)
if defaultCalibFile == "":
defaultCalibFile = currCalibFile
if defaultCalibFile != currCalibFile:
# This is important, the calibration file must be unique
raise Exception("Found two distinct calibration files: " + defaultCalibFile + \
" and " + currCalibFile)
logger.info("Default calibration file: " + defaultCalibFile)
if options.inputCalCamera != "":
defaultCalibFile = options.inputCalCamera
logger.info("Using instead the user-provided: " + defaultCalibFile)
# Find the first image in the range
jpegIndex = icebridge_common.csvIndexFile(jpegFolder)
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex,
prependFolder = True)
if options.startFrame not in jpegFrameDict.keys():
raise Exception("Could not find jpeg image for frame: " + options.startFrame)
firstImage = jpegFrameDict[options.startFrame]
# Create the RPC file before optimization
rpcCalibFile = os.path.join(processedFolder, os.path.basename(defaultCalibFile))
rpcCalibFile = rpcCalibFile.replace(".tsai", "_INIT_RPC.tsai")
logger.info("Will approximate camera model " + defaultCalibFile + " with " + \
options.outputModelType + " model " + rpcCalibFile)
os.system("mkdir -p " + os.path.dirname(rpcCalibFile))
cmd = "convert_pinhole_model --input-file " + firstImage + ' --camera-file ' + \
defaultCalibFile + ' --output-type ' + options.outputModelType + \
' --sample-spacing 50 -o ' + rpcCalibFile
logger.info(cmd)
os.system(cmd)
# Use this one from now on
options.inputCalCamera = rpcCalibFile
# Return the modified values
return (options, cameraFolder, navCameraFolder, processedFolder)
def correctFireballDems(fireballFolder, corrFireballFolder, startFrame,
stopFrame, isNorth, skipValidate, logger):
'''Fix the header problem in Fireball DEMs'''
logger.info('Correcting Fireball DEMs ...')
# Read the existing DEMs
fireballIndexPath = icebridge_common.csvIndexFile(fireballFolder)
if not os.path.exists(fireballIndexPath):
raise Exception("Error: Missing fireball index file: " +
fireballIndexPath + ".")
(fireballFrameDict, fireballUrlDict) = \
icebridge_common.readIndexFile(fireballIndexPath, prependFolder = True)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(
os.path.dirname(fireballFolder), startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(
validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
# Loop through all the input images
os.system('mkdir -p ' + corrFireballFolder)
badFiles = False
for frame in sorted(fireballFrameDict.keys()):
# Skip if outside the frame range
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
inputPath = fireballFrameDict[frame]
if not icebridge_common.isDEM(inputPath):
continue
outputPath = os.path.join(corrFireballFolder,
os.path.basename(inputPath))
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # very vebose
continue
if icebridge_common.isValidImage(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath)
validFilesSet.add(outputPath) # mark it as validated
continue
# Run the correction script
execPath = asp_system_utils.which('correct_icebridge_l3_dem')
cmd = (('%s %s %s %d') % (execPath, inputPath, outputPath, isNorth))
logger.info(cmd)
# TODO: Run this as a subprocess and check the return code
os.system(cmd)
# Check if the output file is good
if not icebridge_common.isValidImage(outputPath):
logger.error('Failed to convert dem file, wiping: ' + inputPath +
' ' + outputPath)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
badFiles = True
else:
if not skipValidate:
validFilesSet.add(outputPath) # mark it as validated
if not skipValidate:
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = icebridge_common.updateValidFilesListFromDisk(
validFilesList, validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
return (not badFiles)
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath +
".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if options.blendToFireball:
fireballFrameDict = icebridge_common.getCorrectedFireballDems(
options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate,
cameraMounting, logger):
'''Convert jpeg images from RGB to single channel.
Returns false if any files failed.'''
badFiles = False
logger.info('Converting input images to grayscale...')
os.system('mkdir -p ' + imageFolder)
# Loop through all the input images
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath +
".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder=True)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(
os.path.dirname(jpegFolder), startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(
validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
# Fast check for missing images. This is fragile, as maybe it gets
# the wrong file with a similar name, but an honest check is very slow.
imageFiles = icebridge_common.getTifs(imageFolder, prependFolder=True)
imageFrameDict = {}
for imageFile in imageFiles:
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if frame < startFrame or frame > stopFrame: continue
imageFrameDict[frame] = imageFile
for frame in sorted(jpegFrameDict.keys()):
inputPath = jpegFrameDict[frame]
# Only deal with frames in range
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if frame in imageFrameDict.keys() and skipValidate:
# Fast, hackish check
continue
if frame not in orthoFrameDict:
logger.info("Error: Could not find ortho image for jpeg frame: " +
str(frame))
# Don't want to throw here. Just ignore the missing ortho
continue
# Make sure the timestamp and frame number are in the output file name
try:
outputPath = icebridge_common.jpegToImageFile(
inputPath, orthoFrameDict[frame])
except Exception, e:
logger.info(str(e))
logger.info("Removing bad file: " + inputPath)
os.system('rm -f ' + inputPath) # will not throw
badFiles = True
continue
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # very verbose
validFilesSet.add(inputPath) # Must have this
continue
if icebridge_common.isValidImage(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath) # verbose
if not skipValidate:
# Mark both the input and the output as validated
validFilesSet.add(inputPath)
validFilesSet.add(outputPath)
continue
# Use ImageMagick tool to convert from RGB to grayscale
# - Some image orientations are rotated to make stereo processing easier.
rotateString = ''
if cameraMounting == 2: # Flight direction towards top of image
rotateString = '-rotate 90'
if cameraMounting == 3: # Flight direction towards bottom of image
rotateString = '-rotate -90'
cmd = ('%s %s -colorspace Gray %s %s') % \
(asp_system_utils.which('convert'), inputPath, rotateString, outputPath)
logger.info(cmd)
# Run command and fetch its output
p = subprocess.Popen(cmd.split(" "),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
badFiles = True
logger.error("Command failed.")
logger.error("Wiping bad files: " + inputPath + " and " +
outputPath + '\n' + output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
if not os.path.exists(outputPath):
badFiles = True
logger.error('Failed to convert jpeg file: ' + inputPath)
logger.error("Wiping bad files: " + inputPath + " and " +
outputPath + '\n' + output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
# Check for corrupted files
if error is not None:
output += error
m = re.match("^.*?premature\s+end", output,
re.IGNORECASE | re.MULTILINE | re.DOTALL)
if m:
badFiles = True
logger.error("Wiping bad files: " + inputPath + " and " +
outputPath + '\n' + output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
def getImageSpacing(orthoFolder, availableFrames, startFrame, stopFrame, forceAllFramesInRange):
'''Find a good image stereo spacing interval that gives us a good
balance between coverage and baseline width.
Also detect all frames where this is a large break after the current frame.'''
logger.info('Computing optimal image stereo interval...')
## With very few cameras this is the only possible way to process them
#if len(availableFrames) < 3 and not forceAllFramesInRange:
# return ([], {}) # No skip, no breaks
# Retrieve a list of the ortho files
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder = True)
# From the dictionary create a sorted list of ortho files in the frame range
breaks = []
largeSkips = {}
orthoFiles = []
for frame in sorted(orthoFrameDict.keys()):
# Only process frames within the range
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
orthoPath = orthoFrameDict[frame]
frame = icebridge_common.getFrameNumberFromFilename(orthoPath)
if not forceAllFramesInRange:
if frame not in availableFrames: # Skip frames we did not compute a camera for
continue
orthoFiles.append(orthoPath)
orthoFiles.sort()
numOrthos = len(orthoFiles)
# First load whatever boxes are there
projectionIndexFile = icebridge_common.projectionBoundsFile(os.path.dirname(orthoFolder))
logger.info("Reading: " + projectionIndexFile)
boundsDict = icebridge_common.readProjectionBounds(projectionIndexFile)
# Get the bounding box and frame number of each ortho image
logger.info('Loading bounding boxes...')
frames = []
updatedBounds = False # will be true if some computation got done
count = 0
for i in range(0, numOrthos):
# This can be slow, so add a progress dialong
count = count + 1
if (count - 1) % 1000 == 0:
logger.info('Progress: ' + str(count) + '/' + str(numOrthos))
thisFrame = icebridge_common.getFrameNumberFromFilename(orthoFiles[i])
if thisFrame not in boundsDict:
imageGeoInfo = asp_geo_utils.getImageGeoInfo(orthoFiles[i], getStats=False)
thisBox = imageGeoInfo['projection_bounds']
boundsDict[thisFrame] = thisBox
updatedBounds = True
frames.append(thisFrame)
# Read this file again, in case some other process modified it in the meantime.
# This won't happen in production mode, but can during testing with partial sequences.
boundsDictRecent = icebridge_common.readProjectionBounds(projectionIndexFile)
for frame in sorted(boundsDictRecent.keys()):
if not frame in boundsDict.keys():
boundsDict[frame] = boundsDictRecent[frame]
updatedBounds = True
# Save the bounds. There is always the danger that two processes will
# do that at the same time, but this is rare, as hopefully we get here
# only once from the manager. It is not a big loss if this file gets messed up.
if updatedBounds:
logger.info("Writing: " + projectionIndexFile)
icebridge_common.writeProjectionBounds(projectionIndexFile, boundsDict)
# Since we are only comparing the image bounding boxes, not their exact corners,
# these ratios are only estimates.
MAX_RATIO = 0.85 # Increase skip until we get below this...
MIN_RATIO = 0.75 # ... but don't go below this value!
NOTRY_RATIO = 0.0001 # Don't bother with overlap amounts less than this (small on purpose)
def getBboxArea(bbox):
'''Return the area of a bounding box in form of (minX, maxX, minY, maxY)'''
width = bbox[1] - bbox[0]
height = bbox[3] - bbox[2]
if (width < 0) or (height < 0):
return 0
return width*height
# Iterate over the frames and find the best stereo frame for each
for i in range(0, numOrthos-1):
thisFrame = frames[i]
thisBox = boundsDict[thisFrame]
thisArea = getBboxArea(thisBox)
interval = 1
while(True):
# Compute intersection area between this and next image
nextFrame = frames[i+interval]
nextBox = boundsDict[nextFrame]
intersect = [max(thisBox[0], nextBox[0]), # Min X
min(thisBox[1], nextBox[1]), # Max X
max(thisBox[2], nextBox[2]), # Min Y
min(thisBox[3], nextBox[3])] # Max Y
area = getBboxArea(intersect)
ratio = 0
if area > 0:
ratio = area / thisArea
if interval == 1: # Cases for the smallest interval...
if ratio < NOTRY_RATIO:
breaks.append(thisFrame) # No match for this frame
logger.info('Detected large break after frame ' + str(thisFrame))
break
if ratio < MIN_RATIO:
break # No reason to try increasing skip amounts for this frame
else: # interval > 1
if ratio < MIN_RATIO: # Went too small, walk back the interval.
interval = interval - 1
break
if ratio > MAX_RATIO: # Too much overlap, increase interval
interval = interval + 1
else: # Overlap is fine, keep this interval.
break
# Handle the case where we go past the end of frames looking for a match.
if i+interval >= len(frames):
interval = interval - 1
break
if interval > 1: # Only record larger than normal intervals.
largeSkips[thisFrame] = interval
logger.info('Detected ' + str(len(breaks)) + ' breaks in image coverage.')
logger.info('Detected ' + str(len(largeSkips)) + ' images with interval > 1.')
return (breaks, largeSkips)
def main(argsIn):
try:
# Sample usage:
# python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \
# --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \
# --num-threads 8 --num-processes 10
usage = '''blend_dems.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
# Run selection
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
# Processing options
parser.add_argument('--bundle-length', dest='bundleLength', default=2,
type=int, help="The number of images to bundle adjust and process " + \
"in a single batch.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument(
'--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This frame will also be processed.')
parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None,
help="Specify a subfolder name where the processing outputs will go. "+\
"The default is no additional folder.")
parser.add_argument(
"--compute-diff-to-prev-dem",
action="store_true",
dest="computeDiffToPrev",
default=False,
help="Compute the absolute difference between the current DEM " +
"and the one before it.")
parser.add_argument("--blend-to-fireball-footprint", action="store_true",
dest="blendToFireball", default=False,
help="Create additional blended DEMs having the same " + \
"footprint as Fireball DEMs.")
# Performance options
parser.add_argument(
'--num-processes',
dest='numProcesses',
default=1,
type=int,
help='The number of simultaneous processes to run.')
parser.add_argument('--num-threads',
dest='numThreads',
default=8,
type=int,
help='The number of threads per process.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
os.system("ulimit -c 0") # disable core dumps
os.system("rm -f core.*") # these keep on popping up
os.system("umask 022") # enforce files be readable by others
if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9:
# Make an exception for 20100422a
raise Exception("The --yyyymmdd field must have length 8 or 9.")
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
os.system('mkdir -p ' + options.outputFolder)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(options.outputFolder, logLevel,
'icebridge_blend_log')
(out, err, status) = asp_system_utils.executeCommand(['uname', '-a'],
suppressOutput=True)
logger.info("Running on machine: " + out)
logger.info(str(argsIn))
processFolder = os.path.join(options.outputFolder, 'processed')
# Handle subfolder option. This is useful for comparing results with different parameters!
if options.processingSubfolder:
processFolder = os.path.join(processFolder,
options.processingSubfolder)
logger.info('Reading from processing subfolder: ' +
options.processingSubfolder)
orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath +
".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if options.blendToFireball:
fireballFrameDict = icebridge_common.getCorrectedFireballDems(
options.outputFolder)
lidarFolder = icebridge_common.getLidarFolder(options.outputFolder)
threadText = ''
if options.numThreads:
threadText = '--threads ' + str(options.numThreads)
redo = False
suppressOutput = True
taskHandles = []
if options.numProcesses > 1:
pool = multiprocessing.Pool(options.numProcesses)
# Bound the frames
sortedFrames = sorted(orthoFrameDict.keys())
if len(sortedFrames) > 0:
if options.startFrame < sortedFrames[0]:
options.startFrame = sortedFrames[0]
if options.stopFrame > sortedFrames[-1] + 1:
options.stopFrame = sortedFrames[-1] + 1
else:
# No ortho files, that means nothing to do
options.startFrame = 0
options.stopFrame = 0
for frame in range(options.startFrame, options.stopFrame):
if not frame in orthoFrameDict:
logger.info("Error: Missing ortho file for frame: " + str(frame) +
".")
continue
orthoFile = orthoFrameDict[frame]
try:
lidarFile = icebridge_common.findMatchingLidarFile(
orthoFile, lidarFolder)
except: # Skip if no lidar file matches this frame
continue
fireballDEM = ""
if options.blendToFireball:
if frame in fireballFrameDict:
fireballDEM = fireballFrameDict[frame]
else:
logger.info("No fireball DEM for frame: " + str(frame))
args = (frame, processFolder, lidarFile, fireballDEM, options,
threadText, redo, suppressOutput)
# Run things sequentially if only one process, to make it easy to debug
if options.numProcesses > 1:
taskHandles.append(pool.apply_async(runBlend, args))
else:
runBlend(*args)
if options.numProcesses > 1:
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
logger,
interactive=False,
quitKey='q',
sleepTime=20)
icebridge_common.stopTaskPool(pool)
