def getImageList(self, prependFolder=False):
'''Return a list containing all the currently stored image files'''
imageFolder = self.getImageFolder()
images = icebridge_common.getTifs(imageFolder)
if prependFolder:
images = [os.path.join(imageFolder, x) for x in images]
images.sort()
return images
def getImageList(self, prependFolder=False):
'''Return a list containing all the currently stored image files'''
imageFolder = self.getImageFolder()
images = icebridge_common.getTifs(imageFolder)
if prependFolder:
images = [os.path.join(imageFolder, x) for x in images]
images.sort()
return images
if not fileList:
logger.error('Unable to find any camera files in ' + calFolder)
return -1
for fileName in fileList:
cameraPath = os.path.join(calFolder, fileName)
# Check if this path is valid
with open(cameraPath, 'r') as f:
for line in f:
if 'fu' in line:
goodFile = True
break
if goodFile:
break
# Get the ortho list
orthoFiles = icebridge_common.getTifs(orthoFolder)
logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.')
# Look up the frame numbers for each ortho file
infoDict = {}
for ortho in orthoFiles:
if ('gray' in ortho) or ('sub' in ortho):
continue
frame = icebridge_common.getFrameNumberFromFilename(ortho)
if frame < options.startFrame or frame > options.stopFrame:
continue
infoDict[frame] = [ortho, '']
# Get the image file list
try:
imageFiles = icebridge_common.getTifs(imageFolder)
def pushByType(run, options, logger, dataType):
# Fetch the ortho index from NSIDC if missing
outputFolder = run.getFolder()
logger.info("Output folder is " + outputFolder)
os.system("mkdir -p " + outputFolder)
# Current directory. It is important to go from /u to the real dir which is /nobackup...
unpackDir = os.path.realpath(os.getcwd())
logger.info("Unpack directory is " + unpackDir)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
fetchIndices(options, logger)
logger.info("Reading ortho index: " + orthoIndexPath)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
# Fetch unarchived folder if missing
if dataType == 'DEM':
unarchivedFolder = run.getAssemblyFolder()
elif dataType == 'ORTHO':
unarchivedFolder = run.getProcessFolder()
else:
raise Exception("Unknown data type: " + dataType)
logger.info("Unarchived data folder is " + unarchivedFolder)
# Especially for ortho, force-fetch each time, as there is no good way
# of checking if we fetched well before.
start_time()
if not archive_functions.fetchProcessedByType(run, unpackDir, logger,
dataType):
return
stop_time("fetching archived data by type: " + dataType, logger)
# Make the output directory at NSIDC
m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd)
if m:
outDir = options.site + "_" + m.group(1) + "." + m.group(
2) + "." + m.group(3)
else:
raise Exception("Could not parse: " + options.yyyymmdd)
# Keep the output directory locally here
localDirPath = os.path.join(outputFolder, dataType, outDir)
os.system("mkdir -p " + localDirPath)
logger.info("Storing the renamed " + dataType + " files in " +
localDirPath)
logger.info("Directory name at NSIDC: " + outDir)
# Read the DEMs and orthos, and copy them to outDir according to the final convention
if dataType == 'DEM':
dataFiles = icebridge_common.getTifs(unarchivedFolder,
prependFolder=True)
else:
dataFiles = glob.glob(
os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif'))
for dataFile in dataFiles:
# Here we use the convention from archive_functions.py for DEMs and from how we store orthos.
if dataType == 'DEM':
m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \
"\.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
else:
m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \
"out-ortho.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
if frameNumber < options.startFrame or frameNumber > options.stopFrame:
continue
# For each data file, copy from the ortho its meta info
if not frameNumber in orthoFrameDict.keys():
# Bugfix: Ortho fetching failed, try again
fetchIndices(options, logger)
logger.info("Re-reading ortho index: " + orthoIndexPath)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if not frameNumber in orthoFrameDict.keys():
# This time there is nothing we can do
raise Exception("Cannot find ortho for frame: " +
str(frameNumber))
orthoFile = orthoFrameDict[frameNumber]
[dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile)
# It is always possible that the ortho file date will be the next day
# after the current flight date, if the flight goes after midnight.
# So it is not unreasonable that options.yyyymmdd != dateString.
if dataType == 'DEM':
outFile = ('IODEM3_%s_%s_%05d_DEM.tif' %
(dateString, timeString, frameNumber))
else:
# TODO: Need to think more of the naming convention.
outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' %
(dateString, timeString, frameNumber))
cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(
localDirPath, outFile)
logger.info(cmd)
os.system(cmd)
# Push the directory to NSIDC
remoteDirPath = os.path.join(
os.path.basename(os.path.dirname(localDirPath)),
os.path.basename(localDirPath))
remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath)
logger.info("Storing at NSIDC in: " + remoteDirPath)
cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \
' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo
logger.info(cmd)
start_time()
(output, err,
status) = asp_system_utils.executeCommand(cmd, suppressOutput=True)
#status = os.system(cmd)
logger.info("LFTP output and error: " + output + ' ' + err)
logger.info("LFTP status: " + str(status))
#if status != 0:
# raise Exception("Problem pushing")
stop_time("push to NSIDC", logger)
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder,
inputCalCamera, cameraLookupPath, noNav,
navCameraFolder, yyyymmdd, site, refDemPath,
cameraFolder, simpleCameras, startFrame,
stopFrame, framesFile, numProcesses, numThreads,
logger):
'''Generate camera models from the ortho files.
Returns false if any files were not generated.'''
logger.info('Generating camera models from ortho images...')
imageFiles = icebridge_common.getTifs(imageFolder)
orthoFiles = icebridge_common.getTifs(orthoFolder)
if navCameraFolder != "":
estimateFiles = icebridge_common.getByExtension(
navCameraFolder, '.tsai')
else:
estimateFiles = []
# See if to process frames from file
filesSet = set()
if framesFile != "":
filesSet = icebridge_common.readLinesInSet(framesFile)
# Make a dictionary of ortho files by frame
# - The orthoFiles list contains _gray.tif as well as the original
# images. Prefer the gray versions because it saves a bit of time
# in the ortho2pinhole process.
orthoFrames = {}
for f in orthoFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
# Record this file if it is the first of this frame or
# if it is the gray version of this frame.
if (frame not in orthoFrames) or ('_gray.tif' in f):
orthoFrames[frame] = f
# Make a dictionary of estimated camera files by frame
estimatedFrames = {}
for f in estimateFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
estimatedFrames[frame] = f
imageFiles.sort()
logger.info('Starting ortho processing pool with ' + str(numProcesses) +
' processes.')
pool = multiprocessing.Pool(numProcesses)
# Loop through all input images
taskHandles = []
outputFiles = []
for imageFile in imageFiles:
# Skip non-image files (including _sub images made by stereo_gui)
# TODO: Use a function here from icebridge_common. Also replace
# all similar locations.
ext = os.path.splitext(imageFile)[1]
if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif'
in imageFile):
continue
# Get associated orthofile
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
if not frame in orthoFrames.keys():
continue
# Find the estimated camera file to use with this ortho frame.
orthoFile = orthoFrames[frame]
try:
estimatedCameraFile = estimatedFrames[frame]
estimatedCameraPath = os.path.join(navCameraFolder,
estimatedCameraFile)
except:
# For now treat this as an error, a missing nav file suggests
# that something is going wrong with the flight!
if not noNav:
logger.error('Missing nav estimated camera for frame ' +
str(frame))
continue
else:
estimatedCameraFile = None
estimatedCameraPath = None
#estimatedCameraFile = None
#estimatedCameraPath = None
# Check output file
inputPath = os.path.join(imageFolder, imageFile)
orthoPath = os.path.join(orthoFolder, orthoFile)
outputCamFile = os.path.join(
cameraFolder, icebridge_common.getCameraFileName(imageFile))
outputFiles.append(outputCamFile)
if os.path.exists(outputCamFile):
logger.info("File exists, skipping: " + outputCamFile)
os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs
continue
# Determine which input camera file will be used for this frame
if inputCalCamera == "":
inputCamFile = getCalibrationFileForFrame(cameraLookupPath,
inputCalFolder, frame,
yyyymmdd, site)
else:
# This logic will force to use a given camera rather than
# looking it up. This is not the usual way of doing things.
inputCamFile = inputCalCamera
if not os.path.exists(inputCalCamera):
raise Exception("Could not find: " + inputCalCamera)
orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath,
outputCamFile, refDemPath, simpleCameras, numThreads)
if numProcesses > 1:
# Add ortho2pinhole command to the task pool
taskHandles.append(
pool.apply_async(cameraFromOrthoWrapper, orthoArgs))
else:
# Single process, more logging info this way
cameraFromOrthoWrapper(*orthoArgs)
# Wait for all the tasks to complete
logger.info('Finished adding ' + str(len(taskHandles)) +
' tasks to the pool.')
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
logger,
interactive=False,
quitKey='q')
# All tasks should be finished
icebridge_common.stopTaskPool(pool)
logger.info('Finished ortho processing.')
# Run a check to see if we got all the output files
for f in outputFiles:
if not os.path.exists(f):
return False
return True
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate,
cameraMounting, logger):
'''Convert jpeg images from RGB to single channel.
Returns false if any files failed.'''
badFiles = False
logger.info('Converting input images to grayscale...')
os.system('mkdir -p ' + imageFolder)
# Loop through all the input images
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
".")
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder=True)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath +
".")
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder=True)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(
os.path.dirname(jpegFolder), startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(
validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
# Fast check for missing images. This is fragile, as maybe it gets
# the wrong file with a similar name, but an honest check is very slow.
imageFiles = icebridge_common.getTifs(imageFolder, prependFolder=True)
imageFrameDict = {}
for imageFile in imageFiles:
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if frame < startFrame or frame > stopFrame: continue
imageFrameDict[frame] = imageFile
for frame in sorted(jpegFrameDict.keys()):
inputPath = jpegFrameDict[frame]
# Only deal with frames in range
if not ((frame >= startFrame) and (frame <= stopFrame)):
continue
if frame in imageFrameDict.keys() and skipValidate:
# Fast, hackish check
continue
if frame not in orthoFrameDict:
logger.info("Error: Could not find ortho image for jpeg frame: " +
str(frame))
# Don't want to throw here. Just ignore the missing ortho
continue
# Make sure the timestamp and frame number are in the output file name
try:
outputPath = icebridge_common.jpegToImageFile(
inputPath, orthoFrameDict[frame])
except Exception, e:
logger.info(str(e))
logger.info("Removing bad file: " + inputPath)
os.system('rm -f ' + inputPath) # will not throw
badFiles = True
continue
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # very verbose
validFilesSet.add(inputPath) # Must have this
continue
if icebridge_common.isValidImage(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath) # verbose
if not skipValidate:
# Mark both the input and the output as validated
validFilesSet.add(inputPath)
validFilesSet.add(outputPath)
continue
# Use ImageMagick tool to convert from RGB to grayscale
# - Some image orientations are rotated to make stereo processing easier.
rotateString = ''
if cameraMounting == 2: # Flight direction towards top of image
rotateString = '-rotate 90'
if cameraMounting == 3: # Flight direction towards bottom of image
rotateString = '-rotate -90'
cmd = ('%s %s -colorspace Gray %s %s') % \
(asp_system_utils.which('convert'), inputPath, rotateString, outputPath)
logger.info(cmd)
# Run command and fetch its output
p = subprocess.Popen(cmd.split(" "),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
badFiles = True
logger.error("Command failed.")
logger.error("Wiping bad files: " + inputPath + " and " +
outputPath + '\n' + output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
if not os.path.exists(outputPath):
badFiles = True
logger.error('Failed to convert jpeg file: ' + inputPath)
logger.error("Wiping bad files: " + inputPath + " and " +
outputPath + '\n' + output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
# Check for corrupted files
if error is not None:
output += error
m = re.match("^.*?premature\s+end", output,
re.IGNORECASE | re.MULTILINE | re.DOTALL)
if m:
badFiles = True
logger.error("Wiping bad files: " + inputPath + " and " +
outputPath + '\n' + output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
def pushByType(run, options, logger, dataType):
# Fetch the ortho index from NSIDC if missing
outputFolder = run.getFolder()
logger.info("Output folder is " + outputFolder)
os.system("mkdir -p " + outputFolder)
# Current directory. It is important to go from /u to the real dir which is /nobackup...
unpackDir = os.path.realpath(os.getcwd())
logger.info("Unpack directory is " + unpackDir)
orthoFolder = icebridge_common.getOrthoFolder(outputFolder)
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
fetchIndices(options, logger)
logger.info("Reading ortho index: " + orthoIndexPath)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
# Fetch unarchived folder if missing
if dataType == 'DEM':
unarchivedFolder = run.getAssemblyFolder()
elif dataType == 'ORTHO':
unarchivedFolder = run.getProcessFolder()
else:
raise Exception("Unknown data type: " + dataType)
logger.info("Unarchived data folder is " + unarchivedFolder)
# Especially for ortho, force-fetch each time, as there is no good way
# of checking if we fetched well before.
start_time()
if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType):
return
stop_time("fetching archived data by type: " + dataType, logger)
# Make the output directory at NSIDC
m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd)
if m:
outDir = options.site + "_" + m.group(1) + "." + m.group(2) + "." + m.group(3)
else:
raise Exception("Could not parse: " + options.yyyymmdd)
# Keep the output directory locally here
localDirPath = os.path.join(outputFolder, dataType, outDir)
os.system("mkdir -p " + localDirPath)
logger.info("Storing the renamed " + dataType + " files in " + localDirPath)
logger.info("Directory name at NSIDC: " + outDir)
# Read the DEMs and orthos, and copy them to outDir according to the final convention
if dataType == 'DEM':
dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True)
else:
dataFiles = glob.glob(os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif'))
for dataFile in dataFiles:
# Here we use the convention from archive_functions.py for DEMs and from how we store orthos.
if dataType == 'DEM':
m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \
"\.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
else:
m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \
"out-ortho.tif$", dataFile)
if not m:
continue
frameNumber = int(m.group(1))
if frameNumber < options.startFrame or frameNumber > options.stopFrame:
continue
# For each data file, copy from the ortho its meta info
if not frameNumber in orthoFrameDict.keys():
# Bugfix: Ortho fetching failed, try again
fetchIndices(options, logger)
logger.info("Re-reading ortho index: " + orthoIndexPath)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
if not frameNumber in orthoFrameDict.keys():
# This time there is nothing we can do
raise Exception("Cannot find ortho for frame: " + str(frameNumber))
orthoFile = orthoFrameDict[frameNumber]
[dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile)
# It is always possible that the ortho file date will be the next day
# after the current flight date, if the flight goes after midnight.
# So it is not unreasonable that options.yyyymmdd != dateString.
if dataType == 'DEM':
outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber))
else:
# TODO: Need to think more of the naming convention.
outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber))
cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(localDirPath, outFile)
logger.info(cmd)
os.system(cmd)
# Push the directory to NSIDC
remoteDirPath = os.path.join(os.path.basename(os.path.dirname(localDirPath)),
os.path.basename(localDirPath))
remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath)
logger.info("Storing at NSIDC in: " + remoteDirPath)
cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \
' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo
logger.info(cmd)
start_time()
(output, err, status) = asp_system_utils.executeCommand(cmd,
suppressOutput = True)
#status = os.system(cmd)
logger.info("LFTP output and error: " + output + ' ' + err)
logger.info("LFTP status: " + str(status))
#if status != 0:
# raise Exception("Problem pushing")
stop_time("push to NSIDC", logger)
def main(argsIn):
# Command line parsing
try:
usage = "usage: camera_models_from_nav.py <image_folder> <ortho_folder> <cal_folder> <nav_folder> <output_folder> [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option('--start-frame',
dest='startFrame',
default=-1,
type='int',
help='The frame number to start processing with.')
parser.add_option('--stop-frame',
dest='stopFrame',
default=999999,
type='int',
help='The frame number to finish processing with.')
parser.add_option("--input-calibration-camera",
dest="inputCalCamera",
default="",
help="Use this input calibrated camera.")
parser.add_option(
'--camera-mounting',
dest='cameraMounting',
default=0,
type='int',
help=
'0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.'
)
(options, args) = parser.parse_args(argsIn)
if len(args) < 5:
print('Error: Missing arguments.')
print(usage)
return -1
imageFolder = os.path.abspath(args[0])
orthoFolder = os.path.abspath(args[1])
calFolder = os.path.abspath(args[2])
navFolder = os.path.abspath(args[3])
outputFolder = os.path.abspath(args[4])
except optparse.OptionError as msg:
raise Usage(msg)
runDir = os.path.dirname(orthoFolder)
os.system("mkdir -p " + runDir)
logLevel = logging.INFO # Make this an option??
logger = icebridge_common.setUpLogger(runDir, logLevel,
'camera_models_from_nav_log')
if not os.path.exists(orthoFolder):
logger.error('Ortho folder ' + orthoFolder + ' does not exist!')
return -1
# Find the nav file
# - There should only be one or two nav files per flight.
fileList = os.listdir(navFolder)
fileList = [x for x in fileList if '.out' in x]
if len(fileList) == 0:
logger.error('No nav files in: ' + navFolder)
return -1
navPath = os.path.join(navFolder, fileList[0])
parsedNavPath = navPath.replace('.out', '.txt')
if not asp_file_utils.fileIsNonZero(navPath):
logger.error('Nav file ' + navPath + ' is invalid!')
return -1
# Create the output file only if it is empty or does not exist
isNonEmpty = asp_file_utils.fileIsNonZero(parsedNavPath)
if not isNonEmpty:
# Initialize the output file as being empty
logger.info("Create empty file: " + parsedNavPath)
open(parsedNavPath, 'w').close()
# Append to the output parsed nav file
for fileName in fileList:
# Convert the nav file from binary to text
navPath = os.path.join(navFolder, fileName)
with open(navPath, 'r') as f:
try:
text = f.readline()
if 'HTML' in text:
# Sometimes the server is down, and instead of the binary nav file
# we are given an html file with an error message.
logger.info("Have invalid nav file: " + navPath)
return -1 # Die in this case!
except UnicodeDecodeError as e:
# Got a binary file, that means likely we are good
pass
cmd = asp_system_utils.which(
'sbet2txt.pl') + ' -q ' + navPath + ' >> ' + parsedNavPath
logger.info(cmd)
if not isNonEmpty:
os.system(cmd)
cameraPath = options.inputCalCamera
if cameraPath == "":
# No input camera file provided, look one up. It does not matter much,
# as later ortho2pinhole will insert the correct intrinsics.
goodFile = False
fileList = os.listdir(calFolder)
fileList = [x for x in fileList if (('.tsai' in x) and ('~' not in x))]
if not fileList:
logger.error('Unable to find any camera files in ' + calFolder)
return -1
for fileName in fileList:
cameraPath = os.path.join(calFolder, fileName)
# Check if this path is valid
with open(cameraPath, 'r') as f:
for line in f:
if 'fu' in line:
goodFile = True
break
if goodFile:
break
# Get the ortho list
orthoFiles = icebridge_common.getTifs(orthoFolder)
logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.')
# Look up the frame numbers for each ortho file
infoDict = {}
for ortho in orthoFiles:
if ('gray' in ortho) or ('sub' in ortho):
continue
frame = icebridge_common.getFrameNumberFromFilename(ortho)
if frame < options.startFrame or frame > options.stopFrame:
continue
infoDict[frame] = [ortho, '']
# Get the image file list
try:
imageFiles = icebridge_common.getTifs(imageFolder)
except Exception as e:
raise Exception(
"Cannot continue with nav generation, will resume later when images are created. This is not a fatal error. "
+ str(e))
logger.info('Found ' + str(len(imageFiles)) + ' image files.')
# Update the second part of each dictionary object
for image in imageFiles:
if ('gray' in image) or ('sub' in image):
continue
frame = icebridge_common.getFrameNumberFromFilename(image)
if frame < options.startFrame or frame > options.stopFrame:
continue
if frame not in infoDict:
logger.info('Image missing ortho file: ' + image)
# don't throw here, that will mess the whole batch, we will recover
# the missing one later.
continue
infoDict[frame][1] = image
os.system('mkdir -p ' + outputFolder)
orthoListFile = os.path.join(
outputFolder, 'ortho_file_list_' + str(options.startFrame) + "_" +
str(options.stopFrame) + '.csv')
# Open the output file for writing
logger.info("Writing: " + orthoListFile)
with open(orthoListFile, 'w') as outputFile:
# Loop through frames in order
for key in sorted(infoDict):
# Write the ortho name and the output camera name to the file
(ortho, image) = infoDict[key]
if not image:
#raise Exception('Ortho missing image file: ' +ortho)
continue
camera = image.replace('.tif', '.tsai')
outputFile.write(ortho + ', ' + camera + '\n')
# Check if we already have all of the output camera files.
haveAllFiles = True
with open(orthoListFile, 'r') as inputFile:
for line in inputFile:
parts = line.split(',')
camPath = os.path.join(outputFolder, parts[1].strip())
if not asp_file_utils.fileIsNonZero(camPath):
logger.info('Missing file -> ' + camPath)
haveAllFiles = False
break
# Call the C++ tool to generate a camera model for each ortho file
if not haveAllFiles:
cmd = (
'nav2cam --input-cam %s --nav-file %s --cam-list %s --output-folder %s --camera-mounting %d'
% (cameraPath, parsedNavPath, orthoListFile, outputFolder,
options.cameraMounting))
logger.info(cmd)
os.system(cmd)
else:
logger.info("All nav files were already generated.")
# Generate a kml file for the nav camera files
kmlPath = os.path.join(outputFolder, 'nav_cameras.kml')
# This is a hack. If we are invoked from a Pleiades node, do not
# create this kml file, as nodes will just overwrite each other.
# This job may happen anyway earlier or later when on the head node.
if not 'PBS_NODEFILE' in os.environ:
try:
tempPath = os.path.join(outputFolder, 'list.txt')
logger.info('Generating nav camera kml file: ' + kmlPath)
os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath)
orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole')
cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath
logger.info(cmd)
asp_system_utils.executeCommand(cmd,
kmlPath,
suppressOutput=True,
redo=False)
os.remove(tempPath)
except Exception as e:
logger.info("Warning: " + str(e))
logger.info('Finished generating camera models from nav!')
return 0
def getCameraModelsFromOrtho(imageFolder, orthoFolder,
inputCalFolder, inputCalCamera,
cameraLookupFile,
noNav,
navCameraFolder,
yyyymmdd, site,
refDemPath, cameraFolder,
simpleCameras,
startFrame, stopFrame,
framesFile,
numProcesses, numThreads, logger):
'''Generate camera models from the ortho files.
Returns false if any files were not generated.'''
logger.info('Generating camera models from ortho images...')
imageFiles = icebridge_common.getTifs(imageFolder)
orthoFiles = icebridge_common.getTifs(orthoFolder)
if navCameraFolder != "":
estimateFiles = icebridge_common.getByExtension(navCameraFolder, '.tsai')
else:
estimateFiles = []
# See if to process frames from file
filesSet = set()
if framesFile != "":
filesSet = icebridge_common.readLinesInSet(framesFile)
# Make a dictionary of ortho files by frame
# - The orthoFiles list contains _gray.tif as well as the original
# images. Prefer the gray versions because it saves a bit of time
# in the ortho2pinhole process.
orthoFrames = {}
for f in orthoFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
# Record this file if it is the first of this frame or
# if it is the gray version of this frame.
if (frame not in orthoFrames) or ('_gray.tif' in f):
orthoFrames[frame] = f
# Make a dictionary of estimated camera files by frame
estimatedFrames = {}
for f in estimateFiles:
frame = icebridge_common.getFrameNumberFromFilename(f)
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
estimatedFrames[frame] = f
imageFiles.sort()
logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.')
pool = multiprocessing.Pool(numProcesses)
# Loop through all input images
taskHandles = []
outputFiles = []
for imageFile in imageFiles:
# Skip non-image files (including _sub images made by stereo_gui)
# TODO: Use a function here from icebridge_common. Also replace
# all similar locations.
ext = os.path.splitext(imageFile)[1]
if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile):
continue
# Get associated orthofile
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if (framesFile != "") and (str(frame) not in filesSet):
continue
if not frame in orthoFrames.keys():
continue
# Find the estimated camera file to use with this ortho frame.
orthoFile = orthoFrames[frame]
try:
estimatedCameraFile = estimatedFrames[frame]
estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile)
except:
# For now treat this as an error, a missing nav file suggests
# that something is going wrong with the flight!
if not noNav:
logger.error('Missing nav estimated camera for frame ' + str(frame))
continue
else:
estimatedCameraFile = None
estimatedCameraPath = None
#estimatedCameraFile = None
#estimatedCameraPath = None
# Check output file
inputPath = os.path.join(imageFolder, imageFile)
orthoPath = os.path.join(orthoFolder, orthoFile)
outputCamFile = os.path.join(cameraFolder,
icebridge_common.getCameraFileName(imageFile))
outputFiles.append(outputCamFile)
if os.path.exists(outputCamFile):
logger.info("File exists, skipping: " + outputCamFile)
os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs
continue
# Determine which input camera file will be used for this frame
if inputCalCamera == "":
inputCamFile = getCalibrationFileForFrame(cameraLookupFile, inputCalFolder,
frame, yyyymmdd, site, logger)
else:
# This logic will force to use a given camera rather than
# looking it up. This is not the usual way of doing things.
inputCamFile = inputCalCamera
if not os.path.exists(inputCalCamera):
raise Exception("Could not find: " + inputCalCamera)
orthoArgs = (inputPath, orthoPath, inputCamFile,
estimatedCameraPath, outputCamFile,
refDemPath, simpleCameras, numThreads)
if numProcesses > 1:
# Add ortho2pinhole command to the task pool
taskHandles.append(pool.apply_async(cameraFromOrthoWrapper, orthoArgs))
else:
# Single process, more logging info this way
cameraFromOrthoWrapper(*orthoArgs)
# Wait for all the tasks to complete
logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.')
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False,
quitKey='q')
# All tasks should be finished
icebridge_common.stopTaskPool(pool)
logger.info('Finished ortho processing.')
# Run a check to see if we got all the output files
for f in outputFiles:
if not os.path.exists(f):
return False
return True
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate,
cameraMounting, logger):
'''Convert jpeg images from RGB to single channel.
Returns false if any files failed.'''
badFiles = False
logger.info('Converting input images to grayscale...')
os.system('mkdir -p ' + imageFolder)
# Loop through all the input images
jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
if not os.path.exists(jpegIndexPath):
raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".")
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath,
prependFolder = True)
# Need the orthos to get the timestamp
orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder))
orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
if not os.path.exists(orthoIndexPath):
raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".")
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath,
prependFolder = True)
if not skipValidate:
validFilesList = icebridge_common.validFilesList(os.path.dirname(jpegFolder),
startFrame, stopFrame)
validFilesSet = set()
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
numInitialValidFiles = len(validFilesSet)
# Fast check for missing images. This is fragile, as maybe it gets
# the wrong file with a similar name, but an honest check is very slow.
imageFiles = icebridge_common.getTifs(imageFolder, prependFolder = True)
imageFrameDict = {}
for imageFile in imageFiles:
frame = icebridge_common.getFrameNumberFromFilename(imageFile)
if frame < startFrame or frame > stopFrame: continue
imageFrameDict[frame] = imageFile
for frame in sorted(jpegFrameDict.keys()):
inputPath = jpegFrameDict[frame]
# Only deal with frames in range
if not ( (frame >= startFrame) and (frame <= stopFrame) ):
continue
if frame in imageFrameDict.keys() and skipValidate:
# Fast, hackish check
continue
if frame not in orthoFrameDict:
logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame))
# Don't want to throw here. Just ignore the missing ortho
continue
# Make sure the timestamp and frame number are in the output file name
try:
outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame])
except Exception as e:
logger.info(str(e))
logger.info("Removing bad file: " + inputPath)
os.system('rm -f ' + inputPath) # will not throw
badFiles = True
continue
# Skip existing valid files
if skipValidate:
if os.path.exists(outputPath):
logger.info("File exists, skipping: " + outputPath)
continue
else:
if outputPath in validFilesSet and os.path.exists(outputPath):
#logger.info('Previously validated: ' + outputPath) # very verbose
validFilesSet.add(inputPath) # Must have this
continue
if icebridge_common.isValidImage(outputPath):
#logger.info("File exists and is valid, skipping: " + outputPath) # verbose
if not skipValidate:
# Mark both the input and the output as validated
validFilesSet.add(inputPath)
validFilesSet.add(outputPath)
continue
# Use ImageMagick tool to convert from RGB to grayscale
# - Some image orientations are rotated to make stereo processing easier.
rotateString = ''
if cameraMounting == 2: # Flight direction towards top of image
rotateString = '-rotate 90 '
if cameraMounting == 3: # Flight direction towards bottom of image
rotateString = '-rotate -90 '
cmd = ('%s %s -colorspace Gray %s%s') % \
(asp_system_utils.which('convert'), inputPath, rotateString, outputPath)
logger.info(cmd)
# Run command and fetch its output
p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
output, error = p.communicate()
if p.returncode != 0:
badFiles = True
logger.error("Command failed.")
logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n'
+ output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
if not os.path.exists(outputPath):
badFiles = True
logger.error('Failed to convert jpeg file: ' + inputPath)
logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n'
+ output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
# Check for corrupted files
if error is not None:
output += error
m = re.match("^.*?premature\s+end", output, re.IGNORECASE|re.MULTILINE|re.DOTALL)
if m:
badFiles = True
logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n'
+ output)
os.system('rm -f ' + inputPath) # will not throw
os.system('rm -f ' + outputPath) # will not throw
if not skipValidate:
# Write to disk the list of validated files, but only if new
# validations happened. First re-read that list, in case a
# different process modified it in the meantime, such as if two
# managers are running at the same time.
numFinalValidFiles = len(validFilesSet)
if numInitialValidFiles != numFinalValidFiles:
validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList,
validFilesSet)
icebridge_common.writeValidFilesList(validFilesList, validFilesSet)
if badFiles:
logger.error("Converstion of JPEGs failed. If any files were corrupted, " +
"they were removed, and need to be re-fetched.")
return (not badFiles)