def fetchAllRunData(yyyymmdd, site, dryRun, startFrame, stopFrame,
maxNumToFetch, fetchNextDay, skipValidate, outputFolder,
jpegFolder, orthoFolder, demFolder, lidarFolder):
'''Download all data needed to process a run'''
logger = logging.getLogger(__name__)
logger.info('Downloading all data for the run! This could take a while.')
baseCommand = (
('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') %
(yyyymmdd, site, startFrame, stopFrame))
if maxNumToFetch >= 0:
baseCommand += ' --max-num-to-fetch ' + str(maxNumToFetch)
if fetchNextDay:
baseCommand += ' --fetch-from-next-day-also'
if skipValidate:
baseCommand += ' --skip-validate'
if dryRun:
baseCommand += ' --dry-run'
jpegCommand = '--type image ' + baseCommand + ' ' + jpegFolder
orthoCommand = '--type ortho ' + baseCommand + ' ' + orthoFolder
demCommand = '--type dem ' + baseCommand + ' ' + demFolder
lidarCommand = '--type lidar ' + baseCommand + ' ' + lidarFolder
# TODO: Handle runs without DEM or ORTHO data.
# Try to do all the downloads one after another
# - On a failure the error message should already be printed.
# - The fetching tool will not redownload existing data.
if fetch_icebridge_data.main(jpegCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(orthoCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(demCommand.split()) < 0:
print 'DEM data is optional, continuing run.'
if fetch_icebridge_data.main(lidarCommand.split()) < 0:
return -1
return (jpegFolder, orthoFolder, demFolder, lidarFolder)
def fetchAllRunData(options, startFrame, stopFrame,
jpegFolder, orthoFolder, fireballFolder, lidarFolder, navFolder):
'''Download all data needed to process a run'''
logger = logging.getLogger(__name__)
logger.info('Downloading data for the run...')
baseCommand = (('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d')
% (options.yyyymmdd, options.site, startFrame, stopFrame))
if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0:
baseCommand += ' --max-num-lidar-to-fetch ' + str(options.maxNumLidarToFetch)
if options.refetchIndex:
baseCommand += ' --refetch-index' # this was not right in older fetched runs
if options.refetchNav:
baseCommand += ' --refetch-nav' # sometimes this was corrupted
if options.stopAfterIndexFetch:
baseCommand += ' --stop-after-index-fetch'
if options.skipValidate:
baseCommand += ' --skip-validate'
if options.ignoreMissingLidar:
baseCommand += ' --ignore-missing-lidar'
if options.dryRun:
baseCommand += ' --dry-run'
jpegCommand = baseCommand + ' ' + jpegFolder
orthoCommand = baseCommand + ' ' + orthoFolder
fireballCommand = baseCommand + ' ' + fireballFolder
lidarCommand = baseCommand + ' ' + lidarFolder
navCommand = baseCommand + ' ' + navFolder
# Try to do all the downloads one after another
# - On a failure the error message should already be printed.
# - The fetching tool will not redownload existing data.
if fetch_icebridge_data.main(jpegCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(orthoCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(fireballCommand.split()) < 0:
logger.info('Fireball DEM data is optional, continuing run.')
if not options.noNavFetch:
if fetch_icebridge_data.main(navCommand.split()) < 0:
return -1
# Skip the lidar fetch if the user requested no lidar files
if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch > 0):
if fetch_icebridge_data.main(lidarCommand.split()) < 0:
return -1
# jpeg and ortho indices must be consistent
if not options.skipValidate:
logger.info("Check for consistency between raw and ortho images.")
jpegIndex = icebridge_common.csvIndexFile(jpegFolder)
orthoIndex = icebridge_common.csvIndexFile(orthoFolder)
(jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex)
(orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex)
for jpegFrame in jpegFrameDict.keys():
if jpegFrame < startFrame or jpegFrame > stopFrame:
continue
if jpegFrame not in orthoFrameDict.keys():
logger.info("Found jpeg frame missing from ortho: " + str(jpegFrame))
#raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame))
for orthoFrame in orthoFrameDict.keys():
if orthoFrame < startFrame or orthoFrame > stopFrame:
continue
if orthoFrame not in jpegFrameDict.keys():
# This can happen, don't die because of it
logger.info("Found ortho frame missing from jpeg: " + str(orthoFrame))
#raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame))
# TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it.
return 0
def fetchAllRunData(options, startFrame, stopFrame, jpegFolder, orthoFolder,
fireballFolder, lidarFolder, navFolder):
'''Download all data needed to process a run'''
logger = logging.getLogger(__name__)
logger.info('Downloading data for the run...')
baseCommand = (
('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') %
(options.yyyymmdd, options.site, startFrame, stopFrame))
if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0:
baseCommand += ' --max-num-lidar-to-fetch ' + str(
options.maxNumLidarToFetch)
if options.refetchIndex:
baseCommand += ' --refetch-index' # this was not right in older fetched runs
if options.refetchNav:
baseCommand += ' --refetch-nav' # sometimes this was corrupted
if options.stopAfterIndexFetch:
baseCommand += ' --stop-after-index-fetch'
if options.skipValidate:
baseCommand += ' --skip-validate'
if options.ignoreMissingLidar:
baseCommand += ' --ignore-missing-lidar'
if options.dryRun:
baseCommand += ' --dry-run'
jpegCommand = baseCommand + ' ' + jpegFolder
orthoCommand = baseCommand + ' ' + orthoFolder
fireballCommand = baseCommand + ' ' + fireballFolder
lidarCommand = baseCommand + ' ' + lidarFolder
navCommand = baseCommand + ' ' + navFolder
# Try to do all the downloads one after another
# - On a failure the error message should already be printed.
# - The fetching tool will not redownload existing data.
if fetch_icebridge_data.main(jpegCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(orthoCommand.split()) < 0:
return -1
if fetch_icebridge_data.main(fireballCommand.split()) < 0:
logger.info('Fireball DEM data is optional, continuing run.')
if not options.noNavFetch:
if fetch_icebridge_data.main(navCommand.split()) < 0:
return -1
# Skip the lidar fetch if the user requested no lidar files
if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch >
0):
if fetch_icebridge_data.main(lidarCommand.split()) < 0:
return -1
# jpeg and ortho indices must be consistent
if not options.skipValidate:
logger.info("Check for consistency between raw and ortho images.")
jpegIndex = icebridge_common.csvIndexFile(jpegFolder)
orthoIndex = icebridge_common.csvIndexFile(orthoFolder)
(jpegFrameDict,
jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex)
(orthoFrameDict,
orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex)
for jpegFrame in jpegFrameDict.keys():
if jpegFrame < startFrame or jpegFrame > stopFrame:
continue
if jpegFrame not in orthoFrameDict.keys():
logger.info("Found jpeg frame missing from ortho: " +
str(jpegFrame))
#raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame))
for orthoFrame in orthoFrameDict.keys():
if orthoFrame < startFrame or orthoFrame > stopFrame:
continue
if orthoFrame not in jpegFrameDict.keys():
# This can happen, don't die because of it
logger.info("Found ortho frame missing from jpeg: " +
str(orthoFrame))
#raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame))
# TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it.
return (jpegFolder, orthoFolder, fireballFolder, lidarFolder)
