def fetchAllRunData(yyyymmdd, site, dryRun, startFrame, stopFrame, maxNumToFetch, fetchNextDay, skipValidate, outputFolder, jpegFolder, orthoFolder, demFolder, lidarFolder): '''Download all data needed to process a run''' logger = logging.getLogger(__name__) logger.info('Downloading all data for the run! This could take a while.') baseCommand = ( ('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') % (yyyymmdd, site, startFrame, stopFrame)) if maxNumToFetch >= 0: baseCommand += ' --max-num-to-fetch ' + str(maxNumToFetch) if fetchNextDay: baseCommand += ' --fetch-from-next-day-also' if skipValidate: baseCommand += ' --skip-validate' if dryRun: baseCommand += ' --dry-run' jpegCommand = '--type image ' + baseCommand + ' ' + jpegFolder orthoCommand = '--type ortho ' + baseCommand + ' ' + orthoFolder demCommand = '--type dem ' + baseCommand + ' ' + demFolder lidarCommand = '--type lidar ' + baseCommand + ' ' + lidarFolder # TODO: Handle runs without DEM or ORTHO data. # Try to do all the downloads one after another # - On a failure the error message should already be printed. # - The fetching tool will not redownload existing data. if fetch_icebridge_data.main(jpegCommand.split()) < 0: return -1 if fetch_icebridge_data.main(orthoCommand.split()) < 0: return -1 if fetch_icebridge_data.main(demCommand.split()) < 0: print 'DEM data is optional, continuing run.' if fetch_icebridge_data.main(lidarCommand.split()) < 0: return -1 return (jpegFolder, orthoFolder, demFolder, lidarFolder)
def fetchAllRunData(options, startFrame, stopFrame, jpegFolder, orthoFolder, fireballFolder, lidarFolder, navFolder): '''Download all data needed to process a run''' logger = logging.getLogger(__name__) logger.info('Downloading data for the run...') baseCommand = (('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') % (options.yyyymmdd, options.site, startFrame, stopFrame)) if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0: baseCommand += ' --max-num-lidar-to-fetch ' + str(options.maxNumLidarToFetch) if options.refetchIndex: baseCommand += ' --refetch-index' # this was not right in older fetched runs if options.refetchNav: baseCommand += ' --refetch-nav' # sometimes this was corrupted if options.stopAfterIndexFetch: baseCommand += ' --stop-after-index-fetch' if options.skipValidate: baseCommand += ' --skip-validate' if options.ignoreMissingLidar: baseCommand += ' --ignore-missing-lidar' if options.dryRun: baseCommand += ' --dry-run' jpegCommand = baseCommand + ' ' + jpegFolder orthoCommand = baseCommand + ' ' + orthoFolder fireballCommand = baseCommand + ' ' + fireballFolder lidarCommand = baseCommand + ' ' + lidarFolder navCommand = baseCommand + ' ' + navFolder # Try to do all the downloads one after another # - On a failure the error message should already be printed. # - The fetching tool will not redownload existing data. if fetch_icebridge_data.main(jpegCommand.split()) < 0: return -1 if fetch_icebridge_data.main(orthoCommand.split()) < 0: return -1 if fetch_icebridge_data.main(fireballCommand.split()) < 0: logger.info('Fireball DEM data is optional, continuing run.') if not options.noNavFetch: if fetch_icebridge_data.main(navCommand.split()) < 0: return -1 # Skip the lidar fetch if the user requested no lidar files if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch > 0): if fetch_icebridge_data.main(lidarCommand.split()) < 0: return -1 # jpeg and ortho indices must be consistent if not options.skipValidate: logger.info("Check for consistency between raw and ortho images.") jpegIndex = icebridge_common.csvIndexFile(jpegFolder) orthoIndex = icebridge_common.csvIndexFile(orthoFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex) for jpegFrame in jpegFrameDict.keys(): if jpegFrame < startFrame or jpegFrame > stopFrame: continue if jpegFrame not in orthoFrameDict.keys(): logger.info("Found jpeg frame missing from ortho: " + str(jpegFrame)) #raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame)) for orthoFrame in orthoFrameDict.keys(): if orthoFrame < startFrame or orthoFrame > stopFrame: continue if orthoFrame not in jpegFrameDict.keys(): # This can happen, don't die because of it logger.info("Found ortho frame missing from jpeg: " + str(orthoFrame)) #raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame)) # TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it. return 0
def fetchAllRunData(options, startFrame, stopFrame, jpegFolder, orthoFolder, fireballFolder, lidarFolder, navFolder): '''Download all data needed to process a run''' logger = logging.getLogger(__name__) logger.info('Downloading data for the run...') baseCommand = ( ('--yyyymmdd %s --site %s --start-frame %d --stop-frame %d') % (options.yyyymmdd, options.site, startFrame, stopFrame)) if options.maxNumLidarToFetch is not None and options.maxNumLidarToFetch >= 0: baseCommand += ' --max-num-lidar-to-fetch ' + str( options.maxNumLidarToFetch) if options.refetchIndex: baseCommand += ' --refetch-index' # this was not right in older fetched runs if options.refetchNav: baseCommand += ' --refetch-nav' # sometimes this was corrupted if options.stopAfterIndexFetch: baseCommand += ' --stop-after-index-fetch' if options.skipValidate: baseCommand += ' --skip-validate' if options.ignoreMissingLidar: baseCommand += ' --ignore-missing-lidar' if options.dryRun: baseCommand += ' --dry-run' jpegCommand = baseCommand + ' ' + jpegFolder orthoCommand = baseCommand + ' ' + orthoFolder fireballCommand = baseCommand + ' ' + fireballFolder lidarCommand = baseCommand + ' ' + lidarFolder navCommand = baseCommand + ' ' + navFolder # Try to do all the downloads one after another # - On a failure the error message should already be printed. # - The fetching tool will not redownload existing data. if fetch_icebridge_data.main(jpegCommand.split()) < 0: return -1 if fetch_icebridge_data.main(orthoCommand.split()) < 0: return -1 if fetch_icebridge_data.main(fireballCommand.split()) < 0: logger.info('Fireball DEM data is optional, continuing run.') if not options.noNavFetch: if fetch_icebridge_data.main(navCommand.split()) < 0: return -1 # Skip the lidar fetch if the user requested no lidar files if (options.maxNumLidarToFetch is None) or (options.maxNumLidarToFetch > 0): if fetch_icebridge_data.main(lidarCommand.split()) < 0: return -1 # jpeg and ortho indices must be consistent if not options.skipValidate: logger.info("Check for consistency between raw and ortho images.") jpegIndex = icebridge_common.csvIndexFile(jpegFolder) orthoIndex = icebridge_common.csvIndexFile(orthoFolder) (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndex) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndex) for jpegFrame in jpegFrameDict.keys(): if jpegFrame < startFrame or jpegFrame > stopFrame: continue if jpegFrame not in orthoFrameDict.keys(): logger.info("Found jpeg frame missing from ortho: " + str(jpegFrame)) #raise Exception ("Found jpeg frame missing from ortho:" + str(jpegFrame)) for orthoFrame in orthoFrameDict.keys(): if orthoFrame < startFrame or orthoFrame > stopFrame: continue if orthoFrame not in jpegFrameDict.keys(): # This can happen, don't die because of it logger.info("Found ortho frame missing from jpeg: " + str(orthoFrame)) #raise Exception ("Found ortho frame missing from jpeg:" + str(orthoFrame)) # TODO: Wipe any ortho and jpeg images not in the index, or at least warn about it. return (jpegFolder, orthoFolder, fireballFolder, lidarFolder)