def get_opasses(self, file, sectorfiles=None):
log.info('Running pass predictor on: ' + str(file))
# Paths sometimes mess this up. Use os.path.basename
fn = DataFileName(os.path.basename(file))
sfn = fn.create_standard(downloadSiteObj=self)
if sectorfiles:
opasses = pass_prediction([sfn.satname], [sfn.sensorname],
None,
None,
sfn.datetime - timedelta(hours=9),
sfn.datetime,
single=True,
force=True,
sectorfiles=sectorfiles)
elif self.sector_file:
opasses = pass_prediction([sfn.satname], [sfn.sensorname],
self.sector_file,
self.sectorlist,
sfn.datetime - timedelta(hours=9),
sfn.datetime,
force=True,
single=True)
else:
opasses = pass_prediction([sfn.satname], [sfn.sensorname],
None,
None,
sfn.datetime - timedelta(hours=9),
sfn.datetime,
force=True,
single=True)
return opasses
def predict_sectors(platform_name, source_name, start_dt, end_dt):
'''
Attempt to find overlapping sectors using the overpass predictor.
It appears that if no overpasses are found we will still continue to process with ALL sectors.
I'm not sure why this would be needed. Shouldn't we always be able to get a sector list here?
'''
opasses = pass_prediction([platform_name], [source_name], None, None, start_dt, end_dt,
force=True, single=True, quiet=True)
if not opasses:
return None
# Since we found overpasses build the sector list
sectorlist = []
check_enddt = df.end_datetime
if df.start_datetime == df.end_datetime:
check_enddt = df.end_datetime + timedelta(hours=2)
log.info('Matching overpasses between '+str(df.start_datetime)+' and '+str(check_enddt)+': ')
for opass in opasses:
if _FileNameBase.is_concurrent_with(opass.startdt, df.start_datetime, opass.enddt, check_enddt):
log.info('\tOverlaps overpass {0}'.format(opass.opass))
sectorlist.extend([yy.lower() for yy in opass.actualsectornames])
return sectorlist
def process_overpass(
satellite,
sensor,
productlist,
sectorlist,
sectorfiles,
extra_dirs,
sector_file,
datelist,
hourlist=None,
data_outpath=None,
product_outpath=None,
list=False,
clean=False,
forceclean=False,
download=False,
queue=None,
mp_max_cpus=1,
allstatic=True,
alldynamic=True,
tc=False,
volcano=False,
quiet=False,
start_datetime=None,
end_datetime=None,
):
if quiet:
log.setLevel(35)
log.interactive('')
opasses = []
old_opasses = []
overall_start_dt = None
overall_end_dt = None
single = False
both = False
if sectorlist:
single = True
both = False
if hourlist == None:
for datestr in datelist:
if sectorlist:
log.interactive(
'Checking for overpasses $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' -s "' +
' '.join(sectorlist) + '" --all')
else:
log.interactive(
'Checking for overpasses $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' --all')
sys.stdout.write('.')
sys.stdout.flush()
start_dt = datetime.strptime(datestr + '0000', '%Y%m%d%H%M')
end_dt = datetime.strptime(datestr + '2359', '%Y%m%d%H%M')
opasses.extend(
pass_prediction([satellite], [sensor],
sector_file,
sectorlist,
start_dt - timedelta(minutes=15),
end_dt + timedelta(minutes=15),
single=single,
both=both,
force=True,
quiet=quiet))
sys.stdout.write('\n')
if opasses and len(opasses) < 200 and len(opasses) != 0:
log.interactive('Available overpasses: ' + bigindent +
bigindent.join(sorted(str(val)
for val in opasses)) + '\n')
elif opasses:
log.interactive(
str(len(opasses)) + ' available overpasses, not listing\n')
return opasses
else:
hourstart = hourlist[0]
if len(hourlist) == 1:
hourend = hourlist[0]
else:
hourend = hourlist[-1]
for datestr in datelist:
if sectorlist and hourlist:
log.interactive(
'Checking for overpasses for $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' -H "' +
' '.join(hourlist) + '" -s "' + ' '.join(sectorlist) +
'" --all')
else:
log.interactive(
'Checking for overpasses for $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' --all')
sys.stdout.write('.')
sys.stdout.flush()
start_dt = datetime.strptime(datestr + hourstart + '00',
'%Y%m%d%H%M')
start_dt = start_dt - timedelta(minutes=15)
if overall_start_dt == None or overall_start_dt > start_dt:
overall_start_dt = start_dt
end_dt = datetime.strptime(datestr + hourend + '59', '%Y%m%d%H%M')
end_dt = end_dt + timedelta(minutes=15)
if overall_end_dt == None or overall_end_dt < end_dt:
overall_end_dt = end_dt
opasses.extend(
pass_prediction([satellite], [sensor],
sector_file,
sectorlist,
start_dt,
end_dt,
single=single,
force=True,
quiet=quiet))
sys.stdout.write('\n')
if opasses and len(opasses) < 20:
log.interactive('Available overpasses: ' + bigindent +
bigindent.join(sorted(str(val)
for val in opasses)) + '\n\n')
elif opasses:
log.interactive(
str(len(opasses)) + ' available overpasses, not listing\n\n')
# Start 8h before start time to make sure we can get the
# sector file entry before
if sensor != 'modis':
overall_start_dt = overall_start_dt - timedelta(minutes=480)
log.info('Overall start and end times: ' + str(overall_start_dt) + ' to ' +
str(overall_end_dt))
if download == True:
log.interactive('queue: ' + str(queue) + '\n\n')
data_type = default_data_type[sensor]
host_type = default_host_type[data_type]
#Can't we do something to minimize the copypaste done here? Hard to maintain...
if (data_type, host_type) in non_qsubbed:
for opass in opasses:
log.info('sectorfiles: ' + str(sectorfiles))
sector_file = sectorfile.open(allstatic=allstatic,
alldynamic=alldynamic,
tc=tc,
start_datetime=opass.startdt -
timedelta(hours=6),
end_datetime=opass.enddt,
one_per_sector=True)
if not sectorfiles:
currsectorfiles = sector_file.names
else:
currsectorfiles = sectorfiles
log.info('currsectorfiles: ' + str(currsectorfiles))
log.interactive('Downloading opass: '******'\n\n')
si = SatSensorInfo(satellite, sensor)
# If they are very long files (ie, full orbit), make
# sure we get the file before the overpass time
startdt = opass.startdt - timedelta(minutes=si.mins_per_file)
downloader(
data_type,
host_type,
sector_file=sector_file,
sectorlist=sectorlist,
sectorfiles=currsectorfiles,
productlist=productlist,
data_outpath=data_outpath,
product_outpath=product_outpath,
start_datetime=startdt,
end_datetime=opass.enddt,
queue=queue,
allstatic=allstatic,
alldynamic=alldynamic,
tc=tc,
volcano=volcano,
#max_connections=8,
max_wait_seconds=None,
)
time.sleep(5)
else:
log.interactive(sectorfiles)
downloader(
data_type,
host_type,
sector_file=sector_file,
sectorlist=sectorlist,
sectorfiles=sectorfiles,
productlist=productlist,
data_outpath=data_outpath,
product_outpath=product_outpath,
start_datetime=overall_start_dt,
end_datetime=overall_end_dt,
queue=queue,
allstatic=allstatic,
alldynamic=alldynamic,
tc=tc,
opasses=opasses,
#max_connections=8,
max_wait_seconds=None,
)
time.sleep(5)
all_files = []
# Reverse=True for newest first
all_files = sorted(find_available_data_files(opasses, start_dt, satellite,
sensor, extra_dirs),
reverse=True)
log.info('Done sorting default')
#shell()
if productlist and 'near-constant-contrast' in productlist:
log.info(' Checking near-constant-contrast files')
# Reverse=True for newest first
all_files = sorted(find_available_data_files(opasses,
start_dt,
satellite,
sensor,
extra_dirs,
prodtype='ncc'),
reverse=True)
file_str = '\n\t'.join(all_files)
log.info('Files found current search time for %s: \n\t%s' %
(str(opasses), file_str))
if not all_files:
log.info('No files available in directories listed above')
log.info(
'To check alternate directories, you can call (replace /sb2/viirs and /sb1/viirs with the paths where data files are available): '
)
infostr = ''
if productlist:
infostr += '-p ' + "'" + ' '.join(productlist) + "'"
if sectorlist:
infostr += '-s ' + "'" + ' '.join(sectorlist) + "'"
log.info(
"process_overpass.py %s %s '%s' -d '/sb2/viirs /sb1/viirs' %s -H '%s'"
% (satellite, sensor, ' '.join(datelist), infostr,
' '.join(hourlist)))
return None
try:
for opass in opasses:
currdatelist = []
day_count = (opass.enddt - opass.startdt).days + 1
for dt in (opass.startdt + timedelta(n) for n in range(day_count)):
currdatelist.append(dt.strftime('%Y%m%d'))
log.info('Checking for existing products to clean... clean: ' +
str(clean) + ' forceclean: ' + str(forceclean))
find_existing_products(sensor, sector_file,
opass.actualsectornames, productlist,
currdatelist, clean, forceclean)
except ProductError, resp:
log.error(str(resp) + ' Check spelling?')
' Appear to be no data files in directory !! Skipping'
)
continue
else:
dfn = DataFileName(os.path.basename(file))
dt = dfn.datetime
if start_datetime and (dfn.datetime < start_datetime
or dfn.datetime > end_datetime):
log.interactive('Outside of time range, skipping')
continue
currsectorlist = []
if not sectorlist:
curropasses = pass_prediction([dfn.satname], [dfn.sensorname],
sector_file,
sectorlist,
dt - timedelta(minutes=15),
dt + timedelta(minutes=15),
single=True,
quiet=quiet)
for opass in curropasses:
log.info(opass)
if dfn.is_concurrent_with(opass.startdt, dt, opass.enddt):
log.interactive('Overlaps overpass ' +
str(opass.opass))
currsectorlist.extend(opass.actualsectornames)
else:
currsectorlist = sectorlist
#log.interactive('process_overpass sectorfiles: '+str(sectorfiles))
#log.interactive('process_overpass currsectorlist: '+str(currsectorlist))
#log.interactive('Data file date: '+fileobj.date)