def latency(
start_datetime,
end_datetime,
sensor,
satellites,
data_providers,
channels,
overall=False,
verbose=True,
):
log.info('\n\n')
nowtime = datetime.utcnow()
total_hours = (end_datetime - start_datetime).days * 24 + (
end_datetime - start_datetime).seconds // 3600
log.info(str(total_hours) + ' hours')
allfiles = []
for sat in satellites:
log.info('Trying sat ' + sat + ' sensor: ' + sensor)
currdata_providers = data_providers
currchannels = channels
if not data_providers:
currdata_providers = ['*']
if not channels:
currchannels = ['*']
for data_provider in currdata_providers:
for channel in currchannels:
allfiles += DataFileName.list_range_of_files(
sat,
sensor,
start_datetime,
end_datetime,
datetime_wildcards={
'%H': '*%H',
'%M': '*',
'%S': '*'
},
data_provider=data_provider,
resolution='*',
channel=channel,
producttype='*',
area='*',
extra='*',
ext='*',
forprocess=False)
if overall:
totalsize, totalnum = calc_latency(
allfiles,
fileclass='DataFileName',
verbose=verbose,
classkeys={
'overall': ['sensorname', 'satname', 'dataprovider'],
})
else:
totalsize, totalnum = calc_latency(
allfiles,
fileclass='DataFileName',
verbose=verbose,
classkeys={
'individual':
['sensorname', 'satname', 'channel', 'dataprovider'],
'overall': ['sensorname', 'satname', 'dataprovider'],
})
log.interactive('Total size on disk for ' + str(totalnum) +
' data files: ' + convert_bytes(totalsize) + ': sensor: ' +
sensor + ' satellites: ' + ', '.join(satellites))
return totalsize, totalnum
def find_available_data_files(opasses,
start_dt,
satellite,
sensor,
extra_dirs,
prodtype=None,
ext='*'):
all_files = []
# runfulldir = None
overall_start_dt = start_dt
overall_end_dt = start_dt
#print opasses
for opass in sorted(opasses, key=operator.attrgetter('basedt')):
log.interactive('Trying opass: '******'*'
prodtype = dfn.sensorinfo.FName['default_producttype']
#print prodtype
# dfn.producttype = prodtype
# runfulldir = dfn.sensorinfo.FName['runfulldir']
# If this is a long overpass, make sure we get the data files
# coming before the overpass time
mins_per_file = dfn.sensorinfo.mins_per_file
startdt = opass.startdt - timedelta(minutes=mins_per_file)
enddt = opass.enddt + timedelta(minutes=mins_per_file)
if startdt < overall_start_dt:
overall_start_dt = startdt
overall_end_dt = enddt
all_files += DataFileName.list_range_of_files(
satellite,
sensor,
startdt,
enddt,
datetime_wildcards={
'%H': '%H',
'%M': '*',
'%S': '*'
},
data_provider='*',
resolution='*',
channel='*',
producttype=prodtype,
area='*',
extra='*',
ext='*',
forprocess=True,
)
all_files = list(set(all_files))
log.interactive(' Total files so far: ' + str(len(all_files)))
if all_files:
log.interactive('Available files:' + bigindent + bigindent.join(
commands.getoutput('ls --full-time -d ' + str(file))
for file in sorted(all_files)) + '\n\n')
return all_files
