def latency( start_datetime, end_datetime, sensor, satellites, data_providers, channels, overall=False, verbose=True, ): log.info('\n\n') nowtime = datetime.utcnow() total_hours = (end_datetime - start_datetime).days * 24 + ( end_datetime - start_datetime).seconds // 3600 log.info(str(total_hours) + ' hours') allfiles = [] for sat in satellites: log.info('Trying sat ' + sat + ' sensor: ' + sensor) currdata_providers = data_providers currchannels = channels if not data_providers: currdata_providers = ['*'] if not channels: currchannels = ['*'] for data_provider in currdata_providers: for channel in currchannels: allfiles += DataFileName.list_range_of_files( sat, sensor, start_datetime, end_datetime, datetime_wildcards={ '%H': '*%H', '%M': '*', '%S': '*' }, data_provider=data_provider, resolution='*', channel=channel, producttype='*', area='*', extra='*', ext='*', forprocess=False) if overall: totalsize, totalnum = calc_latency( allfiles, fileclass='DataFileName', verbose=verbose, classkeys={ 'overall': ['sensorname', 'satname', 'dataprovider'], }) else: totalsize, totalnum = calc_latency( allfiles, fileclass='DataFileName', verbose=verbose, classkeys={ 'individual': ['sensorname', 'satname', 'channel', 'dataprovider'], 'overall': ['sensorname', 'satname', 'dataprovider'], }) log.interactive('Total size on disk for ' + str(totalnum) + ' data files: ' + convert_bytes(totalsize) + ': sensor: ' + sensor + ' satellites: ' + ', '.join(satellites)) return totalsize, totalnum
def find_available_data_files(opasses, start_dt, satellite, sensor, extra_dirs, prodtype=None, ext='*'): all_files = [] # runfulldir = None overall_start_dt = start_dt overall_end_dt = start_dt #print opasses for opass in sorted(opasses, key=operator.attrgetter('basedt')): log.interactive('Trying opass: '******'*' prodtype = dfn.sensorinfo.FName['default_producttype'] #print prodtype # dfn.producttype = prodtype # runfulldir = dfn.sensorinfo.FName['runfulldir'] # If this is a long overpass, make sure we get the data files # coming before the overpass time mins_per_file = dfn.sensorinfo.mins_per_file startdt = opass.startdt - timedelta(minutes=mins_per_file) enddt = opass.enddt + timedelta(minutes=mins_per_file) if startdt < overall_start_dt: overall_start_dt = startdt overall_end_dt = enddt all_files += DataFileName.list_range_of_files( satellite, sensor, startdt, enddt, datetime_wildcards={ '%H': '%H', '%M': '*', '%S': '*' }, data_provider='*', resolution='*', channel='*', producttype=prodtype, area='*', extra='*', ext='*', forprocess=True, ) all_files = list(set(all_files)) log.interactive(' Total files so far: ' + str(len(all_files))) if all_files: log.interactive('Available files:' + bigindent + bigindent.join( commands.getoutput('ls --full-time -d ' + str(file)) for file in sorted(all_files)) + '\n\n') return all_files