def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
# Use filename field for filename_datetime if it is available.
dfn = DataFileName(os.path.basename(glob(os.path.join(fname, '*'))[0]))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
metadata['top']['start_datetime'] = sdfn.datetime
metadata['top']['end_datetime'] = sdfn.datetime
metadata['top']['dataprovider'] = 'nesdisstar'
metadata['top']['platform_name'] = sdfn.satname
metadata['top']['source_name'] = 'seviri'
# MUST be set on readers that sector at read time.
# Affects how reading/processing is done in driver.py
metadata['top']['sector_definition'] = sector_definition
metadata['top']['SECTOR_ON_READ'] = True
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
# chans == [] specifies we don't want to read ANY data, just metadata.
# chans == None specifies that we are not specifying a channel list,
# and thus want ALL channels.
if chans == []:
# If NO CHANNELS were specifically requested, just return at this
# point with the metadata fields populated. A dummy SciFile dataset
# will be created with only metadata. This is for checking what
# platform/source combination we are using, etc.
return
outdir = os.path.join(gpaths['LOCALSCRATCH'],
os.path.dirname(sdfn.name))
self.decompress_msg(fname, outdir, chans)
try:
global_data = Scene(platform_name="Meteosat-8",
sensor="seviri",
reader="hrit_msg",
start_time=sdfn.datetime,
base_dir=outdir)
except TypeError:
global_data = Scene(
filenames=glob(os.path.join(outdir, '*')),
reader="hrit_msg",
filter_parameters={'start_time': sdfn.datetime})
metadata['top']['start_datetime'] = global_data.start_time
metadata['top']['end_datetime'] = global_data.end_time
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
# Loop through the variables found in the current dataset
# The dataset_info dictionary maps the geoips varname to the
# varname found in the original datafile
for geoipsvarname, spvarname in self.dataset_info[dsname].items():
# If we requested specific channels, and the current channel
# is not in the list, skip this variable.
if chans and geoipsvarname not in chans:
continue
# Read the current channel data into the datavars dictionary
log.info(' Initializing ' + dsname + ' channel "' +
spvarname + '" from file into SciFile channel: "' +
geoipsvarname + '"...')
global_data.load([spvarname])
# Read spvarname from the original datafile into datavars[dsname][geoipsvarname]
ad = sector_definition.area_definition
log.info(' Sectoring data to ' + ad.name + ' ...')
sectored_data = global_data.resample(ad)
for spvarname in sectored_data.datasets.keys():
for dsname in self.dataset_info.keys():
for geoipsvarname in self.dataset_info[dsname].keys():
if self.dataset_info[dsname][
geoipsvarname] == spvarname.name:
if 'Longitude' not in gvars[dsname].keys():
log.info(' Saving Longitude to gvars')
gvars[dsname]['Longitude'] = np.ma.array(
ad.get_lonlats()[0])
if 'Latitude' not in gvars[dsname].keys():
log.info(' Saving Latitude to gvars')
gvars[dsname]['Latitude'] = np.ma.array(
ad.get_lonlats()[1])
if 'SunZenith' not in gvars[dsname].keys():
from geoips.scifile.solar_angle_calc import satnav
log.info(
' Using satnav, can only calculate Sun Zenith angles'
)
gvars[dsname]['SunZenith'] = satnav(
'SunZenith', metadata['top']['start_datetime'],
gvars[dsname]['Longitude'],
gvars[dsname]['Latitude'])
self.set_variable_metadata(metadata, dsname,
geoipsvarname)
try:
datavars[dsname][geoipsvarname] =\
np.ma.array(sectored_data.datasets[spvarname.name].data,
mask=sectored_data.datasets[spvarname.name].mask)
log.warning('Sectored variable %s ' %
(spvarname.name))
except AttributeError:
log.warning(
'Variable %s does not contain a mask, masking invalid values! Might take longer'
% (spvarname.name))
datavars[dsname][geoipsvarname] =\
np.ma.masked_invalid(sectored_data.datasets[spvarname.name].data)
def read(self,fname,datavars,gvars,metadata,chans=None,sector_definition=None):
# Use the appropriate python based reader for opening the current data
# type. This package must be installed on the system, and imported above
# in the "Installed Libraries" import section.
df = ncdf.Dataset(str(fname), 'r')
# Uncomment this shell statement in order to interrogate the data file
# and determine what attributes and fields need to be read from the
# data file.
#print 'Entering IPython shell in '+self.name+' for development purposes'
#shell()
# Grab necessary metadata that will populate the _finfo,
# _dsinfo, and _varinfo properties on the SciFile object.
# The available fields for varinfo can be found in
# scifile/containers.py at the beginning of the file.
# Additional options available with the metadata dictionary
# can be found in the comments with the dataset_info property above.
metadata['top']['start_datetime'] = datetime.strptime(df.time_coverage_start.split('.')[0],'%Y-%m-%dT%H:%M:%S')
# Note an apparent bug in productfilename uses end_datetime as filename.
# For now just leave out end_datetime (it automatically gets set to start_datetime
# in scifile if undefined)
# Ooops, this might have messed up pass predictor
metadata['top']['end_datetime'] = datetime.strptime(df.time_coverage_end.split('.')[0],'%Y-%m-%dT%H:%M:%S')
metadata['top']['dataprovider'] = 'unknown'
# DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of
# Commerce
if 'DOC/NOAA/NESDIS/OSPO' in df.institution:
metadata['top']['dataprovider'] = 'noaa-nesdis-ospo'
elif 'NOAA' in df.institution and 'NESDIS' in df.institution:
metadata['top']['dataprovider'] = 'noaanesdis'
elif 'NOAA' in df.institution:
metadata['top']['dataprovider'] = 'noaa'
metadata['top']['filename_datetime'] = metadata['top']['start_datetime']
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
# platform_name and source_name MUST match values found
# in SensorInfo_classes and SatInfo_classes in utils/satellite_info.py.
# Those are the keys used throughout GeoIPS for determining what data
# type we are working with. If opening the SatSensorInfo object fails,
# raise an Error and stop operation.
# source_name = 'amsr2'
# platform_name = 'gcom-w1'
metadata['top']['platform_name'] = df.platform_name.lower()
metadata['top']['source_name'] = df.instrument_name.lower()
si = SatSensorInfo(metadata['top']['platform_name'],metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError('Unrecognized platform and source name combination: '+metadata['top']['platform_name']+' '+metadata['top']['source_name'])
# Use filename field for filename_datetime if it is available.
# Else, just use the start_datetime we found from the data
# above. Note we ALWAYS want to have a default if DataFileName
# is not defined. We do not want to rely on having our specific
# internal filename format in order to process, but if we have
# additional information available from the data filename, we
# can use it.
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# chans == [] specifies we don't want to read ANY data, just metadata.
# chans == None specifies that we are not specifying a channel list,
# and thus want ALL channels.
if chans == []:
# If NO CHANNELS were specifically requested, just return at this
# point with the metadata fields populated. A dummy SciFile dataset
# will be created with only metadata. This is for checking what
# platform/source combination we are using, etc.
return
# Set up the dictionaries of variables that will go in each dataset.
# datavars: actual channel data
# gvars: geolocation variable data
# specifically named:
# Latitude (REQUIRED),
# Longitude (REQUIRED), and
# SunZenith (optional, required for day/night
# discrimination)
# Each data variable array and geolocation variable array of a
# specific dataset_name MUST be the same shape
# datavars[dataset_name][geoips_varname] = geoips_varname_channel_data
# gvars[dataset_name]['Latitude'] = dataset_name_lat_numpy_array
# gvars[dataset_name]['Longitude'] = dataset_name_lon_numpy_array
# *OPTIONAL* gvars[dataset_name]['SunZenith'] = dataset_name_sunzenith_numpy_array
# geoips_varname_channel_data.shape == dataset_name_lat_numpy_array.shape
# == dataset_name_lon_array.shape == dataset_name_sunzenith_numpy_array.shape
# Only datavars and gvars with the same shape can go in the same dataset.
# See additional datavars and gvars dictionary structure information
# found in the comments above, with the dataset_info property of this reader.
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
# Loop through the variables found in the current dataset
# The dataset_info dictionary maps the geoips varname to the
# varname found in the original datafile
for geoipsvarname,ncvarname in self.dataset_info[dsname].items():
# If we requested specific channels, and the current channel
# is not in the list, skip this variable.
if chans and geoipsvarname not in chans:
continue
# Read the current channel data into the datavars dictionary
log.info(' Reading '+dsname+' channel "'+ncvarname+'" from file into SciFile channel: "'+geoipsvarname+'"...')
# Read ncvarname from the original datafile into datavars[dsname][geoipsvarname]
datavars[dsname][geoipsvarname] = np.ma.masked_equal(df.variables[ncvarname][...],df.variables[ncvarname]._FillValue)
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
# Loop through the variables found in the current dataset
# The gvar_info dictionary maps the geoips varname to the
# varname found in the original datafile
for geoipsvarname,ncvarname in self.gvar_info[dsname].items():
# Read the current channel data into the datavars dictionary
log.info(' Reading '+dsname+' channel "'+ncvarname+'" from file into SciFile channel: "'+geoipsvarname+'"...')
# Read ncvarname from the original datafile into datavars[dsname][geoipsvarname]
gvars[dsname][geoipsvarname] = np.ma.masked_equal(df.variables[ncvarname][...],df.variables[ncvarname]._FillValue)
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
class data_grid:
#indir='devnull'...this looks for the directory that the file is in
dtg = '9999999999'
mapvars = {}
nz = 0
nzp1 = 0
nnest = 0
delx0 = 0
dely0 = 0
nest = [0]
class Nest(object):
def __init__(self, nx, ny, ii, jj, iref, jref, tcmx, tcmy, delx,
dely):
self.nx = nx
self.ny = ny
self.ii = ii
self.jj = jj
self.iref = iref
self.jref = jref
self.tcmx = tcmx
self.tcmy = tcmy
self.delx = delx
self.dely = dely
# define the model grid
model_grid = {}
model_grid['im'] = 277
model_grid['jm'] = 229
model_grid['lm'] = 60
model_grid['num_bytes'] = model_grid['im'] * model_grid['jm'] * 4
#Constant files for geolocation and header
latitude = '/SATPROJECT/users/projects3/users/laflash/outdirs/nrtdata/realtime/longterm_files/COAMPS_metadata/latitu_sfc_000000_000000_3a0277x0229_2018092400_00000000_fcstfld'
longitude = '/SATPROJECT/users/projects3/users/laflash/outdirs/nrtdata/realtime/longterm_files/COAMPS_metadata/longit_sfc_000000_000000_3a0277x0229_2018092400_00000000_fcstfld'
header = '/SATPROJECT/users/projects3/users/laflash/outdirs/nrtdata/realtime/longterm_files/COAMPS_metadata/datahd_sfc_000000_000000_1a2000x0001_2018090918_00000000_infofld'
#def main(file, lat_file, lon_file, cdtg, fld, level, model_grid, image_dir):
# istat = 0
# lat, istat = seek_field(lat_file, model_grid, 1)
# lon, istat = seek_field(lon_file, model_grid, 1)
#
# data, stat = seek_field(file, model_grid, level)
#
# title = ( "%s lvl:%.2i %s %s" % (fld.upper(), int(level), cdtg, tau) )
# level_name = ( "l%.2i" % int(level) )
# image_name = '_'.join(["ascos1", "2a", cdtg, fld, level_name, tau])
#
# plot_global(data, lat, lon, title, image_name,
# clabel=plot_parm[fld]['units'],
# range=[plot_parm[fld]['min'],plot_parm[fld]['max']])
def seek_field(filename):
print 'Reading file...'
#metadata
datafilename = filename.split('/')[-1].split('_')
wxparameter = datafilename[0]
level_type = datafilename[1]
lvl1 = datafilename[2]
lvl1 = "%06.1f" % (float(lvl1))
lvl2 = datafilename[3]
if wxparameter == 'latitu' or wxparameter == 'longit':
lvl2 = 1
else:
lvl2 = "%06.1f" % (float(lvl2))
imest_and_gridlevels = datafilename[4]
dtg = datafilename[5]
filetype = datafilename[6]
record_length = model_grid['num_bytes']
# top to bottom
offset = (model_grid['lm'] - int(float(lvl2))) * record_length
# bottom to top
offset = (int(float(lvl2)) - 1) * record_length
#offset = (1 - int(float(lvl2))) * record_length
# binary file read
if os.path.isfile(filename):
f = open(filename, 'rb')
f.seek(offset)
data = np.fromstring(f.read(model_grid['num_bytes']),
dtype='float32')
if sys.byteorder == 'little':
data = data.byteswap()
data = data.reshape(model_grid['jm'], model_grid['im'])
data = np.ma.masked_equal(data, -990.99)
istat = 0
else:
print "missing file"
print filename
data = [[-999.99] * model_grid['im']] * model_grid['jm']
istat = -1
return data, wxparameter, level_type, lvl1, lvl2, dtg, filetype
def read_coamps_header(filename):
#"%s/datahd_sfc_000000_000000_1a2000x0001_%s_00000000_infofld"%(indir, dtg)
if os.path.isfile(filename): #might not need
#data_grid.indir = indir#might not need
f = open(filename, 'rb')
datahd = f.read()
datahd = list(datahd.split())
# separate occasional values with no space between them
for j in range(len(datahd)):
val = datahd[j]
if len(val) > 13:
i1 = 0
k = 0
for i in range(len(val) - 1):
if val[i:i + 1] == 'E':
newval = val[i1:i + 4]
if i + 4 < 15:
datahd[j] = newval
else:
datahd.insert(j + k, newval)
k = k + 1
i1 = i + 4
data_grid.mapvars['nproj'] = float(datahd[2])
data_grid.mapvars['stdlat1'] = float(datahd[3])
data_grid.mapvars['stdlat2'] = float(datahd[4])
data_grid.mapvars['stdlon'] = float(datahd[5])
data_grid.mapvars['reflat'] = float(datahd[6])
data_grid.mapvars['reflon'] = float(datahd[7])
data_grid.nz = int(float(datahd[1]))
data_grid.nzp1 = int(float(datahd[1])) + 1
data_grid.nnest = int(float(datahd[10]))
data_grid.delx0 = float(datahd[9])
data_grid.dely0 = float(datahd[8])
nn = 1
while nn <= data_grid.nnest:
ng = 30 + (nn - 1) * 30
nx = int(float(datahd[ng - 1]))
ny = int(float(datahd[ng + 0]))
ii = float(datahd[ng + 1])
jj = float(datahd[ng + 2])
iref = float(datahd[ng + 3])
jref = float(datahd[ng + 4])
tcmx = float(datahd[ng + 27])
tcmy = float(datahd[ng + 28])
delx = float(datahd[ng + 6])
dely = float(datahd[ng + 7])
data_grid.nest.append(
Nest(nx, ny, ii, jj, iref, jref, tcmx, tcmy, delx,
dely))
nn = nn + 1
# vertical indices
nz = data_grid.nz
dsigm = np.array(datahd[500:500 + nz]).astype(np.float)
data_grid.sigw = np.append(
np.flipud(np.cumsum(np.flipud(dsigm))), [0.0])
data_grid.sigm = datahd[800:800 + nz]
data_grid.ztop = data_grid.sigw[0]
data, wxparameter, level_type, lvl1, lvl2, dtg, filetype = seek_field(
fname)
metadata['top']['level'] = lvl2
metadata['top']['start_datetime'] = datetime.strptime(dtg, '%Y%m%d%H')
metadata['top']['end_datetime'] = datetime.strptime(dtg, '%Y%m%d%H')
metadata['top']['dataprovider'] = 'NRL'
metadata['top']['filename_datetime'] = metadata['top'][
'start_datetime']
metadata['top']['platform_name'] = 'model'
metadata['top']['source_name'] = 'coampsieee'
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
if chans == []:
return
# def rdata (filename)#parmnm, lvltyp, lev1, lev2, inest, dtg, tau, indir='def', outtyp='fcstfld', im=-1, jm=-1):
#
# # read coamps binary flat file
#
# if indir == 'def': indir = data_grid.indir
# if im == -1: im = data_grid.nest[inest].nx
# if jm == -1: jm = data_grid.nest[inest].ny
# if inest == 0:
# #global
# filename = "%s/%s_%s_%06.1f_%06.1f_glob%03dx%03d_%s_%04d0000_%s" \
# %(indir, parmnm, lvltyp, lev1, lev2, im, jm, dtg, tau, outtyp)
# else:
# #COAMPS
# filename = "%s/%s_%s_%06d_%06d_%1da%04dx%04d_%s_%04d0000_%s" \
# %(indir, parmnm, lvltyp, lev1, lev2, inest, im, jm, dtg, tau, outtyp)
#
# # print "Reading %s"%filename
# num_bytes = im*jm*4
#
# offset = 0
#
# # binary file read
# if os.path.isfile(filename):
# f = open( filename, 'rb' )
# f.seek( offset )
# data = np.fromstring(f.read(num_bytes), dtype='float32')
# # COAMPS irsat values are little_endian all others are big
# if sys.byteorder == 'little':
# if parmnm != 'irrcmp':
# data = data.byteswap()
# data = data.reshape(jm, im)
# data = np.ma.masked_equal(data, -990.99)
# f.close()
# istat = 0
# else:
# print "MISSING %s"%parmnm
# print filename
# data = [[-999.99] * im] * jm
# istat = -1
# return data, istat
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
for geoipsvarname, dfvarname in self.dataset_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
fillvalue = data.fill_value
datavars[dsname][geoipsvarname] = np.ma.masked_equal(
data, fillvalue)
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
for geoipsvarname, dfvarname in self.gvar_info[dsname].items():
if dfvarname == 'latitu':
geolog, wxparameter, level_type, lvl1, lvl2, dtg, filetype = seek_field(
latitude)
if dfvarname == 'longit':
geolog, wxparameter, level_type, lvl1, lvl2, dtg, filetype = seek_field(
longitude)
xx, yy = geolog.shape
for aa in range(xx):
for bb in range(yy):
if geolog[aa, bb] > 180:
geolog[aa, bb] = geolog[aa, bb] - 360
fillvalue = geolog.fill_value
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
gvars[dsname][geoipsvarname] = np.ma.masked_equal(
geolog, fillvalue)
class TESTLOCAL_TESTABI_LOCALDATA(LocalDataSite):
''' Subclass of LocalDataSite for initiating processing of
locally available ABI data.
Data_type and host_type are used as keys for running downloader
./downloaders/downloader.py abi local
host is only used for display purposes for LocalDataSite -
set to the actual remote site for FTP/HTTP. '''
data_type = 'testabi'
host_type = 'testlocal'
host = 'localdirs '
# List the base_dirs that are set in utils/satellite_info.py
# These will show up for logging purposes when you run
# ./downloaders/downloader.py
sensor_info = SatSensorInfo('goes16','abi')
for currdir in sensor_info.OrigFNames:
if 'base_dir' in currdir.keys() and currdir['base_dir']:
host += ' '+currdir['base_dir']
def __init__(self,downloadactive=True,bandlist=None,**kwargs):
''' Required __init__ method, set up downloader attributes:
queue parameters
* queue set to $DEFAULT_QUEUE by default in downloader.py
* (None if DEFAULT_QUEUE not set)
* can set the num cpus and mem per cpu here.
GeoIPS processing parameters
* self.run_geoips defaults to False!
External processing scripts
* self.pp_script : if defined, will run non-GeoIPS processing'''
super(TESTLOCAL_TESTABI_LOCALDATA,self).__init__(downloadactive,bandlist=bandlist,**kwargs)
self.run_geoips = True
self.mp_max_cpus = 8
self.mp_mem_per_cpu = 25
#self.pp_script = os.getenv('PROCSDIR')+'/h8/h8_postprocs'
def get_final_filename(self,file):
''' Keep original filename, but use standard path
The default for this method uses GeoIPS standard
filename and GeoIPS standard path. '''
fn = DataFileName(os.path.basename(file))
sdfn = fn.create_standard(downloadSiteObj=self)
return os.path.dirname(sdfn.name)+'/'+os.path.basename(file)
def sort_files(self,filelist):
# OR_ABI-L1b-RadF-M3C02_G16_s20170642036100_e20170642046467_c20170642046500.nc
# OR_ABI-L1b-RadC-M3C03_G16_s20171070042189_e20171070044562_c20171070045005.nc
# OR_SEIS-L1b-EHIS_G16_s20170622315250_e20170622315250_c20170622320253.nc
# DataFileName way too slow, need to fix that. For now, just
# sort directly
filelist.sort(cmp,key=lambda x:os.path.basename(x).split('_')[3],reverse=True)
return filelist
def getfile(self,remotefile,localfile):
log.info('Just creating symlink from original '+remotefile+' to final '+localfile)
if not self.downloadactive:
log.info(' *** nodownload set, not moving pushed file %s ' % remotefile)
log.info(' *** to localfile %s' % localfile)
else:
log.info(' *** moving pushed file %s ' % remotefile)
log.info(' *** to localfile %s' % localfile)
os.symlink(remotefile,localfile)
def run_on_files(self,final_file):
''' The default for this method runs on every file that
is downloaded. Overriding for ABI to only process when
a directory contains 16 files. Also allow for limiting
the frequency we actually try to process if we can't
keep up '''
runtype = 'RadF'
dirname = os.path.dirname(final_file)
# List all RadF files
listthesefiles = dirname+'/*{0}*'.format(runtype)
files = glob(listthesefiles)
num_files = len(files)
log.info(' In run_on_files TESTLOCAL_TESTABI '+str(num_files)+' files in directory '+listthesefiles)
# Limit frequency we actually process
#dfn = DataFileName(os.path.basename(final_file)).create_standard()
#if dfn.datetime.minute != 0 and dfn.datetime.minute != 30:
# log.info('ONLY RUNNING 0 30 MINUTES FOR NOW. Skipping processing')
# return []
# Once we get 16 files, and the current file is RadF, kick off processing
if num_files == 16 and runtype in final_file:
return [final_file]
else:
return []
def getfilelist(self,start_datetime,end_datetime):
# Data locations are specified in $GEOIPS/utils/satellite_info.py
# and $GEOIPS/utils/path/datafilename.py
files = []
for dir in self.sensor_info.OrigFNames:
if 'base_dir' in dir.keys() and dir['base_dir']:
# Files go in base_dir/YYYYMMDD/OR_*.nc
# where basedir is $BIGDATA/incoming
log.info(' Finding appropriate files in '+dir['base_dir']+'/*/OR_*.nc ...')
currfiles = glob(dir['base_dir']+'/*/OR_*.nc')
files += currfiles
return files
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
fileobj = ncdf.Dataset(str(fname), 'r')
if chans and not (set(fileobj.variables.keys()) & set(chans)):
log.info('Did not request any variables from ' + fname +
', only need ' + str(chans) + ', returning empty list')
return
#Read the date and time
# Terascan stores the "pass_date" as days since Jan 1 1900 (inclusive), and
# "start_time" as seconds since midnight.
sdt = datetime.strptime('1899-12-31', '%Y-%m-%d') + timedelta(
days=int(fileobj.getncattr('pass_date')),
seconds=float(fileobj.getncattr('start_time')))
if not metadata['top'][
'start_datetime'] or sdt < metadata['top']['start_datetime']:
metadata['top']['start_datetime'] = sdt
satellite = fileobj.getncattr('satellite')
for sat in all_sats_for_sensor('gvar'):
si = SatSensorInfo(sat)
if si.tscan_tle_name == satellite:
metadata['top']['platform_name'] = si.geoips_satname
# Terascan stores sensor as gvissr, we want to call it gvar.
metadata['top']['source_name'] = 'gvar'
metadata['top']['NO_GRANULE_COMPOSITES'] = True
# Not sure why I added this.
metadata['top']['NON_SECTORABLE'] = True
# Create dummy dataset with metadata if we specifically requested no channels
if chans == []:
# Will set up empty dataset in def read
# Only need varinfo, everything else empty
return
log.info('Reading ' + fname + ' Requested channels: ' + str(chans))
#Read the geolocation data in the native resolution from Terascan.
log.info(' Reading longitudes...')
lon = fileobj.variables['longitude'][...]
log.info(' Reading latitudes...')
lat = fileobj.variables['latitude'][...]
log.info(' Reading solar zenith angles...')
zen = fileobj.variables['sun_zenith'][...]
# Find which dataset this channel should be in
for dslabel in self.dataset_info.keys():
if set(fileobj.variables.keys()) & set(self.dataset_info[dslabel]):
datasettag = dslabel
alldata = []
#Read the actual data
for chn in fileobj.variables.keys():
if chans != None and chn not in chans:
if chn not in ['latitude', 'longitude', 'sun_zenith']:
log.info(' Did not request variable ' + chn +
', skipping...')
continue
if 'Latitude' not in gvars[datasettag].keys():
gvars[datasettag]['Latitude'] = lat
gvars[datasettag]['Longitude'] = lon
gvars[datasettag]['SunZenith'] = zen
log.info(' Reading ' + datasettag + ' data: ' + chn + '...')
data = fileobj.variables[chn][...]
if hasattr(data, 'fill_value'):
datavars[datasettag][chn] = data
else:
# I think if there are no masked values, tdfnc does not create a
# mask. So this should work fine... Just nothing will get masked...
if data.min() > -32768:
maskval = -32768
else:
maskval = 100 * data.min()
datavars[datasettag][chn] = np.ma.masked_equal(data, maskval)
fileobj.close()
return
def process_overpass(
satellite,
sensor,
productlist,
sectorlist,
sectorfiles,
extra_dirs,
sector_file,
datelist,
hourlist=None,
data_outpath=None,
product_outpath=None,
list=False,
clean=False,
forceclean=False,
download=False,
queue=None,
mp_max_cpus=1,
allstatic=True,
alldynamic=True,
tc=False,
volcano=False,
quiet=False,
start_datetime=None,
end_datetime=None,
):
if quiet:
log.setLevel(35)
log.interactive('')
opasses = []
old_opasses = []
overall_start_dt = None
overall_end_dt = None
single = False
both = False
if sectorlist:
single = True
both = False
if hourlist == None:
for datestr in datelist:
if sectorlist:
log.interactive(
'Checking for overpasses $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' -s "' +
' '.join(sectorlist) + '" --all')
else:
log.interactive(
'Checking for overpasses $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' --all')
sys.stdout.write('.')
sys.stdout.flush()
start_dt = datetime.strptime(datestr + '0000', '%Y%m%d%H%M')
end_dt = datetime.strptime(datestr + '2359', '%Y%m%d%H%M')
opasses.extend(
pass_prediction([satellite], [sensor],
sector_file,
sectorlist,
start_dt - timedelta(minutes=15),
end_dt + timedelta(minutes=15),
single=single,
both=both,
force=True,
quiet=quiet))
sys.stdout.write('\n')
if opasses and len(opasses) < 200 and len(opasses) != 0:
log.interactive('Available overpasses: ' + bigindent +
bigindent.join(sorted(str(val)
for val in opasses)) + '\n')
elif opasses:
log.interactive(
str(len(opasses)) + ' available overpasses, not listing\n')
return opasses
else:
hourstart = hourlist[0]
if len(hourlist) == 1:
hourend = hourlist[0]
else:
hourend = hourlist[-1]
for datestr in datelist:
if sectorlist and hourlist:
log.interactive(
'Checking for overpasses for $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' -H "' +
' '.join(hourlist) + '" -s "' + ' '.join(sectorlist) +
'" --all')
else:
log.interactive(
'Checking for overpasses for $GEOIPS/geoips/process_overpass.py '
+ satellite + ' ' + sensor + ' ' + datestr + ' --all')
sys.stdout.write('.')
sys.stdout.flush()
start_dt = datetime.strptime(datestr + hourstart + '00',
'%Y%m%d%H%M')
start_dt = start_dt - timedelta(minutes=15)
if overall_start_dt == None or overall_start_dt > start_dt:
overall_start_dt = start_dt
end_dt = datetime.strptime(datestr + hourend + '59', '%Y%m%d%H%M')
end_dt = end_dt + timedelta(minutes=15)
if overall_end_dt == None or overall_end_dt < end_dt:
overall_end_dt = end_dt
opasses.extend(
pass_prediction([satellite], [sensor],
sector_file,
sectorlist,
start_dt,
end_dt,
single=single,
force=True,
quiet=quiet))
sys.stdout.write('\n')
if opasses and len(opasses) < 20:
log.interactive('Available overpasses: ' + bigindent +
bigindent.join(sorted(str(val)
for val in opasses)) + '\n\n')
elif opasses:
log.interactive(
str(len(opasses)) + ' available overpasses, not listing\n\n')
# Start 8h before start time to make sure we can get the
# sector file entry before
if sensor != 'modis':
overall_start_dt = overall_start_dt - timedelta(minutes=480)
log.info('Overall start and end times: ' + str(overall_start_dt) + ' to ' +
str(overall_end_dt))
if download == True:
log.interactive('queue: ' + str(queue) + '\n\n')
data_type = default_data_type[sensor]
host_type = default_host_type[data_type]
#Can't we do something to minimize the copypaste done here? Hard to maintain...
if (data_type, host_type) in non_qsubbed:
for opass in opasses:
log.info('sectorfiles: ' + str(sectorfiles))
sector_file = sectorfile.open(allstatic=allstatic,
alldynamic=alldynamic,
tc=tc,
start_datetime=opass.startdt -
timedelta(hours=6),
end_datetime=opass.enddt,
one_per_sector=True)
if not sectorfiles:
currsectorfiles = sector_file.names
else:
currsectorfiles = sectorfiles
log.info('currsectorfiles: ' + str(currsectorfiles))
log.interactive('Downloading opass: '******'\n\n')
si = SatSensorInfo(satellite, sensor)
# If they are very long files (ie, full orbit), make
# sure we get the file before the overpass time
startdt = opass.startdt - timedelta(minutes=si.mins_per_file)
downloader(
data_type,
host_type,
sector_file=sector_file,
sectorlist=sectorlist,
sectorfiles=currsectorfiles,
productlist=productlist,
data_outpath=data_outpath,
product_outpath=product_outpath,
start_datetime=startdt,
end_datetime=opass.enddt,
queue=queue,
allstatic=allstatic,
alldynamic=alldynamic,
tc=tc,
volcano=volcano,
#max_connections=8,
max_wait_seconds=None,
)
time.sleep(5)
else:
log.interactive(sectorfiles)
downloader(
data_type,
host_type,
sector_file=sector_file,
sectorlist=sectorlist,
sectorfiles=sectorfiles,
productlist=productlist,
data_outpath=data_outpath,
product_outpath=product_outpath,
start_datetime=overall_start_dt,
end_datetime=overall_end_dt,
queue=queue,
allstatic=allstatic,
alldynamic=alldynamic,
tc=tc,
opasses=opasses,
#max_connections=8,
max_wait_seconds=None,
)
time.sleep(5)
all_files = []
# Reverse=True for newest first
all_files = sorted(find_available_data_files(opasses, start_dt, satellite,
sensor, extra_dirs),
reverse=True)
log.info('Done sorting default')
#shell()
if productlist and 'near-constant-contrast' in productlist:
log.info(' Checking near-constant-contrast files')
# Reverse=True for newest first
all_files = sorted(find_available_data_files(opasses,
start_dt,
satellite,
sensor,
extra_dirs,
prodtype='ncc'),
reverse=True)
file_str = '\n\t'.join(all_files)
log.info('Files found current search time for %s: \n\t%s' %
(str(opasses), file_str))
if not all_files:
log.info('No files available in directories listed above')
log.info(
'To check alternate directories, you can call (replace /sb2/viirs and /sb1/viirs with the paths where data files are available): '
)
infostr = ''
if productlist:
infostr += '-p ' + "'" + ' '.join(productlist) + "'"
if sectorlist:
infostr += '-s ' + "'" + ' '.join(sectorlist) + "'"
log.info(
"process_overpass.py %s %s '%s' -d '/sb2/viirs /sb1/viirs' %s -H '%s'"
% (satellite, sensor, ' '.join(datelist), infostr,
' '.join(hourlist)))
return None
try:
for opass in opasses:
currdatelist = []
day_count = (opass.enddt - opass.startdt).days + 1
for dt in (opass.startdt + timedelta(n) for n in range(day_count)):
currdatelist.append(dt.strftime('%Y%m%d'))
log.info('Checking for existing products to clean... clean: ' +
str(clean) + ' forceclean: ' + str(forceclean))
find_existing_products(sensor, sector_file,
opass.actualsectornames, productlist,
currdatelist, clean, forceclean)
except ProductError, resp:
log.error(str(resp) + ' Check spelling?')
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
f1 = open(fname, 'rb')
#READ HEARDER
sw_rev = np.fromstring(f1.read(2),
dtype=np.dtype('short')).byteswap()[0]
endian, fileid = np.fromstring(f1.read(2),
dtype=np.dtype('int8')).byteswap()
rev = np.fromstring(f1.read(4), dtype=np.dtype('int32')).byteswap()
year = np.fromstring(f1.read(4), dtype=np.dtype('int32')).byteswap()
jday = np.fromstring(f1.read(2), dtype=np.dtype('short')).byteswap()
hour, minu = np.fromstring(f1.read(2),
dtype=np.dtype('int8')).byteswap()
satid, nsdr = np.fromstring(f1.read(4),
dtype=np.dtype('short')).byteswap()
spare1, spare2, spare3 = np.fromstring(
f1.read(3), dtype=np.dtype('int8')).byteswap()
proc_stat_flags = np.fromstring(f1.read(1),
dtype=np.dtype('int8')).byteswap()
spare4 = np.fromstring(f1.read(4), dtype=np.dtype('int32')).byteswap()
#Need to set up time to be read in by the metadata (year and jday are arrays)
time = '%04d%03d%02d%02d' % (year[0], jday[0], hour, minu)
nbytes = 28 #bytes that have been read in
#Read scan records at 512-byte boundaries
nfiller = 512 - (
nbytes % 512
) # skip nfiller bytes so that the scan header will start at the 513th byte of the data records,
filler_bytes = np.fromstring(f1.read(nfiller),
dtype=np.dtype('int8')).byteswap()
# Rev 6A of the SSMIS SDR software changed the scalling of channel 12-16 to 100 (it was 10 before this change)
# effective with orbit rev 12216 for F-16 and thereafter for all future satellites
rev6a = 1
if satid == 1 and rev[0] < 12216:
rev6a = 0
if satid == 1:
satid = 'f16'
elif satid == 2:
satid = 'f17'
elif satid == 3:
satid = 'f18'
elif satid == 4:
satid = 'f19'
else:
return False
#Enter metadata
metadata['top']['start_datetime'] = datetime.strptime(time, '%Y%j%H%M')
metadata['top']['end_datetime'] = datetime.strptime(time, '%Y%j%H%M')
metadata['top']['dataprovider'] = 'DMSP'
metadata['top']['filename_datetime'] = metadata['top'][
'start_datetime']
metadata['top']['platform_name'] = satid
metadata['top']['source_name'] = 'ssmis'
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
if chans == []:
return
bad_value = -999
for nn in range(nsdr): #loop number of sdr data records
log.info(' Reading sdr #' + str(nn) + ' of ' + str(nsdr))
nbytes = 0
#SCAN HEADER
syncword = np.fromstring(f1.read(4),
dtype=np.dtype('int32')).byteswap()
scan_year = np.fromstring(f1.read(4),
dtype=np.dtype('int32')).byteswap()
scan_jday = np.fromstring(f1.read(2),
dtype=np.dtype('short')).byteswap()
scan_hour, scan_minu = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
scan = np.fromstring(f1.read(4),
dtype=np.dtype('int32')).byteswap()
nscan_imager, nscan_enviro, nscan_las, nscan_uas = np.fromstring(
f1.read(4), dtype=np.dtype('int8')).byteswap()
start_scantime_imager = np.fromstring(
f1.read(112), dtype=np.dtype('int32')).byteswap()
scenecounts_imager = np.fromstring(
f1.read(28), dtype=np.dtype('uint8')).byteswap()
start_scantime_enviro = np.fromstring(
f1.read(96), dtype=np.dtype('int32')).byteswap()
scenecounts_enviro = np.fromstring(
f1.read(24), dtype=np.dtype('uint8')).byteswap()
start_scantime_las = np.fromstring(
f1.read(32), dtype=np.dtype('int32')).byteswap()
scenecounts_las = np.fromstring(
f1.read(8), dtype=np.dtype('uint8')).byteswap()
start_scantime_uas = np.fromstring(
f1.read(16), dtype=np.dtype('int32')).byteswap()
scenecounts_uas = np.fromstring(
f1.read(4), dtype=np.dtype('uint8')).byteswap()
spare = np.fromstring(f1.read(20),
dtype=np.dtype('int32')).byteswap()
nbytes += 360 #total bytes of the scan header
nscan0 = scan - 1
#-----------------------------------------------------------------------------------------
try:
imager_read = np.ma.zeros((nscan_imager, 180))
np.ma.masked_all_like(imager_read)
except:
print 'Shell dropped for imager_read'
if scenecounts_imager[0] < 0:
print "IMAGER is negative"
lt = np.ma.masked_values(imager_read, bad_value)
lg = np.ma.masked_values(imager_read, bad_value)
ch08 = np.ma.masked_values(imager_read, bad_value)
ch09 = np.ma.masked_values(imager_read, bad_value)
ch10 = np.ma.masked_values(imager_read, bad_value)
ch11 = np.ma.masked_values(imager_read, bad_value)
ch17 = np.ma.masked_values(imager_read, bad_value)
ch18 = np.ma.masked_values(imager_read, bad_value)
surf = np.ma.masked_values(imager_read, bad_value)
rain = np.ma.masked_values(imager_read, bad_value)
#IMAGER READ DATA
for ii in range(nscan_imager):
if start_scantime_imager[ii] == -999:
print 'value of imager scan is %d' % ii
continue
for jj in range(scenecounts_imager[ii]):
imager_lat, imager_lon, imager_scene = np.fromstring(
f1.read(6), dtype=np.dtype('short')).byteswap()
imager_surf, imager_rain = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
imager_ch08, imager_ch09, imager_ch10, imager_ch11, imager_ch17, imager_ch18 = np.fromstring(
f1.read(12), dtype=np.dtype('short')).byteswap()
nbytes += 20
k = 180 * (nscan0 + ii) + jj
lat = 0.01 * imager_lat
lon = 0.01 * imager_lon
try:
lt[ii][jj] = lat
lg[ii][jj] = lon
ch08[ii][jj] = imager_ch08 #150 Ghz
ch09[ii][jj] = imager_ch09 #183+-7
ch10[ii][jj] = imager_ch10 #183+-3
ch11[ii][jj] = imager_ch11 #183+-1
ch17[ii][jj] = imager_ch17 #91V
ch18[ii][jj] = imager_ch18 #91H
surf[ii][jj] = imager_surf
rain[ii][jj] = imager_rain
except:
print 'Failed setting arrays in scan_imager'
if 'Latitude' not in gvars['IMAGER'].keys():
gvars['IMAGER']['Latitude'] = lt
gvars['IMAGER']['Longitude'] = lg
datavars['IMAGER']['ch08'] = ch08
datavars['IMAGER']['ch09'] = ch09
datavars['IMAGER']['ch10'] = ch10
datavars['IMAGER']['ch11'] = ch11
datavars['IMAGER']['ch17'] = ch17
datavars['IMAGER']['ch18'] = ch18
datavars['IMAGER']['surface'] = surf
datavars['IMAGER']['rain'] = rain
else:
gvars['IMAGER']['Latitude'] = np.ma.vstack(
(gvars['IMAGER']['Latitude'], lt))
gvars['IMAGER']['Longitude'] = np.ma.vstack(
(gvars['IMAGER']['Longitude'], lg))
datavars['IMAGER']['ch08'] = np.ma.vstack(
(datavars['IMAGER']['ch08'], ch08))
datavars['IMAGER']['ch09'] = np.ma.vstack(
(datavars['IMAGER']['ch09'], ch09))
datavars['IMAGER']['ch10'] = np.ma.vstack(
(datavars['IMAGER']['ch10'], ch10))
datavars['IMAGER']['ch11'] = np.ma.vstack(
(datavars['IMAGER']['ch11'], ch11))
datavars['IMAGER']['ch17'] = np.ma.vstack(
(datavars['IMAGER']['ch17'], ch17))
datavars['IMAGER']['ch18'] = np.ma.vstack(
(datavars['IMAGER']['ch18'], ch18))
datavars['IMAGER']['surface'] = np.ma.vstack(
(datavars['IMAGER']['surface'], surf))
datavars['IMAGER']['rain'] = np.ma.vstack(
(datavars['IMAGER']['rain'], rain))
gvars['IMAGER']['Latitude'] = np.ma.masked_values(
gvars['IMAGER']['Latitude'], bad_value)
gvars['IMAGER']['Longitude'] = np.ma.masked_values(
gvars['IMAGER']['Longitude'], bad_value)
datavars['IMAGER']['ch08'] = np.ma.masked_values(
datavars['IMAGER']['ch08'], bad_value)
datavars['IMAGER']['ch09'] = np.ma.masked_values(
datavars['IMAGER']['ch09'], bad_value)
datavars['IMAGER']['ch10'] = np.ma.masked_values(
datavars['IMAGER']['ch10'], bad_value)
datavars['IMAGER']['ch11'] = np.ma.masked_values(
datavars['IMAGER']['ch11'], bad_value)
datavars['IMAGER']['ch17'] = np.ma.masked_values(
datavars['IMAGER']['ch17'], bad_value)
datavars['IMAGER']['ch18'] = np.ma.masked_values(
datavars['IMAGER']['ch18'], bad_value)
datavars['IMAGER']['surface'] = np.ma.masked_values(
datavars['IMAGER']['surface'], bad_value)
datavars['IMAGER']['rain'] = np.ma.masked_values(
datavars['IMAGER']['rain'], bad_value)
#-----------------------------------------------------------------------------------------
enviro_read = np.ma.zeros((nscan_enviro, 90))
np.ma.masked_all_like(enviro_read)
if scenecounts_enviro[0] < 0:
print "ENVIRO is negative"
lt = np.ma.masked_equal(enviro_read, bad_value)
lg = np.ma.masked_equal(enviro_read, bad_value)
ch12 = np.ma.masked_equal(enviro_read, bad_value)
ch13 = np.ma.masked_equal(enviro_read, bad_value)
ch14 = np.ma.masked_equal(enviro_read, bad_value)
ch15 = np.ma.masked_equal(enviro_read, bad_value)
ch16 = np.ma.masked_equal(enviro_read, bad_value)
ch15_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch16_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch17_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch18_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch17_5x4 = np.ma.masked_equal(enviro_read, bad_value)
ch18_5x4 = np.ma.masked_equal(enviro_read, bad_value)
#ENVIRO READ DATA
for ii in range(nscan_enviro):
if ii % 2 == 0: #for odd scan numbers
if start_scantime_enviro[ii] == -999:
print 'value of enviro odd scan is %d' % ii
continue
for jj in range(scenecounts_enviro[ii]):
enviroodd_lat, enviroodd_lon, enviroodd_scene = np.fromstring(
f1.read(6), dtype=np.dtype('short')).byteswap()
enviroodd_seaice, enviroodd_surf = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
enviroodd_ch12, enviroodd_ch13, enviroodd_ch14, enviroodd_ch15, enviroodd_ch16, enviroodd_ch15_5x5, enviroodd_ch16_5x5, enviroodd_ch17_5x5, enviroodd_ch18_5x5, enviroodd_ch17_5x4, enviroodd_ch18_5x4 = np.fromstring(
f1.read(22), dtype=np.dtype('short')).byteswap()
enviroodd_rain1, enviroodd_rain2 = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
edr_bitflags = np.fromstring(
f1.read(4), dtype=np.dtype('int32')).byteswap()
nbytes += 36
lat = 0.01 * enviroodd_lat
lon = 0.01 * enviroodd_lon
lt[ii][jj] = lat
lg[ii][jj] = lon
if rev6a == 1:
ch12[ii][jj] = enviroodd_ch12 #19H
ch13[ii][jj] = enviroodd_ch13 #19V
ch14[ii][jj] = enviroodd_ch14 #22V
ch15[ii][jj] = enviroodd_ch15 #37H
ch16[ii][jj] = enviroodd_ch16 #37V
ch15_5x5[ii][jj] = enviroodd_ch15_5x5
ch16_5x5[ii][jj] = enviroodd_ch16_5x5
ch17_5x5[ii][jj] = enviroodd_ch17_5x5
ch18_5x5[ii][jj] = enviroodd_ch18_5x5
ch17_5x4[ii][jj] = enviroodd_ch17_5x4
ch18_5x4[ii][jj] = enviroodd_ch18_5x4
else:
ch12[ii][jj] = 10 * enviroodd_ch12
ch13[ii][jj] = 10 * enviroodd_ch13
ch14[ii][jj] = 10 * enviroodd_ch14
ch15[ii][jj] = 10 * enviroodd_ch15
ch16[ii][jj] = 10 * enviroodd_ch16
ch15_5x5[ii][jj] = 10 * enviroodd_ch15_5x5
ch16_5x5[ii][jj] = 10 * enviroodd_ch16_5x5
ch17_5x5[ii][jj] = 10 * enviroodd_ch17_5x5
ch18_5x5[ii][jj] = 10 * enviroodd_ch18_5x5
ch17_5x4[ii][jj] = 10 * enviroodd_ch17_5x4
ch18_5x4[ii][jj] = 10 * enviroodd_ch18_5x4
if ii % 2 == 1: # for even scan numbers
if start_scantime_enviro[ii] == -999:
print 'value of enviro even scan is %d' % ii
continue
for jj in range(scenecounts_enviro[ii]):
enviroeven_lat, enviroeven_lon, enviroeven_scene = np.fromstring(
f1.read(6), dtype=np.dtype('short')).byteswap()
enviroeven_seaice, enviroeven_surf = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
enviroeven_ch12, enviroeven_ch13, enviroeven_ch14, enviroeven_ch15, enviroeven_ch16 = np.fromstring(
f1.read(10), dtype=np.dtype('short')).byteswap()
nbytes += 18
lat = 0.01 * enviroeven_lat
lon = 0.01 * enviroeven_lon
lt[ii][jj] = lat
lg[ii][jj] = lon
if rev6a == 1:
ch12[ii][jj] = enviroeven_ch12
ch13[ii][jj] = enviroeven_ch13
ch14[ii][jj] = enviroeven_ch14
ch15[ii][jj] = enviroeven_ch15
ch16[ii][jj] = enviroeven_ch16
else:
ch12[ii][jj] = 10 * enviroeven_ch12
ch13[ii][jj] = 10 * enviroeven_ch13
ch14[ii][jj] = 10 * enviroeven_ch14
ch15[ii][jj] = 10 * enviroeven_ch15
ch16[ii][jj] = 10 * enviroeven_ch16
if 'Latitude' not in gvars['ENVIRO'].keys():
gvars['ENVIRO']['Latitude'] = lt
gvars['ENVIRO']['Longitude'] = lg
datavars['ENVIRO']['ch12'] = ch12
datavars['ENVIRO']['ch13'] = ch13
datavars['ENVIRO']['ch14'] = ch14
datavars['ENVIRO']['ch15'] = ch15
datavars['ENVIRO']['ch16'] = ch16
datavars['ENVIRO']['ch15_5x5'] = ch15_5x5
datavars['ENVIRO']['ch16_5x5'] = ch16_5x5
datavars['ENVIRO']['ch17_5x5'] = ch17_5x5
datavars['ENVIRO']['ch18_5x5'] = ch18_5x5
datavars['ENVIRO']['ch17_5x4'] = ch17_5x4
datavars['ENVIRO']['ch18_5x4'] = ch18_5x4
else:
gvars['ENVIRO']['Latitude'] = np.ma.vstack(
(gvars['ENVIRO']['Latitude'], lt))
gvars['ENVIRO']['Longitude'] = np.ma.vstack(
(gvars['ENVIRO']['Longitude'], lg))
datavars['ENVIRO']['ch12'] = np.ma.vstack(
(datavars['ENVIRO']['ch12'], ch12))
datavars['ENVIRO']['ch13'] = np.ma.vstack(
(datavars['ENVIRO']['ch13'], ch13))
datavars['ENVIRO']['ch14'] = np.ma.vstack(
(datavars['ENVIRO']['ch14'], ch14))
datavars['ENVIRO']['ch15'] = np.ma.vstack(
(datavars['ENVIRO']['ch15'], ch15))
datavars['ENVIRO']['ch16'] = np.ma.vstack(
(datavars['ENVIRO']['ch16'], ch16))
datavars['ENVIRO']['ch15_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch15_5x5'], ch15_5x5))
datavars['ENVIRO']['ch16_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch16_5x5'], ch16_5x5))
datavars['ENVIRO']['ch17_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch17_5x5'], ch17_5x5))
datavars['ENVIRO']['ch18_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch18_5x5'], ch18_5x5))
datavars['ENVIRO']['ch17_5x4'] = np.ma.vstack(
(datavars['ENVIRO']['ch17_5x4'], ch17_5x4))
datavars['ENVIRO']['ch18_5x4'] = np.ma.vstack(
(datavars['ENVIRO']['ch18_5x4'], ch18_5x4))
gvars['ENVIRO']['Latitude'] = np.ma.masked_equal(
gvars['ENVIRO']['Latitude'], bad_value)
gvars['ENVIRO']['Longitude'] = np.ma.masked_equal(
gvars['ENVIRO']['Longitude'], bad_value)
datavars['ENVIRO']['ch12'] = np.ma.masked_equal(
datavars['ENVIRO']['ch12'], bad_value)
datavars['ENVIRO']['ch13'] = np.ma.masked_equal(
datavars['ENVIRO']['ch13'], bad_value)
datavars['ENVIRO']['ch14'] = np.ma.masked_equal(
datavars['ENVIRO']['ch14'], bad_value)
datavars['ENVIRO']['ch15'] = np.ma.masked_equal(
datavars['ENVIRO']['ch15'], bad_value)
datavars['ENVIRO']['ch16'] = np.ma.masked_equal(
datavars['ENVIRO']['ch16'], bad_value)
datavars['ENVIRO']['ch15_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch15_5x5'], bad_value)
datavars['ENVIRO']['ch16_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch16_5x5'], bad_value)
datavars['ENVIRO']['ch17_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch17_5x5'], bad_value)
datavars['ENVIRO']['ch18_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch18_5x5'], bad_value)
datavars['ENVIRO']['ch17_5x4'] = np.ma.masked_equal(
datavars['ENVIRO']['ch17_5x4'], bad_value)
datavars['ENVIRO']['ch18_5x4'] = np.ma.masked_equal(
datavars['ENVIRO']['ch18_5x4'], bad_value)
#-----------------------------------------------------------------------------------------
las_read = np.ma.zeros((nscan_las, 60))
np.ma.masked_all_like(las_read)
if scenecounts_las[0] < 0:
print "LAS is negative"
lt = np.ma.masked_equal(las_read, bad_value)
lg = np.ma.masked_equal(las_read, bad_value)
ch01_3x3 = np.ma.masked_equal(las_read, bad_value)
ch02_3x3 = np.ma.masked_equal(las_read, bad_value)
ch03_3x3 = np.ma.masked_equal(las_read, bad_value)
ch04_3x3 = np.ma.masked_equal(las_read, bad_value)
ch05_3x3 = np.ma.masked_equal(las_read, bad_value)
ch06_3x3 = np.ma.masked_equal(las_read, bad_value)
ch07_3x3 = np.ma.masked_equal(las_read, bad_value)
ch08_5x5 = np.ma.masked_equal(las_read, bad_value)
ch09_5x5 = np.ma.masked_equal(las_read, bad_value)
ch10_5x5 = np.ma.masked_equal(las_read, bad_value)
ch11_5x5 = np.ma.masked_equal(las_read, bad_value)
ch18_5x5 = np.ma.masked_equal(las_read, bad_value)
ch24_3x3 = np.ma.masked_equal(las_read, bad_value)
height_1000mb = np.ma.masked_equal(las_read, bad_value)
surf = np.ma.masked_equal(las_read, bad_value)
#LAS READ DATA
for ii in range(nscan_las):
if start_scantime_las[ii] == -999:
print 'value of las scan is %d' % ii
continue
for jj in range(scenecounts_las[ii]):
try:
las_lati, las_long, las_ch01_3x3, las_ch02_3x3, las_ch03_3x3, las_ch04_3x3, las_ch05_3x3, las_ch06_3x3, las_ch07_3x3, las_ch08_5x5, las_ch09_5x5, las_ch10_5x5, las_ch11_5x5, las_ch18_5x5, las_ch24_3x3, las_height_1000mb, las_surf = np.fromstring(
f1.read(34), dtype=np.dtype('short')).byteswap()
las_tqflag, las_hqflag = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
las_terrain, las_scene = np.fromstring(
f1.read(4), dtype=np.dtype('short')).byteswap()
except:
continue
lat = 0.01 * las_lati
lon = 0.01 * las_long
nbytes += 40
lt[ii][jj] = lat
lg[ii][jj] = lon
ch01_3x3[ii][jj] = las_ch01_3x3 #50.3 V
ch02_3x3[ii][jj] = las_ch02_3x3 #52.8 V
ch03_3x3[ii][jj] = las_ch03_3x3 #53.60V
ch04_3x3[ii][jj] = las_ch04_3x3 #54.4 V
ch05_3x3[ii][jj] = las_ch05_3x3 #55.5 V
ch06_3x3[ii][jj] = las_ch06_3x3 #57.3 RCP
ch07_3x3[ii][jj] = las_ch07_3x3 #59.4 RCP
ch08_5x5[ii][jj] = las_ch08_5x5 #150 H
ch09_5x5[ii][jj] = las_ch09_5x5 #183.31+-7 H
ch10_5x5[ii][jj] = las_ch10_5x5 #183.31+-3 H
ch11_5x5[ii][jj] = las_ch11_5x5 #183.31+-1 H
ch18_5x5[ii][jj] = las_ch18_5x5 #91 H
ch24_3x3[ii][jj] = las_ch24_3x3 #60.79+-36+-0.05 RCP
height_1000mb[ii][jj] = las_height_1000mb
surf[ii][jj] = las_surf
if 'Latitude' not in gvars['LAS'].keys():
gvars['LAS']['Latitude'] = lt
gvars['LAS']['Longitude'] = lg
datavars['LAS']['ch01_3x3'] = ch01_3x3
datavars['LAS']['ch02_3x3'] = ch02_3x3
datavars['LAS']['ch03_3x3'] = ch03_3x3
datavars['LAS']['ch04_3x3'] = ch04_3x3
datavars['LAS']['ch05_3x3'] = ch05_3x3
datavars['LAS']['ch06_3x3'] = ch06_3x3
datavars['LAS']['ch07_3x3'] = ch07_3x3
datavars['LAS']['ch08_5x5'] = ch08_5x5
datavars['LAS']['ch09_5x5'] = ch09_5x5
datavars['LAS']['ch10_5x5'] = ch10_5x5
datavars['LAS']['ch11_5x5'] = ch11_5x5
datavars['LAS']['ch18_5x5'] = ch18_5x5
datavars['LAS']['ch24_3x3'] = ch24_3x3
datavars['LAS']['height_1000mb'] = height_1000mb
datavars['LAS']['surf'] = surf
else:
gvars['LAS']['Latitude'] = np.ma.vstack(
(gvars['LAS']['Latitude'], lt))
gvars['LAS']['Longitude'] = np.ma.vstack(
(gvars['LAS']['Longitude'], lg))
datavars['LAS']['ch01_3x3'] = np.ma.vstack(
(datavars['LAS']['ch01_3x3'], ch01_3x3))
datavars['LAS']['ch02_3x3'] = np.ma.vstack(
(datavars['LAS']['ch02_3x3'], ch02_3x3))
datavars['LAS']['ch03_3x3'] = np.ma.vstack(
(datavars['LAS']['ch03_3x3'], ch03_3x3))
datavars['LAS']['ch04_3x3'] = np.ma.vstack(
(datavars['LAS']['ch04_3x3'], ch04_3x3))
datavars['LAS']['ch05_3x3'] = np.ma.vstack(
(datavars['LAS']['ch05_3x3'], ch05_3x3))
datavars['LAS']['ch06_3x3'] = np.ma.vstack(
(datavars['LAS']['ch06_3x3'], ch06_3x3))
datavars['LAS']['ch07_3x3'] = np.ma.vstack(
(datavars['LAS']['ch07_3x3'], ch07_3x3))
datavars['LAS']['ch08_5x5'] = np.ma.vstack(
(datavars['LAS']['ch08_5x5'], ch08_5x5))
datavars['LAS']['ch09_5x5'] = np.ma.vstack(
(datavars['LAS']['ch09_5x5'], ch09_5x5))
datavars['LAS']['ch10_5x5'] = np.ma.vstack(
(datavars['LAS']['ch10_5x5'], ch10_5x5))
datavars['LAS']['ch11_5x5'] = np.ma.vstack(
(datavars['LAS']['ch11_5x5'], ch11_5x5))
datavars['LAS']['ch18_5x5'] = np.ma.vstack(
(datavars['LAS']['ch18_5x5'], ch18_5x5))
datavars['LAS']['ch24_3x3'] = np.ma.vstack(
(datavars['LAS']['ch24_3x3'], ch24_3x3))
datavars['LAS']['height_1000mb'] = np.ma.vstack(
(datavars['LAS']['height_1000mb'], height_1000mb))
datavars['LAS']['surf'] = np.ma.vstack(
(datavars['LAS']['surf'], surf))
gvars['LAS']['Latitude'] = np.ma.masked_equal(
gvars['LAS']['Latitude'], bad_value)
gvars['LAS']['Longitude'] = np.ma.masked_equal(
gvars['LAS']['Longitude'], bad_value)
datavars['LAS']['ch01_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch01_3x3'], bad_value)
datavars['LAS']['ch02_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch02_3x3'], bad_value)
datavars['LAS']['ch03_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch03_3x3'], bad_value)
datavars['LAS']['ch04_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch04_3x3'], bad_value)
datavars['LAS']['ch05_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch05_3x3'], bad_value)
datavars['LAS']['ch06_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch06_3x3'], bad_value)
datavars['LAS']['ch07_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch07_3x3'], bad_value)
datavars['LAS']['ch08_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch08_5x5'], bad_value)
datavars['LAS']['ch09_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch09_5x5'], bad_value)
datavars['LAS']['ch10_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch10_5x5'], bad_value)
datavars['LAS']['ch11_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch11_5x5'], bad_value)
datavars['LAS']['ch18_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch18_5x5'], bad_value)
datavars['LAS']['ch24_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch24_3x3'], bad_value)
datavars['LAS']['height_1000mb'] = np.ma.masked_equal(
datavars['LAS']['height_1000mb'], bad_value)
datavars['LAS']['surf'] = np.ma.masked_equal(
datavars['LAS']['surf'], bad_value)
#---------------------------------------------------------------------------------
uas_read = np.ma.zeros((nscan_uas, 30))
np.ma.masked_all_like(uas_read)
if scenecounts_uas[0] < 0:
print "UAS is negative"
lt = np.ma.masked_equal(uas_read, bad_value)
lg = np.ma.masked_equal(uas_read, bad_value)
ch19_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch20_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch21_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch22_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch23_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch24_6x6 = np.ma.masked_equal(uas_read, bad_value)
sceneu = np.ma.masked_equal(uas_read, bad_value)
tqflag = np.ma.masked_equal(uas_read, bad_value)
#UAS READ DATA
for ii in range(nscan_uas):
if start_scantime_uas[ii] == -999:
print 'value of uas scan is %d' % ii
continue
for jj in range(scenecounts_uas[ii]):
uas_lat, uas_lon, uas_ch19_6x6, uas_ch20_6x6, uas_ch21_6x6, uas_ch22_6x6, uas_ch23_6x6, uas_ch24_6x6, uas_scene, uas_tqflag = np.fromstring(
f1.read(20), dtype=np.dtype('short')).byteswap()
uas_field, uas_bdotk2 = np.fromstring(
f1.read(8), dtype=np.dtype('int32')).byteswap()
nbytes += 28
lat = 0.01 * uas_lat
lon = 0.01 * uas_lon
lt[ii][jj] = lat
lg[ii][jj] = lon
ch19_6x6[ii][jj] = uas_ch19_6x6 #63.28+-0.28 RCP GHz
ch20_6x6[ii][jj] = uas_ch20_6x6 #60.79+-0.36 RCP
ch21_6x6[ii][jj] = uas_ch21_6x6 #60.79+-0.36+-0.002 RCP
ch22_6x6[ii][jj] = uas_ch22_6x6 #60.79+-0.36+-0.0055 RCP
ch23_6x6[ii][jj] = uas_ch23_6x6 #60.79+-0.36+-0.0016 RCP
ch24_6x6[ii][jj] = uas_ch24_6x6 #60.79+-0.36+-0.050 RCP
sceneu[ii][jj] = uas_scene
tqflag[ii][jj] = uas_tqflag
if 'Latitude' not in gvars['UAS'].keys():
gvars['UAS']['Latitude'] = lt
gvars['UAS']['Longitude'] = lg
datavars['UAS']['ch19_6x6'] = ch19_6x6
datavars['UAS']['ch20_6x6'] = ch20_6x6
datavars['UAS']['ch21_6x6'] = ch21_6x6
datavars['UAS']['ch22_6x6'] = ch22_6x6
datavars['UAS']['ch23_6x6'] = ch23_6x6
datavars['UAS']['ch24_6x6'] = ch24_6x6
datavars['UAS']['scene'] = sceneu
datavars['UAS']['uas_tqflag'] = tqflag
else:
gvars['UAS']['Latitude'] = np.ma.vstack(
(gvars['UAS']['Latitude'], lt))
gvars['UAS']['Longitude'] = np.ma.vstack(
(gvars['UAS']['Longitude'], lg))
datavars['UAS']['ch19_6x6'] = np.ma.vstack(
(datavars['UAS']['ch19_6x6'], ch19_6x6))
datavars['UAS']['ch20_6x6'] = np.ma.vstack(
(datavars['UAS']['ch20_6x6'], ch20_6x6))
datavars['UAS']['ch21_6x6'] = np.ma.vstack(
(datavars['UAS']['ch21_6x6'], ch21_6x6))
datavars['UAS']['ch22_6x6'] = np.ma.vstack(
(datavars['UAS']['ch22_6x6'], ch22_6x6))
datavars['UAS']['ch23_6x6'] = np.ma.vstack(
(datavars['UAS']['ch23_6x6'], ch23_6x6))
datavars['UAS']['ch24_6x6'] = np.ma.vstack(
(datavars['UAS']['ch24_6x6'], ch24_6x6))
datavars['UAS']['scene'] = np.ma.vstack(
(datavars['UAS']['scene'], sceneu))
datavars['UAS']['uas_tqflag'] = np.ma.vstack(
(datavars['UAS']['uas_tqflag'], tqflag))
gvars['UAS']['Latitude'] = np.ma.masked_equal(
gvars['UAS']['Latitude'], bad_value)
gvars['UAS']['Longitude'] = np.ma.masked_equal(
gvars['UAS']['Longitude'], bad_value)
datavars['UAS']['ch19_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch19_6x6'], bad_value)
datavars['UAS']['ch20_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch20_6x6'], bad_value)
datavars['UAS']['ch21_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch21_6x6'], bad_value)
datavars['UAS']['ch22_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch22_6x6'], bad_value)
datavars['UAS']['ch23_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch23_6x6'], bad_value)
datavars['UAS']['ch24_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch24_6x6'], bad_value)
datavars['UAS']['scene'] = np.ma.masked_equal(
datavars['UAS']['scene'], bad_value)
datavars['UAS']['uas_tqflag'] = np.ma.masked_equal(
datavars['UAS']['uas_tqflag'], bad_value)
print 'nfiller=', nfiller
nfiller = 512 - (
nbytes % 512
) # nfiller bytes to be skipped so that the next scan header will start at the 513th byte.
try:
filler_bytes = np.fromstring(
f1.read(nfiller), dtype=np.dtype('int8')).byteswap()[0]
except:
continue
f1.close()
#-----------------------------------------------------------------------------------------------------
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
for geoipsvarname, dfvarname in self.dataset_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
#shell()
data = datavars[dsname][geoipsvarname]
fillvalue = -999
datavars[dsname][geoipsvarname] = (
np.ma.masked_equal(data, fillvalue) / 100) + 273.15
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
for geoipsvarname, dfvarname in self.gvar_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
#shell()
data = gvars[dsname][geoipsvarname]
fillvalue = -999
gvars[dsname][geoipsvarname] = np.ma.masked_equal(
data, fillvalue)
def xarray_from_scifile(datafile, area_def=None, dsnames=None, varnames=None, roi=None):
''' Create xarray dataset from existing scifile datafile object. This includes the standard expected attributes
Args:
datafile (SciFile) : SciFile object with standard data / metadata fields, used to populate xarray
dsnames (:obj:`list`, optional) : List of strings indicating desired datasets to include. Default is None,
which indicates including *all* datasets.
varnames (:obj:`list`, optional) : List of strings indicating desired variables to include. Default is None
which indicates including *all* variables.
roi (:obj:`int`, optional) : Radius of Influence for interpolation, in meters. Default is None, which
results in attempting to use radius_of_influence from metadata.
Returns:
xarray Dataset object with attributes and variables set appropriately
'''
import xarray
xarray_datasets = [xarray.Dataset()]
for dsname in datafile.datasets.keys():
if dsnames and dsname not in dsnames:
continue
for varname in datafile.datasets[dsname].variables.keys():
if varnames and varname not in varnames:
continue
currvar = datafile.datasets[dsname].variables[varname]
orig_dims = list(currvar.shape)
for xarray_dataset in xarray_datasets:
if len(xarray_dataset.variables.keys()) == 0 or xarray_dataset.dims.values() == orig_dims:
xarray_dataset[varname] = xarray.DataArray(currvar)
LOG.info('Added %s variable to xarray dataset dims %s', varname, xarray_dataset[varname].dims)
if 'timestamp' in varname:
from numpy import datetime64
xarray_dataset[varname] = (xarray_dataset[varname]*10**9).astype(datetime64)
for varname in datafile.datasets[dsname].geolocation_variables.keys():
if varnames and varname not in varnames:
continue
xvarname = varname
if varname == 'Latitude':
xvarname = 'latitude'
if varname == 'Longitude':
xvarname = 'longitude'
currvar = datafile.datasets[dsname].geolocation_variables[varname]
orig_dims = list(currvar.shape)
for xarray_dataset in xarray_datasets:
if len(xarray_dataset.variables.keys()) == 0 or xarray_dataset.dims.values() == orig_dims:
xarray_dataset[xvarname] = xarray.DataArray(currvar)
LOG.info('Added %s variable to xarray dataset dims %s', xvarname, xarray_dataset[xvarname].dims)
for xarray_dataset in xarray_datasets:
xarray_dataset.attrs['start_datetime'] = datafile.start_datetime
xarray_dataset.attrs['end_datetime'] = datafile.end_datetime
xarray_dataset.attrs['source_name'] = datafile.source_name
xarray_dataset.attrs['platform_name'] = datafile.platform_name
xarray_dataset.attrs['data_provider'] = datafile.dataprovider
LOG.info('Added standard attributes to xarray dataset with dims %s', xarray_dataset['latitude'].dims)
roivar = 'interpolation_radius_of_influence'
newroi = None
if roi:
LOG.info('Added PASSED radius of influence %s to xarray attrs', roi)
newroi = roi
elif 'interpolation_radius_of_influence' in datafile.metadata['top'].keys():
LOG.info('Added scifile READER radius of influence %s to xarray attrs', datafile.metadata['top'][roivar])
newroi = datafile.metadata['top'][roivar]
else:
try:
from geoips.utils.satellite_info import SatSensorInfo
sensor_info = SatSensorInfo(datafile.platform_name, datafile.source_name)
newroi = sensor_info.interpolation_radius_of_influence
LOG.info('Added SENSORINFO radius of influence %s to xarray attrs', newroi)
except (KeyError, AttributeError):
# If we are registering a non-sat dataset, SatSensorInfo is not defined, and ROI not defined.
# so open default SatSensorInfo object (which will have default ROI)
from geoips.utils.satellite_info import SatSensorInfo
sensor_info = SatSensorInfo()
newroi = sensor_info.interpolation_radius_of_influence
LOG.info(' Using DEFAULT SATELLITE_INFO radius of influence: '+str(newroi))
if area_def is not None and hasattr(area_def, 'pixel_size_x') and hasattr(area_def, 'pixel_size_y'):
if area_def.pixel_size_x > newroi or area_def.pixel_size_y > newroi:
LOG.info(' Using sector radius of influence: '+str(area_def.pixel_size_x*1.5)+' or '+str(area_def.pixel_size_y*1.5)+', not sensor/product: '+str(newroi))
newroi = area_def.pixel_size_x * 1.5 if area_def.pixel_size_x > area_def.pixel_size_y else area_def.pixel_size_y * 1.5
for xarray_dataset in xarray_datasets:
LOG.info(' Set roi to %s for xarray_dataset with dimensions %s', newroi, xarray_dataset['latitude'].dims)
xarray_dataset.attrs[roivar] = newroi
return xarray_datasets
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
# Since the variables are all contained within separate files, we need to allow for
# a loop to cycle through the files
df = pg.open(fname)
temp = None
for vars in MODEL_GRIB_Reader.dataset_info.values():
for var in vars.values():
try:
temp = df.select(name=var)[0] #grabs the first file
except:
if var == '238 (instant)' or var == '133 (instant)' or var == '222 (instant)' or var == '218 (instant)' or var == '221 (instant)':
temptemp = df.select()
if var in str(temptemp):
temp = temptemp[0]
continue
#temp = df.select(name = 'Temperature')[0]
#print 'Entering IPython shell in '+self.name+' for development purposes'
#shell()
dt = '{:08d}{:04d}'.format(temp.validityDate, temp.validityTime)
print dt
if not temp:
log.warning('Unable to read from file ' + fname +
' no matching select for vars')
return
if temp.validityDate:
metadata['top']['start_datetime'] = datetime.strptime(
dt, '%Y%m%d%H%M')
metadata['top']['end_datetime'] = datetime.strptime(
dt, '%Y%m%d%H%M')
else:
metadata['top']['start_datetime'] = temp.analDate
metadata['top']['end_datetime'] = temp.analDate
metadata['top']['dataprovider'] = temp.centre
metadata['top']['filename_datetime'] = metadata['top'][
'start_datetime']
metadata['top']['platform_name'] = 'model'
#metadata['top']['source_name'] = 'model'
metadata['top']['tau'] = temp.startStep
metadata['top']['level'] = temp.level
if 'COAMPS' in fname:
metadata['top']['source_name'] = 'coamps'
else:
metadata['top']['source_name'] = 'navgem'
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
if chans == []:
return
new = None
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
for geoipsvarname, dfvarname in self.dataset_info[dsname].items():
try:
new = df.select(
name=dfvarname) #,typeOfLevel='isobaricInhPa')
except:
continue
for newest in new:
newest
data = newest.values
fillvalue = newest.missingValue
level = newest.level
#shell()
if newest.units == str('m s**-1'):
data = data * 1.94384
log.info(' Reading ' + dsname + ' channel "' +
dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + str(level) + '" ...')
datavars[dsname][geoipsvarname +
str(level)] = np.ma.masked_equal(
data, fillvalue)
shape = datavars[dsname][geoipsvarname + str(level)].shape
if not new:
log.warning('Unable to read from file ' + fname +
' no matching selects')
return
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
for geoipsvarname, dfvarname in self.gvar_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
for newest in new:
newest
data = newest[dfvarname]
fillvalue = newest.missingValue
#shell()
if data.size == newest.getNumberOfValues:
data = np.reshape(data, shape)
#if data.shape == (1038240,):
# data= np.reshape(data,(721,1440))
#elif data.shape == (259920,):
# data= np.reshape(data,(361,720))
#elif data.shape == (76454,):
# data= np.reshape(data,(254, 301))
gvars[dsname][geoipsvarname] = np.ma.masked_equal(
data, fillvalue)