def get_opasses(self, file, sectorfiles=None):
log.info('Running pass predictor on: ' + str(file))
# Paths sometimes mess this up. Use os.path.basename
fn = DataFileName(os.path.basename(file))
sfn = fn.create_standard(downloadSiteObj=self)
if sectorfiles:
opasses = pass_prediction([sfn.satname], [sfn.sensorname],
None,
None,
sfn.datetime - timedelta(hours=9),
sfn.datetime,
single=True,
force=True,
sectorfiles=sectorfiles)
elif self.sector_file:
opasses = pass_prediction([sfn.satname], [sfn.sensorname],
self.sector_file,
self.sectorlist,
sfn.datetime - timedelta(hours=9),
sfn.datetime,
force=True,
single=True)
else:
opasses = pass_prediction([sfn.satname], [sfn.sensorname],
None,
None,
sfn.datetime - timedelta(hours=9),
sfn.datetime,
force=True,
single=True)
return opasses
def read(self,fname,datavars,gvars,metadata,chans=None,sector_definition=None):
fileobj = h5py.File(str(fname), mode='r')
header = fileobj.attrs['FileHeader']
metadata['top']['platform_name'] = get_header_info(header,'SatelliteName').lower()
metadata['top']['source_name'] = get_header_info(header,'InstrumentName').lower()
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['dataprovider'] = sdfn.dataprovider
else:
metadata['top']['dataprovider'] = 'Unknown'
for dsname in self.dataset_info.keys():
londata = fileobj[dsname]['Longitude'].value
latdata = fileobj[dsname]['Latitude'].value
yyyy = fileobj[dsname]['ScanTime']['Year'].value
mm = fileobj[dsname]['ScanTime']['Month'].value
dd = fileobj[dsname]['ScanTime']['DayOfMonth'].value
hh = fileobj[dsname]['ScanTime']['Hour'].value
mn = fileobj[dsname]['ScanTime']['Minute'].value
ss = fileobj[dsname]['ScanTime']['Second'].value
metadata['top']['start_datetime'] = datetime.strptime("%04d%02d%02d%02d%02d%02d"%(yyyy[0],mm[0],dd[0],hh[0],mn[0],ss[0]),'%Y%m%d%H%M%S')
# Note an apparent bug in productfilename uses end_datetime as filename.
# For now just leave out end_datetime (it automatically gets set to start_datetime
# in scifile if undefined)
# Ooops, this might have messed up pass predictor
metadata['top']['end_datetime'] = datetime.strptime("%04d%02d%02d%02d%02d%02d"%(yyyy[-1],mm[-1],dd[-1],hh[-1],mn[-1],ss[-1]),'%Y%m%d%H%M%S')
metadata['top']['filename_datetime'] = metadata['top']['start_datetime']
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
if chans == []:
return
numchans = fileobj[dsname]['Tb'].value.shape[2]
tbdatalist = np.dsplit(fileobj[dsname]['Tb'].value,numchans)
tbdata = []
ii= 0
for data in tbdatalist:
for currgeoipsvarname,ncbandnum in self.dataset_info[dsname].items():
if ii == ncbandnum:
geoipsvarname = currgeoipsvarname
if not chans or geoipsvarname in chans:
datavars[dsname][geoipsvarname] = np.ma.masked_equal(np.squeeze(data),-999)
ii += 1
gvars[dsname]['Latitude'] = latdata
gvars[dsname]['Longitude'] = londata
metadata['gvars'][dsname]['Longitude'] = _empty_varinfo.copy()
metadata['gvars'][dsname]['Latitude'] = _empty_varinfo.copy()
metadata['gvars'][dsname]['Longitude']['nomask'] = True
metadata['gvars'][dsname]['Latitude']['nomask'] = True
return
def get_final_filename(self,file):
''' Keep original filename, but use standard path
The default for this method uses GeoIPS standard
filename and GeoIPS standard path. '''
fn = DataFileName(os.path.basename(file))
sdfn = fn.create_standard(downloadSiteObj=self)
return os.path.dirname(sdfn.name)+'/'+os.path.basename(file)
def get_logfname(self, file, geoips=True, postfix=None):
if os.path.isdir(file):
dfn = DataFileName(os.path.basename(glob(file + '/*')[0]))
else:
dfn = DataFileName(os.path.basename(file))
sdfn = dfn.create_standard(downloadSiteObj=self)
return sdfn.create_logfile(geoips=geoips)
def get_final_filename(self, file):
'''Eventually this will be standard for everything in Site.py
for now test with rscat/viirs only. This returns a string'''
fn = DataFileName(os.path.basename(file))
sdfn = fn.create_standard(downloadSiteObj=self)
if self.get_final_ext():
sdfn.ext = self.get_final_ext()
return sdfn.name
def get_filenamedatetime(self, file):
'''Eventually this will be standard for everything in Site.py
for now test with rscat/viirs only. This returns a datetime
object, not a FileName object.'''
fn = DataFileName(os.path.basename(file))
if fn:
sfn = fn.create_standard(downloadSiteObj=self)
else:
return None
return sfn.datetime
def opass_overlaps(obj, opass, fnstr):
'''Eventually this will be standard for everything in Site.py
for now test with rscat/viirs only. '''
# Paths sometimes mess this up. Use os.path.basename
fn = DataFileName(os.path.basename(fnstr))
sfn = fn.create_standard(downloadSiteObj=obj)
sfn_enddt = sfn.datetime + timedelta(
minutes=fn.sensorinfo.mins_per_file)
#log.info('opass: '******'sfn: '+str(sfn)+' sfn_enddt: '+str(sfn_enddt))
overlaps = sfn.is_concurrent_with(opass.startdt, sfn.datetime,
opass.enddt, sfn_enddt)
#log.info('overlaps: '+str(overlaps))
return overlaps
def run_on_files(self, final_file):
''' The default for this method runs on every file that
is downloaded. Overriding for ABI to only process when
a directory contains 16 files. Also allow for limiting
the frequency we actually try to process if we can't
keep up '''
runtype = 'RadF'
dirname = os.path.dirname(final_file)
# List all RadF files
listthesefiles = dirname + '/*{0}*'.format(runtype)
files = glob(listthesefiles)
num_files = len(files)
log.info(' In run_on_files TESTLOCAL_TESTABI ' + str(num_files) +
' files in directory ' + listthesefiles)
# Limit frequency we actually process
dfn = DataFileName(os.path.basename(final_file)).create_standard()
if dfn.datetime.minute != 0 and dfn.datetime.minute != 30:
log.info('ONLY RUNNING 0 30 MINUTES FOR NOW. Skipping processing')
return []
# Once we get 16 files, and the current file is RadF, kick off processing
if num_files == 16 and runtype in final_file:
return [final_file]
else:
return []
def getfile(self, remotefile, localfile):
processingfile = localfile + '.processing'
ff = open(processingfile, 'w')
ff.close
log.info('Touching temporary file: ' +
os.path.basename(processingfile))
temp_filename = DataFileName(localfile).create_scratchfile()
temp_filename.makedirs()
temp_fnstr = temp_filename.name
if not self.downloadactive:
log.info(
' *** nodownload set, not downloading remotefile %s ' %
remotefile)
log.info(' *** to localfile %s' % localfile)
else:
log.info(' *** grabbing remotefile %s ' % remotefile)
log.info(' *** to localfile %s' % localfile)
self.wget_file(remotefile, temp_fnstr)
self.move_to_final(temp_fnstr, processingfile, localfile)
def sort_files(self, filelist):
'''Override sort_files function found in Site.py. '''
#print filelist
#log.info(' Creating list of DataFileName objects to sort')
newfilelist = [DataFileName(file) for file in filelist]
#log.info(' Sorting list')
newfilelist.sort(cmp, key=lambda x: x.datetime, reverse=True)
#date_pattern = re.compile('RS.*\.([0-9]{11})')
#log.debug('filelist: '+str(filelist))
#filelist.sort(cmp,key=lambda file:date_pattern.search(file).group(1),reverse=True)
#log.debug('filelist: '+str(filelist))
#log.info(' Creating list of string objects to return')
filelist = [file.name for file in newfilelist]
return filelist
def convert(self, file, final_filename):
'''The convert method is called from the postproc method. The default
method in Site.py just moves from file to final_filename. If there are
actual conversions that have to happen, this will likely be overriden
in the subclasses of Site ([host_type].py, [host_type]_[data_type].py)'''
log.info('Using Site convert')
dfn = DataFileName(os.path.basename(file))
if os.path.isfile(file):
# if file is gzipped, unzip first
if hasattr(self, 'untgzfiles') and self.untgzfiles and (
'tar.gz' in dfn.ext or 'tgz' in dfn.ext):
log.info(' ****untgzing file ' + file)
return self.untgz(file)
elif hasattr(self,
'unbz2files') and self.unbz2files and ('.bz2'
in dfn.ext):
log.info(' ****bunzip2 file ' + file)
retval = os.system('bunzip2 ' + file)
log.info(' ****moving file to ' + final_filename)
if os.path.isfile(os.path.splitext(file)[0]):
shutil.move(os.path.splitext(file)[0], final_filename)
return [final_filename]
elif retval != 0:
log.info(' ****Final file failed!')
return []
elif 'gz' in dfn.ext and 'tgz' not in dfn.ext:
log.info(' ****gunzipping file ' + file)
retval = os.system('gunzip ' + file)
log.info(' ****moving file to ' + final_filename)
if os.path.isfile(os.path.splitext(file)[0]):
shutil.move(os.path.splitext(file)[0], final_filename)
return [final_filename]
elif retval != 0:
log.info(' ****Final file failed!')
return []
else:
log.info(' ****moving file to ' + final_filename)
shutil.move(file, final_filename)
return [final_filename]
else:
log.info('File ' + file + ' does not exist, not moving')
return []
def postproc(self,
file,
geoips_args=None,
forceprocess=False,
noprocess=False):
'''The postproc method is called from downloaders.Sites.Site.get
After a file has been successfully downloaded, postproc is
called on that file to do any necessary conversions, post
processing, etc.
This should not be overridden in the subclasses, but methods that are
called from postproc can be overridden to customize:
run_on_files:
Check for existence of certain files before kicking off processing
Also can put in pre-geoips processing steps in here (to either run on
individual file, or group of files. Note this runs serially, so
don't put anything in here that takes a long time!)
convert:
perform special conversions (by default gunzips and moves to final_filename)
get_final_filename:
by default uses GeoIPS standard data filename (can override to use original, etc)
pre_legacy_procs:
No one uses this - called before ppscript, if returns True, then run ppscript
Defaults to just returning True
pp_script:
attribute set on subclass that is the legacy postprocs call.
If set, pp_script will be run
'''
final_filename = self.get_final_filename(file)
#log.info('Using Site postproc')
# The default version of this (found above in Site.py) just moves
# file to final_filename. Can be overriden in host/data classes
# to actually convert the file, not just change filename...
if not glob(final_filename) or forceprocess:
if forceprocess:
log.interactive(
'Force running convert/process (called immediately after successful download?)'
)
else:
log.interactive('Final file: ' + final_filename +
' did not exist, reconverting/reprocessing')
# convert method returns a list of files that need processing -
# amsu converter, for instance, creates multiple files from a
# single input file
for final_file in self.convert(file, final_filename):
if noprocess:
log.info(
'SKIPPING processing on file, downloader called with --noprocess '
+ final_file)
continue
# If self.pp_script is defined in [host_type].py or
# [host_type]_[data_type].py, run postprocs
runpp = True
try:
log.info(' ****qsubbing ' +
os.path.basename(self.pp_script) + ' ' +
final_file)
except (NameError, AttributeError):
log.info(' ****No non-GeoIPS postprocs defined')
runpp = False
if runpp:
for file in self.run_on_files(final_file):
log.info(
' ****Running non-GeoIPS postprocessing on ' +
os.path.basename(file))
if self.pre_legacy_procs(file):
# pp_script may have arguments, so split on spaces and just use the
# basename of the first argument (script name) for logname
log_fname = self.get_logfname(file, geoips=False)
log_fname.makedirs()
resource_list = None
qsub(self.pp_script + ' ' + file +
os.getenv('LEGACY_PROCSDIR_CALL_AFTER'), [],
queue=self.queue,
name=log_fname.qsubname,
resource_list=resource_list,
outfile=log_fname.name,
suffix='log')
dfn = DataFileName(os.path.basename(final_file))
sdfn = dfn.create_standard(downloadSiteObj=self)
if self.run_geoips == True:
# self.run_on_files defaults to returning final_file
# can override run_on_files function in host_type/data_type
# subclass (ie, for modis where we must have a specific
# set of files before we can run driver)
for file in self.run_on_files(final_file):
log_fname = self.get_logfname(file)
log_fname.makedirs()
log.info(log_fname.name)
# geoips_args are set in downloader based on sector_file information (static vs dynamic),
# and passed arguments for sectorfiles / sectorlist.
# Do NOT run pass predictor in downloader anymore - now it is run in driver.
# but we still need to make sure we allow for passing a list of sectors to downloader...
if not geoips_args:
#arglist = [file,'--queue batch@kahuna','-s "'+' '.join(sectorlist)+'"']
arglist = [file, '--queue batch@kahuna']
# Currently setting mp_max_cpus and mp_mem_per_cpu in individual Site inits
if hasattr(self, 'mp_max_cpus'):
arglist += [
'--mp_max_cpus ' +
str(int(self.mp_max_cpus))
]
else:
dfn = DataFileName(os.path.basename(file))
if dfn.sensorinfo.FName['runfulldir']:
geoips_args.addarg(os.path.dirname(file))
else:
geoips_args.addarg(file)
log.info(geoips_args.options)
# Currently setting mp_max_cpus and mp_mem_per_cpu in individual Site inits
if hasattr(self, 'mp_max_cpus'):
geoips_args.addopt('mp_max_cpus',
str(int(self.mp_max_cpus)))
arglist = geoips_args.get_arglist()
# Remove the file we just added so we can
# add the next one
geoips_args.delarg(0)
# Currently setting mp_max_cpus and mp_mem_per_cpu in individual Site inits
if hasattr(self, 'mp_max_cpus'):
resource_list = os.getenv('PBS_RESOURCE_LIST_SELECT')+str(int(self.mp_max_cpus))+\
os.getenv('PBS_RESOURCE_LIST_MEM')+str(int(self.mp_mem_per_cpu*self.mp_max_cpus))+'gb'+\
os.getenv('PBS_RESOURCE_LIST_QLIST')
elif hasattr(self, 'pbs_resource_list'):
resource_list = self.pbs_resource_list
else:
resource_list = None
if hasattr(self, 'geoips_executable'):
geoips_executable = self.geoips_executable
else:
geoips_executable = gpaths[
'GEOIPS'] + '/geoips/driver.py'
qsub(
geoips_executable,
arglist,
queue=self.queue,
#name='GW'+self.data_type+'_'+self.host_type,
name=log_fname.qsubname,
resource_list=resource_list,
outfile=log_fname.name)
def getsinglefilelist(self,
start_time,
end_time,
searchstring,
login=True,
subdir=None):
if subdir is not None:
fullurlpath = self.baseurl + '/' + subdir
else:
fullurlpath = self.baseurl
log.info('')
log.info('fullurlpath: ' + str(fullurlpath) + '/' + str(searchstring))
indexhtmlfile = DataFileName()
indexhtmlfile.satname = 'wgetsite'
indexhtmlfile.sensorname = self.data_type
indexhtmlfile.dataprovider = self.host_type
# Hmm, FileName object should probably set this when datetime is set?
dt = datetime.utcnow()
indexhtmlfile.time = dt.strftime(indexhtmlfile.datetime_fields['time'])
indexhtmlfile.date = dt.strftime(indexhtmlfile.datetime_fields['date'])
indexhtmlfile.extra = 'index'
indexhtmlfile.ext = 'html'
indexhtmlfile = indexhtmlfile.create_scratchfile()
indexhtmlfile.makedirs()
indexhtmlfnstr = indexhtmlfile.name
log.info(indexhtmlfnstr)
# rcj 13DEC2018 this part doesn't need to run for lance modis data
# it is handled in lance_modis.py getfilelist
if hasattr(self,
'host') and (self.host == 'nrt3.modaps.eosdis.nasa.gov'
or self.host == 'nrt4.modaps.eosdis.nasa.gov'):
pass
#everything that is not lance modis that uses this script should still pass through here
else:
htmlfilelist = open(self.wget_file(fullurlpath,
indexhtmlfnstr)).readlines()
#getfiles = self.getLinksFromHTML(htmlfilelist,r'''.*a href="GAASP-MBT_v"."r"."GW"."_s[0-9]{14}".*''')
#log.info(htmlfilelist)
links = self.getLinksFromHTML(htmlfilelist, searchstring)
# This is defined in Site.py - finding the files in the file list is
# common between HTTP and FTP (getting the lists differs, but sorting
# through the list and returning the desired files is shared)
return self.find_files_in_range(links,
start_time,
end_time,
urlpath=fullurlpath)
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
f1 = open(fname, 'rb')
#READ HEARDER
sw_rev = np.fromstring(f1.read(2),
dtype=np.dtype('short')).byteswap()[0]
endian, fileid = np.fromstring(f1.read(2),
dtype=np.dtype('int8')).byteswap()
rev = np.fromstring(f1.read(4), dtype=np.dtype('int32')).byteswap()
year = np.fromstring(f1.read(4), dtype=np.dtype('int32')).byteswap()
jday = np.fromstring(f1.read(2), dtype=np.dtype('short')).byteswap()
hour, minu = np.fromstring(f1.read(2),
dtype=np.dtype('int8')).byteswap()
satid, nsdr = np.fromstring(f1.read(4),
dtype=np.dtype('short')).byteswap()
spare1, spare2, spare3 = np.fromstring(
f1.read(3), dtype=np.dtype('int8')).byteswap()
proc_stat_flags = np.fromstring(f1.read(1),
dtype=np.dtype('int8')).byteswap()
spare4 = np.fromstring(f1.read(4), dtype=np.dtype('int32')).byteswap()
#Need to set up time to be read in by the metadata (year and jday are arrays)
time = '%04d%03d%02d%02d' % (year[0], jday[0], hour, minu)
nbytes = 28 #bytes that have been read in
#Read scan records at 512-byte boundaries
nfiller = 512 - (
nbytes % 512
) # skip nfiller bytes so that the scan header will start at the 513th byte of the data records,
filler_bytes = np.fromstring(f1.read(nfiller),
dtype=np.dtype('int8')).byteswap()
# Rev 6A of the SSMIS SDR software changed the scalling of channel 12-16 to 100 (it was 10 before this change)
# effective with orbit rev 12216 for F-16 and thereafter for all future satellites
rev6a = 1
if satid == 1 and rev[0] < 12216:
rev6a = 0
if satid == 1:
satid = 'f16'
elif satid == 2:
satid = 'f17'
elif satid == 3:
satid = 'f18'
elif satid == 4:
satid = 'f19'
else:
return False
#Enter metadata
metadata['top']['start_datetime'] = datetime.strptime(time, '%Y%j%H%M')
metadata['top']['end_datetime'] = datetime.strptime(time, '%Y%j%H%M')
metadata['top']['dataprovider'] = 'DMSP'
metadata['top']['filename_datetime'] = metadata['top'][
'start_datetime']
metadata['top']['platform_name'] = satid
metadata['top']['source_name'] = 'ssmis'
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
if chans == []:
return
bad_value = -999
for nn in range(nsdr): #loop number of sdr data records
log.info(' Reading sdr #' + str(nn) + ' of ' + str(nsdr))
nbytes = 0
#SCAN HEADER
syncword = np.fromstring(f1.read(4),
dtype=np.dtype('int32')).byteswap()
scan_year = np.fromstring(f1.read(4),
dtype=np.dtype('int32')).byteswap()
scan_jday = np.fromstring(f1.read(2),
dtype=np.dtype('short')).byteswap()
scan_hour, scan_minu = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
scan = np.fromstring(f1.read(4),
dtype=np.dtype('int32')).byteswap()
nscan_imager, nscan_enviro, nscan_las, nscan_uas = np.fromstring(
f1.read(4), dtype=np.dtype('int8')).byteswap()
start_scantime_imager = np.fromstring(
f1.read(112), dtype=np.dtype('int32')).byteswap()
scenecounts_imager = np.fromstring(
f1.read(28), dtype=np.dtype('uint8')).byteswap()
start_scantime_enviro = np.fromstring(
f1.read(96), dtype=np.dtype('int32')).byteswap()
scenecounts_enviro = np.fromstring(
f1.read(24), dtype=np.dtype('uint8')).byteswap()
start_scantime_las = np.fromstring(
f1.read(32), dtype=np.dtype('int32')).byteswap()
scenecounts_las = np.fromstring(
f1.read(8), dtype=np.dtype('uint8')).byteswap()
start_scantime_uas = np.fromstring(
f1.read(16), dtype=np.dtype('int32')).byteswap()
scenecounts_uas = np.fromstring(
f1.read(4), dtype=np.dtype('uint8')).byteswap()
spare = np.fromstring(f1.read(20),
dtype=np.dtype('int32')).byteswap()
nbytes += 360 #total bytes of the scan header
nscan0 = scan - 1
#-----------------------------------------------------------------------------------------
try:
imager_read = np.ma.zeros((nscan_imager, 180))
np.ma.masked_all_like(imager_read)
except:
print 'Shell dropped for imager_read'
if scenecounts_imager[0] < 0:
print "IMAGER is negative"
lt = np.ma.masked_values(imager_read, bad_value)
lg = np.ma.masked_values(imager_read, bad_value)
ch08 = np.ma.masked_values(imager_read, bad_value)
ch09 = np.ma.masked_values(imager_read, bad_value)
ch10 = np.ma.masked_values(imager_read, bad_value)
ch11 = np.ma.masked_values(imager_read, bad_value)
ch17 = np.ma.masked_values(imager_read, bad_value)
ch18 = np.ma.masked_values(imager_read, bad_value)
surf = np.ma.masked_values(imager_read, bad_value)
rain = np.ma.masked_values(imager_read, bad_value)
#IMAGER READ DATA
for ii in range(nscan_imager):
if start_scantime_imager[ii] == -999:
print 'value of imager scan is %d' % ii
continue
for jj in range(scenecounts_imager[ii]):
imager_lat, imager_lon, imager_scene = np.fromstring(
f1.read(6), dtype=np.dtype('short')).byteswap()
imager_surf, imager_rain = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
imager_ch08, imager_ch09, imager_ch10, imager_ch11, imager_ch17, imager_ch18 = np.fromstring(
f1.read(12), dtype=np.dtype('short')).byteswap()
nbytes += 20
k = 180 * (nscan0 + ii) + jj
lat = 0.01 * imager_lat
lon = 0.01 * imager_lon
try:
lt[ii][jj] = lat
lg[ii][jj] = lon
ch08[ii][jj] = imager_ch08 #150 Ghz
ch09[ii][jj] = imager_ch09 #183+-7
ch10[ii][jj] = imager_ch10 #183+-3
ch11[ii][jj] = imager_ch11 #183+-1
ch17[ii][jj] = imager_ch17 #91V
ch18[ii][jj] = imager_ch18 #91H
surf[ii][jj] = imager_surf
rain[ii][jj] = imager_rain
except:
print 'Failed setting arrays in scan_imager'
if 'Latitude' not in gvars['IMAGER'].keys():
gvars['IMAGER']['Latitude'] = lt
gvars['IMAGER']['Longitude'] = lg
datavars['IMAGER']['ch08'] = ch08
datavars['IMAGER']['ch09'] = ch09
datavars['IMAGER']['ch10'] = ch10
datavars['IMAGER']['ch11'] = ch11
datavars['IMAGER']['ch17'] = ch17
datavars['IMAGER']['ch18'] = ch18
datavars['IMAGER']['surface'] = surf
datavars['IMAGER']['rain'] = rain
else:
gvars['IMAGER']['Latitude'] = np.ma.vstack(
(gvars['IMAGER']['Latitude'], lt))
gvars['IMAGER']['Longitude'] = np.ma.vstack(
(gvars['IMAGER']['Longitude'], lg))
datavars['IMAGER']['ch08'] = np.ma.vstack(
(datavars['IMAGER']['ch08'], ch08))
datavars['IMAGER']['ch09'] = np.ma.vstack(
(datavars['IMAGER']['ch09'], ch09))
datavars['IMAGER']['ch10'] = np.ma.vstack(
(datavars['IMAGER']['ch10'], ch10))
datavars['IMAGER']['ch11'] = np.ma.vstack(
(datavars['IMAGER']['ch11'], ch11))
datavars['IMAGER']['ch17'] = np.ma.vstack(
(datavars['IMAGER']['ch17'], ch17))
datavars['IMAGER']['ch18'] = np.ma.vstack(
(datavars['IMAGER']['ch18'], ch18))
datavars['IMAGER']['surface'] = np.ma.vstack(
(datavars['IMAGER']['surface'], surf))
datavars['IMAGER']['rain'] = np.ma.vstack(
(datavars['IMAGER']['rain'], rain))
gvars['IMAGER']['Latitude'] = np.ma.masked_values(
gvars['IMAGER']['Latitude'], bad_value)
gvars['IMAGER']['Longitude'] = np.ma.masked_values(
gvars['IMAGER']['Longitude'], bad_value)
datavars['IMAGER']['ch08'] = np.ma.masked_values(
datavars['IMAGER']['ch08'], bad_value)
datavars['IMAGER']['ch09'] = np.ma.masked_values(
datavars['IMAGER']['ch09'], bad_value)
datavars['IMAGER']['ch10'] = np.ma.masked_values(
datavars['IMAGER']['ch10'], bad_value)
datavars['IMAGER']['ch11'] = np.ma.masked_values(
datavars['IMAGER']['ch11'], bad_value)
datavars['IMAGER']['ch17'] = np.ma.masked_values(
datavars['IMAGER']['ch17'], bad_value)
datavars['IMAGER']['ch18'] = np.ma.masked_values(
datavars['IMAGER']['ch18'], bad_value)
datavars['IMAGER']['surface'] = np.ma.masked_values(
datavars['IMAGER']['surface'], bad_value)
datavars['IMAGER']['rain'] = np.ma.masked_values(
datavars['IMAGER']['rain'], bad_value)
#-----------------------------------------------------------------------------------------
enviro_read = np.ma.zeros((nscan_enviro, 90))
np.ma.masked_all_like(enviro_read)
if scenecounts_enviro[0] < 0:
print "ENVIRO is negative"
lt = np.ma.masked_equal(enviro_read, bad_value)
lg = np.ma.masked_equal(enviro_read, bad_value)
ch12 = np.ma.masked_equal(enviro_read, bad_value)
ch13 = np.ma.masked_equal(enviro_read, bad_value)
ch14 = np.ma.masked_equal(enviro_read, bad_value)
ch15 = np.ma.masked_equal(enviro_read, bad_value)
ch16 = np.ma.masked_equal(enviro_read, bad_value)
ch15_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch16_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch17_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch18_5x5 = np.ma.masked_equal(enviro_read, bad_value)
ch17_5x4 = np.ma.masked_equal(enviro_read, bad_value)
ch18_5x4 = np.ma.masked_equal(enviro_read, bad_value)
#ENVIRO READ DATA
for ii in range(nscan_enviro):
if ii % 2 == 0: #for odd scan numbers
if start_scantime_enviro[ii] == -999:
print 'value of enviro odd scan is %d' % ii
continue
for jj in range(scenecounts_enviro[ii]):
enviroodd_lat, enviroodd_lon, enviroodd_scene = np.fromstring(
f1.read(6), dtype=np.dtype('short')).byteswap()
enviroodd_seaice, enviroodd_surf = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
enviroodd_ch12, enviroodd_ch13, enviroodd_ch14, enviroodd_ch15, enviroodd_ch16, enviroodd_ch15_5x5, enviroodd_ch16_5x5, enviroodd_ch17_5x5, enviroodd_ch18_5x5, enviroodd_ch17_5x4, enviroodd_ch18_5x4 = np.fromstring(
f1.read(22), dtype=np.dtype('short')).byteswap()
enviroodd_rain1, enviroodd_rain2 = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
edr_bitflags = np.fromstring(
f1.read(4), dtype=np.dtype('int32')).byteswap()
nbytes += 36
lat = 0.01 * enviroodd_lat
lon = 0.01 * enviroodd_lon
lt[ii][jj] = lat
lg[ii][jj] = lon
if rev6a == 1:
ch12[ii][jj] = enviroodd_ch12 #19H
ch13[ii][jj] = enviroodd_ch13 #19V
ch14[ii][jj] = enviroodd_ch14 #22V
ch15[ii][jj] = enviroodd_ch15 #37H
ch16[ii][jj] = enviroodd_ch16 #37V
ch15_5x5[ii][jj] = enviroodd_ch15_5x5
ch16_5x5[ii][jj] = enviroodd_ch16_5x5
ch17_5x5[ii][jj] = enviroodd_ch17_5x5
ch18_5x5[ii][jj] = enviroodd_ch18_5x5
ch17_5x4[ii][jj] = enviroodd_ch17_5x4
ch18_5x4[ii][jj] = enviroodd_ch18_5x4
else:
ch12[ii][jj] = 10 * enviroodd_ch12
ch13[ii][jj] = 10 * enviroodd_ch13
ch14[ii][jj] = 10 * enviroodd_ch14
ch15[ii][jj] = 10 * enviroodd_ch15
ch16[ii][jj] = 10 * enviroodd_ch16
ch15_5x5[ii][jj] = 10 * enviroodd_ch15_5x5
ch16_5x5[ii][jj] = 10 * enviroodd_ch16_5x5
ch17_5x5[ii][jj] = 10 * enviroodd_ch17_5x5
ch18_5x5[ii][jj] = 10 * enviroodd_ch18_5x5
ch17_5x4[ii][jj] = 10 * enviroodd_ch17_5x4
ch18_5x4[ii][jj] = 10 * enviroodd_ch18_5x4
if ii % 2 == 1: # for even scan numbers
if start_scantime_enviro[ii] == -999:
print 'value of enviro even scan is %d' % ii
continue
for jj in range(scenecounts_enviro[ii]):
enviroeven_lat, enviroeven_lon, enviroeven_scene = np.fromstring(
f1.read(6), dtype=np.dtype('short')).byteswap()
enviroeven_seaice, enviroeven_surf = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
enviroeven_ch12, enviroeven_ch13, enviroeven_ch14, enviroeven_ch15, enviroeven_ch16 = np.fromstring(
f1.read(10), dtype=np.dtype('short')).byteswap()
nbytes += 18
lat = 0.01 * enviroeven_lat
lon = 0.01 * enviroeven_lon
lt[ii][jj] = lat
lg[ii][jj] = lon
if rev6a == 1:
ch12[ii][jj] = enviroeven_ch12
ch13[ii][jj] = enviroeven_ch13
ch14[ii][jj] = enviroeven_ch14
ch15[ii][jj] = enviroeven_ch15
ch16[ii][jj] = enviroeven_ch16
else:
ch12[ii][jj] = 10 * enviroeven_ch12
ch13[ii][jj] = 10 * enviroeven_ch13
ch14[ii][jj] = 10 * enviroeven_ch14
ch15[ii][jj] = 10 * enviroeven_ch15
ch16[ii][jj] = 10 * enviroeven_ch16
if 'Latitude' not in gvars['ENVIRO'].keys():
gvars['ENVIRO']['Latitude'] = lt
gvars['ENVIRO']['Longitude'] = lg
datavars['ENVIRO']['ch12'] = ch12
datavars['ENVIRO']['ch13'] = ch13
datavars['ENVIRO']['ch14'] = ch14
datavars['ENVIRO']['ch15'] = ch15
datavars['ENVIRO']['ch16'] = ch16
datavars['ENVIRO']['ch15_5x5'] = ch15_5x5
datavars['ENVIRO']['ch16_5x5'] = ch16_5x5
datavars['ENVIRO']['ch17_5x5'] = ch17_5x5
datavars['ENVIRO']['ch18_5x5'] = ch18_5x5
datavars['ENVIRO']['ch17_5x4'] = ch17_5x4
datavars['ENVIRO']['ch18_5x4'] = ch18_5x4
else:
gvars['ENVIRO']['Latitude'] = np.ma.vstack(
(gvars['ENVIRO']['Latitude'], lt))
gvars['ENVIRO']['Longitude'] = np.ma.vstack(
(gvars['ENVIRO']['Longitude'], lg))
datavars['ENVIRO']['ch12'] = np.ma.vstack(
(datavars['ENVIRO']['ch12'], ch12))
datavars['ENVIRO']['ch13'] = np.ma.vstack(
(datavars['ENVIRO']['ch13'], ch13))
datavars['ENVIRO']['ch14'] = np.ma.vstack(
(datavars['ENVIRO']['ch14'], ch14))
datavars['ENVIRO']['ch15'] = np.ma.vstack(
(datavars['ENVIRO']['ch15'], ch15))
datavars['ENVIRO']['ch16'] = np.ma.vstack(
(datavars['ENVIRO']['ch16'], ch16))
datavars['ENVIRO']['ch15_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch15_5x5'], ch15_5x5))
datavars['ENVIRO']['ch16_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch16_5x5'], ch16_5x5))
datavars['ENVIRO']['ch17_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch17_5x5'], ch17_5x5))
datavars['ENVIRO']['ch18_5x5'] = np.ma.vstack(
(datavars['ENVIRO']['ch18_5x5'], ch18_5x5))
datavars['ENVIRO']['ch17_5x4'] = np.ma.vstack(
(datavars['ENVIRO']['ch17_5x4'], ch17_5x4))
datavars['ENVIRO']['ch18_5x4'] = np.ma.vstack(
(datavars['ENVIRO']['ch18_5x4'], ch18_5x4))
gvars['ENVIRO']['Latitude'] = np.ma.masked_equal(
gvars['ENVIRO']['Latitude'], bad_value)
gvars['ENVIRO']['Longitude'] = np.ma.masked_equal(
gvars['ENVIRO']['Longitude'], bad_value)
datavars['ENVIRO']['ch12'] = np.ma.masked_equal(
datavars['ENVIRO']['ch12'], bad_value)
datavars['ENVIRO']['ch13'] = np.ma.masked_equal(
datavars['ENVIRO']['ch13'], bad_value)
datavars['ENVIRO']['ch14'] = np.ma.masked_equal(
datavars['ENVIRO']['ch14'], bad_value)
datavars['ENVIRO']['ch15'] = np.ma.masked_equal(
datavars['ENVIRO']['ch15'], bad_value)
datavars['ENVIRO']['ch16'] = np.ma.masked_equal(
datavars['ENVIRO']['ch16'], bad_value)
datavars['ENVIRO']['ch15_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch15_5x5'], bad_value)
datavars['ENVIRO']['ch16_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch16_5x5'], bad_value)
datavars['ENVIRO']['ch17_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch17_5x5'], bad_value)
datavars['ENVIRO']['ch18_5x5'] = np.ma.masked_equal(
datavars['ENVIRO']['ch18_5x5'], bad_value)
datavars['ENVIRO']['ch17_5x4'] = np.ma.masked_equal(
datavars['ENVIRO']['ch17_5x4'], bad_value)
datavars['ENVIRO']['ch18_5x4'] = np.ma.masked_equal(
datavars['ENVIRO']['ch18_5x4'], bad_value)
#-----------------------------------------------------------------------------------------
las_read = np.ma.zeros((nscan_las, 60))
np.ma.masked_all_like(las_read)
if scenecounts_las[0] < 0:
print "LAS is negative"
lt = np.ma.masked_equal(las_read, bad_value)
lg = np.ma.masked_equal(las_read, bad_value)
ch01_3x3 = np.ma.masked_equal(las_read, bad_value)
ch02_3x3 = np.ma.masked_equal(las_read, bad_value)
ch03_3x3 = np.ma.masked_equal(las_read, bad_value)
ch04_3x3 = np.ma.masked_equal(las_read, bad_value)
ch05_3x3 = np.ma.masked_equal(las_read, bad_value)
ch06_3x3 = np.ma.masked_equal(las_read, bad_value)
ch07_3x3 = np.ma.masked_equal(las_read, bad_value)
ch08_5x5 = np.ma.masked_equal(las_read, bad_value)
ch09_5x5 = np.ma.masked_equal(las_read, bad_value)
ch10_5x5 = np.ma.masked_equal(las_read, bad_value)
ch11_5x5 = np.ma.masked_equal(las_read, bad_value)
ch18_5x5 = np.ma.masked_equal(las_read, bad_value)
ch24_3x3 = np.ma.masked_equal(las_read, bad_value)
height_1000mb = np.ma.masked_equal(las_read, bad_value)
surf = np.ma.masked_equal(las_read, bad_value)
#LAS READ DATA
for ii in range(nscan_las):
if start_scantime_las[ii] == -999:
print 'value of las scan is %d' % ii
continue
for jj in range(scenecounts_las[ii]):
try:
las_lati, las_long, las_ch01_3x3, las_ch02_3x3, las_ch03_3x3, las_ch04_3x3, las_ch05_3x3, las_ch06_3x3, las_ch07_3x3, las_ch08_5x5, las_ch09_5x5, las_ch10_5x5, las_ch11_5x5, las_ch18_5x5, las_ch24_3x3, las_height_1000mb, las_surf = np.fromstring(
f1.read(34), dtype=np.dtype('short')).byteswap()
las_tqflag, las_hqflag = np.fromstring(
f1.read(2), dtype=np.dtype('int8')).byteswap()
las_terrain, las_scene = np.fromstring(
f1.read(4), dtype=np.dtype('short')).byteswap()
except:
continue
lat = 0.01 * las_lati
lon = 0.01 * las_long
nbytes += 40
lt[ii][jj] = lat
lg[ii][jj] = lon
ch01_3x3[ii][jj] = las_ch01_3x3 #50.3 V
ch02_3x3[ii][jj] = las_ch02_3x3 #52.8 V
ch03_3x3[ii][jj] = las_ch03_3x3 #53.60V
ch04_3x3[ii][jj] = las_ch04_3x3 #54.4 V
ch05_3x3[ii][jj] = las_ch05_3x3 #55.5 V
ch06_3x3[ii][jj] = las_ch06_3x3 #57.3 RCP
ch07_3x3[ii][jj] = las_ch07_3x3 #59.4 RCP
ch08_5x5[ii][jj] = las_ch08_5x5 #150 H
ch09_5x5[ii][jj] = las_ch09_5x5 #183.31+-7 H
ch10_5x5[ii][jj] = las_ch10_5x5 #183.31+-3 H
ch11_5x5[ii][jj] = las_ch11_5x5 #183.31+-1 H
ch18_5x5[ii][jj] = las_ch18_5x5 #91 H
ch24_3x3[ii][jj] = las_ch24_3x3 #60.79+-36+-0.05 RCP
height_1000mb[ii][jj] = las_height_1000mb
surf[ii][jj] = las_surf
if 'Latitude' not in gvars['LAS'].keys():
gvars['LAS']['Latitude'] = lt
gvars['LAS']['Longitude'] = lg
datavars['LAS']['ch01_3x3'] = ch01_3x3
datavars['LAS']['ch02_3x3'] = ch02_3x3
datavars['LAS']['ch03_3x3'] = ch03_3x3
datavars['LAS']['ch04_3x3'] = ch04_3x3
datavars['LAS']['ch05_3x3'] = ch05_3x3
datavars['LAS']['ch06_3x3'] = ch06_3x3
datavars['LAS']['ch07_3x3'] = ch07_3x3
datavars['LAS']['ch08_5x5'] = ch08_5x5
datavars['LAS']['ch09_5x5'] = ch09_5x5
datavars['LAS']['ch10_5x5'] = ch10_5x5
datavars['LAS']['ch11_5x5'] = ch11_5x5
datavars['LAS']['ch18_5x5'] = ch18_5x5
datavars['LAS']['ch24_3x3'] = ch24_3x3
datavars['LAS']['height_1000mb'] = height_1000mb
datavars['LAS']['surf'] = surf
else:
gvars['LAS']['Latitude'] = np.ma.vstack(
(gvars['LAS']['Latitude'], lt))
gvars['LAS']['Longitude'] = np.ma.vstack(
(gvars['LAS']['Longitude'], lg))
datavars['LAS']['ch01_3x3'] = np.ma.vstack(
(datavars['LAS']['ch01_3x3'], ch01_3x3))
datavars['LAS']['ch02_3x3'] = np.ma.vstack(
(datavars['LAS']['ch02_3x3'], ch02_3x3))
datavars['LAS']['ch03_3x3'] = np.ma.vstack(
(datavars['LAS']['ch03_3x3'], ch03_3x3))
datavars['LAS']['ch04_3x3'] = np.ma.vstack(
(datavars['LAS']['ch04_3x3'], ch04_3x3))
datavars['LAS']['ch05_3x3'] = np.ma.vstack(
(datavars['LAS']['ch05_3x3'], ch05_3x3))
datavars['LAS']['ch06_3x3'] = np.ma.vstack(
(datavars['LAS']['ch06_3x3'], ch06_3x3))
datavars['LAS']['ch07_3x3'] = np.ma.vstack(
(datavars['LAS']['ch07_3x3'], ch07_3x3))
datavars['LAS']['ch08_5x5'] = np.ma.vstack(
(datavars['LAS']['ch08_5x5'], ch08_5x5))
datavars['LAS']['ch09_5x5'] = np.ma.vstack(
(datavars['LAS']['ch09_5x5'], ch09_5x5))
datavars['LAS']['ch10_5x5'] = np.ma.vstack(
(datavars['LAS']['ch10_5x5'], ch10_5x5))
datavars['LAS']['ch11_5x5'] = np.ma.vstack(
(datavars['LAS']['ch11_5x5'], ch11_5x5))
datavars['LAS']['ch18_5x5'] = np.ma.vstack(
(datavars['LAS']['ch18_5x5'], ch18_5x5))
datavars['LAS']['ch24_3x3'] = np.ma.vstack(
(datavars['LAS']['ch24_3x3'], ch24_3x3))
datavars['LAS']['height_1000mb'] = np.ma.vstack(
(datavars['LAS']['height_1000mb'], height_1000mb))
datavars['LAS']['surf'] = np.ma.vstack(
(datavars['LAS']['surf'], surf))
gvars['LAS']['Latitude'] = np.ma.masked_equal(
gvars['LAS']['Latitude'], bad_value)
gvars['LAS']['Longitude'] = np.ma.masked_equal(
gvars['LAS']['Longitude'], bad_value)
datavars['LAS']['ch01_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch01_3x3'], bad_value)
datavars['LAS']['ch02_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch02_3x3'], bad_value)
datavars['LAS']['ch03_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch03_3x3'], bad_value)
datavars['LAS']['ch04_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch04_3x3'], bad_value)
datavars['LAS']['ch05_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch05_3x3'], bad_value)
datavars['LAS']['ch06_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch06_3x3'], bad_value)
datavars['LAS']['ch07_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch07_3x3'], bad_value)
datavars['LAS']['ch08_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch08_5x5'], bad_value)
datavars['LAS']['ch09_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch09_5x5'], bad_value)
datavars['LAS']['ch10_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch10_5x5'], bad_value)
datavars['LAS']['ch11_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch11_5x5'], bad_value)
datavars['LAS']['ch18_5x5'] = np.ma.masked_equal(
datavars['LAS']['ch18_5x5'], bad_value)
datavars['LAS']['ch24_3x3'] = np.ma.masked_equal(
datavars['LAS']['ch24_3x3'], bad_value)
datavars['LAS']['height_1000mb'] = np.ma.masked_equal(
datavars['LAS']['height_1000mb'], bad_value)
datavars['LAS']['surf'] = np.ma.masked_equal(
datavars['LAS']['surf'], bad_value)
#---------------------------------------------------------------------------------
uas_read = np.ma.zeros((nscan_uas, 30))
np.ma.masked_all_like(uas_read)
if scenecounts_uas[0] < 0:
print "UAS is negative"
lt = np.ma.masked_equal(uas_read, bad_value)
lg = np.ma.masked_equal(uas_read, bad_value)
ch19_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch20_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch21_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch22_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch23_6x6 = np.ma.masked_equal(uas_read, bad_value)
ch24_6x6 = np.ma.masked_equal(uas_read, bad_value)
sceneu = np.ma.masked_equal(uas_read, bad_value)
tqflag = np.ma.masked_equal(uas_read, bad_value)
#UAS READ DATA
for ii in range(nscan_uas):
if start_scantime_uas[ii] == -999:
print 'value of uas scan is %d' % ii
continue
for jj in range(scenecounts_uas[ii]):
uas_lat, uas_lon, uas_ch19_6x6, uas_ch20_6x6, uas_ch21_6x6, uas_ch22_6x6, uas_ch23_6x6, uas_ch24_6x6, uas_scene, uas_tqflag = np.fromstring(
f1.read(20), dtype=np.dtype('short')).byteswap()
uas_field, uas_bdotk2 = np.fromstring(
f1.read(8), dtype=np.dtype('int32')).byteswap()
nbytes += 28
lat = 0.01 * uas_lat
lon = 0.01 * uas_lon
lt[ii][jj] = lat
lg[ii][jj] = lon
ch19_6x6[ii][jj] = uas_ch19_6x6 #63.28+-0.28 RCP GHz
ch20_6x6[ii][jj] = uas_ch20_6x6 #60.79+-0.36 RCP
ch21_6x6[ii][jj] = uas_ch21_6x6 #60.79+-0.36+-0.002 RCP
ch22_6x6[ii][jj] = uas_ch22_6x6 #60.79+-0.36+-0.0055 RCP
ch23_6x6[ii][jj] = uas_ch23_6x6 #60.79+-0.36+-0.0016 RCP
ch24_6x6[ii][jj] = uas_ch24_6x6 #60.79+-0.36+-0.050 RCP
sceneu[ii][jj] = uas_scene
tqflag[ii][jj] = uas_tqflag
if 'Latitude' not in gvars['UAS'].keys():
gvars['UAS']['Latitude'] = lt
gvars['UAS']['Longitude'] = lg
datavars['UAS']['ch19_6x6'] = ch19_6x6
datavars['UAS']['ch20_6x6'] = ch20_6x6
datavars['UAS']['ch21_6x6'] = ch21_6x6
datavars['UAS']['ch22_6x6'] = ch22_6x6
datavars['UAS']['ch23_6x6'] = ch23_6x6
datavars['UAS']['ch24_6x6'] = ch24_6x6
datavars['UAS']['scene'] = sceneu
datavars['UAS']['uas_tqflag'] = tqflag
else:
gvars['UAS']['Latitude'] = np.ma.vstack(
(gvars['UAS']['Latitude'], lt))
gvars['UAS']['Longitude'] = np.ma.vstack(
(gvars['UAS']['Longitude'], lg))
datavars['UAS']['ch19_6x6'] = np.ma.vstack(
(datavars['UAS']['ch19_6x6'], ch19_6x6))
datavars['UAS']['ch20_6x6'] = np.ma.vstack(
(datavars['UAS']['ch20_6x6'], ch20_6x6))
datavars['UAS']['ch21_6x6'] = np.ma.vstack(
(datavars['UAS']['ch21_6x6'], ch21_6x6))
datavars['UAS']['ch22_6x6'] = np.ma.vstack(
(datavars['UAS']['ch22_6x6'], ch22_6x6))
datavars['UAS']['ch23_6x6'] = np.ma.vstack(
(datavars['UAS']['ch23_6x6'], ch23_6x6))
datavars['UAS']['ch24_6x6'] = np.ma.vstack(
(datavars['UAS']['ch24_6x6'], ch24_6x6))
datavars['UAS']['scene'] = np.ma.vstack(
(datavars['UAS']['scene'], sceneu))
datavars['UAS']['uas_tqflag'] = np.ma.vstack(
(datavars['UAS']['uas_tqflag'], tqflag))
gvars['UAS']['Latitude'] = np.ma.masked_equal(
gvars['UAS']['Latitude'], bad_value)
gvars['UAS']['Longitude'] = np.ma.masked_equal(
gvars['UAS']['Longitude'], bad_value)
datavars['UAS']['ch19_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch19_6x6'], bad_value)
datavars['UAS']['ch20_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch20_6x6'], bad_value)
datavars['UAS']['ch21_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch21_6x6'], bad_value)
datavars['UAS']['ch22_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch22_6x6'], bad_value)
datavars['UAS']['ch23_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch23_6x6'], bad_value)
datavars['UAS']['ch24_6x6'] = np.ma.masked_equal(
datavars['UAS']['ch24_6x6'], bad_value)
datavars['UAS']['scene'] = np.ma.masked_equal(
datavars['UAS']['scene'], bad_value)
datavars['UAS']['uas_tqflag'] = np.ma.masked_equal(
datavars['UAS']['uas_tqflag'], bad_value)
print 'nfiller=', nfiller
nfiller = 512 - (
nbytes % 512
) # nfiller bytes to be skipped so that the next scan header will start at the 513th byte.
try:
filler_bytes = np.fromstring(
f1.read(nfiller), dtype=np.dtype('int8')).byteswap()[0]
except:
continue
f1.close()
#-----------------------------------------------------------------------------------------------------
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
for geoipsvarname, dfvarname in self.dataset_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
#shell()
data = datavars[dsname][geoipsvarname]
fillvalue = -999
datavars[dsname][geoipsvarname] = (
np.ma.masked_equal(data, fillvalue) / 100) + 273.15
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
for geoipsvarname, dfvarname in self.gvar_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
#shell()
data = gvars[dsname][geoipsvarname]
fillvalue = -999
gvars[dsname][geoipsvarname] = np.ma.masked_equal(
data, fillvalue)
def latency(
start_datetime,
end_datetime,
sensor,
satellites,
data_providers,
channels,
overall=False,
verbose=True,
):
log.info('\n\n')
nowtime = datetime.utcnow()
total_hours = (end_datetime - start_datetime).days * 24 + (
end_datetime - start_datetime).seconds // 3600
log.info(str(total_hours) + ' hours')
allfiles = []
for sat in satellites:
log.info('Trying sat ' + sat + ' sensor: ' + sensor)
currdata_providers = data_providers
currchannels = channels
if not data_providers:
currdata_providers = ['*']
if not channels:
currchannels = ['*']
for data_provider in currdata_providers:
for channel in currchannels:
allfiles += DataFileName.list_range_of_files(
sat,
sensor,
start_datetime,
end_datetime,
datetime_wildcards={
'%H': '*%H',
'%M': '*',
'%S': '*'
},
data_provider=data_provider,
resolution='*',
channel=channel,
producttype='*',
area='*',
extra='*',
ext='*',
forprocess=False)
if overall:
totalsize, totalnum = calc_latency(
allfiles,
fileclass='DataFileName',
verbose=verbose,
classkeys={
'overall': ['sensorname', 'satname', 'dataprovider'],
})
else:
totalsize, totalnum = calc_latency(
allfiles,
fileclass='DataFileName',
verbose=verbose,
classkeys={
'individual':
['sensorname', 'satname', 'channel', 'dataprovider'],
'overall': ['sensorname', 'satname', 'dataprovider'],
})
log.interactive('Total size on disk for ' + str(totalnum) +
' data files: ' + convert_bytes(totalsize) + ': sensor: ' +
sensor + ' satellites: ' + ', '.join(satellites))
return totalsize, totalnum
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
class data_grid:
#indir='devnull'...this looks for the directory that the file is in
dtg = '9999999999'
mapvars = {}
nz = 0
nzp1 = 0
nnest = 0
delx0 = 0
dely0 = 0
nest = [0]
class Nest(object):
def __init__(self, nx, ny, ii, jj, iref, jref, tcmx, tcmy, delx,
dely):
self.nx = nx
self.ny = ny
self.ii = ii
self.jj = jj
self.iref = iref
self.jref = jref
self.tcmx = tcmx
self.tcmy = tcmy
self.delx = delx
self.dely = dely
# define the model grid
model_grid = {}
model_grid['im'] = 277
model_grid['jm'] = 229
model_grid['lm'] = 60
model_grid['num_bytes'] = model_grid['im'] * model_grid['jm'] * 4
#Constant files for geolocation and header
latitude = '/SATPROJECT/users/projects3/users/laflash/outdirs/nrtdata/realtime/longterm_files/COAMPS_metadata/latitu_sfc_000000_000000_3a0277x0229_2018092400_00000000_fcstfld'
longitude = '/SATPROJECT/users/projects3/users/laflash/outdirs/nrtdata/realtime/longterm_files/COAMPS_metadata/longit_sfc_000000_000000_3a0277x0229_2018092400_00000000_fcstfld'
header = '/SATPROJECT/users/projects3/users/laflash/outdirs/nrtdata/realtime/longterm_files/COAMPS_metadata/datahd_sfc_000000_000000_1a2000x0001_2018090918_00000000_infofld'
#def main(file, lat_file, lon_file, cdtg, fld, level, model_grid, image_dir):
# istat = 0
# lat, istat = seek_field(lat_file, model_grid, 1)
# lon, istat = seek_field(lon_file, model_grid, 1)
#
# data, stat = seek_field(file, model_grid, level)
#
# title = ( "%s lvl:%.2i %s %s" % (fld.upper(), int(level), cdtg, tau) )
# level_name = ( "l%.2i" % int(level) )
# image_name = '_'.join(["ascos1", "2a", cdtg, fld, level_name, tau])
#
# plot_global(data, lat, lon, title, image_name,
# clabel=plot_parm[fld]['units'],
# range=[plot_parm[fld]['min'],plot_parm[fld]['max']])
def seek_field(filename):
print 'Reading file...'
#metadata
datafilename = filename.split('/')[-1].split('_')
wxparameter = datafilename[0]
level_type = datafilename[1]
lvl1 = datafilename[2]
lvl1 = "%06.1f" % (float(lvl1))
lvl2 = datafilename[3]
if wxparameter == 'latitu' or wxparameter == 'longit':
lvl2 = 1
else:
lvl2 = "%06.1f" % (float(lvl2))
imest_and_gridlevels = datafilename[4]
dtg = datafilename[5]
filetype = datafilename[6]
record_length = model_grid['num_bytes']
# top to bottom
offset = (model_grid['lm'] - int(float(lvl2))) * record_length
# bottom to top
offset = (int(float(lvl2)) - 1) * record_length
#offset = (1 - int(float(lvl2))) * record_length
# binary file read
if os.path.isfile(filename):
f = open(filename, 'rb')
f.seek(offset)
data = np.fromstring(f.read(model_grid['num_bytes']),
dtype='float32')
if sys.byteorder == 'little':
data = data.byteswap()
data = data.reshape(model_grid['jm'], model_grid['im'])
data = np.ma.masked_equal(data, -990.99)
istat = 0
else:
print "missing file"
print filename
data = [[-999.99] * model_grid['im']] * model_grid['jm']
istat = -1
return data, wxparameter, level_type, lvl1, lvl2, dtg, filetype
def read_coamps_header(filename):
#"%s/datahd_sfc_000000_000000_1a2000x0001_%s_00000000_infofld"%(indir, dtg)
if os.path.isfile(filename): #might not need
#data_grid.indir = indir#might not need
f = open(filename, 'rb')
datahd = f.read()
datahd = list(datahd.split())
# separate occasional values with no space between them
for j in range(len(datahd)):
val = datahd[j]
if len(val) > 13:
i1 = 0
k = 0
for i in range(len(val) - 1):
if val[i:i + 1] == 'E':
newval = val[i1:i + 4]
if i + 4 < 15:
datahd[j] = newval
else:
datahd.insert(j + k, newval)
k = k + 1
i1 = i + 4
data_grid.mapvars['nproj'] = float(datahd[2])
data_grid.mapvars['stdlat1'] = float(datahd[3])
data_grid.mapvars['stdlat2'] = float(datahd[4])
data_grid.mapvars['stdlon'] = float(datahd[5])
data_grid.mapvars['reflat'] = float(datahd[6])
data_grid.mapvars['reflon'] = float(datahd[7])
data_grid.nz = int(float(datahd[1]))
data_grid.nzp1 = int(float(datahd[1])) + 1
data_grid.nnest = int(float(datahd[10]))
data_grid.delx0 = float(datahd[9])
data_grid.dely0 = float(datahd[8])
nn = 1
while nn <= data_grid.nnest:
ng = 30 + (nn - 1) * 30
nx = int(float(datahd[ng - 1]))
ny = int(float(datahd[ng + 0]))
ii = float(datahd[ng + 1])
jj = float(datahd[ng + 2])
iref = float(datahd[ng + 3])
jref = float(datahd[ng + 4])
tcmx = float(datahd[ng + 27])
tcmy = float(datahd[ng + 28])
delx = float(datahd[ng + 6])
dely = float(datahd[ng + 7])
data_grid.nest.append(
Nest(nx, ny, ii, jj, iref, jref, tcmx, tcmy, delx,
dely))
nn = nn + 1
# vertical indices
nz = data_grid.nz
dsigm = np.array(datahd[500:500 + nz]).astype(np.float)
data_grid.sigw = np.append(
np.flipud(np.cumsum(np.flipud(dsigm))), [0.0])
data_grid.sigm = datahd[800:800 + nz]
data_grid.ztop = data_grid.sigw[0]
data, wxparameter, level_type, lvl1, lvl2, dtg, filetype = seek_field(
fname)
metadata['top']['level'] = lvl2
metadata['top']['start_datetime'] = datetime.strptime(dtg, '%Y%m%d%H')
metadata['top']['end_datetime'] = datetime.strptime(dtg, '%Y%m%d%H')
metadata['top']['dataprovider'] = 'NRL'
metadata['top']['filename_datetime'] = metadata['top'][
'start_datetime']
metadata['top']['platform_name'] = 'model'
metadata['top']['source_name'] = 'coampsieee'
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
if chans == []:
return
# def rdata (filename)#parmnm, lvltyp, lev1, lev2, inest, dtg, tau, indir='def', outtyp='fcstfld', im=-1, jm=-1):
#
# # read coamps binary flat file
#
# if indir == 'def': indir = data_grid.indir
# if im == -1: im = data_grid.nest[inest].nx
# if jm == -1: jm = data_grid.nest[inest].ny
# if inest == 0:
# #global
# filename = "%s/%s_%s_%06.1f_%06.1f_glob%03dx%03d_%s_%04d0000_%s" \
# %(indir, parmnm, lvltyp, lev1, lev2, im, jm, dtg, tau, outtyp)
# else:
# #COAMPS
# filename = "%s/%s_%s_%06d_%06d_%1da%04dx%04d_%s_%04d0000_%s" \
# %(indir, parmnm, lvltyp, lev1, lev2, inest, im, jm, dtg, tau, outtyp)
#
# # print "Reading %s"%filename
# num_bytes = im*jm*4
#
# offset = 0
#
# # binary file read
# if os.path.isfile(filename):
# f = open( filename, 'rb' )
# f.seek( offset )
# data = np.fromstring(f.read(num_bytes), dtype='float32')
# # COAMPS irsat values are little_endian all others are big
# if sys.byteorder == 'little':
# if parmnm != 'irrcmp':
# data = data.byteswap()
# data = data.reshape(jm, im)
# data = np.ma.masked_equal(data, -990.99)
# f.close()
# istat = 0
# else:
# print "MISSING %s"%parmnm
# print filename
# data = [[-999.99] * im] * jm
# istat = -1
# return data, istat
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
for geoipsvarname, dfvarname in self.dataset_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
fillvalue = data.fill_value
datavars[dsname][geoipsvarname] = np.ma.masked_equal(
data, fillvalue)
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
for geoipsvarname, dfvarname in self.gvar_info[dsname].items():
if dfvarname == 'latitu':
geolog, wxparameter, level_type, lvl1, lvl2, dtg, filetype = seek_field(
latitude)
if dfvarname == 'longit':
geolog, wxparameter, level_type, lvl1, lvl2, dtg, filetype = seek_field(
longitude)
xx, yy = geolog.shape
for aa in range(xx):
for bb in range(yy):
if geolog[aa, bb] > 180:
geolog[aa, bb] = geolog[aa, bb] - 360
fillvalue = geolog.fill_value
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
gvars[dsname][geoipsvarname] = np.ma.masked_equal(
geolog, fillvalue)
def find_available_data_files(opasses,
start_dt,
satellite,
sensor,
extra_dirs,
prodtype=None,
ext='*'):
all_files = []
# runfulldir = None
overall_start_dt = start_dt
overall_end_dt = start_dt
#print opasses
for opass in sorted(opasses, key=operator.attrgetter('basedt')):
log.interactive('Trying opass: '******'*'
prodtype = dfn.sensorinfo.FName['default_producttype']
#print prodtype
# dfn.producttype = prodtype
# runfulldir = dfn.sensorinfo.FName['runfulldir']
# If this is a long overpass, make sure we get the data files
# coming before the overpass time
mins_per_file = dfn.sensorinfo.mins_per_file
startdt = opass.startdt - timedelta(minutes=mins_per_file)
enddt = opass.enddt + timedelta(minutes=mins_per_file)
if startdt < overall_start_dt:
overall_start_dt = startdt
overall_end_dt = enddt
all_files += DataFileName.list_range_of_files(
satellite,
sensor,
startdt,
enddt,
datetime_wildcards={
'%H': '%H',
'%M': '*',
'%S': '*'
},
data_provider='*',
resolution='*',
channel='*',
producttype=prodtype,
area='*',
extra='*',
ext='*',
forprocess=True,
)
all_files = list(set(all_files))
log.interactive(' Total files so far: ' + str(len(all_files)))
if all_files:
log.interactive('Available files:' + bigindent + bigindent.join(
commands.getoutput('ls --full-time -d ' + str(file))
for file in sorted(all_files)) + '\n\n')
return all_files
# return None
# if files:
if list == False:
file_num = 0
for file in all_files:
file_num = file_num + 1
log.interactive('\n\n\n')
log.interactive('Starting processing for file ' + str(file_num) +
' of ' + str(len(all_files)) + ': ' + file +
'\n\n')
if os.path.isdir(file):
try:
dfn = DataFileName(glob(file + '/*')[0])
except IndexError:
log.interactive(
' Appear to be no data files in directory !! Skipping'
)
continue
else:
dfn = DataFileName(os.path.basename(file))
dt = dfn.datetime
if start_datetime and (dfn.datetime < start_datetime
or dfn.datetime > end_datetime):
log.interactive('Outside of time range, skipping')
continue
currsectorlist = []
if not sectorlist:
curropasses = pass_prediction([dfn.satname], [dfn.sensorname],
def makedirs(self, file):
dfn = DataFileName(os.path.basename(file))
sdfn = dfn.create_standard(downloadSiteObj=self)
sdfn.makedirs()
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
'''
Reads all topo data in the region defined by the area definition.
'''
self.area_definition = sector_definition.area_definition
self.shape = (43200, 21600)
crnr_1, crnr_2 = self.corner_coords
inds = ((crnr_1[0], crnr_2[0], 1), (crnr_1[1], crnr_2[1], 1))
try:
xstart, xend, xstep = inds[0]
ystart, yend, ystep = inds[1]
shape = (yend - ystart, xend - xstart)
dims = 2
#If that fails, try for one dimension
except TypeError:
start, end, step = inds
shape = (end - start, )
dims = 1
storage_dtype = np.dtype('>i2')
fileobj = open(fname, mode='r')
#If we are only slicing in one dimension
if dims == 1:
data_array = self.read_slice(fileobj, start, shape, storage_dtype)
elif dims == 2:
lineshape = (xend - xstart, )
data_array = np.ma.empty(shape, dtype=np.dtype(storage_dtype))
newline = 0
for line in range(ystart, yend, ystep):
start = xstart + line * self.shape[0]
linedata = self.read_slice(fileobj, start, lineshape,
storage_dtype)
data_array[newline, :] = linedata
newline += 1
fileobj.close()
datavars['TOPO']['topo'] = data_array
#Create lons and lats
pixel_size = 30 / 3600.0 #30 arc seconds
lonline = np.ma.arange(*inds[0]) * pixel_size
gvars['TOPO']['Longitude'] = np.ma.vstack(
[lonline for num in range(*inds[1])])
latsamp = np.ma.arange(*inds[1]) * pixel_size
gvars['TOPO']['Latitude'] = np.ma.hstack(
[latsamp[np.newaxis].T for num in range(*inds[0])])
#Convert lons to -180 to 180
western = gvars['TOPO']['Longitude'] > 180
gvars['TOPO']['Longitude'][
western] = gvars['TOPO']['Longitude'][western] - 360
#Convert lats to -90 to 90
southern = gvars['TOPO']['Latitude'] > 90
northern = gvars['TOPO']['Latitude'] <= 90
gvars['TOPO']['Latitude'][southern] = -(
gvars['TOPO']['Latitude'][southern] - 90)
gvars['TOPO']['Latitude'][northern] = np.abs(
gvars['TOPO']['Latitude'][northern] - 90)
# Grab necessary metadata that will populate the _finfo,
# _dsinfo, and _varinfo properties on the SciFile object.
# These get passed along when initializing the Variable
# instance, then get propagated up to the dsinfo and finfo
# levels.
# The available fields for varinfo can be found in
# scifile/containers.py at the beginning of the file.
try:
sdfn = DataFileName(os.path.basename(fname)).create_standard()
metadata['top']['start_datetime'] = sdfn.datetime
metadata['top']['dataprovider'] = sdfn.dataprovider
except:
# Set an arbitrary time on the data... Not used for anything anyway...
metadata['top']['start_datetime'] = datetime.strptime(
'20150410.000000', '%Y%m%d.%H%M%S')
metadata['top']['dataprovider'] = None
metadata['top']['end_datetime'] = metadata['top']['start_datetime']
#metadata['TOPO']['platform_name'] = sdfn.satname
metadata['top']['platform_name'] = None
# metadata['TOPO']['filename_datetime'] = varinfo['start_datetime']
metadata['top']['source_name'] = None
return
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
# Use filename field for filename_datetime if it is available.
dfn = DataFileName(os.path.basename(glob(os.path.join(fname, '*'))[0]))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
metadata['top']['start_datetime'] = sdfn.datetime
metadata['top']['end_datetime'] = sdfn.datetime
metadata['top']['dataprovider'] = 'nesdisstar'
metadata['top']['platform_name'] = sdfn.satname
metadata['top']['source_name'] = 'seviri'
# MUST be set on readers that sector at read time.
# Affects how reading/processing is done in driver.py
metadata['top']['sector_definition'] = sector_definition
metadata['top']['SECTOR_ON_READ'] = True
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
# chans == [] specifies we don't want to read ANY data, just metadata.
# chans == None specifies that we are not specifying a channel list,
# and thus want ALL channels.
if chans == []:
# If NO CHANNELS were specifically requested, just return at this
# point with the metadata fields populated. A dummy SciFile dataset
# will be created with only metadata. This is for checking what
# platform/source combination we are using, etc.
return
outdir = os.path.join(gpaths['LOCALSCRATCH'],
os.path.dirname(sdfn.name))
self.decompress_msg(fname, outdir, chans)
try:
global_data = Scene(platform_name="Meteosat-8",
sensor="seviri",
reader="hrit_msg",
start_time=sdfn.datetime,
base_dir=outdir)
except TypeError:
global_data = Scene(
filenames=glob(os.path.join(outdir, '*')),
reader="hrit_msg",
filter_parameters={'start_time': sdfn.datetime})
metadata['top']['start_datetime'] = global_data.start_time
metadata['top']['end_datetime'] = global_data.end_time
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
# Loop through the variables found in the current dataset
# The dataset_info dictionary maps the geoips varname to the
# varname found in the original datafile
for geoipsvarname, spvarname in self.dataset_info[dsname].items():
# If we requested specific channels, and the current channel
# is not in the list, skip this variable.
if chans and geoipsvarname not in chans:
continue
# Read the current channel data into the datavars dictionary
log.info(' Initializing ' + dsname + ' channel "' +
spvarname + '" from file into SciFile channel: "' +
geoipsvarname + '"...')
global_data.load([spvarname])
# Read spvarname from the original datafile into datavars[dsname][geoipsvarname]
ad = sector_definition.area_definition
log.info(' Sectoring data to ' + ad.name + ' ...')
sectored_data = global_data.resample(ad)
for spvarname in sectored_data.datasets.keys():
for dsname in self.dataset_info.keys():
for geoipsvarname in self.dataset_info[dsname].keys():
if self.dataset_info[dsname][
geoipsvarname] == spvarname.name:
if 'Longitude' not in gvars[dsname].keys():
log.info(' Saving Longitude to gvars')
gvars[dsname]['Longitude'] = np.ma.array(
ad.get_lonlats()[0])
if 'Latitude' not in gvars[dsname].keys():
log.info(' Saving Latitude to gvars')
gvars[dsname]['Latitude'] = np.ma.array(
ad.get_lonlats()[1])
if 'SunZenith' not in gvars[dsname].keys():
from geoips.scifile.solar_angle_calc import satnav
log.info(
' Using satnav, can only calculate Sun Zenith angles'
)
gvars[dsname]['SunZenith'] = satnav(
'SunZenith', metadata['top']['start_datetime'],
gvars[dsname]['Longitude'],
gvars[dsname]['Latitude'])
self.set_variable_metadata(metadata, dsname,
geoipsvarname)
try:
datavars[dsname][geoipsvarname] =\
np.ma.array(sectored_data.datasets[spvarname.name].data,
mask=sectored_data.datasets[spvarname.name].mask)
log.warning('Sectored variable %s ' %
(spvarname.name))
except AttributeError:
log.warning(
'Variable %s does not contain a mask, masking invalid values! Might take longer'
% (spvarname.name))
datavars[dsname][geoipsvarname] =\
np.ma.masked_invalid(sectored_data.datasets[spvarname.name].data)
def read(self,fname,datavars,gvars,metadata,chans=None,sector_definition=None):
# Use the appropriate python based reader for opening the current data
# type. This package must be installed on the system, and imported above
# in the "Installed Libraries" import section.
df = ncdf.Dataset(str(fname), 'r')
# Uncomment this shell statement in order to interrogate the data file
# and determine what attributes and fields need to be read from the
# data file.
#print 'Entering IPython shell in '+self.name+' for development purposes'
#shell()
# Grab necessary metadata that will populate the _finfo,
# _dsinfo, and _varinfo properties on the SciFile object.
# The available fields for varinfo can be found in
# scifile/containers.py at the beginning of the file.
# Additional options available with the metadata dictionary
# can be found in the comments with the dataset_info property above.
metadata['top']['start_datetime'] = datetime.strptime(df.time_coverage_start.split('.')[0],'%Y-%m-%dT%H:%M:%S')
# Note an apparent bug in productfilename uses end_datetime as filename.
# For now just leave out end_datetime (it automatically gets set to start_datetime
# in scifile if undefined)
# Ooops, this might have messed up pass predictor
metadata['top']['end_datetime'] = datetime.strptime(df.time_coverage_end.split('.')[0],'%Y-%m-%dT%H:%M:%S')
metadata['top']['dataprovider'] = 'unknown'
# DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of
# Commerce
if 'DOC/NOAA/NESDIS/OSPO' in df.institution:
metadata['top']['dataprovider'] = 'noaa-nesdis-ospo'
elif 'NOAA' in df.institution and 'NESDIS' in df.institution:
metadata['top']['dataprovider'] = 'noaanesdis'
elif 'NOAA' in df.institution:
metadata['top']['dataprovider'] = 'noaa'
metadata['top']['filename_datetime'] = metadata['top']['start_datetime']
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
# platform_name and source_name MUST match values found
# in SensorInfo_classes and SatInfo_classes in utils/satellite_info.py.
# Those are the keys used throughout GeoIPS for determining what data
# type we are working with. If opening the SatSensorInfo object fails,
# raise an Error and stop operation.
# source_name = 'amsr2'
# platform_name = 'gcom-w1'
metadata['top']['platform_name'] = df.platform_name.lower()
metadata['top']['source_name'] = df.instrument_name.lower()
si = SatSensorInfo(metadata['top']['platform_name'],metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError('Unrecognized platform and source name combination: '+metadata['top']['platform_name']+' '+metadata['top']['source_name'])
# Use filename field for filename_datetime if it is available.
# Else, just use the start_datetime we found from the data
# above. Note we ALWAYS want to have a default if DataFileName
# is not defined. We do not want to rely on having our specific
# internal filename format in order to process, but if we have
# additional information available from the data filename, we
# can use it.
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# chans == [] specifies we don't want to read ANY data, just metadata.
# chans == None specifies that we are not specifying a channel list,
# and thus want ALL channels.
if chans == []:
# If NO CHANNELS were specifically requested, just return at this
# point with the metadata fields populated. A dummy SciFile dataset
# will be created with only metadata. This is for checking what
# platform/source combination we are using, etc.
return
# Set up the dictionaries of variables that will go in each dataset.
# datavars: actual channel data
# gvars: geolocation variable data
# specifically named:
# Latitude (REQUIRED),
# Longitude (REQUIRED), and
# SunZenith (optional, required for day/night
# discrimination)
# Each data variable array and geolocation variable array of a
# specific dataset_name MUST be the same shape
# datavars[dataset_name][geoips_varname] = geoips_varname_channel_data
# gvars[dataset_name]['Latitude'] = dataset_name_lat_numpy_array
# gvars[dataset_name]['Longitude'] = dataset_name_lon_numpy_array
# *OPTIONAL* gvars[dataset_name]['SunZenith'] = dataset_name_sunzenith_numpy_array
# geoips_varname_channel_data.shape == dataset_name_lat_numpy_array.shape
# == dataset_name_lon_array.shape == dataset_name_sunzenith_numpy_array.shape
# Only datavars and gvars with the same shape can go in the same dataset.
# See additional datavars and gvars dictionary structure information
# found in the comments above, with the dataset_info property of this reader.
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
# Loop through the variables found in the current dataset
# The dataset_info dictionary maps the geoips varname to the
# varname found in the original datafile
for geoipsvarname,ncvarname in self.dataset_info[dsname].items():
# If we requested specific channels, and the current channel
# is not in the list, skip this variable.
if chans and geoipsvarname not in chans:
continue
# Read the current channel data into the datavars dictionary
log.info(' Reading '+dsname+' channel "'+ncvarname+'" from file into SciFile channel: "'+geoipsvarname+'"...')
# Read ncvarname from the original datafile into datavars[dsname][geoipsvarname]
datavars[dsname][geoipsvarname] = np.ma.masked_equal(df.variables[ncvarname][...],df.variables[ncvarname]._FillValue)
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
# Loop through the variables found in the current dataset
# The gvar_info dictionary maps the geoips varname to the
# varname found in the original datafile
for geoipsvarname,ncvarname in self.gvar_info[dsname].items():
# Read the current channel data into the datavars dictionary
log.info(' Reading '+dsname+' channel "'+ncvarname+'" from file into SciFile channel: "'+geoipsvarname+'"...')
# Read ncvarname from the original datafile into datavars[dsname][geoipsvarname]
gvars[dsname][geoipsvarname] = np.ma.masked_equal(df.variables[ncvarname][...],df.variables[ncvarname]._FillValue)
def read(self,
fname,
datavars,
gvars,
metadata,
chans=None,
sector_definition=None):
# Since the variables are all contained within separate files, we need to allow for
# a loop to cycle through the files
df = pg.open(fname)
temp = None
for vars in MODEL_GRIB_Reader.dataset_info.values():
for var in vars.values():
try:
temp = df.select(name=var)[0] #grabs the first file
except:
if var == '238 (instant)' or var == '133 (instant)' or var == '222 (instant)' or var == '218 (instant)' or var == '221 (instant)':
temptemp = df.select()
if var in str(temptemp):
temp = temptemp[0]
continue
#temp = df.select(name = 'Temperature')[0]
#print 'Entering IPython shell in '+self.name+' for development purposes'
#shell()
dt = '{:08d}{:04d}'.format(temp.validityDate, temp.validityTime)
print dt
if not temp:
log.warning('Unable to read from file ' + fname +
' no matching select for vars')
return
if temp.validityDate:
metadata['top']['start_datetime'] = datetime.strptime(
dt, '%Y%m%d%H%M')
metadata['top']['end_datetime'] = datetime.strptime(
dt, '%Y%m%d%H%M')
else:
metadata['top']['start_datetime'] = temp.analDate
metadata['top']['end_datetime'] = temp.analDate
metadata['top']['dataprovider'] = temp.centre
metadata['top']['filename_datetime'] = metadata['top'][
'start_datetime']
metadata['top']['platform_name'] = 'model'
#metadata['top']['source_name'] = 'model'
metadata['top']['tau'] = temp.startStep
metadata['top']['level'] = temp.level
if 'COAMPS' in fname:
metadata['top']['source_name'] = 'coamps'
else:
metadata['top']['source_name'] = 'navgem'
si = SatSensorInfo(metadata['top']['platform_name'],
metadata['top']['source_name'])
if not si:
from ..scifileexceptions import SciFileError
raise SciFileError(
'Unrecognized platform and source name combination: ' +
metadata['top']['platform_name'] + ' ' +
metadata['top']['source_name'])
dfn = DataFileName(os.path.basename(fname))
if dfn:
sdfn = dfn.create_standard()
metadata['top']['filename_datetime'] = sdfn.datetime
# Tells driver to NOT try to sector this data.
metadata['top']['NON_SECTORABLE'] = True
if chans == []:
return
new = None
# Loop through each dataset name found in the dataset_info property above.
for dsname in self.dataset_info.keys():
for geoipsvarname, dfvarname in self.dataset_info[dsname].items():
try:
new = df.select(
name=dfvarname) #,typeOfLevel='isobaricInhPa')
except:
continue
for newest in new:
newest
data = newest.values
fillvalue = newest.missingValue
level = newest.level
#shell()
if newest.units == str('m s**-1'):
data = data * 1.94384
log.info(' Reading ' + dsname + ' channel "' +
dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + str(level) + '" ...')
datavars[dsname][geoipsvarname +
str(level)] = np.ma.masked_equal(
data, fillvalue)
shape = datavars[dsname][geoipsvarname + str(level)].shape
if not new:
log.warning('Unable to read from file ' + fname +
' no matching selects')
return
# Loop through each dataset name found in the gvar_info property above.
for dsname in self.gvar_info.keys():
for geoipsvarname, dfvarname in self.gvar_info[dsname].items():
log.info(' Reading ' + dsname + ' channel "' + dfvarname +
'" from file into SciFile channel: "' +
geoipsvarname + '"...')
for newest in new:
newest
data = newest[dfvarname]
fillvalue = newest.missingValue
#shell()
if data.size == newest.getNumberOfValues:
data = np.reshape(data, shape)
#if data.shape == (1038240,):
# data= np.reshape(data,(721,1440))
#elif data.shape == (259920,):
# data= np.reshape(data,(361,720))
#elif data.shape == (76454,):
# data= np.reshape(data,(254, 301))
gvars[dsname][geoipsvarname] = np.ma.masked_equal(
data, fillvalue)
