def read_001002(WAVstr, filename):
import ncepbufr
import numpy as np
from datetime import datetime
dtstr = 'YEAR MNTH DAYS HOUR MINU'
lcstr = 'CLONH CLATH'
bufr = ncepbufr.open(filename)
#bufr.dump_table('prepbufr.table')
cnt = 0
datestmp = [] #[YYYY MM DD HH]
loc = [] #[lon lat]
wav = []
while bufr.advance() == 0: # loop over messages.
# print(bufr.msg_counter, bufr.msg_type, bufr.msg_date, bufr.receipt_time)
while bufr.load_subset() == 0: # loop over subsets in message.
#bufr.dump_subset('xx102.dumped',append=True)
wavdum = bufr.read_subset(WAVstr).squeeze()
#wavdum = bufr.read_subset(SSTstr, seq=True).squeeze()-273.15
if np.isfinite(wavdum).any():
wav = np.append(wav, wavdum)
datestmp = np.append(datestmp,
bufr.read_subset(dtstr).squeeze(),
axis=0)
loc = np.append(loc, bufr.read_subset(lcstr).squeeze())
cnt += 1
print(cnt, ' Observations are imported', 'np.shape(wav) = ',
np.shape(wav), wavdum)
#if bufr.msg_counter == 200 : break
bufr.close()
loc = np.reshape(loc, (cnt, int(np.array(np.shape(loc)) / cnt)))
wav = np.reshape(wav, (cnt, int(np.array(np.shape(wav)) / cnt)))
datestmp = np.reshape(datestmp,
(cnt, int(np.array(np.shape(datestmp)) / cnt)))
d = dict()
d['longitude'] = loc[:, 0]
d['latitude'] = loc[:, 1]
cnt = 0
for i in WAVstr.split(' '):
d[i] = wav[:, cnt]
cnt += 1
d['date'] = []
for i in range(len(datestmp)):
d['date'].append(
datetime(year=int(datestmp[i, 0]),
month=int(datestmp[i, 1]),
day=int(datestmp[i, 2]),
hour=int(datestmp[i, 3]),
minute=int(datestmp[i, 4])))
return d
def read_ncep_bufr(fname, convert_uv, offset):
"""
Reads NCEP BUFR type files,
which include the local table as the first message.
Extracts and converts height and wind speed/direction.
"""
data_exists = False
empty_array = (np.zeros((3, )), np.zeros((3, )), np.zeros((3, )))
offset_seconds = 3600. * offset
if '2017' in fname:
return empty_array
try:
bufr = ncepbufr.open(fname)
while bufr.advance() == 0:
while bufr.load_subset() == 0:
elev = bufr.read_subset('GELV')[0][0]
ftim = bufr.read_subset('FTIM')[0][0]
if ftim == offset_seconds:
data_exists = True
pressure_pa = bufr.read_subset('PRES')[0]
u_wind = bufr.read_subset('UWND')[0]
v_wind = bufr.read_subset('VWND')[0]
temperature = bufr.read_subset('TMDB')[0]
specific_humidity = bufr.read_subset('SPFH')[0]
if not data_exists:
pressure_pa = np.zeros((50, ))
u_wind = np.zeros((50, ))
v_wind = np.zeros((50, ))
temperature = np.zeros((50, ))
specific_humidity = np.zeros((50, ))
elev = 0.
bufr.close()
pressure_hpa = pressure_pa / 100.
height_standard = np.rint(
profiler_metr.pressure_to_height(pressure_hpa, elev))
height_hyps = profiler_metr.hypsometric(specific_humidity, temperature,
pressure_hpa, elev)
if convert_uv:
direction = np.around(profiler_metr.wind_direction(u_wind, v_wind),
decimals=1)
speed = np.around(profiler_metr.wind_speed(u_wind, v_wind),
decimals=2)
return zip(height_hyps, speed, direction)
else:
return (height_hyps, u_wind, v_wind)
except IOError as err:
print 'IO error reading NCEP BUFR file %s, skipping for now' % err
APP.logger.exception(err)
return empty_array
except IndexError as err:
print 'Index error while reading NCEP BUFR file %s, skipping for now' % err
reload(ncepbufr)
APP.logger.exception(err)
return empty_array
except Exception as err:
print 'Unknown error: %s, skipping' % err
reload(ncepbufr)
APP.logger.exception(err)
return empty_array
def read_bufrsfc(fileaname):
''' Read specific SFC observations from one PrepBufr file
filter out missing values
'''
exp_reports = [
'ADPSFC',
]
exp_mnemonic = ['SID', 'XOB', 'YOB', 'DHR']
exp_tdmnem = ['TDO']
exp_tcmnem = ['TOB']
exp_uvmnem = ['UOB', 'VOB']
tcobs = []
tdobs = []
uvobs = []
outfile = 'rap.2019052018.adpsfc.txt'
outtable = 'rap.table'
bufr = ncepbufr.open(filename)
#bufr.advance()
#bufr.load_subset()
#bufr.dump_subset(outfile)
#temp = bufr.read_subset('SSTH').squeeze()-273.15 # convert from Kelvin to Celsius
#bufr.dump_table(outtable)
#bufr.print_table()
#msgs=bufr.inventory()
while bufr.advance() == 0:
#print(bufr.msg_counter,bufr.msg_type,bufr.msg_date, bufr.receipt_time,bufr.subsets)
if bufr.msg_type in exp_reports:
while bufr.load_subset() == 0:
tco = bufr.read_subset('TOB')
tdo = bufr.read_subset('TDO')
#tdo = bufr.read_subset('QOB')
uvo = bufr.read_subset('UOB VOB')
hdr = bufr.read_subset('SID XOB YOB DHR').squeeze()
if not np.ma.is_masked(tco):
tcobs.append((struct.pack('d', hdr[0]), hdr[1], hdr[2],
hdr[3], tco[0][0]))
if not np.ma.is_masked(tdo):
tdobs.append((struct.pack('d', hdr[0]), hdr[1], hdr[2],
hdr[3], tdo[0][0]))
if not np.ma.is_masked(uvo):
uvobs.append((struct.pack('d', hdr[0]), hdr[1], hdr[2],
hdr[3], uvo[0][0], uvo[1][0]))
#print(f"hdr={hdr},tc={tco},td={tdo},uob={uob},vob={vob}")
bufr.close()
obs = {'TOB': tcobs, 'TDO': tdobs, 'UVO': uvobs}
return obs
def BfilePreprocess(BufrFname, Obs):
# This routine will read the BUFR file and figure out how many observations
# will be read when recording data.
bufr = ncepbufr.open(BufrFname)
# The number of observations will be equal to the total number of subsets
# contained in the selected messages.
NumObs = 0
Obs.start_msg_selector()
while (Obs.select_next_msg(bufr)):
NumObs += Obs.msg_obs_count(bufr)
bufr.close()
return [ NumObs, Obs.num_msg_selected, Obs.num_msg_mtype ]
from __future__ import print_function
import ncepbufr
import numpy as np
# run test.py first to create prepbufr.table file.
hdstr = 'SID XOB YOB DHR TYP ELV SAID T29'
obstr = 'POB QOB TOB ZOB UOB VOB PWO MXGS HOVI CAT PRSS TDO PMO'
qcstr = 'PQM QQM TQM ZQM WQM NUL PWQ PMQ'
oestr = 'POE QOE TOE NUL WOE NUL PWE'
# open prepbufr file.
bufr = ncepbufr.open('data/prepbufr2', 'w', table='prepbufr.table')
idate = 2010050700 # cycle time: YYYYMMDDHH
subset = 'ADPSFC' # surface land (SYNOPTIC, METAR) reports
bufr.open_message(subset, idate)
hdr = bufr.missing_value * np.ones(len(hdstr.split()), np.float)
hdr[0] = np.fromstring('KTKI ', dtype=np.float)[0]
hdr[1] = 263.4
hdr[2] = 33.2
hdr[3] = -0.1
hdr[4] = 287
hdr[5] = 179
# encode header for wind obs
bufr.write_subset(hdr, hdstr)
# set obs, qcf, oer for wind
obs = bufr.missing_value * np.ones(len(obstr.split()), np.float)
oer = bufr.missing_value * np.ones(len(oestr.split()), np.float)
qcf = bufr.missing_value * np.ones(len(qcstr.split()), np.float)
from __future__ import print_function
import ncepbufr
import numpy as np
hdstr='SID XOB YOB DHR TYP ELV SAID T29'
obstr='POB QOB TOB ZOB UOB VOB PWO MXGS HOVI CAT PRSS TDO PMO'
qcstr='PQM QQM TQM ZQM WQM NUL PWQ PMQ'
oestr='POE QOE TOE NUL WOE NUL PWE'
# open prepbufr file.
bufr = ncepbufr.open('prepbufr2','w',table='prepbufr.table')
idate=2010050700 # cycle time: YYYYMMDDHH
subset='ADPSFC' # surface land (SYNOPTIC, METAR) reports
bufr.open_message(subset, idate)
hdr = bufr.missing_value*np.ones(len(hdstr.split()),np.float)
hdr[0] = np.fromstring('KTKI ',dtype=np.float)[0]
hdr[1]=263.4; hdr[2]=33.2; hdr[3] = -0.1; hdr[4]=287; hdr[5]=179
# encode header for wind obs
bufr.write_subset(hdr,hdstr)
# set obs, qcf, oer for wind
obs = bufr.missing_value*np.ones(len(obstr.split()),np.float)
oer = bufr.missing_value*np.ones(len(oestr.split()),np.float)
qcf = bufr.missing_value*np.ones(len(qcstr.split()),np.float)
obs[0]=985.2; obs[4]=-2.8; obs[5]=-7.7; obs[7]=6.0
qcf[0]=2.0 ; qcf[4]=2.0; oer[4] = 1.6
# encode wind obs
bufr.write_subset(obs,obstr)
# encode wind ob err
bufr.write_subset(oer,oestr)
from __future__ import print_function
import ncepbufr
hdrstr = 'YEAR MNTH DAYS HOUR MINU PCCF ELRC SAID PTID GEODU'
# read gpsro file.
bufr = ncepbufr.open('data/gpsbufr')
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr = bufr.read_subset(hdrstr).squeeze()
yyyymmddhh = '%04i%02i%02i%02i%02i' % tuple(hdr[0:5])
satid = int(hdr[7])
ptid = int(hdr[8])
nreps_this_ROSEQ2 = bufr.read_subset('{ROSEQ2}').squeeze()
nreps_this_ROSEQ1 = len(nreps_this_ROSEQ2)
data1b = bufr.read_subset('ROSEQ1', seq=True) # bending angle
data2a = bufr.read_subset('ROSEQ3', seq=True) # refractivity
levs_bend = data1b.shape[1]
levs_ref = data2a.shape[1]
if levs_ref != levs_bend:
print('skip report due to bending angle/refractivity mismatch')
continue
print('sat id,platform transitter id, levels, yyyymmddhhmm =',\
satid,ptid,levs_ref,yyyymmddhh)
print('k, height, lat, lon, ref, bend:')
for k in range(levs_ref):
rlat = data1b[0, k]
rlon = data1b[1, k]
def main():
tic = time.clock()
# Mnemonics for getting data from the bufr file.
hdstr = 'SSTN CLON CLAT SELV ANEL YEAR MNTH DAYS HOUR MINU QCRW ANAZ' #PRFR' # MGPT'
obstr = 'DIST125M DMVR DVSW' #NL2RW--level 2 radial wind.
#obstr2='STDM SUPLON SUPLAT HEIT RWND RWAZ RSTD' #RWSOB--radial wind super ob.
#1. INITIALIZING SOME BASIC LISTS AND GETTING INPUT DATA.
i = 0
sids = []
lons = []
lats = []
l2rw = []
anel = []
anaz = []
dist125m = []
ymdhm = []
radii = []
PRF = []
OBS_FILE = sys.argv[1]
message_type = sys.argv[2]
date = sys.argv[3]
STAID = sys.argv[4]
OBS_FILECPY = OBS_FILE + "_cpy"
anel0 = float(sys.argv[5])
del_anel = 0.25
maxanaz = 0
#2. READ PREPBUFR FILE.
b = 'false' # used for breaking the loop.
bufr = ncepbufr.open(OBS_FILE) # bufr file for reading.
bufrcpy = ncepbufr.open(
OBS_FILECPY) # bufr file for reading ahead and determining maxanaz.
bufr.dump_table('l2rwbufr.table') # dump table to file.
while bufr.advance() == 0: # loop over messages.
#bufr has been advanced, now if reading concurrently, bufrcpy should be behind by one. But sometimes
#bufrcpy could be ahead of where we need it (but only by one step) since we read it forward.
#We don't want to advance it in that case.
if ((bufr.msg_counter - bufrcpy.msg_counter) == 1): bufrcpy.advance()
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
if (bufr.msg_type == message_type):
while bufr.load_subset() == 0: # loop over subsets in message.
hdr = bufr.read_subset(
hdstr).squeeze() # parse hdstr='SSTN CLON ... etc.'
station_id = hdr[0].tostring() # convert SSTN to string.
station_id = station_id.strip(
) # remove white space from SSTN.
#*********************
bufrcpy.load_subset() #**********************
hdrcpy = bufrcpy.read_subset(hdstr).squeeze() #
#*******************************************
if (station_id == STAID
): # comes from input. used for picking a single SSTN.
if (hdr[4] >= anel0 - del_anel and hdr[4] <=
anel0 + del_anel): # read an elevation angle.
obs = bufr.read_subset(obstr).squeeze(
) # parse obstr='DIST125M DMVR DVSW'
#*******************************************
obscpy = bufrcpy.read_subset(obstr).squeeze() #
#*******************************************
print(bufr.msg_counter, 'SSTN,CLON,CLAT,ANAL,ANAZ : ',
station_id, hdr[1], hdr[2], hdr[4], hdr[11])
i = i + 1 #number of observations counter.
sids.append(station_id) # station ids
l2rw.append(obs[1]) # level 2 radial winds
anel.append(hdr[4]) # elevation angles
anaz.append(hdr[11]) # azimuthal angles
radii.append(obs[0] * 125) #distances in units of 1 m
ymdhm.append(int(hdr[9]))
#It is not sufficient to check for len(anaz) == 360. Sometimes, it is missing.
#***********************************************************************************************************
if (len(anaz) == 355): #pause to read ahead.
while bufrcpy.load_subset(
) == 0: # read ahead with bufrcpy
hdrcpy = bufrcpy.read_subset(hdstr).squeeze()
maxanaz = max(maxanaz, int(np.ceil(
hdrcpy[11]))) #need hdrcpy[11] rounded up.
tries = 0
while tries < 10 and bufrcpy.advance(
) == 0: # look no. tries messages ahead.
tries += 1
while bufrcpy.load_subset() == 0:
if (bufrcpy.msg_type == message_type):
station_id = hdrcpy[0].tostring(
) # convert SSTN to string.
station_id = station_id.strip(
) # remove white space from SSTN.
if (station_id == STAID):
if (hdrcpy[4] >= anel0 - del_anel
and hdrcpy[4] <=
anel0 + del_anel):
hdrcpy = bufrcpy.read_subset(
hdstr).squeeze()
maxanaz = max(
maxanaz,
int(np.ceil(hdrcpy[11])))
print(bufrcpy.msg_counter,
'cpySSTN,DATE/TIME : ',
station_id, hdrcpy[5:10],
hdrcpy[4], hdrcpy[11],
len(anaz))
print(maxanaz)
if (len(anaz) >= 355): print(len(anaz), maxanaz)
#***********************************************************************************************************
if (len(anaz) == maxanaz and len(anaz) > 355):
while (len(anaz) <
360): #fill in some missing data points
l2rw.append(obs[1] * 0 + -999.)
anel.append(anel[-1])
anaz.append(anaz[-1] + 1)
radii.append(radii[-1])
ymdhm.append(ymdhm[-1])
while (
len(anaz) < 361
): # fill in the final beam with nan mean between 1st and 359th
l2rw.append(l2rw[0])
anel.append(anel[0])
anaz.append(anaz[-1] + 1)
radii.append(radii[0])
ymdhm.append(ymdhm[0])
b = 'true'
if (b == 'true'): break # stop reading after all anaz's read.
bufr.close()
print("number of obs: ", i)
MM = str(ymdhm[-1])
print(np.min(anel), np.max(anel))
print(np.min(ymdhm), np.max(ymdhm))
toc = time.clock() # check how long reading the data in took.
sec = str(toc - tic)
print('time it took to run: ' + sec + ' seconds.')
#3. FIND THE MAX/MIN VALUES OF THE DIST125M AND THETA ARRAYS.
maxRadii = 100000 # set max of max distances to about 100 mi.
minRadii = 0 # min of min distances.
#4. INITIALIZE THE POLAR PLOT AS ALL MISSING DATA (-999).
fig = plt.figure(figsize=(8, 8)) # 8" x 8" seems plenty large.
ax = fig.add_subplot(111)
# ax = fig.add_subplot(111,polar=True) # we would like it to be a polar plot.
# ax.set_theta_zero_location("N") # set theta zero location to point North.
# ax.set_theta_direction(-1) # set theta to increase clockwise (-1).
theta, r = np.meshgrid(anaz, np.arange(minRadii, maxRadii,
250.)) # create meshgrid
theta = deg2rad(theta) # convert theta from degrees to radians.
rw = np.zeros(shape=(len(theta),
len(r[0]))) # initialize the polar plot with all 0's.
rw.fill(-999) # change all values to missing data (-999).
#PRF_new=np.zeros(shape=(len(theta),len(r[0])))
#PRF_new.fill(-999)
r = r.T
rw = rw.T #; PRF_new.T # transpose r and rw for later manipulation.
#5. POPULATE THE EMPTY RW ARRAY WITH ACTUAL VALUES.
for i in range(len(anaz)): # for every azimuth angle ...
sys.stdout.write('\r' + str(i) + '/' + str(len(anaz)))
sys.stdout.flush()
for j in range(
len(radii[i])
): # ... loop over every observation distance from radar ...
for k in range(
len(r[i])
): # ... and loop over an equally 125m spaced array ...
if (radii[i][j] == r[i][k]
): # ... if the observation dist = dist125m ...
rw[i][k] = l2rw[i][
j] # ... assign the value of the observation to that index.
#PRF_new[i][k]=PRF[i][j]
break # speeds things up by about 50-60%.
rw[np.isnan(
rw)] = -999 # the masked values are converted to nan. set nan to -999.
print(np.max(rw)) # check that it is not nan.
ax.hist(rw[rw != -999.], bins=41)
plt.title('Doppler Velocity \n Station ID: '+STAID\
+' Scan Angle: '+str(anel[0])\
+' Date: '+date+MM.zfill(2),fontsize=15,y=1.12) # add a useful title.
plt.xlabel('velocity (m/s)')
plt.ylabel('number of obs')
ax.grid(True)
plt.show() # make the plot.
plt.savefig('./'+STAID+'_'+str(anel[0])+'_'+date+MM.zfill(2)+'_hist.png'\
,bbox_inches='tight') # save figure.
#7. CALCULATE SOME STATS
toc = time.clock() # check to see how long it took to run.
sec = str(toc - tic)
print('time it took to run: ' + sec + ' seconds.')
def read_snd(filename, outtable=None):
bufr = ncepbufr.open(filename)
#bufr.advance()
#bufr.load_subset()
#bufr.dump_subset(outfile)
#temp = bufr.read_subset('SSTH').squeeze()-273.15 # convert from Kelvin to Celsius
if outtable is not None: bufr.dump_table(outtable)
#bufr.print_table()
#msgs=bufr.inventory()
stations = {}
while bufr.advance() == 0:
station = {}
if bufr.msg_type == "ADPUPA":
#print(bufr.msg_counter,bufr.msg_type,bufr.msg_date, bufr.receipt_time,bufr.subsets)
msgtime = datetime.strptime(str(bufr.msg_date), '%Y%m%d%H')
while bufr.load_subset() == 0:
sco = bufr.read_subset('SID').squeeze()
typ = bufr.read_subset('TYP').squeeze()
xob = bufr.read_subset('XOB').squeeze()
yob = bufr.read_subset('YOB').squeeze()
dhr = bufr.read_subset('DHR').squeeze()
tob = bufr.read_subset('TOB').squeeze()
tdo = bufr.read_subset('TDO').squeeze()
pob = bufr.read_subset('POB').squeeze()
uob = bufr.read_subset('UOB').squeeze()
vob = bufr.read_subset('VOB').squeeze()
#ddo = bufr.read_subset('DDO').squeeze()
#sso = bufr.read_subset('SOB').squeeze()
sid = struct.pack('d', sco).decode("utf-8")
obt = msgtime + timedelta(hours=dhr.compressed()[0])
print(sid, typ, bufr.msg_date, dhr.compressed())
if sid not in stations.keys():
stations[sid] = {
'lat': yob,
'lon': xob if xob < 180 else xob - 360.,
'time': obt
}
if typ < 200: # 100-199 for mass observations
#stations[sid]['ps'] = pso.tolist(-9999.) * units.hPa
#stations[sid]['T'] = tco.tolist(-9999.) * units.degC
#stations[sid]['Td'] = tdo.tolist(-9999.) * units.degC
mask = np.any([tob.mask, tdo.mask], axis=0)
pob.mask = mask
tob.mask = mask
tdo.mask = mask
pso = pob.compressed() # remove missing tob or tdo levels
tds = tdo.compressed()
tso = tob.compressed()
stations[sid]['ps'] = pso * units.hPa
stations[sid]['T'] = tso * units.degC
stations[sid]['Td'] = tds * units.degC
else: # 200-299 for wind observations
mask = np.any([uob.mask, vob.mask], axis=0)
pob.mask = mask
uob.mask = mask
vob.mask = mask
pso = pob.compressed() # remove missing uob or vob levels
uso = uob.compressed()
vso = vob.compressed()
stations[sid]['pv'] = pso * units.hPa
stations[sid]['u'] = uso * units.meter / units.second
stations[sid]['v'] = vso * units.meter / units.second
#stations[sid]['dir'] = ddo.tolist(-9999.) * units.degree
#stations[sid]['spd'] = sso.tolist(-9999.) * units.meter/units.second
# stations.append(station)
#if np.ma.is_masked(tco[0]): continue
#else: break
#print(tco)
#print(tdo)
#print(pso)
#print(uvo)
#sys.exit(0)
#bufr.dump_subset(outfile,append=True)
bufr.close()
return stations
def __init__(self, log, version_str, h5_fname, record):
self.log = log
self.h5_fname = h5_fname
# Determine the bufr file time range that would include this record
# Use it to create the file name
# Go through the upper limits of the time ranges in seconds
sec_of_day = record.sec + record.min * 60 + record.hour * 3600
for test_sec in self.SEARCH_HOURS:
if sec_of_day <= test_sec:
break
if test_sec == self.HOUR24:
end_hour = 3
else:
end_hour = test_sec // 3600
self.dt_end = datetime.datetime(record.year, record.month, record.day,
end_hour, 0, 0)
if test_sec == self.HOUR24:
# end_hour is 3 on the next day
self.dt_end = self.dt_end + datetime.timedelta(days=1)
self.dt_start = self.dt_end - datetime.timedelta(hours=6)
# Create a time stamp for the end of the gfs file
self.dt_gfs_end = self.dt_end - datetime.timedelta(minutes=45)
dt_center = self.dt_end - datetime.timedelta(hours=3)
# Use the dt_center fields to create the gfs bufr file name
# The gfs file will be written first, so the next few object variables
# are set accordingly. They will all be changed when it is time to
# write the gdas file.
self.bufr_file = (
'gfs.t{:02d}z.tempd.{:d}{:02d}{:02d}{}.bufr_d'.format(
dt_center.hour, dt_center.year, dt_center.month, dt_center.day,
version_str))
# Determine the relative lower level directory path this file should
# end up in.
self.bufr_subdirs = 'gfs.{:d}{:02d}{:02d}/{:02d}'.format(
dt_center.year, dt_center.month, dt_center.day, dt_center.hour)
# But, initially leave out the subdirs and give the file name a
# .<hdf5_file_name>.part extension. This will be the file's path while
# it is being written. The <hdf5_file_name> is there to keep the file
# name from colliding with any bufr file of the same name that might be
# created by a different converter
self.bufr_path_part = os.path.join(
os.environ['BUFR_OUTPUT_DIR'],
self.bufr_file + '.' + self.h5_fname + '.part')
# See if the .part file already exists
if os.path.isfile(self.bufr_path_part):
# Probably a BUFR error on the previous run
raise BufrError(self.bufr_path_part)
# Open the bufr file
self.log.info('Opening: %s', self.bufr_path_part)
self.bufr = ncepbufr.open(self.bufr_path_part,
'w',
table='tempest-D.table')
# The file data-hour is needed when the first bufr message is created
self.datehour = int('{:d}{:02d}{:02d}{:02d}'.format(
dt_center.year, dt_center.month, dt_center.day, dt_center.hour))
# Open the first bufr message
# Again, the message length limit will cause messages to be closed and
# new ones opened automatically, all with the same date-hour.
self.bufr.open_message(TD_record.subset, self.datehour)
self.subsetcnt = 0
# Write this first record to the file
self.write_a_record(record)
def write_record(self, record):
# First, the date/time should be within this bufr file's time range
if not (record.datetime() > self.dt_start
and record.datetime() <= self.dt_end):
raise ValueError((
'It is incorrect for this record: %s, angle_idx: %d, line_idx:'
' %d, to be written to this BUFR file: %s') %
(record.datetime().isoformat(), record.sangle_idx,
record.sline_idx, os.path.basename(
self.bufr_path)))
if record.datetime() <= self.dt_gfs_end:
# The file name better still start with 'gfs'
if 'gfs' != os.path.basename(self.bufr_path_part)[0:3]:
raise ValueError(
('This record: %s, angle_idx: %d, line_idx:'
' %d, should be written to the gfs file instead of: %s') %
(record.datetime().isoformat(), record.sangle_idx,
record.sline_idx, os.path.basename(self.bufr_path_part)))
elif 'gfs' == os.path.basename(self.bufr_path_part)[0:3]:
# It's time to switch to the gdas file
self.bufr.close_message()
self.bufr.close()
self.log.info('All %d subsets written to %s', self.subsetcnt,
os.path.basename(self.bufr_path_part))
# Move this to the appropriate subdirs without the .part extensions
bufr_dir = os.path.join(os.environ['BUFR_OUTPUT_DIR'],
self.bufr_subdirs)
os.makedirs(bufr_dir, exist_ok=True)
gfs_path = os.path.join(bufr_dir, self.bufr_file)
os.rename(self.bufr_path_part, gfs_path)
# Switch to the gdas file
self.bufr_subdirs = self.bufr_subdirs.replace('gfs.', 'gdas.', 1)
self.bufr_file = self.bufr_file.replace('gfs.', 'gdas.', 1)
self.bufr_path_part = os.path.join(
os.environ['BUFR_OUTPUT_DIR'],
self.bufr_file + '.' + self.h5_fname + '.part')
# Make sure it doesn't already exist
if os.path.isfile(self.bufr_path_part):
raise ValueError((
'This record: %s, angle_idx: %s, line_idx:'
' %s, should be the first record written to the gdas file,'
' but the gdas file "%s" already exists') %
record.datetime().isoformat(),
record.sangle_idx, record.sline_idx,
os.path.basename(self.bufr_path_part))
shutil.copy(gfs_path, self.bufr_path_part)
# Open the gdas bufr file for appending
self.log.info('Opening: %s for appending', self.bufr_path_part)
self.bufr = ncepbufr.open(self.bufr_path_part, 'a')
# Open the first bufr message
self.bufr.open_message(TD_record.subset, self.datehour)
# Write the record to the current bufr file
self.write_a_record(record)
def main():
tic = time.clock()
# Mnemonics for getting data from the bufr file.
hdstr = 'SSTN CLON CLAT SELV ANEL YEAR MNTH DAYS HOUR MINU QCRW ANAZ' #PRFR' # MGPT'
obstr = 'DIST HREF'
#obstr2='STDM SUPLON SUPLAT HEIT RWND RWAZ RSTD' #RWSOB--radial wind super ob.
#1. INITIALIZING SOME BASIC LISTS AND GETTING INPUT DATA.
i = 0
sids = []
lons = []
lats = []
l2rw = []
anel = []
anaz = []
dist125m = []
ymdhm = []
radii = []
PRF = []
OBS_FILE = sys.argv[1]
message_type = sys.argv[2]
date = sys.argv[3]
STAID = sys.argv[4]
OBS_FILECPY = OBS_FILE + "_cpy"
anel0 = float(sys.argv[5])
del_anel = 0.25
maxanaz = 0
#2. READ PREPBUFR FILE.
b = 'false' # used for breaking the loop.
bufr = ncepbufr.open(OBS_FILE) # bufr file for reading.
bufrcpy = ncepbufr.open(
OBS_FILECPY) # bufr file for reading ahead and determining maxanaz.
bufr.dump_table('l2rwbufr.table') # dump table to file.
while bufr.advance() == 0: # loop over messages.
#bufr has been advanced, now if reading concurrently, bufrcpy should be behind by one. But sometimes
#bufrcpy could be ahead of where we need it (but only by one step) since we read it forward.
#We don't want to advance it in that case.
if ((bufr.msg_counter - bufrcpy.msg_counter) == 1): bufrcpy.advance()
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
if (bufr.msg_type == message_type):
while bufr.load_subset() == 0: # loop over subsets in message.
hdr = bufr.read_subset(
hdstr).squeeze() # parse hdstr='SSTN CLON ... etc.'
station_id = hdr[0].tostring() # convert SSTN to string.
station_id = station_id.strip(
) # remove white space from SSTN.
#*********************
bufrcpy.load_subset() #**********************
hdrcpy = bufrcpy.read_subset(hdstr).squeeze() #
#*******************************************
if (station_id == STAID
): # comes from input. used for picking a single SSTN.
if (hdr[4] >= anel0 - del_anel and hdr[4] <=
anel0 + del_anel): # read an elevation angle.
obs = bufr.read_subset(obstr).squeeze(
) # parse obstr='DIST125M DMVR DVSW'
#*******************************************
obscpy = bufrcpy.read_subset(obstr).squeeze() #
#*******************************************
print(bufr.msg_counter, 'SSTN,CLON,CLAT,ANAL,ANAZ : ',
station_id, hdr[1], hdr[2], hdr[4], hdr[11])
i = i + 1 #number of observations counter.
sids.append(station_id) # station ids
l2rw.append(obs[1]) # level 2 radial winds
anel.append(hdr[4]) # elevation angles
anaz.append(hdr[11]) # azimuthal angles
radii.append(obs[0]) #distances in units of 1 m
ymdhm.append(int(hdr[9]))
#It is not sufficient to check for len(anaz) == 360. Sometimes, it is missing.
#***********************************************************************************************************
if (len(anaz) == 355): #pause to read ahead.
while bufrcpy.load_subset(
) == 0: # read ahead with bufrcpy
hdrcpy = bufrcpy.read_subset(hdstr).squeeze()
maxanaz = max(maxanaz, int(np.ceil(
hdrcpy[11]))) #need hdrcpy[11] rounded up.
tries = 0
while tries < 10 and bufrcpy.advance(
) == 0: # look no. tries messages ahead.
tries += 1
while bufrcpy.load_subset() == 0:
if (bufrcpy.msg_type == message_type):
station_id = hdrcpy[0].tostring(
) # convert SSTN to string.
station_id = station_id.strip(
) # remove white space from SSTN.
if (station_id == STAID):
if (hdrcpy[4] >= anel0 - del_anel
and hdrcpy[4] <=
anel0 + del_anel):
hdrcpy = bufrcpy.read_subset(
hdstr).squeeze()
maxanaz = max(
maxanaz,
int(np.ceil(hdrcpy[11])))
print(bufrcpy.msg_counter,
'cpySSTN,DATE/TIME : ',
station_id, hdrcpy[5:10],
hdrcpy[4], hdrcpy[11],
len(anaz))
print(maxanaz)
if (len(anaz) >= 355): print(len(anaz), maxanaz)
#***********************************************************************************************************
if (len(anaz) == maxanaz and len(anaz) > 355):
while (len(anaz) <
360): #fill in some missing data points
l2rw.append(obs[1] * 0 + -999.)
anel.append(anel[-1])
anaz.append(anaz[-1] + 1)
radii.append(radii[-1])
ymdhm.append(ymdhm[-1])
while (
len(anaz) < 361
): # fill in the final beam with nan mean between 1st and 359th
l2rw.append(l2rw[0])
anel.append(anel[0])
anaz.append(anaz[-1] + 1)
radii.append(radii[0])
ymdhm.append(ymdhm[0])
b = 'true'
if (b == 'true'): break # stop reading after all anaz's read.
bufr.close()
print("number of obs: ", i)
MM = str(ymdhm[-1])
print(np.min(anel), np.max(anel))
print(np.min(ymdhm), np.max(ymdhm))
toc = time.clock() # check how long reading the data in took.
sec = str(toc - tic)
print('time it took to run: ' + sec + ' seconds.')
#3. FIND THE MAX/MIN VALUES OF THE DIST125M AND THETA ARRAYS.
maxRadii = 100000 # set max of max distances to about 100 mi.
minRadii = 0 # min of min distances.
#4. INITIALIZE THE POLAR PLOT AS ALL MISSING DATA (-999).
fig = plt.figure(figsize=(8, 8)) # 8" x 8" seems plenty large.
ax = fig.add_subplot(111,
polar=True) # we would like it to be a polar plot.
ax.set_theta_zero_location("N") # set theta zero location to point North.
ax.set_theta_direction(-1) # set theta to increase clockwise (-1).
theta, r = np.meshgrid(anaz, np.arange(minRadii, maxRadii,
1000.)) # create meshgrid
theta = deg2rad(theta) # convert theta from degrees to radians.
rw = np.zeros(shape=(len(theta),
len(r[0]))) # initialize the polar plot with all 0's.
rw.fill(-999) # change all values to missing data (-999).
#PRF_new=np.zeros(shape=(len(theta),len(r[0])))
#PRF_new.fill(-999)
r = r.T
rw = rw.T #; PRF_new.T # transpose r and rw for later manipulation.
#5. POPULATE THE EMPTY RW ARRAY WITH ACTUAL VALUES.
for i in range(len(anaz)): # for every azimuth angle ...
sys.stdout.write('\r' + str(i) + '/' + str(len(anaz)))
sys.stdout.flush()
for j in range(
len(radii[i])
): # ... loop over every observation distance from radar ...
for k in range(
len(r[i])
): # ... and loop over an equally 125m spaced array ...
if (radii[i][j] == r[i][k]
): # ... if the observation dist = dist125m ...
rw[i][k] = l2rw[i][
j] # ... assign the value of the observation to that index.
#PRF_new[i][k]=PRF[i][j]
break # speeds things up by about 50-60%.
rw[np.isnan(
rw)] = -999 # the masked values are converted to nan. set nan to -999.
print(np.max(rw)) # check that it is not nan.
calc_variance = True
figname = './' + str(STAID) + 'stats' + str(date)
if (calc_variance):
f1 = open(figname, 'w+')
rw_stdev = rw[rw != -999.].std()
rw_mean = rw[rw != -999.].mean()
rw_var = rw[rw != -999.].var()
f1.write(
"The output here describes some statistics for the radial wind obs \n"
)
f1.write("Obs file={} \n".format(OBS_FILE))
f1.write("date ={} \n".format(date))
f1.write("Station ={} \n".format(STAID))
f1.write("STDEV ={} \n".format(rw_stdev))
f1.write("MEAN ={} \n".format(rw_mean))
f1.write("VAR ={} \n".format(rw_var))
f1.close()
#6. FINISH MAKING THE POLAR PLOT WITH THE FILLED IN VALUES.
if (False):
cmap = plt.cm.jet # use the jet colormap.
cmap.set_under('white') # set the -999 values to white.
mesh = ax.pcolormesh(theta,
r.T,
rw.T,
shading='flat',
cmap=cmap,
vmin=-40,
vmax=40) # plot the data.
elif (True): #for reflectivity
clevs = [
5., 10., 15., 20., 25., 30., 35., 40., 45., 50., 55., 60., 65.,
70., 75.
]
cmap = ncepy.mrms_radarmap()
mesh = ax.pcolormesh(theta,
r.T,
rw.T,
shading='flat',
cmap=cmap,
vmin=5,
vmax=75) # plot the data.
cmap.set_under('#999999')
else:
c = mcolors.ColorConverter().to_rgb
cmap = make_colormap([
c('deepskyblue'),
c('navy'),
0.20, # light blue to dark blue
c('#02ff02'),
c('#003500'),
0.47, # bright green to dark green
c('#809e80'),
c('white'),
0.50, # gray with green tint to white
c('white'),
c('#9e8080'),
0.53, # white to gray with red tint
c('#350000'),
c('#ff0000'),
0.80, # dark red to bright red
c('salmon'),
c('yellow')
]) # salmon to yellow
cmap.set_under('#999999')
cmap.set_over('purple')
mesh = ax.pcolormesh(theta,
r.T,
rw.T,
shading='flat',
cmap=cmap,
vmin=-40,
vmax=40) # plot the data.
cbar = fig.colorbar(mesh, shrink=0.85, pad=0.10, ax=ax) # add a colorbar.
cbar.set_label(
'$dBZ$') # radial wind data is in units of meters per second.
plt.title('Doppler Reflectity \n Station ID: '+STAID\
+' Scan Angle: '+str(anel[0])\
+' Date: '+date+MM.zfill(2),fontsize=15,y=1.12) # add a useful title.
ax.grid(True)
plt.show() # make the plot.
plt.savefig('./REF_'+STAID+'_'+str(anel[0])+'_'+date+MM.zfill(2)+'.png'\
,bbox_inches='tight') # save figure.
#7. CALCULATE SOME STATS
toc = time.clock() # check to see how long it took to run.
sec = str(toc - tic)
print('time it took to run: ' + sec + ' seconds.')
from __future__ import print_function
import ncepbufr
hdstr1 ='SAID SIID FOVN YEAR MNTH DAYS HOUR MINU SECO CLAT CLON CLATH CLONH HOLS'
hdstr2 ='SAZA SOZA BEARAZ SOLAZI'
# read amsua radiance file.
bufr = ncepbufr.open('1bamua')
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr1 = bufr.read_subset(hdstr1).squeeze()
hdr2 = bufr.read_subset(hdstr2).squeeze()
yyyymmddhhss ='%04i%02i%02i%02i%02i%02i' % tuple(hdr1[3:9])
# for satellite id, see common code table c-5
# (http://www.emc.ncep.noaa.gov/mmb/data_processing/common_tbl_c1-c5.htm#c-5)
# for sensor id, see common code table c-8
# (http://www.emc.ncep.noaa.gov/mmb/data_processing/common_tbl_c8-c14.htm#c-8)
print('sat id,sensor id lat, lon, yyyymmddhhmmss =',int(hdr1[0]),\
int(hdr1[1]),hdr1[9],hdr1[10],yyyymmddhhss)
obs = bufr.read_subset('TMBR',rep=True).squeeze()
nchanl = len(obs)
for k in range(nchanl):
print('channel, tb =',k+1,obs[k])
# only loop over first 4 subsets
if bufr.msg_counter == 4: break
bufr.close()
print(" Total number of messages that match obs type {0:s}: {1:d}".format(ObsType, TotalMsgs))
print(" Number of messages selected: {0:d}".format(NumMsgs))
print(" Number of observations selected: {0:d}".format(NumObs))
print("")
# Now that we have the number of observations we will be recording, set the dimension
# size in the obs object. Note the set_nobs() method needs to be called before creating
# netcdf variables.
Obs.set_nobs(NumObs)
# Create the dimensions and variables in the netCDF file in preparation for
# recording the selected observations.
nc = Dataset(NetcdfFname, 'w', format='NETCDF4')
Obs.create_nc_datasets(nc)
# Fill in the dimension variables with the coordinate values. Just using dummy values
# for now which are 1..n where n is the size of the coorespoding dimension.
Obs.fill_coords(nc)
# Open the BUFR file and initialize the file object.
bufr = ncepbufr.open(BufrFname)
# Run the conversion
Obs.convert(bufr, nc)
# Clean up
bufr.close()
nc.sync()
nc.close()
from __future__ import print_function
import ncepbufr
hdstr1 = 'SAID SIID FOVN YEAR MNTH DAYS HOUR MINU SECO CLAT CLON CLATH CLONH HOLS'
hdstr2 = 'SAZA SOZA BEARAZ SOLAZI'
# read amsua radiance file.
bufr = ncepbufr.open('data/1bamua')
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr1 = bufr.read_subset(hdstr1).squeeze()
hdr2 = bufr.read_subset(hdstr2).squeeze()
yyyymmddhhss = '%04i%02i%02i%02i%02i%02i' % tuple(hdr1[3:9])
# for satellite id, see common code table c-5
# (http://www.emc.ncep.noaa.gov/mmb/data_processing/common_tbl_c1-c5.htm#c-5)
# for sensor id, see common code table c-8
# (http://www.emc.ncep.noaa.gov/mmb/data_processing/common_tbl_c8-c14.htm#c-8)
print('sat id,sensor id lat, lon, yyyymmddhhmmss =',int(hdr1[0]),\
int(hdr1[1]),hdr1[9],hdr1[10],yyyymmddhhss)
obs = bufr.read_subset('TMBR', rep=True).squeeze()
nchanl = len(obs)
for k in range(nchanl):
print('channel, tb =', k + 1, obs[k])
# only loop over first 4 subsets
if bufr.msg_counter == 4: break
bufr.close()
import ncepbufr
bufr = ncepbufr.open('amsuabufr')
bufr.dump_table('amsuabufr.table')
bufr.close()
filetime = datefrombufr
##datefrombufr=re.findall(r'(\d{10})', radbufr_filename) #finds ten digit number in the string preceded by PERIOD... ## EXAMPLE BUFR /pan2/projects/gfsenkf/whitaker/cfsr_dumps/2005022506/gdas/1bamua.gdas.2005022506
print('datefrombufr=', datefrombufr)
print('filetime=', filetime)
#########################################################################################################
#########################################################################################################
#########################################################################################################
hdstr1 = 'SAID FOVN YEAR MNTH DAYS HOUR MINU SECO CLAT CLON HOLS'
hdstr2 = 'SAZA SOZA BEARAZ SOLAZI'
obstr = 'TMBR'
####################################################
# read amsua radiance file.
bufr = ncepbufr.open(radbufr_filename)
####################################################
####################################################
####################################################
nhd1 = len(hdstr1.split())
nhd2 = len(hdstr2.split())
nhd = nhd1 + nhd2
###################################################
nob = 0
nob1 = 0
nrec_count = 0
# VARIABLES FROM INPUT BUFR FILE
DUMsaid = []
DUMfovn = []
from __future__ import print_function
import ncepbufr
import numpy as np
# read prepbufr file.
hdstr = 'SID XOB YOB DHR TYP ELV SAID T29'
obstr = 'POB QOB TOB ZOB UOB VOB PWO MXGS HOVI CAT PRSS TDO PMO'
qcstr = 'PQM QQM TQM ZQM WQM NUL PWQ PMQ'
oestr = 'POE QOE TOE NUL WOE NUL PWE'
bufr = ncepbufr.open('data/prepbufr')
while bufr.advance() == 0: # loop over messages.
while bufr.load_subset() == 0: # loop over subsets in message.
hdr = bufr.read_subset(hdstr).squeeze()
station_id = hdr[0].tostring()
lon = hdr[1]
lat = hdr[2]
station_type = int(hdr[4])
obs = bufr.read_subset(obstr)
nlevs = obs.shape[-1]
oer = bufr.read_subset(oestr)
qcf = bufr.read_subset(qcstr)
# stop after first 2 messages.
if bufr.msg_counter == 2: break
# check data
# station_id,lon,lat,time,station_type,nlevs
assert station_id.rstrip() == b'91925'
np.testing.assert_almost_equal(lon, 220.97)
np.testing.assert_almost_equal(lat, -9.8)
assert station_type == 220
assert nlevs == 41
obs_str = '%s' % obs[:, -1]
from __future__ import print_function
import ncepbufr
hdrstr = 'SAID CLAT CLON YEAR MNTH DAYS HOUR MINU SWCM SAZA GCLONG SCCF SWQM'
obstr = 'HAMD PRLC WDIR WSPD'
qcstr = 'OGCE GNAP PCCF'
# read satellite wind file.
bufr = ncepbufr.open('satwndbufr')
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr = bufr.read_subset(hdrstr).squeeze()
yyyymmddhh = '%04i%02i%02i%02i%02i' % tuple(hdr[3:8])
satid = int(hdr[0])
windtype = int(hdr[8])
lat = hdr[1]
lon = hdr[2]
qm = hdr[12]
obdata = bufr.read_subset(obstr).squeeze()
print('satid, wind type, lat, lon, press, qcflg, time, speed, dir =',\
satid,windtype,lat,lon,obdata[1],qm,yyyymmddhh,obdata[3],obdata[2])
# only loop over first 4 subsets
if bufr.msg_counter == 4: break
bufr.close()
def main():
tic = time.clock()
# Mnemonics for getting data from the bufr file.
hdstr= 'SSTN CLON CLAT SELV ANEL YEAR MNTH DAYS HOUR MINU QCRW ANAZ' #PRFR' # MGPT'
obstr= 'DIST125M DMVR DVSW' # PRFR' #NL2RW--level 2 radial wind.
obstr2='STDM SUPLON SUPLAT HEIT RWND RWAZ RSTD' #RWSOB--radial wind super ob.
#1. INITIALIZING SOME BASIC LISTS AND GETTING INPUT DATA.
sids=[]; lons=[]; lats=[]; l2rw=[]; anel=[]; anaz=[]; dist125m=[]; ymdhm=[]; radii=[]; PRF=[]
#3. FIND THE MAX/MIN VALUES OF THE DIST125M AND THETA ARRAYS.
maxRadii=100000 # set max of max distances to about 100 mi.
minRadii=0 # min of min distances.
#4. INITIALIZE THE POLAR PLOT AS ALL MISSING DATA (-999).
theta,r = np.meshgrid(anaz,np.arange(minRadii,maxRadii,250.)) # create meshgrid
theta=deg2rad(theta) # convert theta from degrees to radians.
l2rwX = np.zeros(shape=(1,361,400))*np.nan #init first dim to 1. we'll inc this later
#INPUT PARAMS ###############
#OBS_FILE='/scratch4/NCEPDEV/meso/save/Donald.E.Lippi/gsi/data/obsfiles/2015103018/rap.t18z.nexrad.tm00.bufr_d'
OBS_FILE='/scratch4/NCEPDEV/meso/save/Donald.E.Lippi/gsi/data/obsfiles/2015103018/nam.t18z.nexrad.tm00.bufr_d'
OBS_FILECPY=OBS_FILE+"_cpy"
message_type1='NC006027' #6010 + 17z = 6027
message_type2='NC006028' #6010 + 18z = 6028
date=2015103018
STAID="@STAID@"
anel0=@anel0@
del_anel=0.25
del_time=@del_time@
#############################
time_check_1=(60. - del_time*60.)
time_check_2=( del_time*60.)
print("Average obs with time window of {} minutes".format(del_time*60.))
n=-1
#2. READ PREPBUFR FILE.
bufr = ncepbufr.open(OBS_FILE) # bufr file for reading.
bufrcpy=ncepbufr.open(OBS_FILECPY) # bufr file for reading ahead and determining maxanaz.
bufr.dump_table('l2rwbufr.table') # dump table to file.
while bufr.advance() == 0: # loop over messages.
#bufr has been advanced, now if reading concurrently, bufrcpy should be behind by one. But sometimes
#bufrcpy could be ahead of where we need it (but only by one step) since we read it forward.
#We don't want to advance it in that case.
if( (bufr.msg_counter-bufrcpy.msg_counter)==1): bufrcpy.advance()
if(bufr.msg_type == message_type1 or bufr.msg_type == message_type2):
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date,time_check_1,time_check_2)
while bufr.load_subset() == 0: # loop over subsets in message.
hdr = bufr.read_subset(hdstr).squeeze() # parse hdstr='SSTN CLON ... etc.'
bufr_minu=hdr[9]
#*********************
bufrcpy.load_subset()#**********************
hdrcpy=bufrcpy.read_subset(hdstr).squeeze()#
bufrcpy_minu=hdrcpy[9]#*********************
#**********************
good=False
if(bufr.msg_type == message_type1 and bufr_minu >= time_check_1): good=True
if(bufr.msg_type == message_type2 and bufr_minu <= time_check_2): good=True
if(good): #looking for next minute to process
station_id = hdr[0].tostring() # convert SSTN to string.
station_id=station_id.strip() # remove white space from SSTN.
if(station_id == STAID): # comes from input. used for picking a single SSTN.
if(hdr[4] >= anel0-del_anel and hdr[4] <= anel0+del_anel): # read an elevation angle.
if(hdr[11] >= 0 and hdr[11] <= 1): # increment n on first azimuth only
n=n+1
maxanaz=0 #initialize/reset. This gets updates around azm 355
if(np.floor(hdr[11]) != len(anaz)):
sids.append(station_id) # station ids
l2rw.append(obs[1]*0 + -999.) # level 2 radial winds
anel.append(anel[-1]+1.) # elevation angles
anaz.append(anaz[-1]) # azimuthal angles
radii.append(radii[-1]) #distances in units of 1 m
ymdhm.append(ymdhm[-1])
obs = bufr.read_subset(obstr).squeeze() # parse obstr='DIST125M DMVR DVSW'
#*******************************************
obscpy=bufrcpy.read_subset(obstr).squeeze()#
#*******************************************
print(bufr.msg_counter,'SSTN,DATE/TIME : ',station_id,hdr[5:10],hdr[4],hdr[11],n,len(anaz))
sids.append(station_id) # station ids
l2rw.append(obs[1]) # level 2 radial winds
anel.append(hdr[4]) # elevation angles
anaz.append(hdr[11]) # azimuthal angles
radii.append(obs[0]*125) #distances in units of 1 m
ymdhm.append(int(hdr[9]))
#It is not sufficient to check for len(anaz) == 360. Sometimes, it is missing.
#***********************************************************************************************************
if(len(anaz) == 355): #pause to read ahead.
while bufrcpy.load_subset() == 0: # read ahead with bufrcpy
hdrcpy=bufrcpy.read_subset(hdstr).squeeze()
bufrcpy_minu=hdrcpy[9]
maxanaz=max(maxanaz,int(np.ceil(hdrcpy[11]))) #need hdrcpy[11] rounded up.
tries=0
while tries < 10 and bufrcpy.advance() == 0: # look no. tries messages ahead.
tries+=1
while bufrcpy.load_subset() == 0:
bufrcpy_minu=hdrcpy[9]
good=False
if(bufrcpy.msg_type == message_type1 and bufrcpy_minu >= time_check_1):
good=True
if(bufrcpy.msg_type == message_type2 and bufrcpy_minu <= time_check_2):
good=True
if(good): #looking for next minute to process
station_id = hdrcpy[0].tostring() # convert SSTN to string.
station_id=station_id.strip() # remove white space from SSTN.
if(station_id == STAID):
if(hdrcpy[4] >= anel0-del_anel and hdrcpy[4] <= anel0+del_anel):
hdrcpy= bufrcpy.read_subset(hdstr).squeeze()
maxanaz=max(maxanaz,int(np.ceil(hdrcpy[11])))
print(bufrcpy.msg_counter,'cpySSTN,DATE/TIME : ',station_id,hdrcpy[5:10],hdrcpy[4],hdrcpy[11],n,len(anaz))
print(maxanaz)
if(len(anaz) >= 355): print(len(anaz),maxanaz)
#***********************************************************************************************************
#if(len(anaz) >= 355 and we've reached the end of the subset with anaz = 355)
if(len(anaz) == maxanaz and len(anaz)> 355):
while(len(anaz) < 360): #fill in some missing data points
l2rw.append(obs[1]*0 + -999.)
anel.append(anel[-1])
anaz.append(anaz[-1]+1)
radii.append(radii[-1])
ymdhm.append(ymdhm[-1])
while(len(anaz) < 361): # fill in the final beam with nan mean between 1st and 359th
#l2rw_first_last=l2rw[0],l2rw[-1]
#l2rw.append(np.nanmean(np.dstack(l2rw_first_last),axis=2).flatten().tolist())
l2rw.append(l2rw[0])
anel.append(anel[0])
anaz.append(anaz[-1]+1)
radii.append(radii[0])
ymdhm.append(ymdhm[0])
MM=str(ymdhm[-1])
print("max/min el. angle: ",np.max(anel),np.min(anel))
print("max/min minute: ",np.max(ymdhm),np.min(ymdhm))
theta,r = np.meshgrid(anaz,np.arange(minRadii,maxRadii,250.)) # create meshgrid
theta=deg2rad(theta)
rw = np.zeros(shape=(len(theta),len(r[0])))# initialize the polar plot with all 0's
rw.fill(-999) # change all values to missing data (-999).
r=r.T; rw=rw.T#; PRF_new.T # transpose r and rw for later manipulation.
#5. POPULATE THE EMPTY RW ARRAY WITH ACTUAL VALUES.
for i in range(360): # for every azimuth angle ...
print(str(i+1)+'/'+str(360))
for j in range(len(radii[i])): # loop over every observation distance from radar
for k in range(len(r[i])): # and loop over an equally 125m spaced array
if(radii[i][j] == r[i][k]): # if the observation dist = dist125m
rw[i][k]=l2rw[i][j] # assign the value of the obsrvtn to that index.
break # speeds things up by about 50-60%.
if(n>0):
newrow=rw*np.nan
l2rwX = np.concatenate((l2rwX,[newrow]),axis=0)
rw[rw==-999.]=np.nan #set -999 to nans
l2rwX[n,:,:]=rw
sids=[]; lons=[]; lats=[]; l2rw=[]; anel=[]; anaz=[]
dist125m=[]; ymdhm=[]; radii=[]; PRF=[]
bufr.close()
rw_mean = np.nanmean(np.dstack(l2rwX),axis=2)#average element wise over time series
print("shape of l2rwX is" +str(np.shape(l2rwX)))
print("type=",type(l2rwX))
rw=rw_mean # overwrite rw with the rw_mean for further processing
rw[np.isnan(rw)] = -999. # the masked values are converted to nan. set nan to -999.
print("shape of rw is" +str(np.shape(rw)))
toc = time.clock() # check how long reading the data in took.
sec = str(toc-tic)
print('time it took to run: '+sec+' seconds.')
#4. INITIALIZE THE POLAR PLOT
fig = plt.figure(figsize=(8,8)) # 8" x 8" seems plenty large.
ax = fig.add_subplot(111,polar=True) # we would like it to be a polar plot.
ax.set_theta_zero_location("N") # set theta zero location to point North.
ax.set_theta_direction(-1) # set theta to increase clockwise (-1).
calc_variance=True
figname='./'+STAID+'_'+str(anel0)+'_'+str(date)+'_'+str(del_time)
if(calc_variance):
f1=open(figname+'.txt','w+')
rw_stdev=rw[rw!=-999.].std()
rw_mean =rw[rw!=-999.].mean()
rw_var =rw[rw!=-999.].var()
f1.write("The output here describes some statistics for the radial wind obs \n")
f1.write("Obs file={} \n".format(OBS_FILE))
f1.write("date ={} \n".format(date))
f1.write("Station ={} \n".format(STAID))
f1.write("STDEV ={} \n".format(rw_stdev))
f1.write("MEAN ={} \n".format(rw_mean))
f1.write("VAR ={} \n".format(rw_var))
f1.close()
#6. FINISH MAKING THE POLAR PLOT WITH THE FILLED IN VALUES.
if(False):
cmap = plt.cm.jet # use the jet colormap.
cmap.set_under('white') # set the -999 values to white.
else:
c = mcolors.ColorConverter().to_rgb
cmap = make_colormap(
[c('deepskyblue'),c('navy') ,0.20, # light blue to dark blue
c('#02ff02') ,c('#003500') ,0.47, # bright green to dark green
c('#809e80') ,c('white') ,0.50, # gray with green tint to white
c('white') ,c('#9e8080') ,0.53, # white to gray with red tint
c('#350000') ,c('#ff0000') ,0.80, # dark red to bright red
c('salmon') ,c('yellow')]) # salmon to yellow
cmap.set_under('#999999')
cmap.set_over('purple')
mesh = ax.pcolormesh(theta,r.T,rw.T,shading='flat',cmap=cmap,vmin=-40,vmax=40) # plot the data.
cbar = fig.colorbar(mesh,shrink=0.85,pad=0.10,ax=ax) # add a colorbar.
cbar.set_label('$m/s$') # radial wind data is in units of meters per second.
plt.title('Doppler Velocity \n Station ID: '+STAID\
+' Scan Angle: '+str(anel0)\
+' Date: '+str(date),fontsize=15,y=1.12) # add a useful title.
ax.grid(True)
plt.show() # make the plot.
plt.savefig(figname+'.png',bbox_inches='tight') # save figure.
#7. CALCULATE SOME STATS
toc = time.clock() # check to see how long it took to run.
sec = str(toc-tic)
print('time it took to run: '+sec+' seconds.')
from __future__ import print_function
import ncepbufr
import numpy as np
hdrstr ='YEAR MNTH DAYS HOUR MINU PCCF ELRC SAID PTID GEODU'
# read gpsro file.
bufr = ncepbufr.open('gpsbufr_new')
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr = bufr.read_subset(hdrstr).squeeze()
yyyymmddhh ='%04i%02i%02i%02i%02i' % tuple(hdr[0:5])
satid = int(hdr[7])
# if satid == 740:
ptid = int(hdr[8])
nreps_this_ROSEQ2 = bufr.read_subset('{ROSEQ2}').squeeze()
nreps_this_ROSEQ1 = len(nreps_this_ROSEQ2)
data1b = bufr.read_subset('ROSEQ1',seq=True) # bending angle
data2a = bufr.read_subset('ROSEQ3',seq=True) # refractivity
levs_bend = data1b.shape[1]
levs_ref = data2a.shape[1]
if levs_ref != levs_bend:
print('skip report due to bending angle/refractivity mismatch')
continue
print('sat id,platform transitter id, levels, yyyymmddhhmm =',\
satid,ptid,levs_ref,yyyymmddhh)
print('k, lat, lon, height, ref, bend:')
for k in range(levs_ref):
# NCEP BUFR
#----------------------------------------------------------------------------------------#
# filename
dtg = '17091712'
filename = 'gdas.satwnd.tm00.bufr_d.' + dtg
# string
hdrstr = 'SAID CLAT CLON YEAR MNTH DAYS HOUR MINU SWCM SAZA GCLONG SCCF SWQM'
obstr = 'HAMD PRLC WDIR WSPD'
qcstr = 'OGCE GNAP PCCF'
lons = []
lats = []
# read satellite wind file.
bufr = ncepbufr.open(filename)
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr = bufr.read_subset(hdrstr).squeeze()
satid = int(hdr[0])
if satid == 173:
yyyymmddhh = '%04i%02i%02i%02i%02i' % tuple(hdr[3:8])
windtype = int(hdr[8])
qm = hdr[12]
obdata = bufr.read_subset(obstr).squeeze()
lat = hdr[1]
lon = hdr[2]
if windtype == 1:
print('satid, wind type, lat, lon, press, qcflg, time, speed, dir =',\
from __future__ import print_function
import ncepbufr
hdstr='SID XOB YOB DHR TYP ELV SAID T29'
obstr='POB QOB TOB ZOB UOB VOB PWO MXGS HOVI CAT PRSS TDO PMO'
qcstr='PQM QQM TQM ZQM WQM NUL PWQ PMQ'
oestr='POE QOE TOE NUL WOE NUL PWE'
# read prepbufr file.
bufr = ncepbufr.open('prepbufr')
bufr.print_table() # print embedded table
bufr.dump_table('prepbufr.table') # dump table to file
while bufr.advance() == 0: # loop over messages.
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
#bufr.read_subset(obstr) # should raise subset not loaded error
while bufr.load_subset() == 0: # loop over subsets in message.
hdr = bufr.read_subset(hdstr).squeeze()
station_id = hdr[0].tostring()
obs = bufr.read_subset(obstr)
nlevs = obs.shape[-1]
oer = bufr.read_subset(oestr)
qcf = bufr.read_subset(qcstr)
print('station_id, lon, lat, time, station_type, levels =',\
station_id,hdr[1],hdr[2],hdr[3],int(hdr[4]),nlevs)
for k in range(nlevs):
if nlevs > 1:
print('level',k+1)
print('obs',obs[:,k])
print('oer',oer[:,k])
print('qcf',qcf[:,k])
MyArgs = ap.parse_args()
filename_in = MyArgs.input_bufr
filename_out = MyArgs.output_bufr
receipt_time_cutoff = MyArgs.receipt_time
filename_ref = MyArgs.reference_bufr
if filename_out == filename_in:
msg = "output file must not have same name as input file"
raise SystemExit(msg)
if receipt_time_cutoff is None: # no receipt time specified
# if filenameref is a file, determine receipt_time_cutoff
# by looking for newest receipt time in the file.
try:
bufr = ncepbufr.open(filename_ref)
except (IOError, TypeError):
msg = 'must specify either receipt_time or reference_bufr'
raise ValueError(msg)
receipt_time_cutoff = -1
while bufr.advance() == 0: # loop over messages.
if bufr.receipt_time > receipt_time_cutoff:
receipt_time_cutoff = bufr.receipt_time
bufr.close()
else:
receipt_time_cutoff = int(receipt_time_cutoff)
print('receipt time cutoff = %s' % receipt_time_cutoff)
bufr = ncepbufr.open(filename_in)
bufrout = ncepbufr.open(filename_out, 'w', bufr)
nmsg = 0
from __future__ import print_function
import ncepbufr
hdrstr ='YEAR MNTH DAYS HOUR MINU PCCF ELRC SAID PTID GEODU'
# read gpsro file.
bufr = ncepbufr.open('gpsbufr')
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr = bufr.read_subset(hdrstr).squeeze()
yyyymmddhh ='%04i%02i%02i%02i%02i' % tuple(hdr[0:5])
satid = int(hdr[7])
ptid = int(hdr[8])
nreps_this_ROSEQ2 = bufr.read_subset('{ROSEQ2}').squeeze()
nreps_this_ROSEQ1 = len(nreps_this_ROSEQ2)
data1b = bufr.read_subset('ROSEQ1',seq=True) # bending angle
data2a = bufr.read_subset('ROSEQ3',seq=True) # refractivity
levs_bend = data1b.shape[1]
levs_ref = data2a.shape[1]
if levs_ref != levs_bend:
print('skip report due to bending angle/refractivity mismatch')
continue
print('sat id,platform transitter id, levels, yyyymmddhhmm =',\
satid,ptid,levs_ref,yyyymmddhh)
print('k, height, lat, lon, ref, bend:')
for k in range(levs_ref):
rlat = data1b[0,k]
rlon = data1b[1,k]
from __future__ import print_function
import ncepbufr
import sys
# print inventory of specified bufr file
bufr = ncepbufr.open(sys.argv[1])
nsubsets = 0
inv = bufr.inventory()
for n,msg in enumerate(inv):
out = (n+1,)+msg
print('message %s: %s %s %s %s subsets' % out)
nsubsets += out[4]
bufr.close()
print('%s total subsets in %s messages' % (nsubsets,len(inv)))
# Grab input arguemnts
ScriptName = os.path.basename(sys.argv[0])
UsageString = "USAGE: {0:s} <prepbufr>".format(ScriptName)
if len(sys.argv) != 2:
print("ERROR: must supply exactly 1 arguments")
print(UsageString)
sys.exit(1)
PbFname = sys.argv[1]
print("Testing BUFR functions")
print(" Input BUFR file: {0:s}".format(PbFname))
# open file and read through contents
bufr = ncepbufr.open(PbFname)
#bufr.print_table()
# prepBUFR mnemonics
Mnemonics = "TOB"
#Mnemonics = "POB QOB TOB ZOB UOB VOB PWO CAT PRSS TDO PMO XDR YDR HRDR"
# BUFR mnemonics
#Mnemonics = "TMDB"
#Mnemonics = "TMDB TMDP WDIR WSPD QMAT QMWN CLAT CLON FLVL YEAR MNTH DAYS HOUR MINU"
while (bufr.advance() == 0):
print(" MSG: {0:d} {1:s} {2:d} ({3:d})".format(
bufr.msg_counter,bufr.msg_type,bufr.msg_date,bufr._subsets()))
from __future__ import print_function
import ncepbufr
# read gpsro file.
bufr = ncepbufr.open('gpsbufr')
bufr.dump_table('gpsbufr.table')
hdr1[3, nlev], hdr1[4, nlev]))
if rtime > receipt_time: receipt_time = rtime
return receipt_time, rsrd, exprsrd
filename = sys.argv[1]
rtime1 = int(sys.argv[2])
rtime2 = int(sys.argv[3])
filenameo = sys.argv[4]
print('receipt time range = %s to %s' % (rtime1, rtime2))
if filenameo == filename:
msg = "output file must not have same name as input file"
raise SystemExit(msg)
bufr = ncepbufr.open(filename)
nsubs = 0
nsubso = 0
nskip1 = 0
nskip2 = 0
print(filename)
bufrout = ncepbufr.open(filenameo, 'w', bufr)
while bufr.advance() == 0: # loop over messages.
# check individual subsets
bufrout.open_message(bufr.msg_type, bufr.msg_date) # open message
while bufr.load_subset() == 0: # loop over subsets in message.
receipt_time, rsrd, exprsrd = get_subset_info(bufr)
if receipt_time > rtime1 and receipt_time <= rtime2:
#rs_bitstring = "{0:b}".format(rsrd)
# skip restricted data with no expiration time
if rsrd == 0 or exprsrd > 0:
from netCDF4 import Dataset
import numpy as np
import sys, datetime
from utils import quantize
hdstr1 ='SAID SIID FOVN YEAR MNTH DAYS HOUR MINU SECO CLAT CLON CLATH CLONH HOLS'
hdstr2 ='SAZA SOZA BEARAZ SOLAZI'
# input and output file names from command line args.
bufr_filename = sys.argv[1]
nc_filename = sys.argv[2]
if bufr_filename == nc_filename:
raise IOError('cannot overwrite input bufr file')
bufr = ncepbufr.open(bufr_filename)
f = Dataset(nc_filename,'w',format='NETCDF4_CLASSIC')
# get number of channels from 1st message in bufr file.
nchanl = None
while bufr.advance() == 0:
while bufr.load_subset() == 0:
obs = bufr.read_subset('TMBR',rep=True).squeeze()
#obs = bufr.read_subset('REHU',rep=True).squeeze()
nchanl = len(obs)
break
if nchanl is not None: break
bufr.rewind()
# create netCDF variables.
obsdim = f.createDimension('nobs',None)
from __future__ import print_function
import ncepbufr
hdstr='SID XOB YOB DHR TYP ELV SAID T29'
obstr='POB QOB TOB ZOB UOB VOB PWO MXGS HOVI CAT PRSS TDO PMO'
qcstr='PQM QQM TQM ZQM WQM NUL PWQ PMQ'
oestr='POE QOE TOE NUL WOE NUL PWE'
# read prepbufr file.
bufr = ncepbufr.open('data/prepbufr')
bufr.print_table() # print embedded table
bufr.dump_table('prepbufr.table') # dump table to file
while bufr.advance() == 0: # loop over messages.
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date, bufr.receipt_time)
#bufr.read_subset(obstr) # should raise subset not loaded error
while bufr.load_subset() == 0: # loop over subsets in message.
hdr = bufr.read_subset(hdstr).squeeze()
station_id = hdr[0].tostring()
obs = bufr.read_subset(obstr)
nlevs = obs.shape[-1]
oer = bufr.read_subset(oestr)
qcf = bufr.read_subset(qcstr)
print('station_id, lon, lat, time, station_type, levels =',\
station_id,hdr[1],hdr[2],hdr[3],int(hdr[4]),nlevs)
for k in range(nlevs):
if nlevs > 1:
print('level',k+1)
print('obs',obs[:,k])
print('oer',oer[:,k])
print('qcf',qcf[:,k])
parser = argparse.ArgumentParser()
parser.add_argument('bufr_file', help='The BUFR file to check')
parser.add_argument(
'input_hdf5_dir',
help='The directory that contains the Tempest-D HDF5 files to check against'
)
#parser.add_argument(
# 'increment',
# help='The size of the steps through the BUFR file pixels'
#)
parser_args = parser.parse_args()
# Open the HDF5 files and get them set up for pixel data checking
checker = PixelChecker(parser_args.input_hdf5_dir)
bufr = ncepbufr.open(parser_args.bufr_file)
while bufr.advance() == 0:
bufr_header = '{:10d}{:6d}{:^10}'.format(bufr.msg_date, bufr.msg_counter,
bufr.msg_type)
log.info(bufr_header)
while bufr.load_subset() == 0:
pixel = TD_record()
pixel.data = {}
#scalarstr1 = 'SAID YEAR MNTH DAYS HOUR MINU SECO CLATH CLONH CHSQ'
# Squeeze out any extra singleton dimensions and fill in any masked
# values with the fill value - 10E10
hdr = bufr.read_subset(pixel.scalarstr1).squeeze().filled()
#log.debug('hdr: %s', hdr)
from __future__ import print_function
import ncepbufr
hdrstr = 'SAID CLAT CLON YEAR MNTH DAYS HOUR MINU SWCM SAZA GCLONG SCCF SWQM'
obstr = 'HAMD PRLC WDIR WSPD'
qcstr = 'OGCE GNAP PCCF'
# read satellite wind file.
bufr = ncepbufr.open('data/satwndbufr')
bufr.print_table()
while bufr.advance() == 0:
print(bufr.msg_counter, bufr.msg_type, bufr.msg_date)
while bufr.load_subset() == 0:
hdr = bufr.read_subset(hdrstr).squeeze()
yyyymmddhh = '%04i%02i%02i%02i%02i' % tuple(hdr[3:8])
satid = int(hdr[0])
windtype = int(hdr[8])
lat = hdr[1]
lon = hdr[2]
qm = hdr[12]
obdata = bufr.read_subset(obstr).squeeze()
print('satid, wind type, lat, lon, press, qcflg, time, speed, dir =',\
satid,windtype,lat,lon,obdata[1],qm,yyyymmddhh,obdata[3],obdata[2])
# only loop over first 4 subsets
if bufr.msg_counter == 4: break
bufr.close()
qchash = hash(bufr.read_subset(qcstr).tostring())
key = '%s %s %s %s' % (bufr.msg_type, hdrhash, obshash, qchash)
nsubset += 1
if key in bufr_dict:
ndup += 1
if verbose:
print('warning: duplicate key for msg type %s' %
bufr.msg_type)
else:
bufr_dict[key] = bufr.msg_counter, nsubset
return bufr_dict, ndup
# create dictionaries with md5 hashes for each message as keys,
# (msg number,subset number) tuple as values.
bufr = ncepbufr.open(filename_in1)
bufr_dict1, ndup = get_bufr_dict(bufr, verbose=verbose, bufr_type=bufr_type)
print('%s duplicate keys found in %s' % (ndup, filename_in1))
bufr.close()
bufr = ncepbufr.open(filename_in2)
bufr_dict2, ndup = get_bufr_dict(bufr, verbose=verbose, bufr_type=bufr_type)
print('%s duplicate keys found in %s' % (ndup, filename_in2))
# find message subsets in bufr 2 that aren't in bufr 1
# uniq_messages is a dict whose keys are message numbers.
# dict entry is a list with unique subset numbers.
ncount = 0
uniq_messages = {}
for bkey in bufr_dict2:
if bkey not in bufr_dict1:
from __future__ import print_function
import ncepbufr
import sys
# dump contents of bufr file to stdout or to a text file.
# Warning: resulting output may be HUGE.
bufr = ncepbufr.open(sys.argv[1])
first_dump = True # after first write, append to existing file.
verbose = False # this produces more readable output.
while bufr.advance() == 0: # loop over messages.
while bufr.load_subset() == 0: # loop over subsets in message.
if len(sys.argv) > 2:
# dump decoded data to a file.
if first_dump:
# clobber file if it exists
bufr.dump_subset(sys.argv[2],verbose=verbose)
first_dump = False
else:
# append to existing file.
bufr.dump_subset(sys.argv[2],append=True,verbose=verbose)
else:
bufr.print_subset(verbose=verbose) # print decoded subset to stdout
bufr.close()
