def run_calcs(df, ctx):
"""Do our maths."""
# Convert sea level pressure to station pressure
df['pressure'] = mcalc.add_height_to_pressure(
df['slp'].values * units('millibars'),
ctx['nt'].sts[ctx['station']]['elevation'] * units('m')).to(
units('millibar'))
# Compute the relative humidity
df['relh'] = mcalc.relative_humidity_from_dewpoint(
df['tmpf'].values * units('degF'), df['dwpf'].values * units('degF'))
# Compute the mixing ratio
df['mixingratio'] = mcalc.mixing_ratio_from_relative_humidity(
df['relh'].values, df['tmpf'].values * units('degF'),
df['pressure'].values * units('millibars'))
# Compute the saturation mixing ratio
df['saturation_mixingratio'] = mcalc.saturation_mixing_ratio(
df['pressure'].values * units('millibars'),
df['tmpf'].values * units('degF'))
df['vapor_pressure'] = mcalc.vapor_pressure(
df['pressure'].values * units('millibars'),
df['mixingratio'].values * units('kg/kg')).to(units('kPa'))
df['saturation_vapor_pressure'] = mcalc.vapor_pressure(
df['pressure'].values * units('millibars'),
df['saturation_mixingratio'].values * units('kg/kg')).to(units('kPa'))
df['vpd'] = df['saturation_vapor_pressure'] - df['vapor_pressure']
group = df.groupby('year')
df = group.aggregate(np.average)
df['dwpf'] = mcalc.dewpoint(df['vapor_pressure'].values * units('kPa')).to(
units('degF')).m
return df
def mcalc_feelslike(tmpf, dwpf, smps):
"""Compute a feels like temperature
Args:
temperature (temperature): The dry bulb temperature
dewpoint (temperature): The dew point temperature
speed (speed): the wind speed
Returns:
temperature (temperature): The feels like temperature
"""
is_not_scalar = isinstance(tmpf.m, (list, tuple, np.ndarray))
rh = mcalc.relative_humidity_from_dewpoint(tmpf, dwpf)
# NB: update this once metpy 0.11 is released
app = mcalc.apparent_temperature(tmpf, rh, smps)
if hasattr(app, "mask"):
if is_not_scalar:
app[app.mask] = tmpf[app.mask]
else:
app = tmpf
elif hasattr(tmpf, "mask"):
app = masked_array(app.m, app.units)
app.mask = tmpf.mask
return app
def get_df(ctx):
"""Figure out what data we need to fetch here"""
nt = NetworkTable(ctx['network'])
ctx['ugc'] = nt.sts[ctx['station']]['ugc_zone']
pgconn = get_dbconn('postgis')
events = read_sql("""
SELECT generate_series(issue, expire, '1 minute'::interval) as valid,
(phenomena ||'.'|| significance) as vtec
from warnings WHERE ugc = %s and (
(phenomena = 'EH' and significance = 'W') or
(phenomena = 'HT' and significance = 'Y')
) ORDER by issue ASC
""",
pgconn,
params=(ctx['ugc'], ),
index_col='valid')
if events.empty:
raise ValueError("No Alerts were found for UGC: %s" % (ctx['ugc'], ))
pgconn = get_dbconn('asos')
obs = read_sql("""
SELECT valid, tmpf::int as tmpf, dwpf::int as dwpf
from alldata where station = %s
and valid > %s and tmpf > 70 and dwpf > -20 ORDER by valid
""",
pgconn,
params=(ctx['station'], str(events.index.values[0])),
index_col='valid')
ctx['title'] = ("%s [%s] (%s to %s)\n"
"Frequency of NWS Heat Headline for %s by Heat Index") % (
nt.sts[ctx['station']]['name'], ctx['station'],
str(events.index.values[0])[:10],
str(obs.index.values[-1])[:10], ctx['ugc'])
relh = mcalc.relative_humidity_from_dewpoint(
obs['tmpf'].values * munits.degF, obs['dwpf'].values * munits.degF)
obs['feel'] = mcalc.heat_index(obs['tmpf'].values * munits.degF,
relh).magnitude
if ctx['opt'] == 'yes':
obs = obs[obs['feel'] > obs['tmpf']]
obs['feel'] = obs['feel'].round(0)
res = obs.join(events)
res.fillna('None', inplace=True)
counts = res[['feel', 'vtec']].groupby(['feel', 'vtec']).size()
df = pd.DataFrame(counts)
df.columns = ['count']
df.reset_index(inplace=True)
ctx['df'] = df.pivot(index='feel', columns='vtec', values='count')
ctx['df'].fillna(0, inplace=True)
ctx['df']['Total'] = ctx['df'].sum(axis=1)
for vtec in ['HT.Y', 'EH.W', 'None']:
if vtec not in ctx['df'].columns:
ctx['df'][vtec] = 0.
ctx['df'][vtec + "%"] = ctx['df'][vtec] / ctx['df']['Total'] * 100.
def mcalc_feelslike(tmpf, dwpf, smps):
"""Compute a feels like temperature
Args:
temperature (temperature): The dry bulb temperature
dewpoint (temperature): The dew point temperature
speed (speed): the wind speed
Returns:
temperature (temperature): The feels like temperature
"""
rh = mcalc.relative_humidity_from_dewpoint(tmpf, dwpf)
hidx = mcalc.heat_index(tmpf, rh, mask_undefined=True)
wcht = mcalc.windchill(tmpf, smps, mask_undefined=True)
# Where heat index is masked, replace with temperature
hidx[hidx.mask] = tmpf[hidx.mask]
# where ever wcht is not masked, replace with wind chill
hidx[~ wcht.mask] = wcht[~ wcht.mask]
return hidx
def mcalc_feelslike(tmpf, dwpf, smps):
"""Compute a feels like temperature
Args:
temperature (temperature): The dry bulb temperature
dewpoint (temperature): The dew point temperature
speed (speed): the wind speed
Returns:
temperature (temperature): The feels like temperature
"""
rh = mcalc.relative_humidity_from_dewpoint(tmpf, dwpf)
hidx = mcalc.heat_index(tmpf, rh, mask_undefined=True)
wcht = mcalc.windchill(tmpf, smps, mask_undefined=True)
# Where heat index is masked, replace with temperature
hidx[hidx.mask] = tmpf[hidx.mask]
# where ever wcht is not masked, replace with wind chill
hidx[~wcht.mask] = wcht[~wcht.mask]
return hidx
def main():
"""Go Main Go."""
pgconn = get_dbconn('asos')
df = read_sql("""
SELECT date_trunc('hour', valid) as ts, avg(tmpf) as temp,
avg(dwpf) as dew from alldata where station = 'DSM'
and extract(month from valid) in (5, 6, 7, 8) and tmpf is not null
and dwpf is not null GROUP by ts
""", pgconn, index_col=None)
df['relh'] = relative_humidity_from_dewpoint(
df['temp'].values * units('degF'),
df['dew'].values * units('degF')
) * 100.
df['year'] = df['ts'].dt.year
counts = df[['year', 'relh']].groupby('year').count()
df2 = df[df['relh'] >= 87.]
counts2 = df2[['year', 'relh']].groupby('year').count()
counts['hits'] = counts2['relh']
counts['freq'] = counts['hits'] / counts['relh']
counts.to_csv('test.csv')
def calc(self):
"""Compute things not usually computed"""
if self.data["relh"] is None and None not in [
self.data["tmpf"],
self.data["dwpf"],
]:
self.data["relh"] = bounded(
mcalc.relative_humidity_from_dewpoint(
self.data["tmpf"] * munits.degF,
self.data["dwpf"] * munits.degF,
).to(munits.percent).magnitude,
0.5,
100.5,
)
if (self.data["dwpf"] is None
and None not in [self.data["tmpf"], self.data["relh"]]
and self.data["relh"] >= 1 and self.data["relh"] <= 100):
self.data["dwpf"] = bounded(
mcalc.dewpoint_from_relative_humidity(
self.data["tmpf"] * munits.degF,
self.data["relh"] * munits.percent,
).to(munits.degF).magnitude,
-100.0,
100.0,
)
if self.data["feel"] is None and None not in [
self.data["tmpf"],
self.data["relh"],
self.data["sknt"],
]:
self.data["feel"] = bounded(
mcalc.apparent_temperature(
self.data["tmpf"] * munits.degF,
self.data["relh"] * munits.percent,
self.data["sknt"] * munits.knots,
).to(munits.degF).magnitude,
-150.0,
200.0,
)
def calc(self):
"""Compute things not usually computed"""
if (self.data['relh'] is None
and None not in [self.data['tmpf'], self.data['dwpf']]):
self.data['relh'] = bounded(
mcalc.relative_humidity_from_dewpoint(
self.data['tmpf'] * munits.degF, self.data['dwpf'] *
munits.degF).to(munits.percent).magnitude, 0.5, 100.5)
if (self.data['dwpf'] is None
and None not in [self.data['tmpf'], self.data['relh']]
and self.data['relh'] >= 1 and self.data['relh'] <= 100):
self.data['dwpf'] = bounded(
mcalc.dewpoint_rh(self.data['tmpf'] * munits.degF,
self.data['relh'] * munits.percent).to(
munits.degF).magnitude, -100., 100.)
if (self.data['feel'] is None and None not in [
self.data['tmpf'], self.data['relh'], self.data['sknt']
]):
self.data['feel'] = bounded(
mcalc.apparent_temperature(
self.data['tmpf'] * munits.degF,
self.data['relh'] * munits.percent, self.data['sknt'] *
munits.knots).to(munits.degF).magnitude, -150., 200.)
def test_relative_humidity_from_dewpoint():
"""Test Relative Humidity calculation."""
assert_almost_equal(
relative_humidity_from_dewpoint(25. * units.degC, 15. * units.degC),
53.80 * units.percent, 2)
def test_relative_humidity_from_dewpoint_xarray():
"""Test Relative Humidity calculation with xarray data arrays."""
temp = xr.DataArray(25., attrs={'units': 'degC'})
dewp = xr.DataArray(15., attrs={'units': 'degC'})
assert_almost_equal(relative_humidity_from_dewpoint(temp, dewp),
53.80 * units.percent, 2)
def test_relative_humidity_from_dewpoint():
"""Test Relative Humidity calculation."""
assert_almost_equal(relative_humidity_from_dewpoint(25. * units.degC, 15. * units.degC),
53.80 * units.percent, 2)
def Station_Synthetical_Forecast_From_Cassandra(
model='ECMWF',
output_dir=None,
t_range=[0,84],
t_gap=3,
points={'lon':[116.3833], 'lat':[39.9]},
initTime=None,
draw_VIS=True,drw_thr=False,
extra_info={
'output_head_name':' ',
'output_tail_name':' ',
'point_name':' '}
):
#+get all the directories needed
try:
dir_rqd=[
"ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_3_HOURS/",
"ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_6_HOURS/",
"ECMWF_HR/TCDC/",
"ECMWF_HR/LCDC/",
"ECMWF_HR/UGRD_100M/",
"ECMWF_HR/VGRD_100M/",
"NWFD_SCMOC/VIS/",
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='RAIN03'),
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='RAIN06'),
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='T2m'),
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='u10m'),
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='v10m'),
]
except KeyError:
raise ValueError('Can not find all required directories needed')
try:
dir_opt=[
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='Td2m')
]
name_opt=['Td2m']
except:
dir_opt=[
utl.Cassandra_dir(data_type='surface',data_source=model,var_name='rh2m')
]
name_opt=['rh2m']
#+get all the directories needed
if(initTime == None):
last_file={model:get_latest_initTime(dir_rqd[0]),
'SCMOC':get_latest_initTime(dir_rqd[6]),
}
else:
last_file={model:initTime[0],
'SCMOC':initTime[1],
}
y_s={model:int('20'+last_file[model][0:2]),
'SCMOC':int('20'+last_file['SCMOC'][0:2])}
m_s={model:int(last_file[model][2:4]),
'SCMOC':int(last_file['SCMOC'][2:4])}
d_s={model:int(last_file[model][4:6]),
'SCMOC':int(last_file['SCMOC'][4:6])}
h_s={model:int(last_file[model][6:8]),
'SCMOC':int(last_file['SCMOC'][6:8])}
fhours = np.arange(t_range[0], t_range[1], t_gap)
for ifhour in fhours:
if (ifhour == fhours[0] ):
time_all=datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour))
else:
time_all=np.append(time_all,datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour)))
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
t2m=utl.get_model_points_gy(dir_rqd[9], filenames, points,allExists=False)
if(name_opt[0] == 'rh2m'):
rh2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False)
Td2m=mpcalc.dewpoint_rh(t2m['data'].values*units('degC'),rh2m['data'].values/100.)
p_vapor=(rh2m['data'].values/100.)*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values))))
if(name_opt[0] == 'Td2m'):
Td2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False)
rh2m=mpcalc.relative_humidity_from_dewpoint(t2m['data'].values* units('degC'),
Td2m['data'].values* units('degC'))
p_vapor=(np.array(rh2m))*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values))))
Td2m=np.array(Td2m['data'].values)* units('degC')
u10m=utl.get_model_points_gy(dir_rqd[10], filenames, points,allExists=False)
v10m=utl.get_model_points_gy(dir_rqd[11], filenames, points,allExists=False)
wsp10m=(u10m['data']**2+v10m['data']**2)**0.5
AT=1.07*t2m['data'].values+0.2*p_vapor-0.65*wsp10m-2.7
if((t_range[1]) > 72):
fhours = np.arange(6, t_range[1], 6)
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
r03=utl.get_model_points_gy(dir_rqd[8], filenames, points,allExists=False)
else:
r03=utl.get_model_points_gy(dir_rqd[7], filenames, points,allExists=False)
fhours = np.arange(t_range[0], t_range[1], t_gap)
filenames = [last_file['SCMOC']+'.'+str(fhour).zfill(3) for fhour in fhours]
VIS=utl.get_model_points_gy(dir_rqd[6], filenames, points,allExists=False,fill_null=True,Null_value=-0.001)
if(last_file['SCMOC'] == last_file[model] and t_range[1] > 72):
fhours = np.append(np.arange(3,72,3),np.arange(72, (t_range[1]), 6))
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] != last_file[model] and t_range[1] > 60):
fhours = np.append(np.arange(3,60,3),np.arange(60, (t_range[1]), 6))
filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] != last_file[model] and t_range[1] <= 60):
fhours = np.arange(t_range[0], t_range[1], t_gap)
filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] == last_file[model] and t_range[1] <= 72):
fhours = np.arange(t_range[0], t_range[1], t_gap)
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
TCDC=utl.get_model_points_gy(dir_rqd[2], filenames2, points,allExists=False)
LCDC=utl.get_model_points_gy(dir_rqd[3], filenames2, points,allExists=False)
u100m=utl.get_model_points_gy(dir_rqd[4], filenames2, points,allExists=False)
v100m=utl.get_model_points_gy(dir_rqd[5], filenames2, points,allExists=False)
wsp100m=(u100m['data']**2+v100m['data']**2)**0.5
if(fhours[-1] < 120):
gust10m=utl.get_model_points_gy(dir_rqd[0], filenames, points,allExists=False)
if(fhours[-1] > 120):
if(last_file['SCMOC'] == last_file[model]):
fhours = np.arange(0, t_range[1], 6)
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] != last_file[model]):
fhours = np.arange(0, t_range[1], 6)
filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours]
gust10m=utl.get_model_points_gy(dir_rqd[1], filenames, points,allExists=False)
sta_graphics.draw_Station_Synthetical_Forecast_From_Cassandra(
t2m=t2m,Td2m=Td2m,AT=AT,u10m=u10m,v10m=v10m,u100m=u100m,v100m=v100m,
gust10m=gust10m,wsp10m=wsp10m,wsp100m=wsp100m,r03=r03,TCDC=TCDC,LCDC=LCDC,
draw_VIS=draw_VIS,VIS=VIS,drw_thr=drw_thr,
time_all=time_all,
model=model,points=points,
output_dir=output_dir,extra_info=extra_info)
def main():
### START OF USER SETTINGS BLOCK ###
# FILE/DATA SETTINGS
# file path to input
datafile = '/home/jgodwin/python/sfc_observations/surface_observations.txt'
timefile = '/home/jgodwin/python/sfc_observations/validtime.txt'
# MAP SETTINGS
# map names (for tracking purposes)
maps = ['CONUS','Texas','Floater 1']
restart_domain = [True,False]
# map boundaries
west = [-120,-108,-108]
east = [-70,-93,-85]
south = [20,25,37]
north = [50,38,52]
# OUTPUT SETTINGS
# save directory for output
savedir = '/var/www/html/images/'
# filenames ("_[variable].png" will be appended, so only a descriptor like "conus" is needed)
savenames = ['conus','texas','floater1']
# TEST MODE SETTINGS
test = False
testnum = 3
### END OF USER SETTINGS BLOCK ###
for i in range(len(maps)):
if test and i != testnum:
continue
print(maps[i])
# create the map projection
cenlon = (west[i] + east[i]) / 2.0
cenlat = (south[i] + north[i]) / 2.0
sparallel = cenlat
if cenlat > 0:
cutoff = -30
flip = False
elif cenlat < 0:
cutoff = 30
flip = True
if restart_domain:
to_proj = ccrs.LambertConformal(central_longitude=cenlon,central_latitude=cenlat,standard_parallels=[sparallel],cutoff=cutoff)
# open the data
vt = open(timefile).read()
with open(datafile) as f:
data = pd.read_csv(f,header=0,names=['siteID','lat','lon','elev','slp','temp','sky','dpt','wx','wdr',\
'wsp'],na_values=-99999)
# filter data by lat/lon
data = data[(data['lat'] >= south[i]-2.0) & (data['lat'] <= north[i]+2.0) & (data['lon'] >= west[i]-2.0)\
& (data['lon'] <= east[i]+2.0)]
# remove questionable data
data = data[(cToF(data['temp']) <= 120) & (cToF(data['dpt']) <= 80)]
# project lat/lon onto final projection
print("Creating map projection.")
lon = data['lon'].values
lat = data['lat'].values
xp, yp, _ = to_proj.transform_points(ccrs.Geodetic(), lon, lat).T
# remove missing data from pressure and interpolate
# we'll give this a try and see if it can help with my CPU credit problem
if restart_domain:
print("Performing Cressman interpolation.")
x_masked, y_masked, pres = remove_nan_observations(xp, yp, data['slp'].values)
slpgridx, slpgridy, slp = interpolate_to_grid(x_masked, y_masked, pres, interp_type='cressman',
minimum_neighbors=1, search_radius=400000,
hres=100000)
# get wind information and remove missing data
wind_speed = (data['wsp'].values * units('knots'))
wind_dir = data['wdr'].values * units.degree
good_indices = np.where((~np.isnan(wind_dir)) & (~np.isnan(wind_speed)))
x_masked = xp[good_indices]
y_masked = yp[good_indices]
wind_speed = wind_speed[good_indices]
wind_dir = wind_dir[good_indices]
u, v = wind_components(wind_speed, wind_dir)
windgridx, windgridy, uwind = interpolate_to_grid(x_masked, y_masked, np.array(u),
interp_type='cressman', search_radius=400000,
hres=100000)
_, _, vwind = interpolate_to_grid(x_masked, y_masked, np.array(v), interp_type='cressman',
search_radius=400000, hres=100000)
# get temperature information
data['temp'] = cToF(data['temp'])
x_masked, y_masked, t = remove_nan_observations(xp, yp, data['temp'].values)
tempx, tempy, temp = interpolate_to_grid(x_masked, y_masked, t, interp_type='cressman',
minimum_neighbors=3, search_radius=200000, hres=18000)
temp = np.ma.masked_where(np.isnan(temp), temp)
# get dewpoint information
data['dpt'] = cToF(data['dpt'])
x_masked,y_masked,td = remove_nan_observations(xp,yp,data['dpt'].values)
dptx,dpty,dewp = interpolate_to_grid(x_masked,y_masked,td,interp_type='cressman',\
minimum_neighbors=3,search_radius=200000,hres=18000)
dewp = np.ma.masked_where(np.isnan(dewp),dewp)
# interpolate wind speed
x_masked,y_masked,wspd = remove_nan_observations(xp,yp,data['wsp'].values)
wspx,wspy,speed = interpolate_to_grid(x_masked,y_masked,wspd,interp_type='cressman',\
minimum_neighbors=3,search_radius=200000,hres=18000)
speed = np.ma.masked_where(np.isnan(speed),speed)
# derived values
# station pressure
data['pres'] = stationPressure(data['slp'],data['elev'])
# theta-E
data['thetae'] = equivalent_potential_temperature(data['pres'].values*units.hPa,data['temp'].values*units.degF,data['dpt'].values*units.degF)
x_masked,y_masked,thetae = remove_nan_observations(xp,yp,data['thetae'].values)
thex,they,thte = interpolate_to_grid(x_masked,y_masked,thetae,interp_type='cressman',\
minimum_neighbors=3,search_radius=200000,hres=18000)
thte = np.ma.masked_where(np.isnan(thte),thte)
# mixing ratio
relh = relative_humidity_from_dewpoint(data['temp'].values*units.degF,data['dpt'].values*units.degF)
mixr = mixing_ratio_from_relative_humidity(relh,data['temp'].values*units.degF,data['pres'].values*units.hPa) * 1000.0
x_masked,y_masked,mixrat = remove_nan_observations(xp,yp,mixr)
mrx,mry,mrat = interpolate_to_grid(x_masked,y_masked,mixrat,interp_type='cressman',\
minimum_neighbors=3,search_radius=200000,hres=18000)
mrat = np.ma.masked_where(np.isnan(mrat),mrat)
# set up the state borders
state_boundaries = cfeature.NaturalEarthFeature(category='cultural',\
name='admin_1_states_provinces_lines',scale='50m',facecolor='none')
# SCALAR VARIABLES TO PLOT
# variable names (will appear in plot title)
variables = ['Temperature','Dewpoint','Wind Speed','Theta-E','Mixing Ratio']
# units (for colorbar label)
unitlabels = ['F','F','kt','K','g/kg']
# list of actual variables to plot
vardata = [temp,dewp,speed,thte,mrat]
# tag in output filename
varplots = ['temp','dewp','wspd','thte','mrat']
# levels: (lower,upper,step)
levs = [[-20,105,5],[30,85,5],[0,70,5],[250,380,5],[0,22,2]]
# colormaps
colormaps = ['hsv_r','Greens','plasma','hsv_r','Greens']
for j in range(len(variables)):
print("\t%s" % variables[j])
fig = plt.figure(figsize=(20, 10))
view = fig.add_subplot(1, 1, 1, projection=to_proj)
# set up the map and plot the interpolated grids
levels = list(range(levs[j][0],levs[j][1],levs[j][2]))
cmap = plt.get_cmap(colormaps[j])
norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True)
labels = variables[j] + " (" + unitlabels[j] + ")"
# add map features
view.set_extent([west[i],east[i],south[i],north[i]])
view.add_feature(state_boundaries,edgecolor='black')
view.add_feature(cfeature.OCEAN,zorder=-1)
view.add_feature(cfeature.COASTLINE,zorder=2)
view.add_feature(cfeature.BORDERS, linewidth=2,edgecolor='black')
# plot the sea-level pressure
cs = view.contour(slpgridx, slpgridy, slp, colors='k', levels=list(range(990, 1034, 4)))
view.clabel(cs, inline=1, fontsize=12, fmt='%i')
# plot the scalar background
mmb = view.pcolormesh(tempx, tempy, vardata[j], cmap=cmap, norm=norm)
fig.colorbar(mmb, shrink=.4, orientation='horizontal', pad=0.02, boundaries=levels, \
extend='both',label=labels)
# plot the wind barbs
view.barbs(windgridx, windgridy, uwind, vwind, alpha=.4, length=5,flip_barb=flip)
# plot title and save
view.set_title('%s (shaded), SLP, and Wind (valid %s)' % (variables[j],vt))
plt.savefig('/var/www/html/images/%s_%s.png' % (savenames[i],varplots[j]),bbox_inches='tight')
# close everything
fig.clear()
view.clear()
plt.close(fig)
f.close()
print("Script finished.")
def test_relative_humidity_from_dewpoint_with_f():
"""Test Relative Humidity accepts temperature in Fahrenheit."""
assert_almost_equal(relative_humidity_from_dewpoint(70. * units.degF, 55. * units.degF),
58.935 * units.percent, 3)
def do(ts):
"""Process this date timestamp"""
asos = get_dbconn('asos', user='******')
iemaccess = get_dbconn('iem')
icursor = iemaccess.cursor()
df = read_sql("""
select station, network, iemid, drct, sknt,
valid at time zone tzname as localvalid,
tmpf, dwpf from
alldata d JOIN stations t on (t.id = d.station)
where (network ~* 'ASOS' or network = 'AWOS')
and valid between %s and %s and t.tzname is not null
and date(valid at time zone tzname) = %s
ORDER by valid ASC
""", asos, params=(ts - datetime.timedelta(days=2),
ts + datetime.timedelta(days=2),
ts.strftime("%Y-%m-%d")), index_col=None)
# derive some parameters
df['relh'] = mcalc.relative_humidity_from_dewpoint(
df['tmpf'].values * munits.degF,
df['dwpf'].values * munits.degF).to(munits.percent)
df['feel'] = mcalc.apparent_temperature(
df['tmpf'].values * munits.degF,
df['relh'].values * munits.percent,
df['sknt'].values * munits.knots
)
df['u'], df['v'] = mcalc.get_wind_components(
df['sknt'].values * munits.knots,
df['drct'].values * munits.deg
)
df['localvalid_lag'] = df.groupby('iemid')['localvalid'].shift(1)
df['timedelta'] = df['localvalid'] - df['localvalid_lag']
ndf = df[pd.isna(df['timedelta'])]
df.loc[ndf.index.values, 'timedelta'] = pd.to_timedelta(
ndf['localvalid'].dt.hour * 3600. +
ndf['localvalid'].dt.minute * 60., unit='s'
)
df['timedelta'] = df['timedelta'] / np.timedelta64(1, 's')
table = "summary_%s" % (ts.year,)
for iemid, gdf in df.groupby('iemid'):
if len(gdf.index) < 6:
# print(" Quorum not meet for %s" % (gdf.iloc[0]['station'], ))
continue
ldf = gdf.copy()
ldf.interpolate(inplace=True)
totsecs = ldf['timedelta'].sum()
avg_rh = clean((ldf['relh'] * ldf['timedelta']).sum() / totsecs, 1,
100)
min_rh = clean(ldf['relh'].min(), 1, 100)
max_rh = clean(ldf['relh'].max(), 1, 100)
uavg = (ldf['u'] * ldf['timedelta']).sum() / totsecs
vavg = (ldf['u'] * ldf['timedelta']).sum() / totsecs
drct = clean(
mcalc.get_wind_dir(uavg * munits.knots, vavg * munits.knots),
0, 360)
avg_sknt = clean(
(ldf['sknt'] * ldf['timedelta']).sum() / totsecs, 0, 150 # arb
)
max_feel = clean(ldf['feel'].max(), -150, 200)
avg_feel = clean(
(ldf['feel'] * ldf['timedelta']).sum() / totsecs, -150, 200
)
min_feel = clean(ldf['feel'].min(), -150, 200)
def do_update():
"""Inline updating"""
icursor.execute("""
UPDATE """ + table + """
SET avg_rh = %s, min_rh = %s, max_rh = %s,
avg_sknt = %s, vector_avg_drct = %s,
min_feel = %s, avg_feel = %s, max_feel = %s
WHERE
iemid = %s and day = %s
""", (avg_rh, min_rh, max_rh, avg_sknt, drct,
min_feel, avg_feel, max_feel,
iemid, ts))
do_update()
if icursor.rowcount == 0:
print(('compute_daily Adding %s for %s %s %s'
) % (table, gdf.iloc[0]['station'], gdf.iloc[0]['network'],
ts))
icursor.execute("""
INSERT into """ + table + """
(iemid, day) values (%s, %s)
""", (iemid, ts))
do_update()
icursor.close()
iemaccess.commit()
iemaccess.close()
def plotter(fdict):
""" Go """
pgconn = get_dbconn("asos")
ctx = get_autoplot_context(fdict, get_description())
station = ctx["zstation"]
highlightyear = ctx["year"]
sdate = datetime.date(ctx["syear"], 1, 1)
edate = datetime.date(ctx["eyear"] + 1, 1, 1)
ytd = ctx["ytd"]
varname = ctx["var"]
inc = ctx["inc"]
doylimiter = get_doylimit(ytd, varname)
tmpflimit = "and tmpf >= 50" if varname != "windchill" else "and tmpf < 50"
if varname not in ["windchill", "heatindex"]:
tmpflimit = ""
df = read_sql(
"SELECT to_char(valid, 'YYYYmmddHH24') as d, avg(tmpf)::int as tmpf, "
"avg(dwpf)::int as dwpf, avg(coalesce(sknt, 0)) as sknt "
f"from alldata WHERE station = %s {tmpflimit} "
"and dwpf <= tmpf and valid > %s and valid < %s and report_type = 2 "
f"{doylimiter} GROUP by d",
pgconn,
params=(station, sdate, edate),
index_col=None,
)
if df.empty:
raise NoDataFound("No Data Found.")
df["year"] = df["d"].apply(lambda x: int(x[:4]))
df2 = df
title2 = VDICT[varname]
compop = np.greater_equal
inctitle = ""
if varname == "heatindex":
df["heatindex"] = (heat_index(
df["tmpf"].values * units("degF"),
relative_humidity_from_dewpoint(
df["tmpf"].values * units("degF"),
df["dwpf"].values * units("degF"),
),
).to(units("degF")).m)
inctitle = " [All Obs Included]"
if inc == "no":
df2 = df[df["heatindex"] > df["tmpf"]]
inctitle = " [Only Additive]"
else:
df2 = df
maxval = int(df2["heatindex"].max() + 1)
LEVELS[varname] = np.arange(80, maxval)
elif varname == "windchill":
compop = np.less_equal
df["year"] = df["d"].apply(lambda x: (int(x[:4]) - 1)
if int(x[4:6]) < 7 else int(x[:4]))
df["windchill"] = (windchill(
df["tmpf"].values * units("degF"),
df["sknt"].values * units("knot"),
).to(units("degF")).m)
inctitle = " [All Obs Included]"
if inc == "no":
df2 = df[df["windchill"] < df["tmpf"]]
inctitle = " [Only Additive]"
else:
df2 = df
minval = int(df2["windchill"].min() - 1)
LEVELS[varname] = np.arange(minval, minval + 51)
else:
maxval = int(df2[varname].max() + 1)
LEVELS[varname] = np.arange(maxval - 31, maxval)
bs = ctx["_nt"].sts[station]["archive_begin"]
if bs is None:
raise NoDataFound("Unknown station metadata.")
minyear = df["year"].min()
maxyear = df["year"].max()
years = float((maxyear - minyear) + 1)
x = []
y = []
y2 = []
fig = plt.figure(figsize=(9, 6))
ax = fig.add_axes([0.1, 0.1, 0.6, 0.8])
yloc = 1.0
xloc = 1.13
yrlabel = ("%s" %
(highlightyear, ) if varname != "windchill" else "%s-%s" %
(highlightyear, highlightyear + 1))
ax.text(xloc + 0.08,
yloc + 0.04,
"Avg:",
transform=ax.transAxes,
color="b")
ax.text(xloc + 0.21,
yloc + 0.04,
yrlabel,
transform=ax.transAxes,
color="r")
df3 = df2[df2["year"] == highlightyear]
for level in LEVELS[varname]:
x.append(level)
y.append(len(df2[compop(df2[varname], level)]) / years)
y2.append(len(df3[compop(df3[varname], level)]))
if level % 2 == 0:
ax.text(xloc, yloc, "%s" % (level, ), transform=ax.transAxes)
ax.text(
xloc + 0.08,
yloc,
"%.1f" % (y[-1], ),
transform=ax.transAxes,
color="b",
)
ax.text(
xloc + 0.21,
yloc,
"%.0f" % (y2[-1], ),
transform=ax.transAxes,
color="r",
)
yloc -= 0.04
ax.text(xloc, yloc, "n=%s" % (len(df2.index), ), transform=ax.transAxes)
for x0, y0, y02 in zip(x, y, y2):
ax.plot([x0, x0], [y0, y02], color="k")
rdf = pd.DataFrame({"level": x, "avg": y, "d%s" % (highlightyear, ): y2})
x = np.array(x, dtype=np.float64)
ax.scatter(x, y, color="b", label="Avg")
ax.scatter(x, y2, color="r", label=yrlabel)
ax.grid(True)
ymax = int(max([max(y), max(y2)]))
ax.set_xlim(x[0] - 0.5, x[-1] + 0.5)
dy = 24 * (int(ymax / 240) + 1)
ax.set_yticks(range(0, ymax, dy))
ax.set_ylim(-0.5, ymax + 5)
ax2 = ax.twinx()
ax2.set_ylim(-0.5, ymax + 5)
ax2.set_yticks(range(0, ymax, dy))
ax2.set_yticklabels(["%.0f" % (s, ) for s in np.arange(0, ymax, dy) / 24])
ax2.set_ylabel("Expressed in 24 Hour Days")
ax.set_ylabel("Hours Per Year")
ax.set_xlabel(r"%s $^\circ$F" % (VDICT[varname], ))
title = "till %s" % (datetime.date.today().strftime("%-d %b"), )
title = "Entire Year" if ytd == "no" else title
ax.set_title(("[%s] %s %s-%s\n"
"%s Histogram (%s)%s") % (
station,
ctx["_nt"].sts[station]["name"],
minyear,
maxyear,
title2,
title,
inctitle,
))
ax.legend(loc="best", scatterpoints=1)
return fig, rdf
def get_weather(self):
#------------------------------------------------------
#------------------------------------------------------
# Load TMY2
if self.file_ext == TMY2EXT:
f = open(self.weatherpath + self.city + self.file_ext, 'r')
# Header read for lat and lon
head = f.readline()
self.lat = int(head[39:41]) + int(head[42:44]) / 60.0
self.lon = int(head[47:50]) + int(head[51:53]) / 60.0
line = f.readline()
ind = 0
while line:
# Process the line
self.tothor[ind] = float(
line[17:21]) #Total horizontal solar Wh/m2
self.dirnorm[ind] = float(
line[23:27]) #Direct normal solar Wh/m2
self.difhor[ind] = float(
line[29:33]) #Diffuse Horizontal Solar Wh/m2
self.tdry[ind] = float(line[67:71]) * 0.1
#tdrybulb (deg C)
self.rhs[ind] = float(line[79:82]) * 0.01
#relative humidity (%)
self.tdew[ind] = float(line[73:77]) * 0.1
#tdew (deg C) to conform with TB code
self.press[ind] = float(line[84:88])
#atmospheric pressure (mbar) mb = 100 Pascals
#self.wind_speed[ind] = float(line[95:98]) * 0.1; #windspeed m/s
#self.wind_dir[ind] = float(line[90:93]); #wind direction azimuth
#self.cloud[ind] = float(line[59:61])/10.0; #Could cover fraction
#wd.ocloud = getint(line,63,2)/10.0; #Opaque cloud cover fraction
#wd.ceilht = getint(line,106,5); #Cloud ceiling height m
# Calculate specfic humidity from dry bulb, dew point, and atm pressure using MetPy
self.huss[ind] = mpcalc.specific_humidity_from_mixing_ratio(
mpcalc.mixing_ratio_from_relative_humidity(
mpcalc.relative_humidity_from_dewpoint(
self.tdry[ind] * units.degC,
self.tdew[ind] * units.degC),
self.tdry[ind] * units.degC,
self.press[ind] * units.mbar))
#Next line
line = f.readline()
ind = ind + 1
f.close()
#------------------------------------------------------
#------------------------------------------------------
# Load TMY3
elif self.file_ext == TMY3EXT:
f = open(
self.weatherpath + TMY3NUMBER[CITY.index(self.city)] +
self.file_ext, 'r')
# Header read for lat and lon
head = f.readline().split(',')
self.lat = float(head[4])
self.lon = float(head[5])
#Burn a line for the second part of the header.
line = f.readline()
line = f.readline()
ind = 0
while line:
line = line.split(',')
if len(line) < 20:
print('line is short!')
# Process the line
self.tothor[ind] = float(
line[4]) #Total horizontal solar Wh/m2
self.dirnorm[ind] = float(line[7]) #Direct normal solar Wh/m2
self.difhor[ind] = float(
line[10]) #Diffuse Horizontal Solar Wh/m2
self.tdry[ind] = float(line[31])
#tdrybulb (deg C)
self.rhs[ind] = float(line[37]) * 0.01
#relative humidity (%)
self.tdew[ind] = float(line[34])
#tdew (deg C) to conform with TB code
self.press[ind] = float(line[40])
#atmospheric pressure (mbar) mb = 100 Pascals
#self.wind_speed[ind] = float(line[46]); #windspeed m/s
#self.wind_dir[ind] = float(line[43]); #wind direction azimuth
# Calculate specfic humidity from dry bulb, dew point, and atm pressure using MetPy
self.huss[ind] = mpcalc.specific_humidity_from_mixing_ratio(
mpcalc.mixing_ratio_from_relative_humidity(
mpcalc.relative_humidity_from_dewpoint(
self.tdry[ind] * units.degC,
self.tdew[ind] * units.degC),
self.tdry[ind] * units.degC,
self.press[ind] * units.mbar))
#Next line
line = f.readline()
ind = ind + 1
f.close()
def append_cfs(res, lon, lat):
"""Append on needed CFS data."""
gridx, gridy = find_ij(lon, lat)
lastyear = max(res["data"].keys())
thisyear = datetime.date.today().year
lastdate = datetime.date(thisyear, 8, 31)
if lastyear != thisyear:
# We don't have any data yet for this year, so we add some
res["data"][thisyear] = {"dates": [], "high": [], "low": [], "rh": []}
else:
# shrug
if res["data"][lastyear]["dates"]:
lastdate = datetime.datetime.strptime(
res["data"][thisyear]["dates"][-1], "%Y-%m-%d"
).date()
# go find the most recent CFS 0z file
valid = datetime.date.today()
attempt = 0
while True:
testfn = valid.strftime("/mesonet/data/iemre/cfs_%Y%m%d00.nc")
if os.path.isfile(testfn):
break
valid -= datetime.timedelta(hours=24)
attempt += 1
if attempt > 9:
return None
try:
nc = ncopen(testfn, timeout=NCOPEN_TIMEOUT)
except Exception as exp:
LOG.error(exp)
return None
if nc is None:
LOG.debug("Failing %s as nc is None", testfn)
return None
high = (
masked_array(nc.variables["high_tmpk"][:, gridy, gridx], units.degK)
.to(units.degF)
.m
)
low = (
masked_array(nc.variables["low_tmpk"][:, gridy, gridx], units.degK)
.to(units.degF)
.m
)
# RH hack
# found ~20% bias with this value, so arb addition for now
rh = (
relative_humidity_from_dewpoint(
masked_array(high, units.degF), masked_array(low, units.degF)
).m
* 100.0
+ 20.0
)
rh = np.where(rh > 95, 95, rh)
entry = res["data"][thisyear]
# lastdate is either August 31 or a date after, so our first forecast
# date is i+1
tidx = daily_offset(lastdate + datetime.timedelta(days=1))
for i in range(tidx, 365):
lts = datetime.date(thisyear, 1, 1) + datetime.timedelta(days=i)
if lts.month in [9, 10, 11]:
entry["dates"].append(lts.strftime("%Y-%m-%d"))
entry["high"].append(_i(high[i]))
entry["low"].append(_i(low[i]))
entry["rh"].append(_i(rh[i]))
return res
#Load wind gust data
da_wg = xr.open_dataset("/g/data/ma05/" + model +
"/v1/forecast/spec/max_wndgust10m/" + year + "/" +
month + "/max_wndgust10m-fc-spec-PT1H-" + model +
"-v1-" + date + ".sub.nc").sel({
"time":
dt.datetime.strptime(valid, "%Y-%m-%d %H:%M") +
dt.timedelta(hours=wg_lead)
}).max_wndgust10m
#Calculate a few thermodynamic variables
dp = get_dp(hur=da_rh, ta=da_ta, dp_mask=False)
ta_unit = units.units.degC * da_ta
dp_unit = units.units.degC * dp
p_unit = units.units.hectopascals * p_3d
hur_unit = mpcalc.relative_humidity_from_dewpoint(ta_unit, dp_unit)*\
100*units.units.percent
q_unit = mpcalc.mixing_ratio_from_relative_humidity(hur_unit,\
ta_unit,p_unit)
#plot
m = Basemap(llcrnrlon=lon.min(),
llcrnrlat=lat.min(),
urcrnrlon=lon.max(),
urcrnrlat=lat.max(),
projection="cyl",
resolution="h")
#omega = wrf.omega(q_unit, ta_unit.to("K"), da_w.values, p_3d*100)
#omega = omega.assign_coords(dim_0=p, dim_2=lon, dim_1=lat)
#omega = xr.where((da_z >= 1000) & (da_z <= 3000), omega, np.nan)
#c = xr.plot.contour(omega.mean("dim_0"), levels=[10,20,50], colors=["#d0d1e6","#3690c0","#034e7b"])
xsect = radisonde_cross_section(stn_list, df)
######################################################################
# Calculate Variables for Plotting
# --------------------------------
#
# Use MetPy to calculate common variables for plotting a cross section,
# specifically potential temperature and mixing ratio
#
potemp = mpcalc.potential_temperature(
xsect['p_grid'] * units('hPa'),
xsect['grid_data']['temperature'] * units('degC'))
relhum = mpcalc.relative_humidity_from_dewpoint(
xsect['grid_data']['temperature'] * units('degC'),
xsect['grid_data']['dewpoint'] * units('degC'))
mixrat = mpcalc.mixing_ratio_from_relative_humidity(
relhum, xsect['grid_data']['temperature'] * units('degC'),
xsect['p_grid'] * units('hPa'))
######################################################################
# Plot Cross Section
# ------------------
#
# Use standard Matplotlib to plot the now 2D cross section grid using the
# data from xsect and those calculated above. Additionally, the actualy
# radiosonde wind observations are plotted as barbs on this plot.
#
def test_relative_humidity_from_dewpoint_with_f():
"""Test Relative Humidity accepts temperature in Fahrenheit."""
assert_almost_equal(
relative_humidity_from_dewpoint(70. * units.degF, 55. * units.degF),
58.935 * units.percent, 3)
def plotter(fdict):
""" Go """
pgconn = get_dbconn('asos')
ctx = get_autoplot_context(fdict, get_description())
station = ctx['zstation']
network = ctx['network']
nt = NetworkTable(network)
year = ctx['year']
varname = ctx['var']
df = read_sql("""
SELECT extract(year from valid) as year,
coalesce(mslp, alti * 33.8639, 1013.25) as slp,
extract(doy from valid) as doy, tmpf, dwpf from alldata
where station = %s and dwpf > -50 and dwpf < 90 and
tmpf > -50 and tmpf < 120 and valid > '1950-01-01'
and report_type = 2
""",
pgconn,
params=(station, ),
index_col=None)
# compute RH
df['relh'] = mcalc.relative_humidity_from_dewpoint(
df['tmpf'].values * units('degF'), df['dwpf'].values * units('degF'))
# saturation vapor pressure
# Convert sea level pressure to station pressure
df['pressure'] = mcalc.add_height_to_pressure(
df['slp'].values * units('millibars'),
nt.sts[station]['elevation'] * units('m')).to(units('millibar'))
# Compute the relative humidity
df['relh'] = mcalc.relative_humidity_from_dewpoint(
df['tmpf'].values * units('degF'), df['dwpf'].values * units('degF'))
# Compute the mixing ratio
df['mixing_ratio'] = mcalc.mixing_ratio_from_relative_humidity(
df['relh'].values, df['tmpf'].values * units('degF'),
df['pressure'].values * units('millibars'))
# Compute the saturation mixing ratio
df['saturation_mixingratio'] = mcalc.saturation_mixing_ratio(
df['pressure'].values * units('millibars'),
df['tmpf'].values * units('degF'))
df['vapor_pressure'] = mcalc.vapor_pressure(
df['pressure'].values * units('millibars'),
df['mixing_ratio'].values * units('kg/kg')).to(units('kPa'))
df['saturation_vapor_pressure'] = mcalc.vapor_pressure(
df['pressure'].values * units('millibars'),
df['saturation_mixingratio'].values * units('kg/kg')).to(units('kPa'))
df['vpd'] = df['saturation_vapor_pressure'] - df['vapor_pressure']
dailymeans = df[['year', 'doy', varname]].groupby(['year', 'doy']).mean()
dailymeans = dailymeans.reset_index()
df2 = dailymeans[['doy', varname]].groupby('doy').describe()
dyear = df[df['year'] == year]
df3 = dyear[['doy', varname]].groupby('doy').describe()
df3[(varname, 'diff')] = df3[(varname, 'mean')] - df2[(varname, 'mean')]
(fig, ax) = plt.subplots(2, 1, figsize=(8, 6))
multiplier = 1000. if varname == 'mixing_ratio' else 10.
ax[0].fill_between(df2[(varname, 'min')].index.values,
df2[(varname, 'min')].values * multiplier,
df2[(varname, 'max')].values * multiplier,
color='gray')
ax[0].plot(df2[(varname, 'mean')].index.values,
df2[(varname, 'mean')].values * multiplier,
label="Climatology")
ax[0].plot(df3[(varname, 'mean')].index.values,
df3[(varname, 'mean')].values * multiplier,
color='r',
label="%s" % (year, ))
ax[0].set_title(("%s [%s]\nDaily Mean Surface %s") %
(station, nt.sts[station]['name'], PDICT[varname]))
lbl = ("Mixing Ratio ($g/kg$)"
if varname == 'mixing_ratio' else PDICT[varname])
ax[0].set_ylabel(lbl)
ax[0].set_xlim(0, 366)
ax[0].set_ylim(bottom=0)
ax[0].set_xticks(
(1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 365))
ax[0].set_xticklabels(calendar.month_abbr[1:])
ax[0].grid(True)
ax[0].legend(loc=2, fontsize=10)
cabove = 'b' if varname == 'mixing_ratio' else 'r'
cbelow = 'r' if cabove == 'b' else 'b'
rects = ax[1].bar(df3[(varname, 'diff')].index.values,
df3[(varname, 'diff')].values * multiplier,
facecolor=cabove,
edgecolor=cabove)
for rect in rects:
if rect.get_height() < 0.:
rect.set_facecolor(cbelow)
rect.set_edgecolor(cbelow)
plunits = '$g/kg$' if varname == 'mixing_ratio' else 'hPa'
ax[1].set_ylabel("%.0f Departure (%s)" % (year, plunits))
ax[1].set_xlim(0, 366)
ax[1].set_xticks(
(1, 32, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 365))
ax[1].set_xticklabels(calendar.month_abbr[1:])
ax[1].grid(True)
return fig, df3
def main():
"""Do Something"""
pgconn = get_dbconn("iem")
icursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
fn = find_file()
nc = ncopen(fn, timeout=300)
stations = chartostring(nc.variables["stationId"][:])
providers = chartostring(nc.variables["dataProvider"][:])
names = chartostring(nc.variables["stationName"][:])
tmpk = nc.variables["temperature"][:]
dwpk = nc.variables["dewpoint"][:]
# Set some data bounds to keep mcalc from complaining
dwpk = np.ma.where(
np.ma.logical_or(np.ma.less(dwpk, 200), np.ma.greater(dwpk, 320)),
np.nan,
dwpk,
)
tmpk = np.ma.where(
np.ma.logical_or(np.ma.less(tmpk, 200), np.ma.greater(tmpk, 320)),
np.nan,
tmpk,
)
relh = (mcalc.relative_humidity_from_dewpoint(
tmpk * units.degK, dwpk * units.degK).magnitude * 100.0)
obtime = nc.variables["observationTime"][:]
pressure = nc.variables["stationPressure"][:]
# altimeter = nc.variables["altimeter"][:]
# slp = nc.variables["seaLevelPressure"][:]
drct = nc.variables["windDir"][:]
smps = nc.variables["windSpeed"][:]
gmps = nc.variables["windGust"][:]
# gmps_drct = nc.variables["windDirMax"][:]
pcpn = nc.variables["precipAccum"][:]
rtk1 = nc.variables["roadTemperature1"][:]
rtk2 = nc.variables["roadTemperature2"][:]
rtk3 = nc.variables["roadTemperature3"][:]
rtk4 = nc.variables["roadTemperature4"][:]
subk1 = nc.variables["roadSubsurfaceTemp1"][:]
nc.close()
db = {}
for recnum, provider in enumerate(providers):
this_station = stations[recnum]
if not provider.endswith("DOT") and provider not in MY_PROVIDERS:
continue
name = names[recnum]
if name == "":
continue
if provider == "MesoWest":
# get the network from the last portion of the name
network = name.split()[-1]
if network != "VTWAC":
continue
else:
network = provider2network(provider)
if network is None:
continue
db[this_station] = {}
ticks = obtime[recnum]
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks)
db[this_station]["ts"] = ts.replace(tzinfo=pytz.utc)
db[this_station]["network"] = network
db[this_station]["pres"] = sanity_check(pressure[recnum], 0, 1000000)
db[this_station]["tmpk"] = sanity_check(tmpk[recnum], 200, 330)
db[this_station]["dwpk"] = sanity_check(dwpk[recnum], 200, 330)
db[this_station]["relh"] = relh[recnum]
db[this_station]["drct"] = sanity_check(drct[recnum], -1, 361)
db[this_station]["smps"] = sanity_check(smps[recnum], -1, 200)
db[this_station]["gmps"] = sanity_check(gmps[recnum], -1, 200)
db[this_station]["rtk1"] = sanity_check(rtk1[recnum], 0, 500)
db[this_station]["rtk2"] = sanity_check(rtk2[recnum], 0, 500)
db[this_station]["rtk3"] = sanity_check(rtk3[recnum], 0, 500)
db[this_station]["rtk4"] = sanity_check(rtk4[recnum], 0, 500)
db[this_station]["subk"] = sanity_check(subk1[recnum], 0, 500)
db[this_station]["pday"] = sanity_check(pcpn[recnum], -1, 5000)
for sid in db:
# print("Processing %s[%s]" % (sid, db[sid]['network']))
iem = Observation(sid, db[sid]["network"], db[sid]["ts"])
if db[sid]["tmpk"] is not None:
iem.data["tmpf"] = temperature(db[sid]["tmpk"], "K").value("F")
if db[sid]["dwpk"] is not None:
iem.data["dwpf"] = temperature(db[sid]["dwpk"], "K").value("F")
if db[sid]["relh"] is not None and db[sid]["relh"] is not np.ma.masked:
iem.data["relh"] = float(db[sid]["relh"])
if db[sid]["drct"] is not None:
iem.data["drct"] = db[sid]["drct"]
if db[sid]["smps"] is not None:
iem.data["sknt"] = speed(db[sid]["smps"], "MPS").value("KT")
if db[sid]["gmps"] is not None:
iem.data["gust"] = speed(db[sid]["gmps"], "MPS").value("KT")
if db[sid]["pres"] is not None:
iem.data["pres"] = (float(db[sid]["pres"]) / 100.00) * 0.02952
if db[sid]["rtk1"] is not None:
iem.data["tsf0"] = temperature(db[sid]["rtk1"], "K").value("F")
if db[sid]["rtk2"] is not None:
iem.data["tsf1"] = temperature(db[sid]["rtk2"], "K").value("F")
if db[sid]["rtk3"] is not None:
iem.data["tsf2"] = temperature(db[sid]["rtk3"], "K").value("F")
if db[sid]["rtk4"] is not None:
iem.data["tsf3"] = temperature(db[sid]["rtk4"], "K").value("F")
if db[sid]["subk"] is not None:
iem.data["rwis_subf"] = temperature(db[sid]["subk"],
"K").value("F")
if db[sid]["pday"] is not None:
iem.data["pday"] = round(
distance(db[sid]["pday"], "MM").value("IN"), 2)
if not iem.save(icursor):
print(("MADIS Extract: %s found new station: %s network: %s"
"") % (fn.split("/")[-1], sid, db[sid]["network"]))
subprocess.call("python sync_stations.py %s" % (fn, ), shell=True)
os.chdir("../../dbutil")
subprocess.call("sh SYNC_STATIONS.sh", shell=True)
os.chdir("../ingestors/madis")
print("...done with sync.")
del iem
icursor.close()
pgconn.commit()
pgconn.close()
def test_relative_humidity_from_dewpoint_xarray():
"""Test Relative Humidity calculation with xarray data arrays."""
temp = xr.DataArray(25., attrs={'units': 'degC'})
dewp = xr.DataArray(15., attrs={'units': 'degC'})
assert_almost_equal(relative_humidity_from_dewpoint(temp, dewp), 53.80 * units.percent, 2)
def wind_rh_according_to_4D_data(
initTime=None,
fhour=6,
day_back=0,
model='ECMWF',
sta_fcs={
'lon': [101.82, 101.32, 101.84, 102.23, 102.2681],
'lat': [28.35, 27.91, 28.32, 27.82, 27.8492],
'altitude': [3600, 3034.62, 3240, 1669, 1941.5],
'name': ['健美乡', '项脚乡', '\n锦屏镇', '\n马道镇', 'S9005 ']
},
draw_zd=True,
levels=[1000, 950, 925, 900, 850, 800, 700, 600, 500],
map_ratio=19 / 9,
zoom_ratio=1,
south_China_sea=False,
area='全国',
city=False,
output_dir=None,
bkgd_type='satellite',
data_source='MICAPS'):
# micaps data directory
if (area != '全国'):
south_China_sea = False
# prepare data
if (area != '全国'):
cntr_pnt, zoom_ratio = utl.get_map_area(area_name=area)
cntr_pnt = np.append(np.mean(sta_fcs['lon']), np.mean(sta_fcs['lat']))
map_extent = [0, 0, 0, 0]
map_extent[0] = cntr_pnt[0] - zoom_ratio * 1 * map_ratio
map_extent[1] = cntr_pnt[0] + zoom_ratio * 1 * map_ratio
map_extent[2] = cntr_pnt[1] - zoom_ratio * 1
map_extent[3] = cntr_pnt[1] + zoom_ratio * 1
bkgd_level = utl.cal_background_zoom_ratio(zoom_ratio)
# micaps data directory
if (data_source == 'MICAPS'):
try:
data_dir = [
utl.Cassandra_dir(data_type='high',
data_source=model,
var_name='HGT',
lvl=''),
utl.Cassandra_dir(data_type='high',
data_source=model,
var_name='RH',
lvl=''),
utl.Cassandra_dir(data_type='high',
data_source=model,
var_name='UGRD',
lvl=''),
utl.Cassandra_dir(data_type='high',
data_source=model,
var_name='VGRD',
lvl=''),
utl.Cassandra_dir(data_type='surface',
data_source=model,
var_name='u10m'),
utl.Cassandra_dir(data_type='surface',
data_source=model,
var_name='v10m'),
utl.Cassandra_dir(data_type='surface',
data_source=model,
var_name='Td2m'),
utl.Cassandra_dir(data_type='surface',
data_source=model,
var_name='T2m')
]
except KeyError:
raise ValueError('Can not find all directories needed')
# get filename
if (initTime != None):
filename = utl.model_filename(initTime, fhour)
else:
filename = utl.filename_day_back_model(day_back=day_back,
fhour=fhour)
initTime = filename[0:8]
# retrieve data from micaps server
gh = MICAPS_IO.get_model_3D_grid(directory=data_dir[0][0:-1],
filename=filename,
levels=levels)
if (gh is None):
return
gh['data'].values = gh['data'].values * 10
rh = MICAPS_IO.get_model_3D_grid(directory=data_dir[1][0:-1],
filename=filename,
levels=levels,
allExists=False)
if rh is None:
return
u = MICAPS_IO.get_model_3D_grid(directory=data_dir[2][0:-1],
filename=filename,
levels=levels,
allExists=False)
if u is None:
return
v = MICAPS_IO.get_model_3D_grid(directory=data_dir[3][0:-1],
filename=filename,
levels=levels,
allExists=False)
if v is None:
return
u10m = MICAPS_IO.get_model_grid(directory=data_dir[4],
filename=filename)
if u10m is None:
return
v10m = MICAPS_IO.get_model_grid(directory=data_dir[5],
filename=filename)
if v10m is None:
return
td2m = MICAPS_IO.get_model_grid(directory=data_dir[6],
filename=filename)
if td2m is None:
return
t2m = MICAPS_IO.get_model_grid(directory=data_dir[7],
filename=filename)
if t2m is None:
return
if (draw_zd == True):
validtime = (datetime.strptime('20' + initTime, '%Y%m%d%H') +
timedelta(hours=fhour)).strftime("%Y%m%d%H")
directory_obs = utl.Cassandra_dir(data_type='surface',
data_source='OBS',
var_name='PLOT_ALL')
try:
zd_sta = MICAPS_IO.get_station_data(filename=validtime +
'0000.000',
directory=directory_obs,
dropna=True,
cache=False)
obs_valid = True
except:
zd_sta = MICAPS_IO.get_station_data(directory=directory_obs,
dropna=True,
cache=False)
obs_valid = False
zd_lon = zd_sta['lon'].values
zd_lat = zd_sta['lat'].values
zd_alt = zd_sta['Alt'].values
zd_u, zd_v = mpcalc.wind_components(
zd_sta['Wind_speed_2m_avg'].values * units('m/s'),
zd_sta['Wind_angle_2m_avg'].values * units.deg)
idx_zd = np.where((zd_lon > map_extent[0])
& (zd_lon < map_extent[1])
& (zd_lat > map_extent[2])
& (zd_lat < map_extent[3]))
zd_sm_lon = zd_lon[idx_zd[0]]
zd_sm_lat = zd_lat[idx_zd[0]]
zd_sm_alt = zd_alt[idx_zd[0]]
zd_sm_u = zd_u[idx_zd[0]]
zd_sm_v = zd_v[idx_zd[0]]
if (data_source == 'CIMISS'):
# get filename
if (initTime != None):
filename = utl.model_filename(initTime, fhour, UTC=True)
else:
filename = utl.filename_day_back_model(day_back=day_back,
fhour=fhour,
UTC=True)
try:
# retrieve data from CMISS server
gh = CIMISS_IO.cimiss_model_3D_grid(
data_code=utl.CMISS_data_code(data_source=model,
var_name='GPH'),
init_time_str='20' + filename[0:8],
valid_time=fhour,
levattrs={
'long_name': 'pressure_level',
'units': 'hPa',
'_CoordinateAxisType': '-'
},
fcst_levels=levels,
fcst_ele="GPH",
units='gpm')
if gh is None:
return
rh = CIMISS_IO.cimiss_model_3D_grid(
data_code=utl.CMISS_data_code(data_source=model,
var_name='RHU'),
init_time_str='20' + filename[0:8],
valid_time=fhour,
levattrs={
'long_name': 'pressure_level',
'units': 'hPa',
'_CoordinateAxisType': '-'
},
fcst_levels=levels,
fcst_ele="RHU",
units='%')
if rh is None:
return
u = CIMISS_IO.cimiss_model_3D_grid(
data_code=utl.CMISS_data_code(data_source=model,
var_name='WIU'),
init_time_str='20' + filename[0:8],
valid_time=fhour,
levattrs={
'long_name': 'pressure_level',
'units': 'hPa',
'_CoordinateAxisType': '-'
},
fcst_levels=levels,
fcst_ele="WIU",
units='m/s')
if u is None:
return
v = CIMISS_IO.cimiss_model_3D_grid(
data_code=utl.CMISS_data_code(data_source=model,
var_name='WIV'),
init_time_str='20' + filename[0:8],
valid_time=fhour,
levattrs={
'long_name': 'pressure_level',
'units': 'hPa',
'_CoordinateAxisType': '-'
},
fcst_levels=levels,
fcst_ele="WIV",
units='m/s')
if v is None:
return
if (model == 'ECMWF'):
td2m = CIMISS_IO.cimiss_model_by_time(
'20' + filename[0:8],
valid_time=fhour,
data_code=utl.CMISS_data_code(data_source=model,
var_name='DPT'),
levattrs={
'long_name': 'height_above_ground',
'units': 'm',
'_CoordinateAxisType': '-'
},
fcst_level=0,
fcst_ele="DPT",
units='K')
if td2m is None:
return
t2m = CIMISS_IO.cimiss_model_by_time(
'20' + filename[0:8],
valid_time=fhour,
data_code=utl.CMISS_data_code(data_source=model,
var_name='TEF2'),
levattrs={
'long_name': 'height_above_ground',
'units': 'm',
'_CoordinateAxisType': '-'
},
fcst_level=0,
fcst_ele="TEF2",
units='K')
if t2m is None:
return
v10m = CIMISS_IO.cimiss_model_by_time(
'20' + filename[0:8],
valid_time=fhour,
data_code=utl.CMISS_data_code(data_source=model,
var_name='WIV10'),
levattrs={
'long_name': 'height_above_ground',
'units': 'm',
'_CoordinateAxisType': '-'
},
fcst_level=0,
fcst_ele="WIV10",
units='m/s')
if v10m is None:
return
u10m = CIMISS_IO.cimiss_model_by_time(
'20' + filename[0:8],
valid_time=fhour,
data_code=utl.CMISS_data_code(data_source=model,
var_name='WIU10'),
levattrs={
'long_name': 'height_above_ground',
'units': 'm',
'_CoordinateAxisType': '-'
},
fcst_level=0,
fcst_ele="WIU10",
units='m/s')
if u10m is None:
return
if (model == 'GRAPES_GFS'):
rh2m = CIMISS_IO.cimiss_model_by_time(
'20' + filename[0:8],
valid_time=fhour,
data_code=utl.CMISS_data_code(data_source=model,
var_name='RHF2'),
levattrs={
'long_name': 'height_above_ground',
'units': 'm',
'_CoordinateAxisType': '-'
},
fcst_level=2,
fcst_ele="RHF2",
units='%')
if rh2m is None:
return
v10m = CIMISS_IO.cimiss_model_by_time(
'20' + filename[0:8],
valid_time=fhour,
data_code=utl.CMISS_data_code(data_source=model,
var_name='WIV10'),
levattrs={
'long_name': 'height_above_ground',
'units': 'm',
'_CoordinateAxisType': '-'
},
fcst_level=10,
fcst_ele="WIV10",
units='m/s')
if v10m is None:
return
u10m = CIMISS_IO.cimiss_model_by_time(
'20' + filename[0:8],
valid_time=fhour,
data_code=utl.CMISS_data_code(data_source=model,
var_name='WIU10'),
levattrs={
'long_name': 'height_above_ground',
'units': 'm',
'_CoordinateAxisType': '-'
},
fcst_level=10,
fcst_ele="WIU10",
units='m/s')
if u10m is None:
return
except KeyError:
raise ValueError('Can not find all data needed')
if (draw_zd == True):
if (initTime == None):
initTime1 = CIMISS_IO.cimiss_get_obs_latest_time(
data_code="SURF_CHN_MUL_HOR")
initTime = (datetime.strptime('20' + initTime1, '%Y%m%d%H') -
timedelta(days=day_back)).strftime("%Y%m%d%H")[2:]
validtime = (datetime.strptime('20' + initTime, '%Y%m%d%H') +
timedelta(hours=fhour)).strftime("%Y%m%d%H")
data_code = utl.CMISS_data_code(data_source='OBS',
var_name='PLOT_sfc')
zd_sta = CIMISS_IO.cimiss_obs_by_time(
times=validtime + '0000',
data_code=data_code,
sta_levels="011,012,013,014",
elements=
"Station_Id_C,Station_Id_d,lat,lon,Alti,TEM,WIN_D_Avg_2mi,WIN_S_Avg_2mi,RHU"
)
obs_valid = True
if (zd_sta is None):
CIMISS_IO.cimiss_get_obs_latest_time(data_code=data_code,
latestTime=6)
zd_sta = CIMISS_IO.cimiss_obs_by_time(directory=directory_obs,
dropna=True,
cache=False)
obs_valid = False
zd_lon = zd_sta['lon'].values
zd_lat = zd_sta['lat'].values
zd_alt = zd_sta['Alti'].values
zd_u, zd_v = mpcalc.wind_components(
zd_sta['WIN_S_Avg_2mi'].values * units('m/s'),
zd_sta['WIN_D_Avg_2mi'].values * units.deg)
idx_zd = np.where((zd_lon > map_extent[0])
& (zd_lon < map_extent[1])
& (zd_lat > map_extent[2])
& (zd_lat < map_extent[3])
& (zd_sta['WIN_S_Avg_2mi'].values < 1000))
zd_sm_lon = zd_lon[idx_zd[0]]
zd_sm_lat = zd_lat[idx_zd[0]]
zd_sm_alt = zd_alt[idx_zd[0]]
zd_sm_u = zd_u[idx_zd[0]]
zd_sm_v = zd_v[idx_zd[0]]
#maskout area
delt_xy = rh['lon'].values[1] - rh['lon'].values[0]
#+ to solve the problem of labels on all the contours
mask1 = (rh['lon'] > map_extent[0] - delt_xy) & (
rh['lon'] < map_extent[1] + delt_xy) & (
rh['lat'] > map_extent[2] - delt_xy) & (rh['lat'] <
map_extent[3] + delt_xy)
mask2 = (u10m['lon'] > map_extent[0] - delt_xy) & (
u10m['lon'] < map_extent[1] + delt_xy) & (
u10m['lat'] > map_extent[2] - delt_xy) & (u10m['lat'] <
map_extent[3] + delt_xy)
#- to solve the problem of labels on all the contours
rh = rh.where(mask1, drop=True)
u = u.where(mask1, drop=True)
v = v.where(mask1, drop=True)
gh = gh.where(mask1, drop=True)
u10m = u10m.where(mask2, drop=True)
v10m = v10m.where(mask2, drop=True)
#prepare interpolator
Ex1 = np.squeeze(u['data'].values).flatten()
Ey1 = np.squeeze(v['data'].values).flatten()
Ez1 = np.squeeze(rh['data'].values).flatten()
z = (np.squeeze(gh['data'].values)).flatten()
coords = np.zeros((np.size(levels), u['lat'].size, u['lon'].size, 3))
coords[..., 1] = u['lat'].values.reshape((1, u['lat'].size, 1))
coords[..., 2] = u['lon'].values.reshape((1, 1, u['lon'].size))
coords = coords.reshape((Ex1.size, 3))
coords[:, 0] = z
interpolator_U = LinearNDInterpolator(coords, Ex1, rescale=True)
interpolator_V = LinearNDInterpolator(coords, Ey1, rescale=True)
interpolator_RH = LinearNDInterpolator(coords, Ez1, rescale=True)
#process sta_fcs 10m wind
coords2 = np.zeros((np.size(sta_fcs['lon']), 3))
coords2[:, 0] = sta_fcs['altitude']
coords2[:, 1] = sta_fcs['lat']
coords2[:, 2] = sta_fcs['lon']
u_sta = interpolator_U(coords2)
v_sta = interpolator_V(coords2)
RH_sta = interpolator_RH(coords2)
wsp_sta = (u_sta**2 + v_sta**2)**0.5
u10m_2D = u10m.interp(lon=('points', sta_fcs['lon']),
lat=('points', sta_fcs['lat']))
v10m_2D = v10m.interp(lon=('points', sta_fcs['lon']),
lat=('points', sta_fcs['lat']))
if (model == 'GRAPES_GFS' and data_source == 'CIMISS'):
rh2m_2D = rh2m.interp(lon=('points', sta_fcs['lon']),
lat=('points', sta_fcs['lat']))['data'].values
else:
td2m_2D = td2m.interp(lon=('points', sta_fcs['lon']),
lat=('points', sta_fcs['lat']))
t2m_2D = t2m.interp(lon=('points', sta_fcs['lon']),
lat=('points', sta_fcs['lat']))
if (data_source == 'MICAPS'):
rh2m_2D = mpcalc.relative_humidity_from_dewpoint(
t2m_2D['data'].values * units('degC'),
td2m_2D['data'].values * units('degC')) * 100
else:
rh2m_2D = mpcalc.relative_humidity_from_dewpoint(
t2m_2D['data'].values * units('kelvin'),
td2m_2D['data'].values * units('kelvin')) * 100
wsp10m_2D = (u10m_2D['data'].values**2 + v10m_2D['data'].values**2)**0.5
winddir10m = mpcalc.wind_direction(u10m_2D['data'].values * units('m/s'),
v10m_2D['data'].values * units('m/s'))
if (np.isnan(wsp_sta).any()):
if (wsp_sta.size == 1):
wsp_sta[np.isnan(wsp_sta)] = np.squeeze(
wsp10m_2D[np.isnan(wsp_sta)])
RH_sta[np.isnan(RH_sta)] = np.squeeze(
np.array(rh2m_2D)[np.isnan(RH_sta)])
else:
wsp_sta[np.isnan(wsp_sta)] = np.squeeze(wsp10m_2D)[np.isnan(
wsp_sta)]
RH_sta[np.isnan(RH_sta)] = np.squeeze(
np.array(rh2m_2D))[np.isnan(RH_sta)]
u_sta, v_sta = mpcalc.wind_components(wsp_sta * units('m/s'), winddir10m)
#process zd_sta 10m wind
zd_fcst_obs = None
if (draw_zd is True):
coords3 = np.zeros((np.size(zd_sm_alt), 3))
coords3[:, 0] = zd_sm_alt
coords3[:, 1] = zd_sm_lat
coords3[:, 2] = zd_sm_lon
u_sm_sta = interpolator_U(coords3)
v_sm_sta = interpolator_V(coords3)
wsp_sm_sta = (u_sm_sta**2 + v_sm_sta**2)**0.5
u10m_sm = u10m.interp(lon=('points', zd_sm_lon),
lat=('points', zd_sm_lat))
v10m_sm = v10m.interp(lon=('points', zd_sm_lon),
lat=('points', zd_sm_lat))
wsp10m_sta = np.squeeze(
(u10m_sm['data'].values**2 + v10m_sm['data'].values**2)**0.5)
winddir10m_sm = mpcalc.wind_direction(
u10m_sm['data'].values * units('m/s'),
v10m_sm['data'].values * units('m/s'))
if (np.isnan(wsp_sm_sta).any()):
wsp_sm_sta[np.isnan(wsp_sm_sta)] = wsp10m_sta[np.isnan(wsp_sm_sta)]
u_sm_sta, v_sm_sta = mpcalc.wind_components(wsp_sm_sta * units('m/s'),
winddir10m_sm)
zd_fcst_obs = {
'lon': zd_sm_lon,
'lat': zd_sm_lat,
'altitude': zd_sm_alt,
'U': np.squeeze(np.array(u_sm_sta)),
'V': np.squeeze(np.array(v_sm_sta)),
'obs_valid': obs_valid,
'U_obs': np.squeeze(np.array(zd_sm_u)),
'V_obs': np.squeeze(np.array(zd_sm_v))
}
#prepare for graphics
sta_fcs_fcst = {
'lon': sta_fcs['lon'],
'lat': sta_fcs['lat'],
'altitude': sta_fcs['altitude'],
'name': sta_fcs['name'],
'RH': np.array(RH_sta),
'U': np.squeeze(np.array(u_sta)),
'V': np.squeeze(np.array(v_sta))
}
fcst_info = gh.coords
local_scale_graphics.draw_wind_rh_according_to_4D_data(
sta_fcs_fcst=sta_fcs_fcst,
zd_fcst_obs=zd_fcst_obs,
fcst_info=fcst_info,
map_extent=map_extent,
draw_zd=draw_zd,
bkgd_type=bkgd_type,
bkgd_level=bkgd_level,
output_dir=None)
plt.xlabel('Dantong data')
plt.ylabel('ERA5')
plt.title('Wind v (m/s)')
#plt.show()
plt.savefig("figs/wind_v_scatter.png")
#RH
ERA5_T = combined_df['t2m'].values * units.degC
ERA5_Td = (combined_df['d2m'].values) * units.degC
Dan_T = combined_df['Temp_C'].values * units.degC
Dan_RH = combined_df['RH'].values / 100 * units('')
Dan_Td = mpcalc.thermo.dewpoint_from_relative_humidity(Dan_T, Dan_RH)
combined_df['Dan_Td'] = Dan_Td
combined_df['RH_ERA5'] = mpcalc.relative_humidity_from_dewpoint(
ERA5_T, ERA5_Td) * 100
combined_df.plot.scatter('RH', 'RH_ERA5')
plt.xlabel('Dantong data')
plt.ylabel('ERA5')
plt.title('RH (%)')
#plt.show()
plt.savefig("figs/RH_scatter.png")
#Tdew
combined_df.plot.scatter('Dan_Td', 'd2m')
plt.xlabel('Dantong data')
plt.ylabel('ERA5')
plt.title('Dew point (C)')
#plt.show()
plt.savefig("figs/Dew_point_scatter.png")
VP = df_selected_time['vaper_pressure'].values * units.hPa
df_selected_time[
'saturation_mixing_ratio'] = mpcalc.mixing_ratio(
VPS, p)
df_selected_time['mixing_ratio'] = mpcalc.mixing_ratio(
VP, p)
MRS = df_selected_time[
'saturation_mixing_ratio'].values * units('g/kg')
MR = df_selected_time['mixing_ratio'].values * units(
'g/kg')
# Calculate RH
#RH = mpcalc.relative_humidity_from_dewpoint(T[0], Td[0])
df_selected_time[
'RH'] = mpcalc.relative_humidity_from_dewpoint(T, Td)
df_selected_time[
'RH_MR'] = mpcalc.relative_humidity_from_mixing_ratio(
MR, T, p)
print('df_selected_time after drop nan.\n{0}'.format(
df_selected_time))
df_selected_time.to_csv(
r'{0}{1}{2}_{3}_solution.csv'.format(
dir_name, save_dir_name, filename_el[-5],
selected_time[:13]))
p = df_selected_time['pressure'].values * units.hPa
T = df_selected_time['temperature'].values * units.degC
Td = df_selected_time['dewpoint'].values * units.degC
def process_metar(mstr, now):
""" Do the METAR Processing """
mtr = None
while mtr is None:
try:
mtr = Metar(mstr, now.month, now.year)
except MetarParserError as exp:
try:
msg = str(exp)
except Exception as exp:
return None
tokens = ERROR_RE.findall(str(exp))
orig_mstr = mstr
if tokens:
for token in tokens[0].split():
mstr = mstr.replace(" %s" % (token, ), "")
if orig_mstr == mstr:
print("Can't fix badly formatted metar: " + mstr)
return None
else:
print("MetarParserError: " + msg)
return None
except Exception as exp:
print("Double Fail: %s %s" % (mstr, exp))
return None
if mtr is None or mtr.time is None:
return None
ob = OB()
ob.metar = mstr[:254]
ob.valid = now
if mtr.temp:
ob.tmpf = mtr.temp.value("F")
if mtr.dewpt:
ob.dwpf = mtr.dewpt.value("F")
if mtr.wind_speed:
ob.sknt = mtr.wind_speed.value("KT")
if mtr.wind_gust:
ob.gust = mtr.wind_gust.value("KT")
# Calc some stuff
if ob.tmpf is not None and ob.dwpf is not None:
ob.relh = relative_humidity_from_dewpoint(
ob.tmpf * units('degF'),
ob.dwpf * units('degF')).to(units('percent')).magnitude
if ob.sknt is not None:
ob.feel = apparent_temperature(ob.tmpf * units('degF'),
ob.relh * units('percent'),
ob.sknt * units('knots')).to(
units('degF')).magnitude
if mtr.wind_dir and mtr.wind_dir.value() != "VRB":
ob.drct = mtr.wind_dir.value()
if mtr.vis:
ob.vsby = mtr.vis.value("SM")
# see pull request #38
if mtr.press and mtr.press != mtr.press_sea_level:
ob.alti = mtr.press.value("IN")
if mtr.press_sea_level:
ob.mslp = mtr.press_sea_level.value("MB")
if mtr.precip_1hr:
ob.p01i = mtr.precip_1hr.value("IN")
# Do something with sky coverage
for i in range(len(mtr.sky)):
(c, h, _) = mtr.sky[i]
setattr(ob, 'skyc%s' % (i + 1), c)
if h is not None:
setattr(ob, 'skyl%s' % (i + 1), h.value("FT"))
if mtr.max_temp_6hr:
ob.max_tmpf_6hr = mtr.max_temp_6hr.value("F")
if mtr.min_temp_6hr:
ob.min_tmpf_6hr = mtr.min_temp_6hr.value("F")
if mtr.max_temp_24hr:
ob.max_tmpf_24hr = mtr.max_temp_24hr.value("F")
if mtr.min_temp_24hr:
ob.min_tmpf_6hr = mtr.min_temp_24hr.value("F")
if mtr.precip_3hr:
ob.p03i = mtr.precip_3hr.value("IN")
if mtr.precip_6hr:
ob.p06i = mtr.precip_6hr.value("IN")
if mtr.precip_24hr:
ob.p24i = mtr.precip_24hr.value("IN")
# Presentwx
if mtr.weather:
pwx = []
for wx in mtr.weather:
val = "".join([a for a in wx if a is not None])
if val == "" or val == len(val) * "/":
continue
pwx.append(val)
ob.wxcodes = pwx
return ob
# OTHER OPTIONAL AXES TO PLOT
# plot_irradiance
# plot_precipitation
# Download the station data
station = '04360' #'04416' # '89606'# '03065' #04201
url, path = url_timeseries(2020, 4, 20, 00, 2020, 4, 27, 10, station)
# yields an error (many not a time entries)
download_and_save(path, url)
df_synop, df_climat = synop_df(path, timeseries=True)
# Temporary variables for ease
temp = df_synop['TT'].values * units('degC')
pres = df_synop['SLP'].values
dewpoint = df_synop['TD'].values * units('degC')
rh = mpcalc.relative_humidity_from_dewpoint(temp, dewpoint) * 100
ws = df_synop['ff'].values
if 'max_gust' in df_synop.columns:
wsmax = df_synop['max_gust'].values
else:
pass
wd = df_synop['dd'].values
date = pd.to_datetime(df_synop['time'].values).tolist()
# ID For Plotting on Meteogram
probe_id = df_synop.Station[0]
data = {
'wind_speed': (np.array(ws) * units('knots')),
'wind_speed_max': (np.array(wsmax) * units('kph')).to(units('knots')),
'wind_direction': np.array(wd) * units('degrees'),
def f(fullname, selected_time):
fullname_el = fullname.split('/')
#fullname_el = fullname.split('\\')
filename = fullname_el[-1]
filename_el = filename.split('_')
save_base_dir_name = '../1data/'
save_dir_name = '{0}{1}/'.format(save_base_dir_name + dir_name, obs_year)
print('site : {0}'.format(dir_name))
if os.path.isfile('{0}{1}_{2}_student.csv'\
.format(save_dir_name, filename_el[-5], selected_time[:13]))\
and os.path.isfile('{0}{1}_{2}_solution.csv'\
.format(save_dir_name, filename_el[-5], selected_time[:13])):
write_log(log_file, '{3} ::: {0}{1}_{2} files are already exist'\
.format(save_dir_name, filename_el[-5], selected_time[:13], datetime.now()))
else:
try:
f = lambda s: selected_time in s
ids = df['time'].apply(f)
df_selected_time = df[ids]
df_selected_time = df_selected_time.sort_values('pressure',
ascending=False)
print('filename : {0}'.format(fullname))
print('df_selected_time.\n{0}'.format(df_selected_time))
df_selected_time.to_csv(r'{0}{1}_{2}_student.csv'\
.format(save_dir_name, filename_el[-5], selected_time[:13]))
#################################################################################
### We will pull the data out of the example dataset into individual variables and
### assign units.
#################################################################################
p = df_selected_time['pressure'].values * units.hPa
T = df_selected_time['temperature'].values * units.degC
Td = df_selected_time['dewpoint'].values * units.degC
wind_speed = df_selected_time['speed'].values * units.knots
wind_dir = df_selected_time['direction'].values * units.degrees
u, v = mpcalc.wind_components(wind_speed, wind_dir)
# Calculate web bulb temperature
df_selected_time['wet_bulb_T'] = mpcalc.wet_bulb_temperature(
p, T, Td)
# Calculate potential temperature
df_selected_time['potential_T'] = mpcalc.potential_temperature(
p, T)
df_selected_time['potential_T_C'] = df_selected_time[
'potential_T'].values - 273.15
# Calculate saturation vaper pressure
df_selected_time[
'saturation_vaper_pressure'] = mpcalc.saturation_vapor_pressure(
T)
df_selected_time[
'vaper_pressure'] = mpcalc.saturation_vapor_pressure(Td)
SVP = df_selected_time[
'saturation_vaper_pressure'].values * units.hPa
VP = df_selected_time['vaper_pressure'].values * units.hPa
# Calculate mixing ratio
df_selected_time['saturation_mixing_ratio'] = mpcalc.mixing_ratio(
SVP, p)
df_selected_time['mixing_ratio'] = mpcalc.mixing_ratio(VP, p)
SMR = df_selected_time['saturation_mixing_ratio'].values * units(
'g/kg')
MR = df_selected_time['mixing_ratio'].values * units('g/kg')
# Calculate relative humidity
df_selected_time['relative_humidity_from_dewpoint'] \
= mpcalc.relative_humidity_from_dewpoint(T, Td)
df_selected_time['relative_humidity_from_mixing_ratio'] \
= mpcalc.relative_humidity_from_mixing_ratio(MR, T, p)
# Calculate virtual temperature
df_selected_time['virtual_temperature'] \
= mpcalc.virtual_temperature(T, MR)
# Calculate virtual potential temperature
df_selected_time['virtual_potential_temperature'] \
= mpcalc.virtual_potential_temperature(p, T, MR)
print('df_selected_time after drop nan.\n{0}'.format(
df_selected_time))
df_selected_time.to_csv(r'{0}{1}_{2}_solution.csv'\
.format(save_dir_name, filename_el[-5], selected_time[:13]))
except Exception as err:
write_log(err_log_file, '{4} ::: {0} with {1}{2} on {3}'\
.format(err, dir_name, filename, selected_time[:13], datetime.now()))
print('Thread {0} is finished'.format(selected_time))
return 0 # Return a dummy value
def merge(atmos, surface):
"""Create a dictionary of data based on these two dataframes
Args:
atmos (DataFrame): atmospherics
surface (DataFrame): surface data
Returns:
dictionary of values
"""
data = {}
# Do what we can with the atmospheric data
for _, row in atmos.iterrows():
nwsli = get_nwsli(row["Rpuid"])
if nwsli is None:
if int(row["Rpuid"]) not in KNOWN_UNKNOWNS:
print(("process_rwis: Unknown Rpuid: %s in atmos"
"") % (row["Rpuid"], ))
continue
if nwsli not in data:
data[nwsli] = {}
# Timestamp
ts = datetime.datetime.strptime(row["DtTm"], "%m/%d/%y %H:%M")
data[nwsli]["valid"] = ts.replace(tzinfo=pytz.UTC)
data[nwsli]["tmpf"] = get_temp(row["AirTemp"])
data[nwsli]["dwpf"] = get_temp(row["Dewpoint"])
if data[nwsli]["tmpf"] is not None and data[nwsli]["dwpf"] is not None:
data[nwsli]["relh"] = (mcalc.relative_humidity_from_dewpoint(
data[nwsli]["tmpf"] * units("degF"),
data[nwsli]["dwpf"] * units("degF"),
).magnitude * 100.0)
# Rh is unused
data[nwsli]["sknt"] = get_speed(row["SpdAvg"])
data[nwsli]["gust"] = get_speed(row["SpdGust"])
if row["DirMin"] not in ["", 32767, np.nan]:
data[nwsli]["drct"] = row["DirMin"]
# DirMax is unused
# Pressure is not reported
# PcIntens
# PcType
# PcRate
if row["PcAccum"] not in ["", -1, 32767, np.nan]:
data[nwsli]["pday"] = row["PcAccum"] * 0.00098425
if row["Visibility"] not in ["", -1, 32767, np.nan]:
data[nwsli]["vsby"] = row["Visibility"] / 1609.344
# Do what we can with the surface data
for _, row in surface.iterrows():
nwsli = get_nwsli(row["Rpuid"])
if nwsli is None:
if int(row["Rpuid"]) not in KNOWN_UNKNOWNS:
print(("process_rwis: Unknown Rpuid: %s in sfc"
"") % (row["Rpuid"], ))
continue
ts = datetime.datetime.strptime(row["DtTm"], "%m/%d/%y %H:%M")
ts = ts.replace(tzinfo=pytz.UTC)
if nwsli not in data:
data[nwsli] = {"valid": ts}
sensorid = int(row["Senid"])
key = "sfvalid%s" % (sensorid, )
data[nwsli][key] = ts
key = "scond%s" % (sensorid, )
data[nwsli][key] = row["sfcond"]
# sftemp -150
key = "tsf%s" % (sensorid, )
data[nwsli][key] = get_temp(row["sftemp"])
# frztemp 32767
# chemfactor 0
# chempct 101
# depth 32767
# icepct 101
# subsftemp NaN
key = "tsub%s" % (sensorid, )
data[nwsli][key] = get_temp(row["subsftemp"])
# waterlevel NaN
# Unnamed: 13 NaN
# Unnamed: 14 NaN
return data
def process_metar(mstr, now):
""" Do the METAR Processing """
mtr = None
while mtr is None:
try:
mtr = Metar(mstr, now.month, now.year)
except MetarParserError as exp:
try:
msg = str(exp)
except Exception as exp:
return None
tokens = ERROR_RE.findall(str(exp))
orig_mstr = mstr
if tokens:
for token in tokens[0].split():
mstr = mstr.replace(" %s" % (token, ), "")
if orig_mstr == mstr:
print("Can't fix badly formatted metar: " + mstr)
return None
else:
print("MetarParserError: "+msg)
return None
except Exception as exp:
print("Double Fail: %s %s" % (mstr, exp))
return None
if mtr is None or mtr.time is None:
return None
ob = OB()
ob.metar = mstr[:254]
ob.valid = now
if mtr.temp:
ob.tmpf = mtr.temp.value("F")
if mtr.dewpt:
ob.dwpf = mtr.dewpt.value("F")
if mtr.wind_speed:
ob.sknt = mtr.wind_speed.value("KT")
if mtr.wind_gust:
ob.gust = mtr.wind_gust.value("KT")
# Calc some stuff
if ob.tmpf is not None and ob.dwpf is not None:
ob.relh = relative_humidity_from_dewpoint(
ob.tmpf * units('degF'),
ob.dwpf * units('degF')
).to(units('percent')).magnitude
if ob.sknt is not None:
ob.feel = apparent_temperature(
ob.tmpf * units('degF'),
ob.relh * units('percent'),
ob.sknt * units('knots')
).to(units('degF')).magnitude
if mtr.wind_dir and mtr.wind_dir.value() != "VRB":
ob.drct = mtr.wind_dir.value()
if mtr.vis:
ob.vsby = mtr.vis.value("SM")
# see pull request #38
if mtr.press and mtr.press != mtr.press_sea_level:
ob.alti = mtr.press.value("IN")
if mtr.press_sea_level:
ob.mslp = mtr.press_sea_level.value("MB")
if mtr.precip_1hr:
ob.p01i = mtr.precip_1hr.value("IN")
# Do something with sky coverage
for i in range(len(mtr.sky)):
(c, h, _) = mtr.sky[i]
setattr(ob, 'skyc%s' % (i+1), c)
if h is not None:
setattr(ob, 'skyl%s' % (i+1), h.value("FT"))
if mtr.max_temp_6hr:
ob.max_tmpf_6hr = mtr.max_temp_6hr.value("F")
if mtr.min_temp_6hr:
ob.min_tmpf_6hr = mtr.min_temp_6hr.value("F")
if mtr.max_temp_24hr:
ob.max_tmpf_24hr = mtr.max_temp_24hr.value("F")
if mtr.min_temp_24hr:
ob.min_tmpf_6hr = mtr.min_temp_24hr.value("F")
if mtr.precip_3hr:
ob.p03i = mtr.precip_3hr.value("IN")
if mtr.precip_6hr:
ob.p06i = mtr.precip_6hr.value("IN")
if mtr.precip_24hr:
ob.p24i = mtr.precip_24hr.value("IN")
# Presentwx
if mtr.weather:
pwx = []
for wx in mtr.weather:
val = "".join([a for a in wx if a is not None])
if val == "" or val == len(val) * "/":
continue
pwx.append(val)
ob.wxcodes = pwx
return ob
def Station_Snow_Synthetical_Forecast_From_Cassandra(
model='ECMWF',
output_dir=None,
t_range=[0,84],
t_gap=3,
points={'lon':[116.3833], 'lat':[39.9]},
initTime=None,
draw_VIS=True,drw_thr=False,
extra_info={
'output_head_name':' ',
'output_tail_name':' ',
'point_name':' '}
):
#+get all the directories needed
try:
dir_rqd=[
"ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_3_HOURS/",
"ECMWF_HR/10_METRE_WIND_GUST_IN_THE_LAST_6_HOURS/",
"ECMWF_HR/SNOD/",
"ECMWF_HR/SDEN/",
"ECMWF_HR/UGRD_100M/",
"ECMWF_HR/VGRD_100M/",
"NWFD_SCMOC/VIS/",
"NCEP_GFS_HR/SNOD/",
"ECMWF_HR/SNOW06/",
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='T2m'),
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='u10m'),
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='v10m'),
'ECMWF_ENSEMBLE/RAW/SNOW06/'
]
except KeyError:
raise ValueError('Can not find all required directories needed')
try:
dir_opt=[
utl.Cassandra_dir(
data_type='surface',data_source=model,var_name='Td2m')
]
name_opt=['Td2m']
except:
dir_opt=[
utl.Cassandra_dir(data_type='surface',data_source=model,var_name='rh2m')
]
name_opt=['rh2m']
#+get all the directories needed
if(initTime == None):
last_file={model:get_latest_initTime(dir_rqd[0]),
'SCMOC':get_latest_initTime(dir_rqd[6]),
}
else:
last_file={model:initTime[0],
'SCMOC':initTime[1],
}
y_s={model:int('20'+last_file[model][0:2]),
'SCMOC':int('20'+last_file['SCMOC'][0:2])}
m_s={model:int(last_file[model][2:4]),
'SCMOC':int(last_file['SCMOC'][2:4])}
d_s={model:int(last_file[model][4:6]),
'SCMOC':int(last_file['SCMOC'][4:6])}
h_s={model:int(last_file[model][6:8]),
'SCMOC':int(last_file['SCMOC'][6:8])}
fhours = np.arange(t_range[0], t_range[1], t_gap)
for ifhour in fhours:
if (ifhour == fhours[0] ):
time_all=datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour))
else:
time_all=np.append(time_all,datetime(y_s['SCMOC'],m_s['SCMOC'],d_s['SCMOC'],h_s['SCMOC'])+timedelta(hours=int(ifhour)))
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
t2m=utl.get_model_points_gy(dir_rqd[9], filenames, points,allExists=False)
if(name_opt[0] == 'rh2m'):
rh2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False)
Td2m=mpcalc.dewpoint_rh(t2m['data'].values*units('degC'),rh2m['data'].values/100.)
p_vapor=(rh2m['data'].values/100.)*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values))))
if(name_opt[0] == 'Td2m'):
Td2m=utl.get_model_points_gy(dir_opt[0], filenames, points,allExists=False)
rh2m=mpcalc.relative_humidity_from_dewpoint(t2m['data'].values* units('degC'),
Td2m['data'].values* units('degC'))
p_vapor=(np.array(rh2m))*6.105*(math.e**((17.27*t2m['data'].values/(237.7+t2m['data'].values))))
Td2m=np.array(Td2m['data'].values)* units('degC')
#SN06_ensm=utl.get_model_points_gy(dir_rqd[12], filenames, points,allExists=False)
'''
for i in range(0,len(SN06_ensm['forecast_period'])):
SN06_std=np.std(np.squeeze(SN06_ensm['data'].values[i,:]))
SN06_mean=np.mean(np.squeeze(SN06_ensm['data'].values[i,:]))
if(i == 0):
SN06_01=norm.pdf(0.01, SN06_mean, SN06_std)
SN06_10=norm.pdf(0.1, SN06_mean, SN06_std)
SN06_25=norm.pdf(0.25, SN06_mean, SN06_std)
SN06_50=norm.pdf(0.5, SN06_mean, SN06_std)
SN06_75=norm.pdf(0.75, SN06_mean, SN06_std)
SN06_90=norm.pdf(0.9, SN06_mean, SN06_std)
SN06_99=norm.pdf(0.99, SN06_mean, SN06_std)
if(i > 0):
SN06_01=[SN06_01,norm.pdf(0.01, SN06_mean, SN06_std)]
SN06_10=[SN06_10,norm.pdf(0.1, SN06_mean, SN06_std)]
SN06_25=[SN06_25,norm.pdf(0.25, SN06_mean, SN06_std)]
SN06_50=[SN06_50,norm.pdf(0.5, SN06_mean, SN06_std)]
SN06_75=[SN06_75,norm.pdf(0.75, SN06_mean, SN06_std)]
SN06_90=[SN06_90,norm.pdf(0.9, SN06_mean, SN06_std)]
SN06_99=[SN06_99,norm.pdf(0.99, SN06_mean, SN06_std)]
SN06_ensm_stc={
'SN06_01'=SN06_01
'SN06_10'=SN06_10
'SN06_25'=SN06_25
'SN06_50'=SN06_50
'SN06_75'=SN06_75
'SN06_90'=SN06_90
'SN06_99'=SN06_99
}
'''
u10m=utl.get_model_points_gy(dir_rqd[10], filenames, points,allExists=False)
v10m=utl.get_model_points_gy(dir_rqd[11], filenames, points,allExists=False)
wsp10m=(u10m['data']**2+v10m['data']**2)**0.5
AT=1.07*t2m['data'].values+0.2*p_vapor-0.65*wsp10m-2.7
#https://en.wikipedia.org/wiki/Wind_chill
TWC=13.12+0.6215*t2m['data'].values-11.37*(wsp10m**0.16)+0.3965*t2m['data'].values*(wsp10m**0.16)
fhours = np.arange(t_range[0], t_range[1], t_gap)
filenames = [last_file['SCMOC']+'.'+str(fhour).zfill(3) for fhour in fhours]
VIS=utl.get_model_points_gy(dir_rqd[6], filenames, points,allExists=False,fill_null=True,Null_value=-0.001)
if(last_file['SCMOC'] == last_file[model] and t_range[1] > 72):
fhours = np.append(np.arange(3,72,3),np.arange(72, (t_range[1]), 6))
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] != last_file[model] and t_range[1] > 60):
fhours = np.append(np.arange(3,60,3),np.arange(60, (t_range[1]), 6))
filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] != last_file[model] and t_range[1] <= 60):
fhours = np.arange(t_range[0], t_range[1], t_gap)
filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] == last_file[model] and t_range[1] <= 72):
fhours = np.arange(t_range[0], t_range[1], t_gap)
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
filenames2 = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
SNOD1=utl.get_model_points_gy(dir_rqd[2], filenames2, points,allExists=False)
SNOD2=utl.get_model_points_gy(dir_rqd[7], filenames2, points,allExists=False)
SDEN=utl.get_model_points_gy(dir_rqd[3], filenames2, points,allExists=False)
SN06=utl.get_model_points_gy(dir_rqd[8], filenames2, points,allExists=False)
u100m=utl.get_model_points_gy(dir_rqd[4], filenames2, points,allExists=False)
v100m=utl.get_model_points_gy(dir_rqd[5], filenames2, points,allExists=False)
wsp100m=(u100m['data']**2+v100m['data']**2)**0.5
if(fhours[-1] < 120):
gust10m=utl.get_model_points_gy(dir_rqd[0], filenames, points,allExists=False)
if(fhours[-1] > 120):
if(last_file['SCMOC'] == last_file[model]):
fhours = np.arange(0, t_range[1], 6)
filenames = [last_file[model]+'.'+str(fhour).zfill(3) for fhour in fhours]
if(last_file['SCMOC'] != last_file[model]):
fhours = np.arange(0, t_range[1], 6)
filenames = [last_file[model]+'.'+str(fhour+12).zfill(3) for fhour in fhours]
gust10m=utl.get_model_points_gy(dir_rqd[1], filenames, points,allExists=False)
sta_graphics.draw_Station_Snow_Synthetical_Forecast_From_Cassandra(
TWC=TWC,AT=AT,u10m=u10m,v10m=v10m,u100m=u100m,v100m=v100m,
gust10m=gust10m,wsp10m=wsp10m,wsp100m=wsp100m,SNOD1=SNOD1,SNOD2=SNOD2,SDEN=SDEN,SN06=SN06,
draw_VIS=draw_VIS,VIS=VIS,drw_thr=drw_thr,
time_all=time_all,
model=model,points=points,
output_dir=output_dir,extra_info=extra_info)
def main():
"""Do Something"""
pgconn = get_dbconn('iem')
icursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
fn = find_file()
try:
nc = netCDF4.Dataset(fn)
except Exception as _:
# File may be in progress of being read, wait a little bit
time.sleep(20)
nc = netCDF4.Dataset(fn)
stations = nc.variables["stationId"][:]
providers = nc.variables["dataProvider"][:]
names = nc.variables["stationName"][:]
tmpk = nc.variables["temperature"][:]
dwpk = nc.variables["dewpoint"][:]
relh = mcalc.relative_humidity_from_dewpoint(
tmpk * units.degK, dwpk * units.degK).magnitude * 100.
tmpk_dd = nc.variables["temperatureDD"][:]
obtime = nc.variables["observationTime"][:]
pressure = nc.variables["stationPressure"][:]
# altimeter = nc.variables["altimeter"][:]
# slp = nc.variables["seaLevelPressure"][:]
drct = nc.variables["windDir"][:]
smps = nc.variables["windSpeed"][:]
gmps = nc.variables["windGust"][:]
# gmps_drct = nc.variables["windDirMax"][:]
pcpn = nc.variables["precipAccum"][:]
rtk1 = nc.variables["roadTemperature1"][:]
rtk2 = nc.variables["roadTemperature2"][:]
rtk3 = nc.variables["roadTemperature3"][:]
rtk4 = nc.variables["roadTemperature4"][:]
subk1 = nc.variables["roadSubsurfaceTemp1"][:]
db = {}
for recnum, provider in enumerate(providers):
this_provider = provider.tostring().replace('\x00', '')
this_station = stations[recnum].tostring().replace('\x00', '')
if (not this_provider.endswith('DOT')
and this_provider not in MY_PROVIDERS):
continue
name = names[recnum].tostring().replace("'", "").replace(
'\x00', '').replace('\xa0', ' ').strip()
if name == '':
continue
if this_provider == 'MesoWest':
# get the network from the last portion of the name
network = name.split()[-1]
if network != 'VTWAC':
continue
else:
network = provider2network(this_provider)
if network is None:
continue
db[this_station] = {}
ticks = obtime[recnum]
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks)
db[this_station]['ts'] = ts.replace(tzinfo=pytz.utc)
db[this_station]['network'] = network
db[this_station]['pres'] = sanity_check(pressure[recnum], 0, 1000000,
-99)
db[this_station]['tmpk'] = sanity_check(tmpk[recnum], 0, 500, -99)
db[this_station]['dwpk'] = sanity_check(dwpk[recnum], 0, 500, -99)
db[this_station]['tmpk_dd'] = tmpk_dd[recnum]
db[this_station]['relh'] = relh[recnum]
db[this_station]['drct'] = sanity_check(drct[recnum], -1, 361, -99)
db[this_station]['smps'] = sanity_check(smps[recnum], -1, 200, -99)
db[this_station]['gmps'] = sanity_check(gmps[recnum], -1, 200, -99)
db[this_station]['rtk1'] = sanity_check(rtk1[recnum], 0, 500, -99)
db[this_station]['rtk2'] = sanity_check(rtk2[recnum], 0, 500, -99)
db[this_station]['rtk3'] = sanity_check(rtk3[recnum], 0, 500, -99)
db[this_station]['rtk4'] = sanity_check(rtk4[recnum], 0, 500, -99)
db[this_station]['subk'] = sanity_check(subk1[recnum], 0, 500, -99)
db[this_station]['pday'] = sanity_check(pcpn[recnum], -1, 5000, -99)
for sid in db:
# print("Processing %s[%s]" % (sid, db[sid]['network']))
iem = Observation(sid, db[sid]['network'], db[sid]['ts'])
# if not iem.load(icursor):
# print 'Missing fp: %s network: %s station: %s' % (fp,
# db[sid]['network'],
# sid)
# subprocess.call("python sync_stations.py %s" % (fp,), shell=True)
# os.chdir("../../dbutil")
# subprocess.call("sh SYNC_STATIONS.sh", shell=True)
# os.chdir("../ingestors/madis")
iem.data['tmpf'] = temperature(db[sid]['tmpk'], 'K').value('F')
iem.data['dwpf'] = temperature(db[sid]['dwpk'], 'K').value('F')
if db[sid]['relh'] >= 0:
iem.data['relh'] = float(db[sid]['relh'])
if db[sid]['drct'] >= 0:
iem.data['drct'] = db[sid]['drct']
if db[sid]['smps'] >= 0:
iem.data['sknt'] = speed(db[sid]['smps'], 'MPS').value('KT')
if db[sid]['gmps'] >= 0:
iem.data['gust'] = speed(db[sid]['gmps'], 'MPS').value('KT')
if db[sid]['pres'] > 0:
iem.data['pres'] = (float(db[sid]['pres']) / 100.00) * 0.02952
if db[sid]['rtk1'] > 0:
iem.data['tsf0'] = temperature(db[sid]['rtk1'], 'K').value('F')
if db[sid]['rtk2'] > 0:
iem.data['tsf1'] = temperature(db[sid]['rtk2'], 'K').value('F')
if db[sid]['rtk3'] > 0:
iem.data['tsf2'] = temperature(db[sid]['rtk3'], 'K').value('F')
if db[sid]['rtk4'] > 0:
iem.data['tsf3'] = temperature(db[sid]['rtk4'], 'K').value('F')
if db[sid]['subk'] > 0:
iem.data['rwis_subf'] = temperature(db[sid]['subk'],
'K').value('F')
if db[sid]['pday'] >= 0:
iem.data['pday'] = round(
distance(db[sid]['pday'], 'MM').value("IN"), 2)
if not iem.save(icursor):
print(("MADIS Extract: %s found new station: %s network: %s"
"") % (fn.split("/")[-1], sid, db[sid]['network']))
subprocess.call("python sync_stations.py %s" % (fn, ), shell=True)
os.chdir("../../dbutil")
subprocess.call("sh SYNC_STATIONS.sh", shell=True)
os.chdir("../ingestors/madis")
print("...done with sync.")
del iem
nc.close()
icursor.close()
pgconn.commit()
pgconn.close()
def merge(atmos, surface):
"""Create a dictionary of data based on these two dataframes
Args:
atmos (DataFrame): atmospherics
surface (DataFrame): surface data
Returns:
dictionary of values
"""
data = {}
# Do what we can with the atmospheric data
for _, row in atmos.iterrows():
nwsli = get_nwsli(row['Rpuid'])
if nwsli is None:
if int(row['Rpuid']) not in KNOWN_UNKNOWNS:
print(('process_rwis: Unknown Rpuid: %s in atmos'
'') % (row['Rpuid'], ))
continue
if nwsli not in data:
data[nwsli] = {}
# Timestamp
ts = datetime.datetime.strptime(row['DtTm'], '%m/%d/%y %H:%M')
data[nwsli]['valid'] = ts.replace(tzinfo=pytz.UTC)
data[nwsli]['tmpf'] = get_temp(row['AirTemp'])
data[nwsli]['dwpf'] = get_temp(row['Dewpoint'])
if data[nwsli]['tmpf'] is not None and data[nwsli]['dwpf'] is not None:
data[nwsli]['relh'] = mcalc.relative_humidity_from_dewpoint(
data[nwsli]['tmpf'] * units('degF'),
data[nwsli]['dwpf'] * units('degF')).magnitude * 100.
# Rh is unused
data[nwsli]['sknt'] = get_speed(row['SpdAvg'])
data[nwsli]['gust'] = get_speed(row['SpdGust'])
if row['DirMin'] not in ['', 32767, np.nan]:
data[nwsli]['drct'] = row['DirMin']
# DirMax is unused
# Pressure is not reported
# PcIntens
# PcType
# PcRate
if row['PcAccum'] not in ['', -1, 32767, np.nan]:
data[nwsli]['pday'] = row['PcAccum'] * 0.00098425
if row['Visibility'] not in ['', -1, 32767, np.nan]:
data[nwsli]['vsby'] = row['Visibility'] / 1609.344
# Do what we can with the surface data
for _, row in surface.iterrows():
nwsli = get_nwsli(row['Rpuid'])
if nwsli is None:
if int(row['Rpuid']) not in KNOWN_UNKNOWNS:
print(('process_rwis: Unknown Rpuid: %s in sfc'
'') % (row['Rpuid'], ))
continue
ts = datetime.datetime.strptime(row['DtTm'], '%m/%d/%y %H:%M')
ts = ts.replace(tzinfo=pytz.UTC)
if nwsli not in data:
data[nwsli] = {'valid': ts}
sensorid = int(row['Senid'])
key = 'sfvalid%s' % (sensorid, )
data[nwsli][key] = ts
key = 'scond%s' % (sensorid, )
data[nwsli][key] = row['sfcond']
# sftemp -150
key = 'tsf%s' % (sensorid, )
data[nwsli][key] = get_temp(row['sftemp'])
# frztemp 32767
# chemfactor 0
# chempct 101
# depth 32767
# icepct 101
# subsftemp NaN
key = 'tsub%s' % (sensorid, )
data[nwsli][key] = get_temp(row['subsftemp'])
# waterlevel NaN
# Unnamed: 13 NaN
# Unnamed: 14 NaN
return data
def main():
load_start = dt.datetime.now()
#Try parsing arguments using argparse
parser = argparse.ArgumentParser(description='wrf non-parallel convective diagnostics processer')
parser.add_argument("-m",help="Model name",required=True)
parser.add_argument("-r",help="Region name (default is aus)",default="aus")
parser.add_argument("-t1",help="Time start YYYYMMDDHH",required=True)
parser.add_argument("-t2",help="Time end YYYYMMDDHH",required=True)
parser.add_argument("-e", help="CMIP5 experiment name (not required if using era5, erai or barra)", default="")
parser.add_argument("--ens", help="CMIP5 ensemble name (not required if using era5, erai or barra)", default="r1i1p1")
parser.add_argument("--group", help="CMIP6 modelling group name", default="")
parser.add_argument("--project", help="CMIP6 modelling intercomparison project", default="CMIP")
parser.add_argument("--ver6hr", help="Version on al33 for 6hr data", default="")
parser.add_argument("--ver3hr", help="Version on al33 for 3hr data", default="")
parser.add_argument("--issave",help="Save output (True or False, default is False)", default="False")
parser.add_argument("--outname",help="Name of saved output. In the form *outname*_*t1*_*t2*.nc. Default behaviour is the model name",default=None)
parser.add_argument("--al33",help="Should data be gathered from al33? Default is False, and data is gathered from r87. If True, then group is required",default="False")
args = parser.parse_args()
#Parse arguments from cmd line and set up inputs (date region model)
model = args.m
region = args.r
t1 = args.t1
t2 = args.t2
issave = args.issave
al33 = args.al33
if args.outname==None:
out_name = model
else:
out_name = args.outname
experiment = args.e
ensemble = args.ens
group = args.group
project = args.project
ver6hr = args.ver6hr
ver3hr = args.ver3hr
if region == "sa_small":
start_lat = -38; end_lat = -26; start_lon = 132; end_lon = 142
elif region == "aus":
start_lat = -44.525; end_lat = -9.975; start_lon = 111.975; end_lon = 156.275
elif region == "global":
start_lat = -70; end_lat = 70; start_lon = -180; end_lon = 179.75
else:
raise ValueError("INVALID REGION\n")
domain = [start_lat,end_lat,start_lon,end_lon]
try:
time = [dt.datetime.strptime(t1,"%Y%m%d%H"),dt.datetime.strptime(t2,"%Y%m%d%H")]
except:
raise ValueError("INVALID START OR END TIME. SHOULD BE YYYYMMDDHH\n")
if issave=="True":
issave = True
elif issave=="False":
issave = False
else:
raise ValueError("\n INVALID ISSAVE...SHOULD BE True OR False")
if al33=="True":
al33 = True
elif al33=="False":
al33 = False
else:
raise ValueError("\n INVALID al33...SHOULD BE True OR False")
#Load data
print("LOADING DATA...")
if model in ["ACCESS1-0","ACCESS1-3","GFDL-CM3","GFDL-ESM2M","CNRM-CM5","MIROC5",\
"MRI-CGCM3","IPSL-CM5A-LR","IPSL-CM5A-MR","GFDL-ESM2G","bcc-csm1-1","MIROC-ESM",\
"BNU-ESM"]:
#Check that t1 and t2 are in the same year
year = np.arange(int(t1[0:4]), int(t2[0:4])+1)
ta, hur, hgt, terrain, p_3d, ps, ua, va, uas, vas, tas, ta2d, tp, lon, lat, \
date_list = read_cmip(model, experiment, \
ensemble, year, domain, cmip_ver=5, al33=al33, group=group, ver6hr=ver6hr, ver3hr=ver3hr)
p = np.zeros(p_3d[0,:,0,0].shape)
tp = tp.astype("float32", order="C")
elif model in ["ACCESS-ESM1-5", "ACCESS-CM2"]:
year = np.arange(int(t1[0:4]), int(t2[0:4])+1)
ta, hur, hgt, terrain, p_3d, ps, ua, va, uas, vas, tas, ta2d, lon, lat, \
date_list = read_cmip(model, experiment,\
ensemble, year, domain, cmip_ver=6, group=group, project=project)
p = np.zeros(p_3d[0,:,0,0].shape)
else:
raise ValueError("Model not recognised")
ta = ta.astype("float32", order="C")
hur = hur.astype("float32", order="C")
hgt = hgt.astype("float32", order="C")
terrain = terrain.astype("float32", order="C")
p = p.astype("float32", order="C")
ps = ps.astype("float32", order="C")
ua = ua.astype("float32", order="C")
va = va.astype("float32", order="C")
uas = uas.astype("float32", order="C")
vas = vas.astype("float32", order="C")
tas= tas.astype("float32", order="C")
ta2d = ta2d.astype("float32", order="C")
lon = lon.astype("float32", order="C")
lat = lat.astype("float32", order="C")
gc.collect()
#This param list was originally given to AD for ERA5 global lightning report
#param = np.array(["mu_cape", "eff_cape","ncape","mu_cin", "muq", "s06", "s0500", "lr700_500", "mhgt", "ta500","tp","cp","laplacian","t_totals"])
#This param list is intended for application to GCMs based on AD ERA5 lightning report
param = np.array(["mu_cape","s06","laplacian","t_totals","ta850","ta500","dp850","tp",\
"muq","lr700_500","mhgt","z500"])
#Set output array
output_data = np.zeros((ps.shape[0], ps.shape[1], ps.shape[2], len(param)))
#Assign p levels to a 3d array, with same dimensions as input variables (ta, hgt, etc.)
#If the 3d p-lvl array already exists, then declare the variable "mdl_lvl" as true.
try:
p_3d;
mdl_lvl = True
full_p3d = p_3d
except:
mdl_lvl = False
p_3d = np.moveaxis(np.tile(p,[ta.shape[2],ta.shape[3],1]),[0,1,2],[1,2,0]).\
astype(np.float32)
print("LOAD TIME..."+str(dt.datetime.now()-load_start))
tot_start = dt.datetime.now()
for t in np.arange(0,ta.shape[0]):
output = np.zeros((1, ps.shape[1], ps.shape[2], len(param)))
cape_start = dt.datetime.now()
print(date_list[t])
if mdl_lvl:
p_3d = full_p3d[t]
dp = get_dp(hur=hur[t], ta=ta[t], dp_mask = False)
#Insert surface arrays, creating new arrays with "sfc" prefix
sfc_ta = np.insert(ta[t], 0, tas[t], axis=0)
sfc_hgt = np.insert(hgt[t], 0, terrain, axis=0)
sfc_dp = np.insert(dp, 0, ta2d[t], axis=0)
sfc_p_3d = np.insert(p_3d, 0, ps[t], axis=0)
sfc_ua = np.insert(ua[t], 0, uas[t], axis=0)
sfc_va = np.insert(va[t], 0, vas[t], axis=0)
#Sort by ascending p
a,temp1,temp2 = np.meshgrid(np.arange(sfc_p_3d.shape[0]) , np.arange(sfc_p_3d.shape[1]),\
np.arange(sfc_p_3d.shape[2]))
sort_inds = np.flip(np.lexsort([np.swapaxes(a,1,0),sfc_p_3d],axis=0), axis=0)
sfc_hgt = np.take_along_axis(sfc_hgt, sort_inds, axis=0)
sfc_dp = np.take_along_axis(sfc_dp, sort_inds, axis=0)
sfc_p_3d = np.take_along_axis(sfc_p_3d, sort_inds, axis=0)
sfc_ua = np.take_along_axis(sfc_ua, sort_inds, axis=0)
sfc_va = np.take_along_axis(sfc_va, sort_inds, axis=0)
sfc_ta = np.take_along_axis(sfc_ta, sort_inds, axis=0)
#Calculate q and wet bulb for pressure level arrays with surface values
sfc_ta_unit = units.units.degC*sfc_ta
sfc_dp_unit = units.units.degC*sfc_dp
sfc_p_unit = units.units.hectopascals*sfc_p_3d
sfc_hur_unit = mpcalc.relative_humidity_from_dewpoint(sfc_ta_unit, sfc_dp_unit)*\
100*units.units.percent
sfc_q_unit = mpcalc.mixing_ratio_from_relative_humidity(sfc_hur_unit,\
sfc_ta_unit,sfc_p_unit)
sfc_q = np.array(sfc_q_unit)
#Now get most-unstable CAPE (max CAPE in vertical, ensuring parcels used are AGL)
cape3d = wrf.cape_3d(sfc_p_3d,sfc_ta+273.15,\
sfc_q,sfc_hgt,\
terrain,ps[t],\
True,meta=False, missing=0)
cape = cape3d.data[0]
cin = cape3d.data[1]
lfc = cape3d.data[2]
lcl = cape3d.data[3]
el = cape3d.data[4]
#Mask values which are below the surface and above 350 hPa AGL
cape[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan
cin[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan
lfc[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan
lcl[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan
el[(sfc_p_3d > ps[t]) | (sfc_p_3d<(ps[t]-350))] = np.nan
#Get maximum (in the vertical), and get cin, lfc, lcl for the same parcel
mu_cape_inds = np.tile(np.nanargmax(cape,axis=0), (cape.shape[0],1,1))
mu_cape = np.take_along_axis(cape, mu_cape_inds, 0)[0]
muq = np.take_along_axis(sfc_q, mu_cape_inds, 0)[0] * 1000
#Calculate other parameters
#Thermo
thermo_start = dt.datetime.now()
lr700_500 = get_lr_p(ta[t], p_3d, hgt[t], 700, 500)
melting_hgt = get_t_hgt(sfc_ta,np.copy(sfc_hgt),0,terrain)
melting_hgt = np.where((melting_hgt < 0) | (np.isnan(melting_hgt)), 0, melting_hgt)
ta500 = get_var_p_lvl(np.copy(sfc_ta), sfc_p_3d, 500)
ta850 = get_var_p_lvl(np.copy(sfc_ta), sfc_p_3d, 850)
dp850 = get_var_p_lvl(np.copy(sfc_dp), sfc_p_3d, 850)
v_totals = ta850 - ta500
c_totals = dp850 - ta500
t_totals = v_totals + c_totals
#Winds
winds_start = dt.datetime.now()
s06 = get_shear_hgt(sfc_ua, sfc_va, np.copy(sfc_hgt), 0, 6000, terrain)
#Laplacian
x, y = np.meshgrid(lon,lat)
dx, dy = mpcalc.lat_lon_grid_deltas(x,y)
if mdl_lvl:
z500 = get_var_p_lvl(hgt[t], p_3d, 500)
laplacian = np.array(mpcalc.laplacian(z500,deltas=[dy,dx])*1e9)
else:
z500 = np.squeeze(hgt[t,p==500])
laplacian = np.array(mpcalc.laplacian(uniform_filter(z500, 4),deltas=[dy,dx])*1e9)
#Fill output
output = fill_output(output, t, param, ps, "mu_cape", mu_cape)
output = fill_output(output, t, param, ps, "muq", muq)
output = fill_output(output, t, param, ps, "s06", s06)
output = fill_output(output, t, param, ps, "lr700_500", lr700_500)
output = fill_output(output, t, param, ps, "ta500", ta500)
output = fill_output(output, t, param, ps, "ta850", ta850)
output = fill_output(output, t, param, ps, "dp850", dp850)
output = fill_output(output, t, param, ps, "mhgt", melting_hgt)
output = fill_output(output, t, param, ps, "tp", tp[t])
output = fill_output(output, t, param, ps, "laplacian", laplacian)
output = fill_output(output, t, param, ps, "t_totals", t_totals)
output = fill_output(output, t, param, ps, "z500", z500)
output_data[t] = output
print("SAVING DATA...")
param_out = []
for param_name in param:
temp_data = output_data[:,:,:,np.where(param==param_name)[0][0]]
param_out.append(temp_data)
#If the mhgt variable is zero everywhere, then it is likely that data has not been read.
#In this case, all values are missing, set to zero.
for t in np.arange(param_out[0].shape[0]):
if param_out[np.where(param=="mhgt")[0][0]][t].max() == 0:
for p in np.arange(len(param_out)):
param_out[p][t] = np.nan
if issave:
save_netcdf(region, model, out_name, date_list, lat, lon, param, param_out, \
out_dtype = "f4", compress=True)
print(dt.datetime.now() - tot_start)
