def pastMonthObs(): #gets the obs from Mesopy from the last month
d = datetime.now() - timedelta(
1
) #going back into last month, because we want to look back on the month that just passed
lastmonth = '%02d' % d.month #so the month is formatted correctly
year = '%04d' % d.year
lastday = '%02d' % d.day
start = str(year) + str(lastmonth) + str('01') + '0000' #start at 00 UTC
end = str(year) + str(lastmonth) + str(
lastday) + '2359' #end at end of month
m = Meso(token=MesoPy_token)
data = []
time = m.timeseries(stid='kmsn', start=start, end=end)
kmsn = time['STATION'][0] #KMSN station
rawtemps = kmsn['OBSERVATIONS']['air_temp_set_1']
rawobstime = kmsn['OBSERVATIONS']['date_time']
time = []
temps = [] #in fahrenheit
for i in range(len(rawobstime)):
t = str(rawobstime[i]).replace('T', ' ').replace('Z', '')
dt = datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
time.append(mdates.date2num(dt))
temps.append('%.2f' % float(rawtemps[i] * 9 / 5 +
32)) #converts from celsius to fahrenheit
return temps, time
def __init__(self,
API_KEY,
station_metadata_file,
wildfire_occurences_file,
wildfire_weather_file,
year_threshold,
state_specified=None,
station_radius_threshold=10):
# Init meso
self.meso = Meso(token=API_KEY)
self.station_metadata = None
# If file is not specified, retrieve data from API
if os.path.isfile(station_metadata_file):
self.station_metadata = pd.read_csv(station_metadata_file)
else:
self.station_metadata = self.get_station_metadata(
station_metadata_file)
# Class Vairables
self.wildfires_df = pd.read_csv(wildfire_occurences_file)
self.year_thresold = year_threshold
self.state_specified = state_specified
self.wildfire_weather_file = wildfire_weather_file
self.station_radius_threshold = station_radius_threshold
self.preprocess_wildfires()
def retrieve_mesowest_observations(meso_token, tm_start, tm_end, glat, glon):
"""
Retrieve observation data from Mesowest and repackage them as a time-indexed
dictionary of lists of observations.
:param meso_token: the mesowest API access token
:param tm_start: the start of the observation window
:param tm_end: the end of the observation window
:param glat: the lattitudes of the grid points
:param glon: the longitudes of the grid points
"""
def decode_meso_time(t):
# example: '2016-03-30T00:30:00Z'
return datetime.strptime(t,
'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
def meso_time(dt):
# example: 201603311600
return '%04d%02d%02d%02d%02d' % (dt.year, dt.month, dt.day, dt.hour,
dt.minute)
# the bbox for mesowest is: (min(lon), min(lat), max(lon), max(lat)).
min_lat, max_lat = np.amin(glat), np.amax(glat)
min_lon, max_lon = np.amin(glon), np.amax(glon)
# retrieve data from Mesowest API (http://mesowest.org/api/)
m = Meso(meso_token)
meso_obss = m.timeseries(meso_time(tm_start - timedelta(minutes=30)),
meso_time(tm_end + timedelta(minutes=30)),
showemptystations='0',
bbox='%g,%g,%g,%g' %
(min_lon, min_lat, max_lon, max_lat),
vars='fuel_moisture')
# repackage all the observations into a time-indexed structure which groups
# observations at the same time together
obs_data = {}
for stinfo in meso_obss['STATION']:
st_lat, st_lon = float(stinfo['LATITUDE']), float(stinfo['LONGITUDE'])
elev = float(stinfo['ELEVATION']) / 3.2808
ngp = find_closest_grid_point(st_lon, st_lat, glon, glat)
dts = [
decode_meso_time(x) for x in stinfo['OBSERVATIONS']['date_time']
]
if 'fuel_moisture_set_1' in stinfo['OBSERVATIONS']:
fms = stinfo['OBSERVATIONS']['fuel_moisture_set_1']
for ts, fm_obs in zip(dts, fms):
if fm_obs is not None:
o = FM10Observation(ts, st_lat, st_lon, elev,
float(fm_obs) / 100., ngp)
obs_t = obs_data.get(ts, [])
obs_t.append(o)
obs_data[ts] = obs_t
return obs_data
def metadata(self):
"""
Retrieve the metadata from Mesowest. Two calls are made to the Mesowest API.
The first call is to get the networks in order to determine the `network` and
`primary_provider`. The second call retrieves the metadata given the `config`
parameters. The two dataframes are combined and inserted into the database
given the :class:`~wxcb.database.Database` instance.
"""
self._logger.info('Obtaining metadata form Mesowest')
m = Meso(token=self.token)
# get the networks associated with the metadata
networks = m.networks()
net = pd.DataFrame(networks['MNET'])
net['MNET_ID'] = net['ID']
# get metadata from Mesowest
self._logger.debug('Obtaining metadata for {}'.format(self.config))
meta = m.metadata(**self.config)
# add the networks to the meta dataframe
mdf = pd.DataFrame(meta['STATION'])
mdf = pd.merge(mdf, net, on='MNET_ID')
# pull out the data from Mesowest into the database format
DF = pd.DataFrame()
for c in self.conversion:
DF[self.conversion[c]] = mdf[c]
# fill in the network conversion
for n in self.network_conversion:
DF[self.network_conversion[n]] = mdf[n]
# elevation is reported in feet, convet to meters
DF['elevation'] = DF['elevation'].astype(float) / 3.28084
# these are the reported lat/long's for the station that may get changed
# down the road due to location errors
DF['reported_lat'] = DF['latitude']
DF['reported_long'] = DF['longitude']
# calculate the UTM coordinates
DF['utm_x'], DF['utm_y'], DF['utm_zone'] = zip(
*DF.apply(utils.df_utm, axis=1))
# add the source to the DF
DF['source'] = 'mesowest'
DF = DF.where((pd.notnull(DF)), None)
# insert the dataframe into the database
self.db.insert_data(DF, 'metadata', description='Mesowest metadata')
def build_pd(stn_id):
key = 'b1c91e501782441d97ac056e2501b5b0'
m = Meso(token=key)
time = m.timeseries(stid=stn_id, start='199801010000', end='201801010000')
ob = time['STATION'][0]
temp = np.expand_dims(np.array(ob['OBSERVATIONS']['air_temp_set_1'],
dtype=np.float),
axis=1)
wind_dir = np.expand_dims(np.array(
ob['OBSERVATIONS']['wind_direction_set_1'], dtype=np.float),
axis=1) * units.degrees
wind_spd = np.expand_dims(np.array(ob['OBSERVATIONS']['wind_speed_set_1'],
dtype=np.float),
axis=1) * 1.9438445
wind_max = np.expand_dims(np.array(
ob['OBSERVATIONS']['peak_wind_speed_set_1'], dtype=np.float),
axis=1) * 1.9438445
u, v = mpcalc.get_wind_components(wind_spd, wind_dir)
rel_hum = np.expand_dims(np.array(
ob['OBSERVATIONS']['relative_humidity_set_1'], dtype=np.float),
axis=1)
dates = ob['OBSERVATIONS']['date_time']
years = float(dates[0][0:4])
months = float(dates[0][5:7])
hours = float(dates[0][8:10])
days = float(dates[0][11:13])
for i in range(len(dates) - 1):
years = np.vstack((years, float(dates[i + 1][0:4])))
months = np.vstack((months, float(dates[i + 1][5:7])))
days = np.vstack((days, float(dates[i + 1][8:10])))
hours = np.vstack((hours, float(dates[i + 1][11:13])))
minutes = np.expand_dims(np.ones(len(hours)) * 55.0, axis=1)
cols = [
'Year', 'Month', 'Day', 'Hour', 'Minutes', 'Temp', 'RH', 'Dir', 'Spd',
'Max', 'U', 'V'
]
data = np.hstack((years, months, days, hours, minutes, temp, rel_hum,
wind_dir, wind_spd, wind_max, u, v))
data = pd.DataFrame(data=data, columns=cols)
pickle.dump(data, open('../Data/pickles/' + stn_id + '_20yr_RAWS.p', 'wb'))
return data
YesterDay = today - one_day
TODATSTR = YesterDay.strftime("%Y%m%d")
TWODAYSTR = ThreeDay.strftime("%Y%m%d")
StartTime = TWODAYSTR + START_UTC
EndTime = TODATSTR + END_UTC
print StartTime,EndTime
# StartTime ='2011900'
# EndTime = '201802191859'
#The two stations that Mike marked one has different current hour temperature and max/mim temperature
#Stations = {"KTYR": 411701,
Stations = {"KCLL": 413901}
#####Parameters for downloading Mesonet data using MesoWest API#####
MyToken = '994a7e628db34fc68503d44c447aaa6f'
m = Meso(token=MyToken)
base_url = 'http://api.mesowest.net/v2/stations/'
query_type = 'timeseries'
csv_format = '&output=csv'
noHighFrequecy = '&hfmetars=0'
#units = '&units=precip|in,temp|F,speed|mph'
#Changed the temperature from F to C
#units = '&units=precip|in,temp|C,speed|mph'
units = '&units=precip|in,temp|F,speed|mph'
#######################################################################
##Loop through each station for downloading and processing Mesonet Data
for STATION,ID in Stations.items():
try:
print STATION,ID
api_string = base_url + query_type + '?' + 'stid=' + STATION + '&start=' + StartTime + \
from datetime import datetime, timedelta
import os
import urllib
import json
from MesoPy import Meso
import csv
import re
import pandas
Workspace = os.getcwd()
Workspace = r"C:\\DEV\\MesoNet"
Climate_Archive = os.path.join(Workspace, "Inputs", "NWS_Station_Data",
"Climate_Obs")
m = Meso(token='994a7e628db34fc68503d44c447aaa6f')
# Set up date stucture for retrieving 7 days of data
today = datetime.today()
one_day = timedelta(days=1)
sixty_day = timedelta(days=60)
hundred_day = timedelta(days=120)
twenty_day = timedelta(days=20)
six_day = timedelta(days=6)
#yesterday = today - one_day
time = "1200"
#startdate = yesterday.strftime("%Y%m%d")
#enddate = today.strftime("%Y%m%d")
# Observation period begins on the beginning (today) and ends on the endday
# The days count back from today to the date specified below
endday = today
def retrieve_wxobs_synopticlabs(
api_key,
data_path,
station_id='knyc',
st_time='201801010000',
ed_time='201801020000',
vbl_list=['date_time', 'air_temp', 'relative_humidity'],
download_new=False):
"""
Function to retrieve timeseries weather observations from an observation site. Uses the
MesoWest/SynopticLabs API to retrieve the observations.
PARAMETERS:
***********
api_key: SynopticLabs api_key.
data_path: Path to directory in which we want to save the observations.
station_id: Four-letter station id.
st_time: Start time for observations, in format 'YYYYMMDDhhmm'.
ed_time: End time for observations, in format 'YYYYMMDDhhmm'.
vbl_list: List of variables to retrieve from SynopticLabs.
download_new: Boolean, re-download data from SynopticLabs if True.
OUTPUTS:
********
data_ts: Data structure returned from API request.
path_name: File path/name for data structure that was just retrieved.
"""
def get_synopticlabs_token(api_key):
request_generate_token = 'http://api.mesowest.net/v2/auth?apikey=' + api_key
api_out = requests.get(request_generate_token).text
token_dict = json.loads(api_out)
token_synopticlabs = token_dict['TOKEN']
return token_synopticlabs
file_name = 'wxobs_' + station_id + '_' + st_time + '_' + ed_time + '.pkl'
if download_new:
token_synopticlabs = get_synopticlabs_token(api_key)
m = Meso(token=token_synopticlabs)
data_ts = m.timeseries(stid=station_id,
start=st_time,
end=ed_time,
vars=vbl_list)
pickle.dump(data_ts, open(os.path.join(data_path, file_name), 'wb'))
else:
try:
data_ts = pickle.load(
open(os.path.join(data_path, file_name), 'rb'))
except:
raise (OSError('File not found: ' + file_name))
return data_ts, os.path.join(data_path, file_name)
def retrieve_mesowest_observations(meso_token, tm_start, tm_end, glat, glon,
ghgt):
"""
Retrieve observation data from Mesowest and repackage them as a time-indexed
dictionary of lists of observations.
:param meso_token: the mesowest API access token or list of them
:param tm_start: the start of the observation window
:param tm_end: the end of the observation window
:param glat: the lattitudes of the grid points
:param glon: the longitudes of the grid points
:param ghgt: the elevation of the grid points
"""
def decode_meso_time(t):
# example: '2016-03-30T00:30:00Z'
return datetime.strptime(t,
'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
def meso_time(dt):
# example: 201603311600
return '%04d%02d%02d%02d%02d' % (dt.year, dt.month, dt.day, dt.hour,
dt.minute)
# the bbox for mesowest is: (min(lon), min(lat), max(lon), max(lat)).
min_lat, max_lat = np.amin(glat), np.amax(glat)
min_lon, max_lon = np.amin(glon), np.amax(glon)
# retrieve data from Mesonet API (http://api.mesowest.net/)
try:
logging.info(
'retrieve_mesowest_observations: retrieving data using MesoDB')
db = mesoDB('ingest/MesoDB', meso_token)
db.update['startTime'] = tm_start - timedelta(minutes=30)
db.update['endTime'] = tm_end + timedelta(minutes=30)
db.params['startTime'] = tm_start - timedelta(minutes=30)
db.params['endTime'] = tm_end + timedelta(minutes=30)
db.params['longitude1'] = min_lon
db.params['longitude2'] = max_lon
db.params['latitude1'] = min_lat
db.params['latitude2'] = max_lat
df = db.get_DB().dropna(subset=['fm10'])
st = db.sites()
if not len(df):
logging.info(
'retrieve_mesowest_observations: no data for the query specified'
)
return {}
logging.info(
'retrieve_mesowest_observations: re-packaging the observations')
obs_data = {}
for stid, data in df.groupby('STID'):
if len(data):
st_data = st.loc[stid]
st_lat, st_lon = float(st_data['LATITUDE']), float(
st_data['LONGITUDE'])
ngp = find_closest_grid_point(st_lon, st_lat, glon, glat)
st_elev = st_data['ELEVATION']
if st_elev is None:
elev = ghgt[ngp]
else:
elev = float(st_data['ELEVATION']) / 3.2808
dts = data.datetime.dt.to_pydatetime()
fms = np.array(data.fm10)
for ts, fm_obs in zip(dts, fms):
if fm_obs is not None:
o = FM10Observation(ts, st_lat, st_lon, elev,
float(fm_obs) / 100., ngp)
obs_t = obs_data.get(ts, [])
obs_t.append(o)
obs_data[ts] = obs_t
return obs_data
except:
logging.warning(
'retrieve_mesowest_observations: failed with exception {}')
logging.info(
'retrieve_mesowest_observations: retrieving data using Meso instead'
)
if isinstance(meso_token, str):
meso_tokens = [meso_token]
n_tokens = 1
else:
meso_tokens = meso_token
n_tokens = len(meso_tokens)
for tn, meso_token in enumerate(meso_tokens):
m = Meso(meso_token)
logging.info("Retrieving fuel moisture from %s to %s" %
(meso_time(tm_start - timedelta(minutes=30)),
meso_time(tm_end + timedelta(minutes=30))))
logging.info("bbox=' %g,%g,%g,%g'" %
(min_lon, min_lat, max_lon, max_lat))
try:
meso_obss = m.timeseries(
meso_time(tm_start - timedelta(minutes=30)),
meso_time(tm_end + timedelta(minutes=30)),
showemptystations='0',
bbox='%g,%g,%g,%g' % (min_lon, min_lat, max_lon, max_lat),
vars='fuel_moisture')
break
except Exception as e:
if tn == n_tokens - 1:
raise MesoPyError(
'Could not connect to the API. Probably the token(s) usage for this month is full.'
)
else:
logging.warning(
'Could not connect to the API. Probably the token usage for this month is full. Trying next token...'
)
if meso_obss is None:
logging.info(
'retrieve_mesowest_observations: Meso.timeseries returned None'
)
return {}
logging.info(
'retrieve_mesowest_observations: re-packaging the observations')
# repackage all the observations into a time-indexed structure which groups
# observations at the same time together
obs_data = {}
for stinfo in meso_obss['STATION']:
st_lat, st_lon = float(stinfo['LATITUDE']), float(
stinfo['LONGITUDE'])
ngp = find_closest_grid_point(st_lon, st_lat, glon, glat)
st_elev = stinfo['ELEVATION']
if st_elev is None:
elev = ghgt[ngp]
else:
elev = float(stinfo['ELEVATION']) / 3.2808
dts = [
decode_meso_time(x)
for x in stinfo['OBSERVATIONS']['date_time']
]
if 'fuel_moisture_set_1' in stinfo['OBSERVATIONS']:
fms = stinfo['OBSERVATIONS']['fuel_moisture_set_1']
for ts, fm_obs in zip(dts, fms):
if fm_obs is not None:
o = FM10Observation(ts, st_lat, st_lon, elev,
float(fm_obs) / 100., ngp)
obs_t = obs_data.get(ts, [])
obs_t.append(o)
obs_data[ts] = obs_t
return obs_data
def surfacePlot(api):
m = Meso(token=MesoPy_token)
stids = [
'KAIG', 'KATW', 'KASX', 'KDLL', 'KOVS', 'KBUU', 'KCLI', 'KEGV', 'KEAU',
'KFLD', 'KCMY', 'KGRB', 'KHYR', 'KJVL', 'KUNU', 'KENW', 'KLSE', 'KRCX',
'KLNL', 'KLNR', 'KMSN', 'KMTW', 'KMFI', 'KMDZ', 'KLUM', 'KRRL', 'KMKE',
'KMRJ', 'K82C', 'TT278', 'KEFT', 'KRNH', 'KOSH', 'KPBH', 'KPVB',
'KPDC', 'KRHI', 'KBCK', 'LAAW3', 'WUEW3', 'KRPD', 'KEZS', 'KSBM',
'KRZN', 'KSTE', 'KSUE', 'KSUW', 'KY51', 'KRYV', 'KUES', 'KPCZ', 'KAUW',
'KY50', 'KETB', 'KISW', 'KARV', 'GDNW3', 'KRGK', 'AFWW3'
] #stations
latest = m.latest(
stid=stids, within='30', vars='air_temp',
units='temp|F') #gets the latest temps (measured over past 30 minutes)
data = [] #a list with 4 columns: lat, long, temps, stid
for ob in latest['STATION']:
data.append((float(ob['LATITUDE']), float(ob['LONGITUDE']),
float(ob['OBSERVATIONS']['air_temp_value_1']['value']),
ob['STID']))
# Now that we have all the data, we can make the figure
fig = plt.figure(figsize=(10, 10))
plt.style.use('ggplot')
ax = plt.axes(projection=ccrs.Mercator())
ax.set_extent([-86.5, -93.2, 41.75, 47.25])
ax.add_feature(
ShapelyFeature(
Reader("ne_50m_admin_1_states_provinces_lakes.shp").geometries(),
ccrs.PlateCarree(),
facecolor='w',
edgecolor='gray',
linewidth=1.5))
county_shpfile = 'c_05ap16.shp' #wi county shapefile
WI = Reader(county_shpfile).records()
for i in WI:
if i.attributes['STATE'] == 'WI':
ax.add_geometries(i.geometry,
ccrs.PlateCarree(),
facecolor='white',
edgecolor='gray')
roads = cfeature.NaturalEarthFeature(category='cultural',
name='roads',
scale='10m',
facecolor='none')
ax.add_feature(roads, edgecolor='red')
ax.add_feature(
ShapelyFeature(Reader("ne_50m_lakes.shp").geometries(),
ccrs.PlateCarree(),
facecolor='lightblue',
edgecolor='gray',
linewidth=1.5))
for lat, lon, temp, stid in data: #plots a marker for each station
plt.plot(lon,
lat,
marker='o',
color='blue',
markersize=10,
alpha=0.7,
transform=ccrs.PlateCarree())
# Transforms for the text function
transform = ccrs.PlateCarree()._as_mpl_transform(ax)
text_transform = offset_copy(transform, units='dots', x=0, y=0)
# Plot temp for each of the markers
for lat, lon, temp, stid in data:
plt.text(lon,
lat,
str(round(temp, 1)) + u'\N{DEGREE SIGN}' + 'F\n\n',
verticalalignment='center',
horizontalalignment='center',
transform=text_transform,
fontsize=11)
uppertext = """Surface Observations
over Wisconsin, taken
in the last 30 minutes.
Data accessed from
MesoWest and
SynopticLabs.\n"""
lowertext = """See more current surface observations here:
http://mesowest.utah.edu/
http://aos.wisc.edu/weather/wx_obs/Surface.html
Learn how to interpret surface observation symbols here:
http://ww2010.atmos.uiuc.edu/%28Gh%29/guides/maps/sfcobs/home.rxml"""
ax.text(0.015,
0.01,
lowertext,
verticalalignment='bottom',
horizontalalignment='left',
transform=ax.transAxes,
fontsize='10.5',
bbox=dict(facecolor='white', alpha=0.5))
ax.text(0.015,
0.13,
uppertext,
verticalalignment='bottom',
horizontalalignment='left',
transform=ax.transAxes,
fontsize='10.5',
bbox=dict(facecolor='white', alpha=0.5))
plt.title('Current Weather Around Wisconsin', fontsize=14)
plt.savefig('WI_Obs.png', bbox_inches='tight')
#plt.show()
tweet = "What's the weather like around WI? Find out more here: http://aos.wisc.edu/weather/wx_obs/Surface.html"
try:
#print 'WI_Obs.png'
api.update_with_media("WI_Obs.png", status=tweet)
except tweepy.error.TweepError:
print("Twitter error raised")
plt.close('all')
import numpy as np import scipy as sp import matplotlib as mpl import matplotlib.pyplot as plt import pandas as pd import xarray as xr import pickle import os import julian import datetime from datetime import timezone import metpy.calc as mpcalc from metpy.cbook import get_test_data from metpy.plots import add_metpy_logo, SkewT from metpy.units import units from scipy.constants import convert_temperature from MesoPy import Meso key = 'b1c91e501782441d97ac056e2501b5b0' m = Meso(token=key) stations = m.metadata(stid='BBEC1') print(stations)
# if v + '_set_1' not in list(response['STATION'][0]['OBSERVATIONS'].keys()):
# raise ValueError('"{}" not available for station {}!'.format(v, stid))
# Get our times and measurements from the full returned dictionary
date_time = response['STATION'][0]['OBSERVATIONS']['date_time']
site_data = {
v: response['STATION'][0]['OBSERVATIONS'][v + '_set_1']
for v in varlist
}
df = pd.DataFrame(index=date_time, data=site_data)
df.index = pd.to_datetime(df.index)
return df
#use mesopy to get the asos meta_data
m = Meso(token=TOKEN)
stations = m.metadata(bbox=[-123.021397, 37.03180, -120.173988, 38.810713])
N_sta_in = stations['SUMMARY']['NUMBER_OF_OBJECTS']
#obtain the data using the function and the urllib and mesowest api. dump it into pickle files for each station
for n, x in enumerate(stations['STATION']):
if x['PERIOD_OF_RECORD']['start'] is not None:
if x['PERIOD_OF_RECORD']['start'][0:4] == '1970':
sta_id = x['STID']
print(sta_id)
print(str(n) + "/" + str(N_sta_in))
first = x['PERIOD_OF_RECORD']['start'][0:4] + x['PERIOD_OF_RECORD'][
'start'][5:7] + x['PERIOD_OF_RECORD']['start'][8:10] + '0000'
last = x['PERIOD_OF_RECORD']['end'][0:4] + x['PERIOD_OF_RECORD'][
'end'][5:7] + x['PERIOD_OF_RECORD']['end'][8:10] + '0000'
if sta_id != 'KMCC' and sta_id != 'KMER' and sta_id != 'KMHR':
def testauth():
badtoken = Meso(token='3030')
badtoken.latest(stid=['kfnl', 'kden', 'ksdf'], within='30')
# Raspberry Pi official 7" touchscreen display.
# Software: Raspbian Jessie
# Python 2.7
# MesoPy 2.0.3
# Kivy
# Program by Dan Stormont
# License: MIT
from datetime import datetime, date, time
from MesoPy import Meso
from pprint import pprint
#TODO-Get rid of these global constants and variables
MESOWEST_TOKEN = '0ee93b568404435a9f98ead284f5fc9c'
_m = Meso(token=MESOWEST_TOKEN)
def display_latest(station_id):
"""Display the latest weather readings from station_id."""
print 'Most recent weather readings:'
print '-----------------------------'
latest_reading = _m.latest(stid=station_id,
units='precip|in,speed|mph,temp|F')
print ('Temperature: '
+ str(latest_reading['STATION'][0]['OBSERVATIONS']
['air_temp_value_1']['value'])
+ ' deg F')
print ('Rainfall today: '
+ str(latest_reading['STATION'][0]['OBSERVATIONS']
#height_meters3 = dy3 * (int(y_dim3) - 1) #Domain Height
# close the wrfout
nc.close()
# reads UW WRF variables and saves them in a dictionary
wrf3 = met.get_wrf_DF('HRRR',datadir, start, end, x_dim3, y_dim3, lat3, lon3)
print("HRRR dictionary is done")
######################################
########## MESOWEST ##########
######################################
# Enter MesoWest token (generate using API key)
m = Meso(token = '23fd1c019ccb4ec3a6946eb3a13c99ad')
# Convert the start and end time to the string format requried by the API
start_t = start.strftime("%Y%m%d%H%M")
end_t = (end - timedelta(hours=1)).strftime("%Y%m%d%H%M")
tz = 'utc' # This is hard coded for now. Local time could be added later.
# String of some MesoWest variables available from this list:
# https://synopticlabs.org/api/mesonet/variables/
#variables = 'air_temp,pressure,wind_speed,wind_direction,relative_humidity,precip_accum'
variables = 'air_temp,pressure,wind_speed,wind_direction,relative_humidity'
var_list = variables.split(",")
# Bounding box (min lon, min lat, max lon, max lat)
# Lat/lon values taken from UW WRF output
import MesoPy
import json
import pandas as pd
import numpy as np
import math
import plotly.express as px
from plotly.subplots import make_subplots
import plotly.graph_objects as go
import chart_studio.plotly as py
# only needed if intending to pull station data again.
from MesoPy import Meso
m = Meso(token='YOUR TOKEN HERE')
# constants
SB_CONST = 5.67e-8
def html_chart(df, height=1200):
"""
make interactive chart.
param df: inpute dataframe
param height: optional plot height
returns: plotly chart
"""
fig = make_subplots(rows=(len(df.columns)),
cols=1,
subplot_titles=df.columns,
shared_xaxes=True,
vertical_spacing=0.007)
def DataChimp(file, start, TList, PList, SWEList, RList, key):
m = Meso(token=key)
def daterange(start_date, end_date):
for n in range(int ((end_date - start_date).days)):
yield start_date + timedelta(n)
def readLast(fileName):
fileName = open(fileName, 'r')
lineList = fileName.readlines()
date = lineList[-1]
if int(date[5:7]) < 10:
if int(date[7:9]) < 10:
dateobj = datetime.strptime(date[:8], '%Y %m %d')
else:
dateobj = datetime.strptime(date[:9], '%Y %m %d')
else:
dateobj = datetime.strptime(date[:10], '%Y %m %d')
return dateobj
def metaPullMeso(station):
try:
meta = m.metadata(stid=station)
x=meta['STATION'][0]
name = x['NAME']
lat = x['LATITUDE']
lon = x['LONGITUDE']
elev = x['ELEV_DEM']
periodStart = x['PERIOD_OF_RECORD']['start']
periodEnd = x['PERIOD_OF_RECORD']['end']
return name, elev, lat, lon, periodStart, periodEnd
except:
return "None","None","None","None","None","None"
def tminPull(station, start_date):
try:
end_d = start_date + timedelta(hours=32)
stats = m.time_stats(stid=station, start=start_date.strftime("%Y%m%d%H%M"),
end=end_d.strftime("%Y%m%d%H%M"), type='all', units="temp|F")
x=stats['STATION'][0]
tmin = x['STATISTICS']['air_temp_set_1']['minimum']
return tmin
except:
return -100.0
def tmaxPull(station, start_date):
try:
end_d = start_date + timedelta(hours=32)
stats = m.time_stats(stid=station, start=start_date.strftime("%Y%m%d%H%M"),
end=end_d.strftime("%Y%m%d%H%M"), type='all', units="temp|F")
x=stats['STATION'][0]
tmax = x['STATISTICS']['air_temp_set_1']['maximum']
return tmax
except:
return -100.0
def precipPull(station, start_date):
try:
end_d = start_date + timedelta(hours=32)
precip = m.precip(stid=station, start=start_date.strftime("%Y%m%d%H%M"),
end=end_d.strftime("%Y%m%d%H%M"), units='precip|in')
y=precip['STATION'][0]
PrecT = y['OBSERVATIONS']['total_precip_value_1']
return PrecT
except:
return -9.0
def PRCPLookUp(station, start):
try:
NRCS = Client('https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL')
data = NRCS.service.getData(
heightDepth = None,
stationTriplets=station,
elementCd='PRCP',
ordinal=1,
duration='DAILY',
getFlags=False,
alwaysReturnDailyFeb29=False,
beginDate=start,
endDate=start)
x = data[0]
prcp = x['values'][0]
return prcp
except:
return -9.0
def SWELookUp(station, start):
try:
NRCS = Client('https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL')
data = NRCS.service.getData(
heightDepth = None,
stationTriplets=station,
elementCd='WTEQ',
ordinal=1,
duration='DAILY',
getFlags=False,
alwaysReturnDailyFeb29=False,
beginDate=start,
endDate=start)
x = data[0]
swe = x['values'][0]
return swe
except:
return -9.0
def runOffPullv2(station, start_date, end_date):
try:
if start_date == end_date:
return -99.0
else:
datetime_formatter = isodate.datetime_isoformat
start = datetime_formatter(start_date)
start = start[0:10]
dvXml = urlopen('https://waterservices.usgs.gov/nwis/dv/?format=waterml&site='
+station+'¶meterCd=00060&startDT='+start+'&endDT='+start)
tree = et.parse(dvXml)
root = tree.getroot()
values = root.findall('.//{http://www.cuahsi.org/waterML/1.1/}values')
for value in values:
returned = value.find('{http://www.cuahsi.org/waterML/1.1/}value').text
if float(returned) < 0:
return -99.0
elif returned is None:
return -99.0
else:
return returned
except:
return -99.0
if os.path.isfile(file) == True:
# if the .dat file exists this updates the file with the information
# from the last entry to the current day
NewStart = readLast(file)
start_date = NewStart+timedelta(days=1)
end_date = datetime(datetime.now().year, datetime.now().month, datetime.now().day,0,0)
if start_date == end_date:
print("No update needed")
else:
file = open(file, 'a')
for single_date in daterange(start_date, end_date):
print("Processing "+single_date)
TminList=[]
TmaxList=[]
PrecipList=[]
SnowList=[]
ROList=[]
print("Processing "+ str(single_date))
# mesowest uses UTC, this adjusts for PST to UTC
start_d = single_date #+ timedelta(hours=8)
for station in TList:
if station == "xxxxx":
TminList.append('-99.9')
TmaxList.append('-99.9')
else:
tmin = tminPull(station, start_d)
TminList.append(tmin)
tmax = tmaxPull(station, start_d)
TmaxList.append(tmax)
for station in PList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = precipPull (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = PRCPLookUp (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
SnowList.append('-9.9')
else:
SWE = SWELookUp(station, single_date)
SnowList.append(SWE)
for station in RList:
if station == "xxxxx":
ROList.append('-9.9')
else:
runoffval = runOffPullv2(station, single_date, end_date)
ROList.append(runoffval)
for (i,val) in enumerate(ROList):
if val == None:
ROList[i] = 0.0
file.write(str(single_date.year)+" "+str(single_date.month)+" "+str(single_date.day)+" "
+str(0)+" "+str(0)+" "+str(0)+" "+" ".join(map(str,TmaxList))+" "+" ".join(map(str,TminList))+
" "+ " ".join(map(str,PrecipList))+" "+" ".join(map(str,SnowList))+" "+" ".join(map(str,ROList))+'\n')
file.close()
else:
## if no .dat file is found this begins the process of creating one
file = open(file,"w")
file.write("// Station metadata:"+'\n')
file.write("// ID Name Type Latitude Longitude Elevation"+'\n')
#creating the metadata header
for station in TList:
a,b,c,d,e,f = metaPullMeso(station)
file.write("// "+str(station)+" "+str(a)+" "+"TMAX "+str(b)+" "+str(c)+" "+str(d)+'\n')
for station in TList:
a,b,c,d,e,f = metaPullMeso(station)
file.write("// "+str(station)+" "+str(a)+" "+"TMIN "+str(b)+" "+str(c)+" "+str(d)+'\n')
for station in PList:
a,b,c,d,e,f = metaPullMeso(station)
file.write("// "+str(station)+" "+str(a)+" "+"PRECIP "+str(b)+" "+str(c)+" "+str(d)+'\n')
#creating the data object IDs
file.write("tmax "+str(len(TList))+'\n')
file.write("tmin "+str(len(TList))+'\n')
file.write("precip "+str(len(PList)+len(SWEList))+'\n')
file.write("snowdepth "+str(len(SWEList))+'\n')
file.write("runoff "+str(len(RList))+'\n')
file.write("########################################"+'\n')
## datetime object provides previous X years/days/months of data
start_date = datetime.strptime(start, '%m/%d/%Y')
start_date = datetime(start_date.year, start_date.month, start_date.day, 0,0)
end_date = datetime(datetime.now().year, datetime.now().month, datetime.now().day, 0, 0)
for single_date in daterange(start_date, end_date):
print("Processing "+str(single_date))
TminList=[]
TmaxList=[]
PrecipList=[]
SnowList=[]
ROList=[]
# mesowest uses UTC, this adjusts for PST to UTC
start_d = single_date
for station in TList:
if station == "xxxxx":
TminList.append('-99.9')
TmaxList.append('-99.9')
else:
tmin = tminPull(station, start_d)
TminList.append(tmin)
tmax = tmaxPull(station, start_d)
TmaxList.append(tmax)
for station in PList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = precipPull (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = PRCPLookUp (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
SnowList.append('-9.9')
else:
SWE = SWELookUp(station, single_date)
SnowList.append(SWE)
for station in RList:
if station == "xxxxx":
ROList.append('-9.9')
else:
runoffval = runOffPullv2(station, single_date, end_date)
ROList.append(runoffval)
for (i,val) in enumerate(ROList):
if val == None:
ROList[i] = 0.0
file.write(str(single_date.year)+" "+str(single_date.month)+" "+str(single_date.day)+" "
+str(0)+" "+str(0)+" "+str(0)+" "+" ".join(map(str,TmaxList))+" "+" ".join(map(str,TminList))+
" "+ " ".join(map(str,PrecipList))+" "+" ".join(map(str,SnowList))+" "+" ".join(map(str,ROList))+'\n')
file.close()
file.close()
#!/usr/bin/env python3
'''
Copyright © 2018 nick3499@github
Title ⟹ 'Current Weather Conditions: MesoPy (MesoWest API Wrapper)'
Hosted ⟹ https://github.com/nick3499/mesowest_latest_ob
ISC License (ISC) ⟹ https://opensource.org/licenses/ISC
MesoWest Python wrapper ⟹ https://github.com/mesowx/MesoPy
API token required ⟹ https://synopticlabs.org/api/guides/?getstarted
Note: noticed data coming from more than one station, e.g. KILROMEO4, AR794.
'''
from MesoPy import Meso # import `Meso()`
m = Meso(token='xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') # pass API `token`
lat = m.latest(stid='AR794') # pass station ID, get `latest()` ob JSON data
s = lat['STATION'][0]['STID'] # station ID
s1 = lat['STATION'][0]['NAME'] # station name
ob = lat['STATION'][0]['OBSERVATIONS'] # JSON path prefix for current ob
s2 = ob['precip_accum_24_hour_value_1']['value'] # precip
s2 = "{0:.1f}".format(s2 * .0393700787) # mm to in
s3 = ob['solar_radiation_value_1']['value'] # solar rad
s4 = ob['wind_gust_value_1']['value'] # wind gust
s4 = "{0:.1f}".format(s4 * 1.1507794) # knots to mph
s5 = ob['dew_point_temperature_value_1d']['value'] # dew point
s5 = "{0:.1f}".format((1.8 * s5) + 32.0) # C to F
s6 = ob['wind_cardinal_direction_value_1d']['value'] # wind cardinal dir
s7 = ob['pressure_value_1d']['value'] # baro psi (KILROMEO4)
s7 = "{0:.1f}".format((s7 * 0.029529987507120267) * .01) # inHg to Mb
s8 = ob['wind_direction_value_1']['value'] # wind dir degrees
s9 = ob['sea_level_pressure_value_1d']['value'] # sea level psi (KILROMEO4)
s9 = "{0:.1f}".format((s9 * 0.029529987507120267) * .01) # inHg to Mb
s10 = ob['precip_accum_since_local_midnight_value_1']['value']
def get_weather_service_wrapper(token):
from MesoPy import Meso
m = Meso(token=token)
return m
import matplotlib.pyplot as plt
import json
import ast
import xray
import pandas as pd
from netCDF4 import Dataset
from datetime import datetime, timedelta
# OS interaction
import sys
import os
####### USER INPUT ##################
## Your Meswest token
# Email MesoWest API <*****@*****.**> to request a API token
m = Meso(token=your token goes here)
## Local Path and file name of output netcdf file
ncfilename = os.path.normpath('YOUR PATH HERE TEST.nc')
## Select Stations
#1) Manually
#sta_id = ['ksea','sno38','alp44','ksmp','keln']
#2) Select stations from lat lon box
stations = m.metadata(bbox=[-121.837006,47.214409,-121.015778,47.547306])
# Get Station sta_id
# Convert List to nuppy array
N_sta_in = stations['SUMMARY']['NUMBER_OF_OBJECTS']
sta_id = np.empty(N_sta_in,dtype='|S10')
def main():
# This is just useful for formatting returned dict
# pp = pprint.PrettyPrinter(indent=2)
# Create instance of Meso object, pass in YOUR token
m = Meso(token='YOUR TOKEN')
# Use to lookup stations, could specify counties or whatever here
# findstationids = m.station_list(state='CO')
# print(findstationids)
# Grab most recent temp (F) ob in last 90 min at each of the below stations
stations = [
'kgxy, kccu, kcos, kden, kgjt, kbdu, kpub, klhx, kspd, kdro, ksbs, keeo, kguc, klic, '
'kstk, kals, ktad'
]
latest = m.latest(stid=stations,
within='90',
vars='air_temp',
units='temp|F')
# create a list to store everything, iterate over the number of objs returned in latest and append
# lat, long, temp, and stid for use later
data = []
[
data.append((float(ob['LATITUDE']), float(ob['LONGITUDE']),
float(ob['OBSERVATIONS']['air_temp_value_1']['value']),
ob['STID'])) for ob in latest['STATION']
]
print(data)
# Create a MapQuest open aerial instance.
map_quest_aerial = cimgt.MapQuestOpenAerial()
# Create a GeoAxes in the tile's projection.
ax = plt.axes(projection=map_quest_aerial.crs)
# Limit the extent of the map to Colorado's borders
ax.set_extent([-102.03, -109.03, 37, 41])
# Add the MapQuest data at zoom level 8.
ax.add_image(map_quest_aerial, 8)
# Plot lat/long pts with below params
for lat, lon, temp, stid in data:
plt.plot(lon,
lat,
marker='o',
color='y',
markersize=1,
alpha=0.7,
transform=ccrs.Geodetic())
# Transforms for the text func we're about to call
geodetic_transform = ccrs.Geodetic()._as_mpl_transform(ax)
text_transform = offset_copy(geodetic_transform, units='dots', x=0, y=0)
# Plot temp and station id for each of the markers
for lat, lon, temp, stid in data:
plt.text(lon,
lat,
stid + '\n' + str(round(temp, 1)) + u' \N{DEGREE SIGN}' + 'F',
verticalalignment='center',
horizontalalignment='center',
transform=text_transform,
fontsize=9,
bbox=dict(facecolor='wheat', alpha=0.5, boxstyle='round'))
plt.title('Current Weather Around Colorado')
plt.show()
from MesoPy import Meso, MesoPyError
from nose.tools import *
m = Meso(token='demotoken')
# Basic Function Tests
def testvars():
var_list = m.variables()
ok_(var_list)
def testmetadata():
stations = m.metadata(radius=['wbb', 5])
ok_(stations)
def testtimeseries():
timeseries = m.timeseries(stid='kfnl',
start='201504261800',
end='201504262300')
ok_(timeseries)
def testclimatology():
climatology = m.climatology(stid='kden',
startclim='04260000',
endclim='04270000',
units='precip|in')
ok_(climatology)
"""
@author: joshclark
This script just demonstrates some example usage for the workshop
"""
from MesoPy import Meso
import pprint
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
# Instance a Meso object by passing in YOUR api_token
m = Meso(token='YOUR TOKEN')
# Fetches the latest obs for Boulder airport within 30 min of now
latest = m.latest(stid='kbdu', within='30', units='ENGLISH')
pp.pprint(latest)
# Let's format a pretty sentence that tells us the Boulder wx
x = latest['STATION'][0]
st_name = x['NAME']
temp = str(
x['OBSERVATIONS']['air_temp_value_1']['value']) + u'\N{DEGREE SIGN}' + 'F'
wind = str(x['OBSERVATIONS']['wind_speed_value_1']['value']) + ' mph'
result = 'The current weather at ' + st_name + ' is ' + temp + ' with a sustained wind of ' + wind
print(result)
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
def retr_wxobs_synopticlabs(api_key,
data_path,
station_id='knyc',
st_time='201801010000',
ed_time='201801020000',
download_new=False):
"""Function to retrieve timeseries weather observations from an observation site. Uses the
MesoWest/SynopticLabs API to retrieve the observations.
**********
PARAMETERS
api_key: SynopticLabs api_key.
data_path: Path to directory in which we want to save the observations.
station_id: Four-letter station id.
st_time: start time for observations, in format 'YYYYMMDDhhmm'
ed_time: end time for observations, in format 'YYYYMMDDhhmm'
OUTPUT:
obs_dict: Dictionary with station attributes, station observations, observation units, and
quality control summary.
path_name: File path/name for data that was just retrieved.
"""
def get_synopticlabs_token(api_key):
request_generate_token = 'http://api.mesowest.net/v2/auth?apikey=' + api_key
api_out = requests.get(request_generate_token).text
token_dict = json.loads(api_out)
token_synopticlabs = token_dict['TOKEN']
return token_synopticlabs
def get_station_attrs(data_ts, station_attrs):
station_info = {}
for attr in station_attrs:
station_info[attr] = data_ts['STATION'][0][attr]
return station_info
def get_station_obs(data_ts, vbl_list):
station_data = {}
station_data[vbl_list[0]] = data_ts['STATION'][0]['OBSERVATIONS'][
vbl_list[0]]
for vbl in vbl_list[1:]:
station_data[vbl] = data_ts['STATION'][0]['OBSERVATIONS'][list(
data_ts['STATION'][0]['SENSOR_VARIABLES'][vbl].keys())[0]]
return station_data
vbl_list = ['date_time', 'air_temp', 'relative_humidity']
station_attrs = ['STID', 'ELEVATION', 'NAME', 'LONGITUDE', 'LATITUDE']
file_name = 'wxobs_' + station_id + '_' + st_time + '_' + ed_time + '.npy'
token_synopticlabs = get_synopticlabs_token(api_key)
if download_new:
# Retrieve the station data from API.
m = Meso(token=token_synopticlabs)
data_ts = m.timeseries(stid=station_id,
start=st_time,
end=ed_time,
vars=vbl_list[1:])
# Put everything in a single dictionary.
obs_dict = {}
obs_dict['station_attrs'] = get_station_attrs(data_ts, station_attrs)
obs_dict['station_obs'] = get_station_obs(data_ts, vbl_list)
obs_dict['units'] = data_ts['UNITS']
obs_dict['qc_summary'] = data_ts['QC_SUMMARY']
# Save the dictionary
np.save(os.path.join('.', 'data', file_name), obs_dict)
else:
# Load the dictionary
try:
obs_dict = np.load(os.path.join('.', 'data', file_name)).item()
except:
raise (OSError('File not found: ' + filename))
return obs_dict, os.path.join(data_path, file_name)
