def __init__(self,
API_KEY,
station_metadata_file,
wildfire_occurences_file,
wildfire_weather_file,
year_threshold,
state_specified=None,
station_radius_threshold=10):
# Init meso
self.meso = Meso(token=API_KEY)
self.station_metadata = None
# If file is not specified, retrieve data from API
if os.path.isfile(station_metadata_file):
self.station_metadata = pd.read_csv(station_metadata_file)
else:
self.station_metadata = self.get_station_metadata(
station_metadata_file)
# Class Vairables
self.wildfires_df = pd.read_csv(wildfire_occurences_file)
self.year_thresold = year_threshold
self.state_specified = state_specified
self.wildfire_weather_file = wildfire_weather_file
self.station_radius_threshold = station_radius_threshold
self.preprocess_wildfires()
def pastMonthObs(): #gets the obs from Mesopy from the last month
d = datetime.now() - timedelta(
1
) #going back into last month, because we want to look back on the month that just passed
lastmonth = '%02d' % d.month #so the month is formatted correctly
year = '%04d' % d.year
lastday = '%02d' % d.day
start = str(year) + str(lastmonth) + str('01') + '0000' #start at 00 UTC
end = str(year) + str(lastmonth) + str(
lastday) + '2359' #end at end of month
m = Meso(token=MesoPy_token)
data = []
time = m.timeseries(stid='kmsn', start=start, end=end)
kmsn = time['STATION'][0] #KMSN station
rawtemps = kmsn['OBSERVATIONS']['air_temp_set_1']
rawobstime = kmsn['OBSERVATIONS']['date_time']
time = []
temps = [] #in fahrenheit
for i in range(len(rawobstime)):
t = str(rawobstime[i]).replace('T', ' ').replace('Z', '')
dt = datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
time.append(mdates.date2num(dt))
temps.append('%.2f' % float(rawtemps[i] * 9 / 5 +
32)) #converts from celsius to fahrenheit
return temps, time
def retrieve_mesowest_observations(meso_token, tm_start, tm_end, glat, glon):
"""
Retrieve observation data from Mesowest and repackage them as a time-indexed
dictionary of lists of observations.
:param meso_token: the mesowest API access token
:param tm_start: the start of the observation window
:param tm_end: the end of the observation window
:param glat: the lattitudes of the grid points
:param glon: the longitudes of the grid points
"""
def decode_meso_time(t):
# example: '2016-03-30T00:30:00Z'
return datetime.strptime(t,
'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
def meso_time(dt):
# example: 201603311600
return '%04d%02d%02d%02d%02d' % (dt.year, dt.month, dt.day, dt.hour,
dt.minute)
# the bbox for mesowest is: (min(lon), min(lat), max(lon), max(lat)).
min_lat, max_lat = np.amin(glat), np.amax(glat)
min_lon, max_lon = np.amin(glon), np.amax(glon)
# retrieve data from Mesowest API (http://mesowest.org/api/)
m = Meso(meso_token)
meso_obss = m.timeseries(meso_time(tm_start - timedelta(minutes=30)),
meso_time(tm_end + timedelta(minutes=30)),
showemptystations='0',
bbox='%g,%g,%g,%g' %
(min_lon, min_lat, max_lon, max_lat),
vars='fuel_moisture')
# repackage all the observations into a time-indexed structure which groups
# observations at the same time together
obs_data = {}
for stinfo in meso_obss['STATION']:
st_lat, st_lon = float(stinfo['LATITUDE']), float(stinfo['LONGITUDE'])
elev = float(stinfo['ELEVATION']) / 3.2808
ngp = find_closest_grid_point(st_lon, st_lat, glon, glat)
dts = [
decode_meso_time(x) for x in stinfo['OBSERVATIONS']['date_time']
]
if 'fuel_moisture_set_1' in stinfo['OBSERVATIONS']:
fms = stinfo['OBSERVATIONS']['fuel_moisture_set_1']
for ts, fm_obs in zip(dts, fms):
if fm_obs is not None:
o = FM10Observation(ts, st_lat, st_lon, elev,
float(fm_obs) / 100., ngp)
obs_t = obs_data.get(ts, [])
obs_t.append(o)
obs_data[ts] = obs_t
return obs_data
def metadata(self):
"""
Retrieve the metadata from Mesowest. Two calls are made to the Mesowest API.
The first call is to get the networks in order to determine the `network` and
`primary_provider`. The second call retrieves the metadata given the `config`
parameters. The two dataframes are combined and inserted into the database
given the :class:`~wxcb.database.Database` instance.
"""
self._logger.info('Obtaining metadata form Mesowest')
m = Meso(token=self.token)
# get the networks associated with the metadata
networks = m.networks()
net = pd.DataFrame(networks['MNET'])
net['MNET_ID'] = net['ID']
# get metadata from Mesowest
self._logger.debug('Obtaining metadata for {}'.format(self.config))
meta = m.metadata(**self.config)
# add the networks to the meta dataframe
mdf = pd.DataFrame(meta['STATION'])
mdf = pd.merge(mdf, net, on='MNET_ID')
# pull out the data from Mesowest into the database format
DF = pd.DataFrame()
for c in self.conversion:
DF[self.conversion[c]] = mdf[c]
# fill in the network conversion
for n in self.network_conversion:
DF[self.network_conversion[n]] = mdf[n]
# elevation is reported in feet, convet to meters
DF['elevation'] = DF['elevation'].astype(float) / 3.28084
# these are the reported lat/long's for the station that may get changed
# down the road due to location errors
DF['reported_lat'] = DF['latitude']
DF['reported_long'] = DF['longitude']
# calculate the UTM coordinates
DF['utm_x'], DF['utm_y'], DF['utm_zone'] = zip(
*DF.apply(utils.df_utm, axis=1))
# add the source to the DF
DF['source'] = 'mesowest'
DF = DF.where((pd.notnull(DF)), None)
# insert the dataframe into the database
self.db.insert_data(DF, 'metadata', description='Mesowest metadata')
def retrieve_mesowest_observations(meso_token, tm_start, tm_end, glat, glon):
"""
Retrieve observation data from Mesowest and repackage them as a time-indexed
dictionary of lists of observations.
:param meso_token: the mesowest API access token
:param tm_start: the start of the observation window
:param tm_end: the end of the observation window
:param glat: the lattitudes of the grid points
:param glon: the longitudes of the grid points
"""
def decode_meso_time(t):
# example: '2016-03-30T00:30:00Z'
return datetime.strptime(t, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
def meso_time(dt):
# example: 201603311600
return '%04d%02d%02d%02d%02d' % (dt.year, dt.month, dt.day, dt.hour, dt.minute)
# the bbox for mesowest is: (min(lon), min(lat), max(lon), max(lat)).
min_lat, max_lat = np.amin(glat), np.amax(glat)
min_lon, max_lon = np.amin(glon), np.amax(glon)
# retrieve data from Mesowest API (http://mesowest.org/api/)
m = Meso(meso_token)
meso_obss = m.timeseries(meso_time(tm_start - timedelta(minutes=30)),
meso_time(tm_end + timedelta(minutes=30)),
showemptystations = '0', bbox='%g,%g,%g,%g' % (min_lon, min_lat, max_lon, max_lat),
vars='fuel_moisture')
# repackage all the observations into a time-indexed structure which groups
# observations at the same time together
obs_data = {}
for stinfo in meso_obss['STATION']:
st_lat, st_lon = float(stinfo['LATITUDE']), float(stinfo['LONGITUDE'])
elev = float(stinfo['ELEVATION']) / 3.2808
ngp = find_closest_grid_point(st_lon, st_lat, glon, glat)
dts = [decode_meso_time(x) for x in stinfo['OBSERVATIONS']['date_time']]
if 'fuel_moisture_set_1' in stinfo['OBSERVATIONS']:
fms = stinfo['OBSERVATIONS']['fuel_moisture_set_1']
for ts,fm_obs in zip(dts,fms):
if fm_obs is not None:
o = FM10Observation(ts,st_lat,st_lon,elev,float(fm_obs)/100.,ngp)
obs_t = obs_data.get(ts, [])
obs_t.append(o)
obs_data[ts] = obs_t
return obs_data
def main():
# This is just useful for formatting returned dict
# pp = pprint.PrettyPrinter(indent=2)
# Create instance of Meso object, pass in YOUR token
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
# Use to lookup stations, could specify counties or whatever here
# findstationids = m.station_list(state='CO')
# print(findstationids)
# Grab most recent temp (F) ob in last 90 min at each of the below stations
stations = ['kgxy, kccu, kcos, kden, kgjt, kbdu, kpub, klhx, kspd, kdro, ksbs, keeo, kguc, klic, '
'kstk, kals, ktad']
latest = m.latest_obs(stid=stations, within='90', vars='air_temp', units='temp|F')
# create a list to store everything, iterate over the number of objs returned in latest and append
# lat, long, temp, and stid for use later
data = []
[data.append((float(ob['LATITUDE']), float(ob['LONGITUDE']), float(ob['OBSERVATIONS']['air_temp_value_1']['value']),
ob['STID'])) for ob in latest['STATION']]
print(data)
# Create a MapQuest open aerial instance.
map_quest_aerial = cimgt.MapQuestOpenAerial()
# Create a GeoAxes in the tile's projection.
ax = plt.axes(projection=map_quest_aerial.crs)
# Limit the extent of the map to Colorado's borders
ax.set_extent([-102.03, -109.03, 37, 41])
# Add the MapQuest data at zoom level 8.
ax.add_image(map_quest_aerial, 8)
# Plot lat/long pts with below params
for lat, lon, temp, stid in data:
plt.plot(lon, lat, marker='o', color='y', markersize=1,
alpha=0.7, transform=ccrs.Geodetic())
# Transforms for the text func we're about to call
geodetic_transform = ccrs.Geodetic()._as_mpl_transform(ax)
text_transform = offset_copy(geodetic_transform, units='dots', x=0, y=0)
# Plot temp and station id for each of the markers
for lat, lon, temp, stid in data:
plt.text(lon, lat, stid + '\n' + str(round(temp, 1)) + u' \N{DEGREE SIGN}' + 'F',
verticalalignment='center', horizontalalignment='center',
transform=text_transform, fontsize=9,
bbox=dict(facecolor='wheat', alpha=0.5, boxstyle='round'))
plt.title('Current Weather Around Colorado')
plt.show()
def build_pd(stn_id):
key = 'b1c91e501782441d97ac056e2501b5b0'
m = Meso(token=key)
time = m.timeseries(stid=stn_id, start='199801010000', end='201801010000')
ob = time['STATION'][0]
temp = np.expand_dims(np.array(ob['OBSERVATIONS']['air_temp_set_1'],
dtype=np.float),
axis=1)
wind_dir = np.expand_dims(np.array(
ob['OBSERVATIONS']['wind_direction_set_1'], dtype=np.float),
axis=1) * units.degrees
wind_spd = np.expand_dims(np.array(ob['OBSERVATIONS']['wind_speed_set_1'],
dtype=np.float),
axis=1) * 1.9438445
wind_max = np.expand_dims(np.array(
ob['OBSERVATIONS']['peak_wind_speed_set_1'], dtype=np.float),
axis=1) * 1.9438445
u, v = mpcalc.get_wind_components(wind_spd, wind_dir)
rel_hum = np.expand_dims(np.array(
ob['OBSERVATIONS']['relative_humidity_set_1'], dtype=np.float),
axis=1)
dates = ob['OBSERVATIONS']['date_time']
years = float(dates[0][0:4])
months = float(dates[0][5:7])
hours = float(dates[0][8:10])
days = float(dates[0][11:13])
for i in range(len(dates) - 1):
years = np.vstack((years, float(dates[i + 1][0:4])))
months = np.vstack((months, float(dates[i + 1][5:7])))
days = np.vstack((days, float(dates[i + 1][8:10])))
hours = np.vstack((hours, float(dates[i + 1][11:13])))
minutes = np.expand_dims(np.ones(len(hours)) * 55.0, axis=1)
cols = [
'Year', 'Month', 'Day', 'Hour', 'Minutes', 'Temp', 'RH', 'Dir', 'Spd',
'Max', 'U', 'V'
]
data = np.hstack((years, months, days, hours, minutes, temp, rel_hum,
wind_dir, wind_spd, wind_max, u, v))
data = pd.DataFrame(data=data, columns=cols)
pickle.dump(data, open('../Data/pickles/' + stn_id + '_20yr_RAWS.p', 'wb'))
return data
def precip_dataframe(m: Meso, start: str, end: str,
**kwargs) -> pandas.DataFrame:
'''
Returns a pandas dataframe with STID as the index and the precip observation as the first column.
:param m: Meso instance for retrieving data.
:param start: start of observation window.
:param end: end of observation window.
:param kwargs: other key word arguments for the API.
:return: the stations with a observation window within 0.05% of the expected window.
The observation window is defined as (end - start). When creating the dataframe we filter out any observations that
do not have that large of a window as they are un representative of the situation. The filter uses last_report and
first_report from the API dictionary to determine if it is valid. (Returning this invalid data may be implemented)
'''
kwargs['timeformat'] = '%s' # force timeformat
df = json_normalize(m.precip(start=start, end=end, **kwargs)['STATION'])
precip_col = np.full(df.shape[0], np.nan, dtype='float64')
epoch_delta_col = np.full(df.shape[0], 0, dtype='int')
start = datetime.strptime(start, '%Y%m%d%H%M')
end = datetime.strptime(end, '%Y%m%d%H%M')
delta = end.timestamp() - start.timestamp()
if 'units' in kwargs:
_accum_label = 'ACCUM_' + str(int(
delta / 86400)).strip() + '_DAYS[' + kwargs['units'] + ']'
else:
_accum_label = 'ACCUM_' + str(int(delta / 86400)).strip() + '_DAYS[mm]'
for i, row in df.iterrows(
): # extract precip observation into it's own column
if len(row['OBSERVATIONS.precipitation']) > 0:
_dict = row['OBSERVATIONS.precipitation'][0]
epoch_delta_col[i] = (int(_dict['last_report']) -
int(_dict['first_report']))
precip_col[i] = _dict['total']
df[_accum_label] = precip_col # precip data column
df['EPOCH_TIMEDELTA'] = epoch_delta_col # used to filter out erroneous time periods
df = df[abs(df['EPOCH_TIMEDELTA'] - delta) < .0005 * delta]
df = df[['STID', _accum_label]].set_index('STID', drop=True)
return df
import numpy as np import scipy as sp import matplotlib as mpl import matplotlib.pyplot as plt import pandas as pd import xarray as xr import pickle import os import julian import datetime from datetime import timezone import metpy.calc as mpcalc from metpy.cbook import get_test_data from metpy.plots import add_metpy_logo, SkewT from metpy.units import units from scipy.constants import convert_temperature from MesoPy import Meso key = 'b1c91e501782441d97ac056e2501b5b0' m = Meso(token=key) stations = m.metadata(stid='BBEC1') print(stations)
def teststationsfunc():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
stations = m.station_list(state='CO', county='Larimer')
ok_('KFNL' == stations['STATION'][1]['STID'])
class WeatherDataGetter:
def __init__(self,
API_KEY,
station_metadata_file,
wildfire_occurences_file,
wildfire_weather_file,
year_threshold,
state_specified=None,
station_radius_threshold=10):
# Init meso
self.meso = Meso(token=API_KEY)
self.station_metadata = None
# If file is not specified, retrieve data from API
if os.path.isfile(station_metadata_file):
self.station_metadata = pd.read_csv(station_metadata_file)
else:
self.station_metadata = self.get_station_metadata(
station_metadata_file)
# Class Vairables
self.wildfires_df = pd.read_csv(wildfire_occurences_file)
self.year_thresold = year_threshold
self.state_specified = state_specified
self.wildfire_weather_file = wildfire_weather_file
self.station_radius_threshold = station_radius_threshold
self.preprocess_wildfires()
def get_5_nearest(self, lat, lon, n):
"""
Takes position in the form of lat, lon and returns n nearest mesowest stations
:param lat: latitude
:param lon: longitude
:param n: number of stations
:return: n nearest stations
"""
all_points = self.station_metadata[['latitude', 'longitude']]
self.station_metadata['distance'] = cdist([(lat, lon)], all_points).T
n_smallest = self.station_metadata.nsmallest(n=n, columns='distance')
n_smallest['miles'] = [
haversine(lon, lat, row['longitude'], row['latitude'])
for _, row in n_smallest.iterrows()
]
n_smallest = n_smallest[
n_smallest['miles'] <= self.station_radius_threshold]
return n_smallest['STID'].tolist()
def preprocess_wildfires(self):
"""
Preprocesses wildfires database, filtering by year_threshold and state (if specified)
:return: None
"""
# Convert dates to datetimes. Add an year column
self.wildfires_df['DISCOVERY_DATE'] = pd.to_datetime(
self.wildfires_df['DISCOVERY_DATE'])
self.wildfires_df['CONT_DATE'] = pd.to_datetime(
self.wildfires_df['CONT_DATE'])
self.wildfires_df['year'] = pd.DatetimeIndex(
self.wildfires_df['DISCOVERY_DATE']).year
# Filter to all records greater than an year and sort row in descending order by date
self.wildfires_df = self.wildfires_df[
self.wildfires_df['DISCOVERY_DATE'].dt.year >=
self.year_thresold].sort_values(by=['DISCOVERY_DATE'],
ascending=False)
# Filter to only a particular state
if self.state_specified:
self.wildfires_df = self.wildfires_df[self.wildfires_df['STATE'] ==
self.state_specified]
def get_station_metadata(self, file_name):
"""
Gets all stations metadata from mesowest API and saves it in a file
:param file_name: file where station data should be stored
:return: None
"""
vars = ['air_temp', 'relative_humidity', 'wind_speed', 'precip_accum']
metadata = self.meso.metadata(country='us',
status='ACTIVE',
var=vars,
obrange='20110101, 20160101')
out = []
for i in range(len(metadata['STATION'])):
try:
out.append([
metadata['STATION'][i]['STID'],
metadata['STATION'][i]['LATITUDE'],
metadata['STATION'][i]['LONGITUDE']
])
except:
pass
df = pd.DataFrame(out, columns=['STID', 'latitude', 'longitude'])
df.to_csv(file_name)
def get_weather_data(self):
"""
Purpose is to get weather data from nearby stations of a fire
:return: None
"""
# TODO: Add last_n_days query functionality
radius = '10'
last_n_days = 7
cnt = 0 # Keeps cnt of rows. Used to periodically write rows to file
stations_data = [
] # Stores data for a few stations till it is written to file
header = True # For dataframe header
write_rows_threshold = 100
columns = [
'FOD_ID', 'STID', 'distance', 'date_time', 'air_temperature',
'relative_humidity', 'wind_speed', 'precipitation'
]
for index, row in self.wildfires_df.iterrows():
if cnt % write_rows_threshold == 0: # Every write_rows_threshold rows, write to file
df = pd.DataFrame(stations_data, columns=columns)
with open(self.wildfire_weather_file, 'a') as f:
df.to_csv(f, header=header)
header = False
stations_data = []
#
start = (row['DISCOVERY_DATE'] - timedelta(days=0)).replace(
hour=12, minute=0).strftime("%Y%m%d%H%M")
lat = row['LATITUDE']
lon = row['LONGITUDE']
days = [start]
station_data = self.get_weather_data_api(lat,
lon,
radius,
row['FOD_ID'],
days,
stids=self.get_5_nearest(
lat=float(lat),
lon=float(lon),
n=5))
print(cnt)
if station_data:
stations_data.extend(station_data)
cnt += 1
def get_weather_data_api(self, lat, lon, radius, wildfire_ID, days, stids):
"""
Queries mesowest API for given lat, lon and radius. Given nearby stations
:param lat: optional, if stid provided
:param lon: optional, if stid provided
:param radius: optional, if stid provided
:param wildfire_ID:
:param days: TODO: for n days. Currently will only query for a single day
:param stids: STIDs
:return: station data
"""
radius_param = str(lat) + ',' + str(lon) + ',' + str(radius)
vars = ['air_temp', 'relative_humidity', 'wind_speed', 'precip_accum']
# Get data for a locations with sepcified radius
data = []
try:
for day in days:
day_data = self.meso.attime(radius=radius_param,
attime=day,
within=60,
stid=stids,
vars=vars)
if not day_data:
return None
data.append(day_data)
except Exception as e:
print(e)
return
if not data:
return
# Query data to generate station_df which would have the following columns:
# STID, Date_time, Var1, Var2...
# We could omit all stations which do not have all the values
# Currently, keeping all stations irrespective of their missing values
# Each occurence of a date_time val is supposed to be a row.
# Here since, we are only querying attime, we will have only 1 date_time field
vars_data = [
'date_time', 'air_temp_value_1', 'relative_humidity_value_1',
'wind_speed_value_1', 'precip_accum_value_1'
]
stations = [station['STID'] for station in data[0]['STATION']]
station_data = []
# We are storing station ID from first day's data.
# Subsequently querying other days data for these stations and storing them
# TODO: Optimize below for loop
for station_id in stations:
for day_idx in range(len(data)):
if data[day_idx]:
temp = []
for station in data[day_idx]['STATION']:
if station['STID'] == station_id:
row = [
wildfire_ID, station['STID'],
station['DISTANCE']
]
for var in vars_data:
if var == 'date_time':
row.append(days[day_idx])
elif var in station['OBSERVATIONS']:
row.append(
station['OBSERVATIONS'][var]['value'])
else:
row.append('')
temp.append(row)
station_data.extend(temp)
return station_data
import matplotlib.pyplot as plt
import json
import ast
import xray
import pandas as pd
from netCDF4 import Dataset
from datetime import datetime, timedelta
# OS interaction
import sys
import os
####### USER INPUT ##################
## Your Meswest token
# Email MesoWest API <*****@*****.**> to request a API token
m = Meso(token=your token goes here)
## Local Path and file name of output netcdf file
ncfilename = os.path.normpath('YOUR PATH HERE TEST.nc')
## Select Stations
#1) Manually
#sta_id = ['ksea','sno38','alp44','ksmp','keln']
#2) Select stations from lat lon box
stations = m.metadata(bbox=[-121.837006,47.214409,-121.015778,47.547306])
# Get Station sta_id
# Convert List to nuppy array
N_sta_in = stations['SUMMARY']['NUMBER_OF_OBJECTS']
sta_id = np.empty(N_sta_in,dtype='|S10')
import MesoPy
import json
import pandas as pd
import numpy as np
import math
import plotly.express as px
from plotly.subplots import make_subplots
import plotly.graph_objects as go
import chart_studio.plotly as py
# only needed if intending to pull station data again.
from MesoPy import Meso
m = Meso(token='YOUR TOKEN HERE')
# constants
SB_CONST = 5.67e-8
def html_chart(df, height=1200):
"""
make interactive chart.
param df: inpute dataframe
param height: optional plot height
returns: plotly chart
"""
fig = make_subplots(rows=(len(df.columns)),
cols=1,
subplot_titles=df.columns,
shared_xaxes=True,
vertical_spacing=0.007)
def surfacePlot(api):
m = Meso(token=MesoPy_token)
stids = ['KAIG', 'KATW', 'KASX', 'KDLL', 'KOVS', 'KBUU', 'KCLI', 'KEGV', 'KEAU', 'KFLD',
'KCMY', 'KGRB', 'KHYR', 'KJVL', 'KUNU', 'KENW', 'KLSE', 'KRCX', 'KLNL', 'KLNR',
'KMSN', 'KMTW', 'KMFI', 'KMDZ', 'KLUM', 'KRRL', 'KMKE', 'KMRJ', 'K82C', 'TT278',
'KEFT', 'KRNH', 'KOSH', 'KPBH', 'KPVB', 'KPDC', 'KRHI', 'KBCK', 'LAAW3', 'WUEW3',
'KRPD', 'KEZS', 'KSBM', 'KRZN', 'KSTE', 'KSUE', 'KSUW', 'KY51', 'KRYV', 'KUES',
'KPCZ', 'KAUW', 'KY50', 'KETB', 'KISW', 'KARV', 'GDNW3', 'KRGK', 'AFWW3'] #stations
latest = m.latest(stid=stids, within='30', vars='air_temp', units='temp|F') #gets the latest temps (measured over past 30 minutes)
data = [] #a list with 4 columns: lat, long, temps, stid
for ob in latest['STATION']:
data.append((float(ob['LATITUDE']), float(ob['LONGITUDE']),
float(ob['OBSERVATIONS']['air_temp_value_1']['value']),ob['STID']))
# Now that we have all the data, we can make the figure
fig = plt.figure(figsize=(10,10))
plt.style.use('ggplot')
ax = plt.axes(projection=ccrs.Mercator())
ax.set_extent([-86.5, -93.2, 41.75, 47.25])
ax.add_feature(ShapelyFeature(Reader("ne_50m_admin_1_states_provinces_lakes.shp").geometries(),
ccrs.PlateCarree(), facecolor = 'w', edgecolor = 'gray', linewidth=1.5))
county_shpfile = 'c_05ap16.shp' #wi county shapefile
WI = Reader(county_shpfile).records()
for i in WI:
if i.attributes['STATE'] == 'WI':
ax.add_geometries(i.geometry, ccrs.PlateCarree(), facecolor='white',edgecolor = 'gray')
roads = cfeature.NaturalEarthFeature(
category='cultural',
name='roads',
scale='10m',
facecolor = 'none')
ax.add_feature(roads, edgecolor='red')
ax.add_feature(ShapelyFeature(Reader("ne_50m_lakes.shp").geometries(),
ccrs.PlateCarree(), facecolor = 'lightblue', edgecolor = 'gray', linewidth=1.5))
for lat, lon, temp, stid in data: #plots a marker for each station
plt.plot(lon, lat, marker='o', color='blue', markersize=10,
alpha=0.7, transform = ccrs.PlateCarree())
# Transforms for the text function
transform = ccrs.PlateCarree()._as_mpl_transform(ax)
text_transform = offset_copy(transform, units='dots', x=0, y=0)
# Plot temp for each of the markers
for lat, lon, temp, stid in data:
plt.text(lon, lat, str(round(temp, 1)) + u'\N{DEGREE SIGN}' + 'F\n\n',
verticalalignment='center', horizontalalignment='center',
transform=text_transform, fontsize=11)
uppertext = """Surface Observations
over Wisconsin, taken
in the last 30 minutes.
Data accessed from
MesoWest and
SynopticLabs.\n"""
lowertext = """See more current surface observations here:
http://mesowest.utah.edu/
http://aos.wisc.edu/weather/wx_obs/Surface.html
Learn how to interpret surface observation symbols here:
http://ww2010.atmos.uiuc.edu/%28Gh%29/guides/maps/sfcobs/home.rxml"""
ax.text(0.015, 0.01, lowertext, verticalalignment = 'bottom', horizontalalignment = 'left',
transform=ax.transAxes, fontsize = '10.5', bbox=dict(facecolor='white', alpha = 0.5))
ax.text(0.015, 0.13, uppertext, verticalalignment = 'bottom', horizontalalignment = 'left',
transform=ax.transAxes, fontsize = '10.5', bbox=dict(facecolor='white', alpha = 0.5))
plt.title('Current Weather Around Wisconsin', fontsize=14)
plt.savefig('WI_Obs.png', bbox_inches = 'tight')
#plt.show()
tweet = "What's the weather like around WI? Find out more here: http://aos.wisc.edu/weather/wx_obs/Surface.html"
try:
#print 'WI_Obs.png'
api.update_with_media("WI_Obs.png", status = tweet)
except tweepy.error.TweepError:
print ("Twitter error raised")
plt.close('all')
def testvarlistfunc():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
var_list = m.variable_list()
ok_(var_list)
"KHDO": 418103,
"KSSF": 418104,
"KCOT": 418402,
"KALI": 418504,
"KBAZ": 418105,
"KCLL": 413901,
# "KCRS": 412001,
"KTYR": 411701,
"KTRL": 419703,
"KDTO": 419603,
"KMWL": 419404
}
#####Parameters for downloading Mesonet data using MesoWest API#####
MyToken = '994a7e628db34fc68503d44c447aaa6f'
m = Meso(token=MyToken)
# base_url = 'http://api.mesowest.net/v2/stations/'
# query_type = 'timeseries'
# csv_format = '&output=csv'
# noHighFrequecy = '&hfmetars=0'
# Station list
#stid: string, optional Single or comma separated list of MesoWest station IDs. e.g. stid = 'kden,kslc,wbb'
StationIDs = []
for key, value in Stations.items():
StationIDs.append(key)
StationIDs = 'KJCT'
#######################################################################################################
def testvarexists():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
var_list = m.variable_list()
ok_('relative_humidity' in var_list['VARIABLES'])
def testlateststrlist():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
latest = m.latest_obs(stid=['kfnl', 'kden', 'ksdf'])
eq_(len(latest['STATION']), 3)
def testprecipfunc():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
precip = m.precipitation_obs(stid='kfnl', start='201504261800', end='201504271200', units='precip|in')
ok_(precip)
def testclimatologyfunc():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
climatology = m.climatology_obs(stid='kden', startclim='04260000', endclim='04270000', units='precip|in')
ok_(climatology)
def testtimeseriesfunc():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
timeseries = m.timeseries_obs(stid='kfnl', start='201504261800', end='201504262300')
ok_(timeseries)
from datetime import datetime, timedelta
import os
import urllib
import json
from MesoPy import Meso
import csv
import re
Workspace = os.getcwd()
Workspace = r"C:\\DEV\\MesoNet"
print(Workspace)
Climate_Archive = os.path.join(Workspace, "Inputs", "NWS_Station_Data",
"Climate_Obs")
m = Meso(token='994a7e628db34fc68503d44c447aaa6f')
# Set up date stucture for retrieving 7 days of data
today = datetime.today()
one_day = timedelta(days=1)
sixty_day = timedelta(days=60)
#yesterday = today - one_day
time = "1200"
#startdate = yesterday.strftime("%Y%m%d")
#enddate = today.strftime("%Y%m%d")
# Observation period begins on the beginning (today) and ends on the endday
# The days count back from today to the date specified below
endday = today
def testbadurlstring():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
latest = m.latest_obs(stid='')
print(latest)
#!/usr/bin/env python3
'''
Copyright © 2018 nick3499@github
Title ⟹ 'Current Weather Conditions: MesoPy (MesoWest API Wrapper)'
Hosted ⟹ https://github.com/nick3499/mesowest_latest_ob
ISC License (ISC) ⟹ https://opensource.org/licenses/ISC
MesoWest Python wrapper ⟹ https://github.com/mesowx/MesoPy
API token required ⟹ https://synopticlabs.org/api/guides/?getstarted
Note: noticed data coming from more than one station, e.g. KILROMEO4, AR794.
'''
from MesoPy import Meso # import `Meso()`
m = Meso(token='xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') # pass API `token`
lat = m.latest(stid='AR794') # pass station ID, get `latest()` ob JSON data
s = lat['STATION'][0]['STID'] # station ID
s1 = lat['STATION'][0]['NAME'] # station name
ob = lat['STATION'][0]['OBSERVATIONS'] # JSON path prefix for current ob
s2 = ob['precip_accum_24_hour_value_1']['value'] # precip
s2 = "{0:.1f}".format(s2 * .0393700787) # mm to in
s3 = ob['solar_radiation_value_1']['value'] # solar rad
s4 = ob['wind_gust_value_1']['value'] # wind gust
s4 = "{0:.1f}".format(s4 * 1.1507794) # knots to mph
s5 = ob['dew_point_temperature_value_1d']['value'] # dew point
s5 = "{0:.1f}".format((1.8 * s5) + 32.0) # C to F
s6 = ob['wind_cardinal_direction_value_1d']['value'] # wind cardinal dir
s7 = ob['pressure_value_1d']['value'] # baro psi (KILROMEO4)
s7 = "{0:.1f}".format((s7 * 0.029529987507120267) * .01) # inHg to Mb
s8 = ob['wind_direction_value_1']['value'] # wind dir degrees
s9 = ob['sea_level_pressure_value_1d']['value'] # sea level psi (KILROMEO4)
s9 = "{0:.1f}".format((s9 * 0.029529987507120267) * .01) # inHg to Mb
s10 = ob['precip_accum_since_local_midnight_value_1']['value']
def testauth():
m = Meso(api_token='3030')
m.latest_obs(stid=['kfnl', 'kden', 'ksdf'])
def DataChimp(file, start, TList, PList, SWEList, RList, key):
m = Meso(token=key)
def daterange(start_date, end_date):
for n in range(int ((end_date - start_date).days)):
yield start_date + timedelta(n)
def readLast(fileName):
fileName = open(fileName, 'r')
lineList = fileName.readlines()
date = lineList[-1]
if int(date[5:7]) < 10:
if int(date[7:9]) < 10:
dateobj = datetime.strptime(date[:8], '%Y %m %d')
else:
dateobj = datetime.strptime(date[:9], '%Y %m %d')
else:
dateobj = datetime.strptime(date[:10], '%Y %m %d')
return dateobj
def metaPullMeso(station):
try:
meta = m.metadata(stid=station)
x=meta['STATION'][0]
name = x['NAME']
lat = x['LATITUDE']
lon = x['LONGITUDE']
elev = x['ELEV_DEM']
periodStart = x['PERIOD_OF_RECORD']['start']
periodEnd = x['PERIOD_OF_RECORD']['end']
return name, elev, lat, lon, periodStart, periodEnd
except:
return "None","None","None","None","None","None"
def tminPull(station, start_date):
try:
end_d = start_date + timedelta(hours=32)
stats = m.time_stats(stid=station, start=start_date.strftime("%Y%m%d%H%M"),
end=end_d.strftime("%Y%m%d%H%M"), type='all', units="temp|F")
x=stats['STATION'][0]
tmin = x['STATISTICS']['air_temp_set_1']['minimum']
return tmin
except:
return -100.0
def tmaxPull(station, start_date):
try:
end_d = start_date + timedelta(hours=32)
stats = m.time_stats(stid=station, start=start_date.strftime("%Y%m%d%H%M"),
end=end_d.strftime("%Y%m%d%H%M"), type='all', units="temp|F")
x=stats['STATION'][0]
tmax = x['STATISTICS']['air_temp_set_1']['maximum']
return tmax
except:
return -100.0
def precipPull(station, start_date):
try:
end_d = start_date + timedelta(hours=32)
precip = m.precip(stid=station, start=start_date.strftime("%Y%m%d%H%M"),
end=end_d.strftime("%Y%m%d%H%M"), units='precip|in')
y=precip['STATION'][0]
PrecT = y['OBSERVATIONS']['total_precip_value_1']
return PrecT
except:
return -9.0
def PRCPLookUp(station, start):
try:
NRCS = Client('https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL')
data = NRCS.service.getData(
heightDepth = None,
stationTriplets=station,
elementCd='PRCP',
ordinal=1,
duration='DAILY',
getFlags=False,
alwaysReturnDailyFeb29=False,
beginDate=start,
endDate=start)
x = data[0]
prcp = x['values'][0]
return prcp
except:
return -9.0
def SWELookUp(station, start):
try:
NRCS = Client('https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL')
data = NRCS.service.getData(
heightDepth = None,
stationTriplets=station,
elementCd='WTEQ',
ordinal=1,
duration='DAILY',
getFlags=False,
alwaysReturnDailyFeb29=False,
beginDate=start,
endDate=start)
x = data[0]
swe = x['values'][0]
return swe
except:
return -9.0
def runOffPullv2(station, start_date, end_date):
try:
if start_date == end_date:
return -99.0
else:
datetime_formatter = isodate.datetime_isoformat
start = datetime_formatter(start_date)
start = start[0:10]
dvXml = urlopen('https://waterservices.usgs.gov/nwis/dv/?format=waterml&site='
+station+'¶meterCd=00060&startDT='+start+'&endDT='+start)
tree = et.parse(dvXml)
root = tree.getroot()
values = root.findall('.//{http://www.cuahsi.org/waterML/1.1/}values')
for value in values:
returned = value.find('{http://www.cuahsi.org/waterML/1.1/}value').text
if float(returned) < 0:
return -99.0
elif returned is None:
return -99.0
else:
return returned
except:
return -99.0
if os.path.isfile(file) == True:
# if the .dat file exists this updates the file with the information
# from the last entry to the current day
NewStart = readLast(file)
start_date = NewStart+timedelta(days=1)
end_date = datetime(datetime.now().year, datetime.now().month, datetime.now().day,0,0)
if start_date == end_date:
print("No update needed")
else:
file = open(file, 'a')
for single_date in daterange(start_date, end_date):
print("Processing "+single_date)
TminList=[]
TmaxList=[]
PrecipList=[]
SnowList=[]
ROList=[]
print("Processing "+ str(single_date))
# mesowest uses UTC, this adjusts for PST to UTC
start_d = single_date #+ timedelta(hours=8)
for station in TList:
if station == "xxxxx":
TminList.append('-99.9')
TmaxList.append('-99.9')
else:
tmin = tminPull(station, start_d)
TminList.append(tmin)
tmax = tmaxPull(station, start_d)
TmaxList.append(tmax)
for station in PList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = precipPull (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = PRCPLookUp (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
SnowList.append('-9.9')
else:
SWE = SWELookUp(station, single_date)
SnowList.append(SWE)
for station in RList:
if station == "xxxxx":
ROList.append('-9.9')
else:
runoffval = runOffPullv2(station, single_date, end_date)
ROList.append(runoffval)
for (i,val) in enumerate(ROList):
if val == None:
ROList[i] = 0.0
file.write(str(single_date.year)+" "+str(single_date.month)+" "+str(single_date.day)+" "
+str(0)+" "+str(0)+" "+str(0)+" "+" ".join(map(str,TmaxList))+" "+" ".join(map(str,TminList))+
" "+ " ".join(map(str,PrecipList))+" "+" ".join(map(str,SnowList))+" "+" ".join(map(str,ROList))+'\n')
file.close()
else:
## if no .dat file is found this begins the process of creating one
file = open(file,"w")
file.write("// Station metadata:"+'\n')
file.write("// ID Name Type Latitude Longitude Elevation"+'\n')
#creating the metadata header
for station in TList:
a,b,c,d,e,f = metaPullMeso(station)
file.write("// "+str(station)+" "+str(a)+" "+"TMAX "+str(b)+" "+str(c)+" "+str(d)+'\n')
for station in TList:
a,b,c,d,e,f = metaPullMeso(station)
file.write("// "+str(station)+" "+str(a)+" "+"TMIN "+str(b)+" "+str(c)+" "+str(d)+'\n')
for station in PList:
a,b,c,d,e,f = metaPullMeso(station)
file.write("// "+str(station)+" "+str(a)+" "+"PRECIP "+str(b)+" "+str(c)+" "+str(d)+'\n')
#creating the data object IDs
file.write("tmax "+str(len(TList))+'\n')
file.write("tmin "+str(len(TList))+'\n')
file.write("precip "+str(len(PList)+len(SWEList))+'\n')
file.write("snowdepth "+str(len(SWEList))+'\n')
file.write("runoff "+str(len(RList))+'\n')
file.write("########################################"+'\n')
## datetime object provides previous X years/days/months of data
start_date = datetime.strptime(start, '%m/%d/%Y')
start_date = datetime(start_date.year, start_date.month, start_date.day, 0,0)
end_date = datetime(datetime.now().year, datetime.now().month, datetime.now().day, 0, 0)
for single_date in daterange(start_date, end_date):
print("Processing "+str(single_date))
TminList=[]
TmaxList=[]
PrecipList=[]
SnowList=[]
ROList=[]
# mesowest uses UTC, this adjusts for PST to UTC
start_d = single_date
for station in TList:
if station == "xxxxx":
TminList.append('-99.9')
TmaxList.append('-99.9')
else:
tmin = tminPull(station, start_d)
TminList.append(tmin)
tmax = tmaxPull(station, start_d)
TmaxList.append(tmax)
for station in PList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = precipPull (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
PrecipList.append('-9.9')
else:
precip = PRCPLookUp (station, start_d)
PrecipList.append(precip)
for station in SWEList:
if station == "xxxxx":
SnowList.append('-9.9')
else:
SWE = SWELookUp(station, single_date)
SnowList.append(SWE)
for station in RList:
if station == "xxxxx":
ROList.append('-9.9')
else:
runoffval = runOffPullv2(station, single_date, end_date)
ROList.append(runoffval)
for (i,val) in enumerate(ROList):
if val == None:
ROList[i] = 0.0
file.write(str(single_date.year)+" "+str(single_date.month)+" "+str(single_date.day)+" "
+str(0)+" "+str(0)+" "+str(0)+" "+" ".join(map(str,TmaxList))+" "+" ".join(map(str,TminList))+
" "+ " ".join(map(str,PrecipList))+" "+" ".join(map(str,SnowList))+" "+" ".join(map(str,ROList))+'\n')
file.close()
file.close()
def retrieve_wxobs_synopticlabs(
api_key,
data_path,
station_id='knyc',
st_time='201801010000',
ed_time='201801020000',
vbl_list=['date_time', 'air_temp', 'relative_humidity'],
download_new=False):
"""
Function to retrieve timeseries weather observations from an observation site. Uses the
MesoWest/SynopticLabs API to retrieve the observations.
PARAMETERS:
***********
api_key: SynopticLabs api_key.
data_path: Path to directory in which we want to save the observations.
station_id: Four-letter station id.
st_time: Start time for observations, in format 'YYYYMMDDhhmm'.
ed_time: End time for observations, in format 'YYYYMMDDhhmm'.
vbl_list: List of variables to retrieve from SynopticLabs.
download_new: Boolean, re-download data from SynopticLabs if True.
OUTPUTS:
********
data_ts: Data structure returned from API request.
path_name: File path/name for data structure that was just retrieved.
"""
def get_synopticlabs_token(api_key):
request_generate_token = 'http://api.mesowest.net/v2/auth?apikey=' + api_key
api_out = requests.get(request_generate_token).text
token_dict = json.loads(api_out)
token_synopticlabs = token_dict['TOKEN']
return token_synopticlabs
file_name = 'wxobs_' + station_id + '_' + st_time + '_' + ed_time + '.pkl'
if download_new:
token_synopticlabs = get_synopticlabs_token(api_key)
m = Meso(token=token_synopticlabs)
data_ts = m.timeseries(stid=station_id,
start=st_time,
end=ed_time,
vars=vbl_list)
pickle.dump(data_ts, open(os.path.join(data_path, file_name), 'wb'))
else:
try:
data_ts = pickle.load(
open(os.path.join(data_path, file_name), 'rb'))
except:
raise (OSError('File not found: ' + file_name))
return data_ts, os.path.join(data_path, file_name)
def get_weather_service_wrapper(token):
from MesoPy import Meso
m = Meso(token=token)
return m
YesterDay = today - one_day
TODATSTR = YesterDay.strftime("%Y%m%d")
TWODAYSTR = ThreeDay.strftime("%Y%m%d")
StartTime = TWODAYSTR + START_UTC
EndTime = TODATSTR + END_UTC
print StartTime,EndTime
# StartTime ='2011900'
# EndTime = '201802191859'
#The two stations that Mike marked one has different current hour temperature and max/mim temperature
#Stations = {"KTYR": 411701,
Stations = {"KCLL": 413901}
#####Parameters for downloading Mesonet data using MesoWest API#####
MyToken = '994a7e628db34fc68503d44c447aaa6f'
m = Meso(token=MyToken)
base_url = 'http://api.mesowest.net/v2/stations/'
query_type = 'timeseries'
csv_format = '&output=csv'
noHighFrequecy = '&hfmetars=0'
#units = '&units=precip|in,temp|F,speed|mph'
#Changed the temperature from F to C
#units = '&units=precip|in,temp|C,speed|mph'
units = '&units=precip|in,temp|F,speed|mph'
#######################################################################
##Loop through each station for downloading and processing Mesonet Data
for STATION,ID in Stations.items():
try:
print STATION,ID
api_string = base_url + query_type + '?' + 'stid=' + STATION + '&start=' + StartTime + \
#logging.info("Start ASOS processing for %s", today.strftime("%Y%m%d"))
#logging.info("Start ASOS processing for %s", datetime.now().strftime("%Y%m%d%H"))
StartTime = TWODAYSTR + UTCHOUR
EndTime = DATESTR + UTCHOUR
print StartTime, EndTime
#For Test Only
StartTime = 201704011955
EndTime = 201704012000
#For ASOS
StartTime = '201704011900'
EndTime = '201704031900'
m = Meso(token=MyToken)
#q = m.timeseries(start=StartTime,end=EndTime,stid='CNST2')#,vars='wind_speed,pressure')
#print(type(q['STATION']))
#print type(q['STATION'])
##for l in q['STATION']:
## for k,v in l.items():
## print k,v
#meso_df = pd.DataFrame.from_dict(q,orient='columns')
#meso_df
#print m.variables()
'''
SENSOR_VARIABLES {u'wind_speed': {u'wind_speed_set_1': {u'position': u''}},
u'date_time': {u'date_time': {}},
u'solar_radiation': {u'solar_radiation_set_1': {u'position': u''}},
u'wind_gust': {u'wind_gust_set_1': {u'position': u''}},
u'pressure': {u'pressure_set_1': {u'position': u''}},
# "KBAZ": 418105,
# "KCLL": 413901,
# #"KCRS": 412001,
# "KTYR": 411701,
# "KTRL": 419703,
# "KDTO": 419603,
# "KMWL": 419404,
# "KMDD": 419207,
# "KVCT": 418203
# }
#Two more stations
# Stations = {"KMDD": 419207,
# "KVCT": 418203}
#####Parameters for downloading Mesonet data using MesoWest API#####
MyToken = '994a7e628db34fc68503d44c447aaa6f'
m = Meso(token=MyToken)
# base_url = 'http://api.mesowest.net/v2/stations/'
# query_type = 'timeseries'
# csv_format = '&output=csv'
# noHighFrequecy = '&hfmetars=0'
#Stations = {"KDHT": 418702}
# Station list
#stid: string, optional Single or comma separated list of MesoWest station IDs. e.g. stid = 'kden,kslc,wbb'
StationIDs = []
# StationIDs = list(Stations.keys())
for key, value in Stations.items():
StationIDs.append(key)
StationIDs = 'KCLL'
#######################################################################################################
def testauth():
badtoken = Meso(token='3030')
badtoken.latest(stid=['kfnl', 'kden', 'ksdf'], within='30')
"""
@author: joshclark
This script just demonstrates some example usage for the workshop
"""
from MesoPy import Meso
import pprint
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
# Instance a Meso object by passing in YOUR api_token
m = Meso(token='YOUR TOKEN')
# Fetches the latest obs for Boulder airport within 30 min of now
latest = m.latest(stid='kbdu', within='30', units='ENGLISH')
pp.pprint(latest)
# Let's format a pretty sentence that tells us the Boulder wx
x = latest['STATION'][0]
st_name = x['NAME']
temp = str(x['OBSERVATIONS']['air_temp_value_1']['value']) + u'\N{DEGREE SIGN}' + 'F'
wind = str(x['OBSERVATIONS']['wind_speed_value_1']['value']) + ' mph'
result = 'The current weather at ' + st_name + ' is ' + temp + ' with a sustained wind of ' + wind
print(result)
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
# Instance a Meso object by passing in YOUR api_token
def surfacePlot(api):
m = Meso(token=MesoPy_token)
stids = [
'KAIG', 'KATW', 'KASX', 'KDLL', 'KOVS', 'KBUU', 'KCLI', 'KEGV', 'KEAU',
'KFLD', 'KCMY', 'KGRB', 'KHYR', 'KJVL', 'KUNU', 'KENW', 'KLSE', 'KRCX',
'KLNL', 'KLNR', 'KMSN', 'KMTW', 'KMFI', 'KMDZ', 'KLUM', 'KRRL', 'KMKE',
'KMRJ', 'K82C', 'TT278', 'KEFT', 'KRNH', 'KOSH', 'KPBH', 'KPVB',
'KPDC', 'KRHI', 'KBCK', 'LAAW3', 'WUEW3', 'KRPD', 'KEZS', 'KSBM',
'KRZN', 'KSTE', 'KSUE', 'KSUW', 'KY51', 'KRYV', 'KUES', 'KPCZ', 'KAUW',
'KY50', 'KETB', 'KISW', 'KARV', 'GDNW3', 'KRGK', 'AFWW3'
] #stations
latest = m.latest(
stid=stids, within='30', vars='air_temp',
units='temp|F') #gets the latest temps (measured over past 30 minutes)
data = [] #a list with 4 columns: lat, long, temps, stid
for ob in latest['STATION']:
data.append((float(ob['LATITUDE']), float(ob['LONGITUDE']),
float(ob['OBSERVATIONS']['air_temp_value_1']['value']),
ob['STID']))
# Now that we have all the data, we can make the figure
fig = plt.figure(figsize=(10, 10))
plt.style.use('ggplot')
ax = plt.axes(projection=ccrs.Mercator())
ax.set_extent([-86.5, -93.2, 41.75, 47.25])
ax.add_feature(
ShapelyFeature(
Reader("ne_50m_admin_1_states_provinces_lakes.shp").geometries(),
ccrs.PlateCarree(),
facecolor='w',
edgecolor='gray',
linewidth=1.5))
county_shpfile = 'c_05ap16.shp' #wi county shapefile
WI = Reader(county_shpfile).records()
for i in WI:
if i.attributes['STATE'] == 'WI':
ax.add_geometries(i.geometry,
ccrs.PlateCarree(),
facecolor='white',
edgecolor='gray')
roads = cfeature.NaturalEarthFeature(category='cultural',
name='roads',
scale='10m',
facecolor='none')
ax.add_feature(roads, edgecolor='red')
ax.add_feature(
ShapelyFeature(Reader("ne_50m_lakes.shp").geometries(),
ccrs.PlateCarree(),
facecolor='lightblue',
edgecolor='gray',
linewidth=1.5))
for lat, lon, temp, stid in data: #plots a marker for each station
plt.plot(lon,
lat,
marker='o',
color='blue',
markersize=10,
alpha=0.7,
transform=ccrs.PlateCarree())
# Transforms for the text function
transform = ccrs.PlateCarree()._as_mpl_transform(ax)
text_transform = offset_copy(transform, units='dots', x=0, y=0)
# Plot temp for each of the markers
for lat, lon, temp, stid in data:
plt.text(lon,
lat,
str(round(temp, 1)) + u'\N{DEGREE SIGN}' + 'F\n\n',
verticalalignment='center',
horizontalalignment='center',
transform=text_transform,
fontsize=11)
uppertext = """Surface Observations
over Wisconsin, taken
in the last 30 minutes.
Data accessed from
MesoWest and
SynopticLabs.\n"""
lowertext = """See more current surface observations here:
http://mesowest.utah.edu/
http://aos.wisc.edu/weather/wx_obs/Surface.html
Learn how to interpret surface observation symbols here:
http://ww2010.atmos.uiuc.edu/%28Gh%29/guides/maps/sfcobs/home.rxml"""
ax.text(0.015,
0.01,
lowertext,
verticalalignment='bottom',
horizontalalignment='left',
transform=ax.transAxes,
fontsize='10.5',
bbox=dict(facecolor='white', alpha=0.5))
ax.text(0.015,
0.13,
uppertext,
verticalalignment='bottom',
horizontalalignment='left',
transform=ax.transAxes,
fontsize='10.5',
bbox=dict(facecolor='white', alpha=0.5))
plt.title('Current Weather Around Wisconsin', fontsize=14)
plt.savefig('WI_Obs.png', bbox_inches='tight')
#plt.show()
tweet = "What's the weather like around WI? Find out more here: http://aos.wisc.edu/weather/wx_obs/Surface.html"
try:
#print 'WI_Obs.png'
api.update_with_media("WI_Obs.png", status=tweet)
except tweepy.error.TweepError:
print("Twitter error raised")
plt.close('all')
"""
@author: joshclark
This script just demonstrates some example usage for the workshop
"""
from MesoPy import Meso
import pprint
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
# Instance a Meso object by passing in YOUR api_token
m = Meso(token='YOUR TOKEN')
# Fetches the latest obs for Boulder airport within 30 min of now
latest = m.latest(stid='kbdu', within='30', units='ENGLISH')
pp.pprint(latest)
# Let's format a pretty sentence that tells us the Boulder wx
x = latest['STATION'][0]
st_name = x['NAME']
temp = str(
x['OBSERVATIONS']['air_temp_value_1']['value']) + u'\N{DEGREE SIGN}' + 'F'
wind = str(x['OBSERVATIONS']['wind_speed_value_1']['value']) + ' mph'
result = 'The current weather at ' + st_name + ' is ' + temp + ' with a sustained wind of ' + wind
print(result)
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
def testgeoparms():
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
m.precipitation_obs(start='201504261800', end='201504271200', units='precip|in')
from MesoPy import Meso, MesoPyError
import json
import sys
#The script creats one file for each station (stationName.json)
stations = ['SCBH1','KKRH1','KTAH1','PLHH1','C0875','KFWH1','MKRH1','SCEH1','SCSH1','WNVH1']
#Mesowest token
x = Meso('ecbfdf783690489eb29d50b18b700827')
for station in range(len(stations)):
orig_stdout = sys.stdout
fileName = stations[station] + ".json"
f = file(fileName, 'w')
sys.stdout = f
returned = x.timeseries_obs(stid=stations[station], start='201401010000', end='201501010500', obtimezone='local', vars='solar_radiation')
#returned = x.latest_obs(stid='KKRH1, SCBH1, KTAH1, PLHH1', obtimezone='local', vars='solar_radiation', within='1440' )
#write the json file
print (json.dumps(returned, indent = 4))
sys.stdout = orig_stdout
f.close()
#Delete the first line of each file, it is necessary because the first line does not make part of the json file
for station in range (len(stations)):
fileName = stations[station] + ".json"
def retrieve_mesowest_observations(meso_token, tm_start, tm_end, glat, glon,
ghgt):
"""
Retrieve observation data from Mesowest and repackage them as a time-indexed
dictionary of lists of observations.
:param meso_token: the mesowest API access token or list of them
:param tm_start: the start of the observation window
:param tm_end: the end of the observation window
:param glat: the lattitudes of the grid points
:param glon: the longitudes of the grid points
:param ghgt: the elevation of the grid points
"""
def decode_meso_time(t):
# example: '2016-03-30T00:30:00Z'
return datetime.strptime(t,
'%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
def meso_time(dt):
# example: 201603311600
return '%04d%02d%02d%02d%02d' % (dt.year, dt.month, dt.day, dt.hour,
dt.minute)
# the bbox for mesowest is: (min(lon), min(lat), max(lon), max(lat)).
min_lat, max_lat = np.amin(glat), np.amax(glat)
min_lon, max_lon = np.amin(glon), np.amax(glon)
# retrieve data from Mesonet API (http://api.mesowest.net/)
try:
logging.info(
'retrieve_mesowest_observations: retrieving data using MesoDB')
db = mesoDB('ingest/MesoDB', meso_token)
db.update['startTime'] = tm_start - timedelta(minutes=30)
db.update['endTime'] = tm_end + timedelta(minutes=30)
db.params['startTime'] = tm_start - timedelta(minutes=30)
db.params['endTime'] = tm_end + timedelta(minutes=30)
db.params['longitude1'] = min_lon
db.params['longitude2'] = max_lon
db.params['latitude1'] = min_lat
db.params['latitude2'] = max_lat
df = db.get_DB().dropna(subset=['fm10'])
st = db.sites()
if not len(df):
logging.info(
'retrieve_mesowest_observations: no data for the query specified'
)
return {}
logging.info(
'retrieve_mesowest_observations: re-packaging the observations')
obs_data = {}
for stid, data in df.groupby('STID'):
if len(data):
st_data = st.loc[stid]
st_lat, st_lon = float(st_data['LATITUDE']), float(
st_data['LONGITUDE'])
ngp = find_closest_grid_point(st_lon, st_lat, glon, glat)
st_elev = st_data['ELEVATION']
if st_elev is None:
elev = ghgt[ngp]
else:
elev = float(st_data['ELEVATION']) / 3.2808
dts = data.datetime.dt.to_pydatetime()
fms = np.array(data.fm10)
for ts, fm_obs in zip(dts, fms):
if fm_obs is not None:
o = FM10Observation(ts, st_lat, st_lon, elev,
float(fm_obs) / 100., ngp)
obs_t = obs_data.get(ts, [])
obs_t.append(o)
obs_data[ts] = obs_t
return obs_data
except:
logging.warning(
'retrieve_mesowest_observations: failed with exception {}')
logging.info(
'retrieve_mesowest_observations: retrieving data using Meso instead'
)
if isinstance(meso_token, str):
meso_tokens = [meso_token]
n_tokens = 1
else:
meso_tokens = meso_token
n_tokens = len(meso_tokens)
for tn, meso_token in enumerate(meso_tokens):
m = Meso(meso_token)
logging.info("Retrieving fuel moisture from %s to %s" %
(meso_time(tm_start - timedelta(minutes=30)),
meso_time(tm_end + timedelta(minutes=30))))
logging.info("bbox=' %g,%g,%g,%g'" %
(min_lon, min_lat, max_lon, max_lat))
try:
meso_obss = m.timeseries(
meso_time(tm_start - timedelta(minutes=30)),
meso_time(tm_end + timedelta(minutes=30)),
showemptystations='0',
bbox='%g,%g,%g,%g' % (min_lon, min_lat, max_lon, max_lat),
vars='fuel_moisture')
break
except Exception as e:
if tn == n_tokens - 1:
raise MesoPyError(
'Could not connect to the API. Probably the token(s) usage for this month is full.'
)
else:
logging.warning(
'Could not connect to the API. Probably the token usage for this month is full. Trying next token...'
)
if meso_obss is None:
logging.info(
'retrieve_mesowest_observations: Meso.timeseries returned None'
)
return {}
logging.info(
'retrieve_mesowest_observations: re-packaging the observations')
# repackage all the observations into a time-indexed structure which groups
# observations at the same time together
obs_data = {}
for stinfo in meso_obss['STATION']:
st_lat, st_lon = float(stinfo['LATITUDE']), float(
stinfo['LONGITUDE'])
ngp = find_closest_grid_point(st_lon, st_lat, glon, glat)
st_elev = stinfo['ELEVATION']
if st_elev is None:
elev = ghgt[ngp]
else:
elev = float(stinfo['ELEVATION']) / 3.2808
dts = [
decode_meso_time(x)
for x in stinfo['OBSERVATIONS']['date_time']
]
if 'fuel_moisture_set_1' in stinfo['OBSERVATIONS']:
fms = stinfo['OBSERVATIONS']['fuel_moisture_set_1']
for ts, fm_obs in zip(dts, fms):
if fm_obs is not None:
o = FM10Observation(ts, st_lat, st_lon, elev,
float(fm_obs) / 100., ngp)
obs_t = obs_data.get(ts, [])
obs_t.append(o)
obs_data[ts] = obs_t
return obs_data
data_dir = pd.concat([
data.loc[data['Dir'] >= dir_threshold1],
data.loc[(data['Dir'] >= 0) & (data['Dir'] <= dir_threshold2)]
])
data_spd = data_dir.loc[(data_dir['Spd'] >= spd_threshold)]
data_rh = data_spd.loc[(data_spd['RH'] <= rh_threshold)]
#remove duplicate days to get list of days only
days = data_rh.drop_duplicates(subset=['Year', 'Month', 'Day'],
keep='first')[['Year', 'Month', 'Day']]
return data_rh, days
key = 'b1c91e501782441d97ac056e2501b5b0'
m = Meso(token=key)
stn_ids = ['BBEC1','OVYC1','NBRC1','NVHC1','WDAC1','BKSC1','KNXC1','KELC1','ATLC1','RSAC1','HWKC1',\
'LKRC1','BNVC1','OAAC1','PLEC1','ONOC1','OKSC1','LTRC1','PIBC1','LVMC1','VAQC1','LVFC1',\
'BNGC1','RRRC1','CISC1','SETC1','RNYC1','DOGC1','SMDC1','TADC1', 'DUCC1','HLLC1','STUC1',\
'PKCC1','SLEC1']
for i in range(len(stn_ids)):
print(stn_ids[i])
filename = 'pickles/' + stn_ids[i] + '_20yr_RAWS.p'
data = pickle.load(open(filename, 'rb'))
diablo_obs, diablo_days = find_diablo_obs(data)
#make scatter plot of the data
plt.figure(i)
plt.scatter(data['Dir'], data['Spd'], 1.5, 'k')
plt.scatter(diablo_obs['Dir'], diablo_obs['Spd'], 3, 'm', edgecolors='m')
from datetime import datetime, timedelta
import os
import urllib
import json
from MesoPy import Meso
import csv
import re
import pandas
Workspace = os.getcwd()
Workspace = r"C:\\DEV\\MesoNet"
Climate_Archive = os.path.join(Workspace, "Inputs", "NWS_Station_Data",
"Climate_Obs")
m = Meso(token='994a7e628db34fc68503d44c447aaa6f')
# Set up date stucture for retrieving 7 days of data
today = datetime.today()
one_day = timedelta(days=1)
sixty_day = timedelta(days=60)
hundred_day = timedelta(days=120)
twenty_day = timedelta(days=20)
six_day = timedelta(days=6)
#yesterday = today - one_day
time = "1200"
#startdate = yesterday.strftime("%Y%m%d")
#enddate = today.strftime("%Y%m%d")
# Observation period begins on the beginning (today) and ends on the endday
# The days count back from today to the date specified below
endday = today
# Raspberry Pi official 7" touchscreen display.
# Software: Raspbian Jessie
# Python 2.7
# MesoPy 2.0.3
# Kivy
# Program by Dan Stormont
# License: MIT
from datetime import datetime, date, time
from MesoPy import Meso
from pprint import pprint
#TODO-Get rid of these global constants and variables
MESOWEST_TOKEN = '0ee93b568404435a9f98ead284f5fc9c'
_m = Meso(token=MESOWEST_TOKEN)
def display_latest(station_id):
"""Display the latest weather readings from station_id."""
print 'Most recent weather readings:'
print '-----------------------------'
latest_reading = _m.latest(stid=station_id,
units='precip|in,speed|mph,temp|F')
print ('Temperature: '
+ str(latest_reading['STATION'][0]['OBSERVATIONS']
['air_temp_value_1']['value'])
+ ' deg F')
print ('Rainfall today: '
+ str(latest_reading['STATION'][0]['OBSERVATIONS']
def main():
# This is just useful for formatting returned dict
# pp = pprint.PrettyPrinter(indent=2)
# Create instance of Meso object, pass in YOUR token
m = Meso(token='YOUR TOKEN')
# Use to lookup stations, could specify counties or whatever here
# findstationids = m.station_list(state='CO')
# print(findstationids)
# Grab most recent temp (F) ob in last 90 min at each of the below stations
stations = [
'kgxy, kccu, kcos, kden, kgjt, kbdu, kpub, klhx, kspd, kdro, ksbs, keeo, kguc, klic, '
'kstk, kals, ktad'
]
latest = m.latest(stid=stations,
within='90',
vars='air_temp',
units='temp|F')
# create a list to store everything, iterate over the number of objs returned in latest and append
# lat, long, temp, and stid for use later
data = []
[
data.append((float(ob['LATITUDE']), float(ob['LONGITUDE']),
float(ob['OBSERVATIONS']['air_temp_value_1']['value']),
ob['STID'])) for ob in latest['STATION']
]
print(data)
# Create a MapQuest open aerial instance.
map_quest_aerial = cimgt.MapQuestOpenAerial()
# Create a GeoAxes in the tile's projection.
ax = plt.axes(projection=map_quest_aerial.crs)
# Limit the extent of the map to Colorado's borders
ax.set_extent([-102.03, -109.03, 37, 41])
# Add the MapQuest data at zoom level 8.
ax.add_image(map_quest_aerial, 8)
# Plot lat/long pts with below params
for lat, lon, temp, stid in data:
plt.plot(lon,
lat,
marker='o',
color='y',
markersize=1,
alpha=0.7,
transform=ccrs.Geodetic())
# Transforms for the text func we're about to call
geodetic_transform = ccrs.Geodetic()._as_mpl_transform(ax)
text_transform = offset_copy(geodetic_transform, units='dots', x=0, y=0)
# Plot temp and station id for each of the markers
for lat, lon, temp, stid in data:
plt.text(lon,
lat,
stid + '\n' + str(round(temp, 1)) + u' \N{DEGREE SIGN}' + 'F',
verticalalignment='center',
horizontalalignment='center',
transform=text_transform,
fontsize=9,
bbox=dict(facecolor='wheat', alpha=0.5, boxstyle='round'))
plt.title('Current Weather Around Colorado')
plt.show()
"""
@author: joshclark
This script just demonstrates some example usage for the workshop
"""
from MesoPy import Meso
import pprint
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
# Instance a Meso object by passing in YOUR api_token
m = Meso(api_token='3428e1e281164762870915d2ae6781b4')
# Fetches the latest obs for Boulder airport within 30 min of now
latest = m.latest_obs(stid='kbdu', within='30', units='ENGLISH')
pp.pprint(latest)
# Let's format a pretty sentence that tells us the Boulder wx
x = latest['STATION'][0]
st_name = x['NAME']
temp = str(x['OBSERVATIONS']['air_temp_value_1']['value']) + u'\N{DEGREE SIGN}' + 'F'
wind = x['OBSERVATIONS']['wind_speed_value_1']['value'] + ' mph'
result = 'The current weather at ' + st_name + ' is ' + temp + ' with a sustained wind of ' + wind
print(result)
# I import Pretty Print to make the returned dictionary look, well, pretty.
pp = pprint.PrettyPrinter(indent=2)
# Instance a Meso object by passing in YOUR api_token
#height_meters3 = dy3 * (int(y_dim3) - 1) #Domain Height
# close the wrfout
nc.close()
# reads UW WRF variables and saves them in a dictionary
wrf3 = met.get_wrf_DF('HRRR',datadir, start, end, x_dim3, y_dim3, lat3, lon3)
print("HRRR dictionary is done")
######################################
########## MESOWEST ##########
######################################
# Enter MesoWest token (generate using API key)
m = Meso(token = '23fd1c019ccb4ec3a6946eb3a13c99ad')
# Convert the start and end time to the string format requried by the API
start_t = start.strftime("%Y%m%d%H%M")
end_t = (end - timedelta(hours=1)).strftime("%Y%m%d%H%M")
tz = 'utc' # This is hard coded for now. Local time could be added later.
# String of some MesoWest variables available from this list:
# https://synopticlabs.org/api/mesonet/variables/
#variables = 'air_temp,pressure,wind_speed,wind_direction,relative_humidity,precip_accum'
variables = 'air_temp,pressure,wind_speed,wind_direction,relative_humidity'
var_list = variables.split(",")
# Bounding box (min lon, min lat, max lon, max lat)
# Lat/lon values taken from UW WRF output
from MesoPy import Meso, MesoPyError
from nose.tools import *
m = Meso(token='demotoken')
# Basic Function Tests
def testvars():
var_list = m.variables()
ok_(var_list)
def testmetadata():
stations = m.metadata(radius=['wbb', 5])
ok_(stations)
def testtimeseries():
timeseries = m.timeseries(stid='kfnl',
start='201504261800',
end='201504262300')
ok_(timeseries)
def testclimatology():
climatology = m.climatology(stid='kden',
startclim='04260000',
endclim='04270000',
units='precip|in')
ok_(climatology)
def testauth():
badtoken = Meso(token='3030')
badtoken.latest(stid=['kfnl', 'kden', 'ksdf'], within='30')
# if v + '_set_1' not in list(response['STATION'][0]['OBSERVATIONS'].keys()):
# raise ValueError('"{}" not available for station {}!'.format(v, stid))
# Get our times and measurements from the full returned dictionary
date_time = response['STATION'][0]['OBSERVATIONS']['date_time']
site_data = {
v: response['STATION'][0]['OBSERVATIONS'][v + '_set_1']
for v in varlist
}
df = pd.DataFrame(index=date_time, data=site_data)
df.index = pd.to_datetime(df.index)
return df
#use mesopy to get the asos meta_data
m = Meso(token=TOKEN)
stations = m.metadata(bbox=[-123.021397, 37.03180, -120.173988, 38.810713])
N_sta_in = stations['SUMMARY']['NUMBER_OF_OBJECTS']
#obtain the data using the function and the urllib and mesowest api. dump it into pickle files for each station
for n, x in enumerate(stations['STATION']):
if x['PERIOD_OF_RECORD']['start'] is not None:
if x['PERIOD_OF_RECORD']['start'][0:4] == '1970':
sta_id = x['STID']
print(sta_id)
print(str(n) + "/" + str(N_sta_in))
first = x['PERIOD_OF_RECORD']['start'][0:4] + x['PERIOD_OF_RECORD'][
'start'][5:7] + x['PERIOD_OF_RECORD']['start'][8:10] + '0000'
last = x['PERIOD_OF_RECORD']['end'][0:4] + x['PERIOD_OF_RECORD'][
'end'][5:7] + x['PERIOD_OF_RECORD']['end'][8:10] + '0000'
if sta_id != 'KMCC' and sta_id != 'KMER' and sta_id != 'KMHR':
