def reduce_density(df,
dens,
south=-90,
north=90,
east=180,
west=-180,
projection='EU'):
df_small = df[(df.latitude >= south) & (df.latitude <= north) &
(df.longitude <= east) & (df.longitude >= west)]
if (projection == 'GR') or (projection == 'Arctic'):
proj = ccrs.LambertConformal(central_longitude=-35,
central_latitude=65,
standard_parallels=[35])
elif projection == 'Antarctica':
proj = ccrs.SouthPolarStereo()
# elif projection == 'Arctic':
# proj = ccrs.NorthPolarStereo()
else:
proj = ccrs.LambertConformal(central_longitude=13,
central_latitude=47,
standard_parallels=[35])
# Use the cartopy map projection to transform station locations to the map
# and then refine the number of stations plotted by setting a 300km radius
point_locs = proj.transform_points(ccrs.PlateCarree(),
df_small['longitude'].values,
df_small['latitude'].values)
df = df_small[reduce_point_density(point_locs, dens)]
if projection == 'Arctic':
proj = ccrs.NorthPolarStereo()
return proj, point_locs, df
def test_reduce_point_density_1d():
r"""Test that reduce_point_density works with 1D points."""
x = np.array([1, 3, 4, 8, 9, 10])
assert_array_equal(reduce_point_density(x, 2.5),
np.array([1, 0, 1, 1, 0, 0], dtype=np.bool))
def test_reduce_point_density_priority(thin_point_data, radius, truth):
r"""Test that reduce_point_density works properly with priority."""
key = np.array(
[8, 6, 2, 8, 6, 4, 4, 8, 8, 6, 3, 4, 3, 0, 7, 4, 3, 2, 3, 3, 9])
assert_array_equal(reduce_point_density(thin_point_data, radius, key),
truth)
def test_reduce_point_density(thin_point_data, radius, truth):
r"""Test that reduce_point_density works."""
assert_array_equal(reduce_point_density(thin_point_data, radius=radius),
truth)
na_values=-99999)
###########################################
# This sample data has *way* too many stations to plot all of them. The number
# of stations plotted will be reduced using `reduce_point_density`.
# Set up the map projection
proj = ccrs.LambertConformal(central_longitude=-95,
central_latitude=35,
standard_parallels=[35])
# Use the cartopy map projection to transform station locations to the map and
# then refine the number of stations plotted by setting a 300km radius
point_locs = proj.transform_points(ccrs.PlateCarree(), data['lon'].values,
data['lat'].values)
data = data[reduce_point_density(point_locs, 300000.)]
###########################################
# Now that we have the data we want, we need to perform some conversions:
#
# - Get wind components from speed and direction
# - Convert cloud fraction values to integer codes [0 - 8]
# - Map METAR weather codes to WMO codes for weather symbols
# Get the wind components, converting from m/s to knots as will be appropriate
# for the station plot.
u, v = get_wind_components(
(data['wind_speed'].values * units('m/s')).to('knots'),
data['wind_dir'].values * units.degree)
# Convert the fraction value into a code of 0-8 and compensate for NaN values,
# Current observations can be downloaded here:
# https://www.mesonet.org/index.php/weather/category/past_data_files
data = pd.read_csv(get_test_data('mesonet_sample.txt'), na_values=' ')
# Drop stations with missing values of data we want
data = data.dropna(how='any', subset=['PRES', 'TAIR', 'TDEW', 'WDIR', 'WSPD'])
###########################################
# The mesonet has so many stations that it would clutter the plot if we used them all.
# The number of stations plotted will be reduced using `reduce_point_density`.
# Reduce the density of observations so the plot is readable
proj = ccrs.LambertConformal(central_longitude=-98)
point_locs = proj.transform_points(ccrs.PlateCarree(), data['LON'].values,
data['LAT'].values)
data = data[mpcalc.reduce_point_density(point_locs, 50 * units.km)]
###########################################
# Now that we have the data we want, we need to perform some conversions:
#
# - First, assign units to the data, as applicable
# - Convert cardinal wind direction to degrees
# - Get wind components from speed and direction
# Read in the data and assign units as defined by the Mesonet
temperature = data['TAIR'].values * units.degF
dewpoint = data['TDEW'].values * units.degF
pressure = data['PRES'].values * units.hPa
wind_speed = data['WSPD'].values * units.mph
wind_direction = data['WDIR']
latitude = data['LAT']
def test_reduce_point_density_units(thin_point_data, radius, truth):
r"""Test that reduce_point_density works with units."""
assert_array_equal(
reduce_point_density(thin_point_data * units.dam,
radius=radius * units.dam), truth)
# Drop rows with missing winds
data = data.dropna(how='any', subset=['wind_dir', 'wind_speed'])
###########################################
# This sample data has *way* too many stations to plot all of them. The number
# of stations plotted will be reduced using `reduce_point_density`.
# Set up the map projection
proj = ccrs.LambertConformal(central_longitude=-95, central_latitude=35,
standard_parallels=[35])
# Use the cartopy map projection to transform station locations to the map and
# then refine the number of stations plotted by setting a 300km radius
point_locs = proj.transform_points(ccrs.PlateCarree(), data['lon'].values, data['lat'].values)
data = data[reduce_point_density(point_locs, 300000.)]
###########################################
# Now that we have the data we want, we need to perform some conversions:
#
# - Get wind components from speed and direction
# - Convert cloud fraction values to integer codes [0 - 8]
# - Map METAR weather codes to WMO codes for weather symbols
# Get the wind components, converting from m/s to knots as will be appropriate
# for the station plot.
u, v = wind_components((data['wind_speed'].values * units('m/s')).to('knots'),
data['wind_dir'].values * units.degree)
# Convert the fraction value into a code of 0-8 and compensate for NaN values,
# which can be used to pull out the appropriate symbol
def main():
### START OF USER SETTINGS BLOCK ###
# FILE/DATA SETTINGS
# file path to input file
datafile = '/home/jgodwin/python/sfc_observations/surface_observations.txt'
timefile = '/home/jgodwin/python/sfc_observations/validtime.txt'
# file path to county shapefile
ctyshppath = '/home/jgodwin/python/sfc_observations/shapefiles/counties/countyl010g.shp'
# file path to ICAO list
icaopath = '/home/jgodwin/python/sfc_observations/icao_list.csv'
icaodf = pd.read_csv(icaopath, index_col='STATION')
# MAP SETTINGS
# map names (doesn't go anywhere (yet), just for tracking purposes)
maps = ['CONUS', 'Texas', 'Tropical Atlantic']
# minimum radius allowed between points (in km)
radius = [100.0, 50.0, 75.0]
# map boundaries (longitude/latitude degrees)
west = [-122, -108, -100]
east = [-73, -93, -60]
south = [23, 25, 10]
north = [50, 38, 35]
restart_projection = [True, False, True]
# use county map? (True/False): warning, counties load slow!
usecounties = [False, False, False]
# OUTPUT SETTINGS
# save directory for output
savedir = '/var/www/html/images/'
# filenames for output
savenames = ['conus.png', 'texas.png', 'atlantic.png']
# TEST MODE SETTINGS
test = False # True/False
testnum = 5 # which map are you testing? corresponds to index in "maps" above
### END OF USER SETTING SECTION ###
# if there are any missing weather codes, add them here
wx_code_map.update({
'-RADZ': 59,
'-TS': 17,
'VCTSSH': 80,
'-SGSN': 77,
'SHUP': 76,
'FZBR': 48,
'FZUP': 76
})
### READ IN DATA / SETUP MAP ###
# read in the valid time file
vt = open(timefile).read()
# read in the data
for i in range(len(maps)):
if test and i != testnum:
continue
with open(datafile) as f:
data = pd.read_csv(f,header=0,names=['siteID','lat','lon','elev','slp','temp','sky','dpt','wx','wdr',\
'wsp'],na_values=-99999)
# drop rows with missing winds
data = data.dropna(how='any', subset=['wdr', 'wsp'])
# remove data not within our domain
data = data[(data['lat'] >= south[i]-2.0) & (data['lat'] <= north[i]+2.0) \
& (data['lon'] >= west[i]-2.0) & (data['lon'] <= east[i]+2.0)]
# filter data (there seems to be one site always reporting a really anomalous temperature
data = data[data['temp'] <= 50]
print("Working on %s" % maps[i])
# set up the map projection central longitude/latitude and the standard parallels
cenlon = (west[i] + east[i]) / 2.0
cenlat = (south[i] + north[i]) / 2.0
sparallel = cenlat
if cenlat > 0:
cutoff = -30
flip = False
elif cenlat < 0:
cutoff = 30
flip = True
# create the projection
if restart_projection:
proj = ccrs.LambertConformal(central_longitude=cenlon,
central_latitude=cenlat,
standard_parallels=[sparallel],
cutoff=cutoff)
point_locs = proj.transform_points(ccrs.PlateCarree(),
data['lon'].values,
data['lat'].values)
data = data[reduce_point_density(point_locs, radius[i] * 1000)]
# state borders
state_boundaries = cfeature.NaturalEarthFeature(category='cultural',\
name='admin_1_states_provinces_lines',scale='50m',facecolor='none')
# county boundaries
if usecounties[i]:
county_reader = shpreader.Reader(ctyshppath)
counties = list(county_reader.geometries())
COUNTIES = cfeature.ShapelyFeature(counties, ccrs.PlateCarree())
### DO SOME CONVERSIONS ###
# get the wind components
u, v = wind_components(data['wsp'].values * units('knots'),
data['wdr'].values * units.degree)
# convert temperature from Celsius to Fahrenheit
data['temp'] = cToF(data['temp'])
data['dpt'] = cToF(data['dpt'])
# convert the cloud fraction value into a code of 0-8 (oktas) and compenate for NaN values
cloud_frac = (8 * data['sky'])
cloud_frac[np.isnan(cloud_frac)] = 10
cloud_frac = cloud_frac.astype(int)
# map weather strings to WMO codes (only use first symbol if multiple are present
data['wx'] = data.wx.str.split('/').str[0] + ''
wx = [
wx_code_map[s.split()[0] if ' ' in s else s]
for s in data['wx'].fillna('')
]
# get the minimum and maximum temperatures in domain
searchdata = data[(data['lat'] >= south[i]) & (data['lat'] <= north[i]) \
& (data['lon'] >= west[i]) & (data['lon'] <= east[i])]
min_temp = searchdata.loc[searchdata['temp'].idxmin()]
max_temp = searchdata.loc[searchdata['temp'].idxmax()]
max_dewp = searchdata.loc[searchdata['dpt'].idxmax()]
# look up the site names for the min/max temp locations
min_temp_loc = icaoLookup(min_temp['siteID'], icaodf)
max_temp_loc = icaoLookup(max_temp['siteID'], icaodf)
max_dewp_loc = icaoLookup(max_dewp['siteID'], icaodf)
text_str = "Min temp: %.0f F at %s (%s)\nMax temp: %.0f F at %s (%s)\nMax dewpoint: %.0f F at %s (%s)"\
% (min_temp['temp'],min_temp['siteID'],min_temp_loc,\
max_temp['temp'],max_temp['siteID'],max_temp_loc,\
max_dewp['dpt'],max_dewp['siteID'],max_dewp_loc)
### PLOTTING SECTION ###
# change the DPI to increase the resolution
plt.rcParams['savefig.dpi'] = 255
# create the figure and an axes set to the projection
fig = plt.figure(figsize=(20, 10))
ax = fig.add_subplot(1, 1, 1, projection=proj)
# add various map elements
ax.add_feature(cfeature.LAND, zorder=-1)
ax.add_feature(cfeature.OCEAN, zorder=-1)
ax.add_feature(cfeature.LAKES, zorder=-1)
ax.add_feature(cfeature.COASTLINE, zorder=2, edgecolor='black')
ax.add_feature(state_boundaries, edgecolor='black')
if usecounties[i]:
ax.add_feature(COUNTIES,
facecolor='none',
edgecolor='gray',
zorder=-1)
ax.add_feature(cfeature.BORDERS, linewidth=2, edgecolor='black')
# set plot bounds
ax.set_extent((west[i], east[i], south[i], north[i]))
### CREATE STATION PLOTS ###
# lat/lon of the station plots
stationplot = StationPlot(ax,data['lon'].values,data['lat'].values,clip_on=True,\
transform=ccrs.PlateCarree(),fontsize=6)
# plot the temperature and dewpoint
stationplot.plot_parameter('NW', data['temp'], color='red')
stationplot.plot_parameter('SW', data['dpt'], color='darkgreen')
# plot the SLP using the standard trailing three digits
stationplot.plot_parameter(
'NE', data['slp'], formatter=lambda v: format(10 * v, '.0f')[-3:])
# plot the sky condition
stationplot.plot_symbol('C', cloud_frac, sky_cover)
# plot the present weather
stationplot.plot_symbol('W', wx, current_weather)
# plot the wind barbs
stationplot.plot_barb(u, v, flip_barb=flip)
# plot the text of the station ID
stationplot.plot_text((2, 0), data['siteID'])
# plot the valid time
plt.title('Surface Observations valid %s' % vt)
# plot the min/max temperature info and draw circle around warmest and coldest obs
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(west[i],
south[i],
text_str,
fontsize=12,
verticalalignment='top',
bbox=props,
transform=ccrs.Geodetic())
projx1, projy1 = proj.transform_point(min_temp['lon'], min_temp['lat'],
ccrs.Geodetic())
ax.add_patch(
matplotlib.patches.Circle(xy=[projx1, projy1],
radius=50000,
facecolor="None",
edgecolor='blue',
linewidth=3,
transform=proj))
projx2, projy2 = proj.transform_point(max_temp['lon'], max_temp['lat'],
ccrs.Geodetic())
ax.add_patch(
matplotlib.patches.Circle(xy=[projx2, projy2],
radius=50000,
facecolor="None",
edgecolor='red',
linewidth=3,
transform=proj))
projx3, projy3 = proj.transform_point(max_dewp['lon'], max_dewp['lat'],
ccrs.Geodetic())
ax.add_patch(
matplotlib.patches.Circle(xy=[projx3, projy3],
radius=30000,
facecolor="None",
edgecolor='green',
linewidth=3,
transform=proj))
# save the figure
outfile_name = savedir + savenames[i]
plt.savefig(outfile_name, bbox_inches='tight')
# clear and close everything
fig.clear()
ax.clear()
plt.close(fig)
f.close()
print("Script finished.")
def test_reduce_point_density_1d():
r"""Test that reduce_point_density works with 1D points."""
x = np.array([1, 3, 4, 8, 9, 10])
assert_array_equal(reduce_point_density(x, 2.5),
np.array([1, 0, 1, 1, 0, 0], dtype=np.bool))
def test_reduce_point_density_priority(thin_point_data, radius, truth):
r"""Test that reduce_point_density works properly with priority."""
key = np.array([8, 6, 2, 8, 6, 4, 4, 8, 8, 6, 3, 4, 3, 0, 7, 4, 3, 2, 3, 3, 9])
assert_array_equal(reduce_point_density(thin_point_data, radius, key), truth)
def test_reduce_point_density(thin_point_data, radius, truth):
r"""Test that reduce_point_density works."""
assert_array_equal(reduce_point_density(thin_point_data, radius=radius), truth)
print(list(data.variables))
# Get the station IDs
station_id = []
for x in data['station_id']:
string = (x.tostring()).decode('utf-8')
station_id.append(string)
print(string)
cloud_frac = (8 * data['cloud_area_fraction'][:])
cloud_frac[np.isnan(cloud_frac)] = 10
cloud_frac = cloud_frac.astype(int)
# Extract weather as strings
weather = chartostring(data['weather'][:])
# Map weather strings to WMO codes, which we can use to convert to symbols
# Only use the first symbol if there are multiple
wx = [wx_code_map[s.split()[0] if ' ' in s else s] for s in weather]
# Get time into a datetime object
time = [datetime.fromtimestamp(t) for t in data['time'][0]]
time = sorted(time)
print(time)
# Set up the map projection
proj = ccrs.LambertConformal(central_longitude=-95,
central_latitude=35,
standard_parallels=[35])
# Use the cartopy map projection to transform station locations to the map and
# then refine the number of stations plotted by setting a 300km radius
point_locs = proj.transform_points(ccrs.PlateCarree(), data['longitude'][:],
data['latitude'][:])
msk = reduce_point_density(point_locs, 100000.)
df= df.dropna(how='any', subset=['wind_from_direction', 'wind_speed'])
df['cloud_area_fraction'] = (df['cloud_area_fraction'] * 8)
df['cloud_area_fraction'] = df['cloud_area_fraction'].replace(np.nan,10).astype(int)
# Get the columns with strings and decode
str_df = df.select_dtypes([np.object])
str_df = str_df.stack().str.decode('utf-8').unstack()
# Replace decoded columns in PlateCarree
for col in str_df:
df[col] = str_df[col]
# Set up the map projection
proj = ccrs.LambertConformal(central_longitude=13, central_latitude=47,
standard_parallels=[35])
# Use the cartopy map projection to transform station locations to the map and
# then refine the number of stations plotted by setting a 300km radius
point_locs = proj.transform_points(ccrs.PlateCarree(), df['longitude'].values, df['latitude'].values)
df = df[reduce_point_density(point_locs, 1000.)]
# Map weather strings to WMO codes, which we can use to convert to symbols
# Only use the first symbol if there are multiple
df['weather'] = df['weather'].replace('-SG','SG')
df['weather'] = df['weather'].replace('FZBR','FZFG')
wx = [wx_code_map[s.split()[0] if ' ' in s else s] for s in df['weather'].fillna('')]
# Get the wind components, converting from m/s to knots as will be appropriate
# for the station plot.
u, v = get_wind_components(((df['wind_speed'].values)*units('m/s')).to('knots'),
(df['wind_from_direction'].values) * units.degree)
cloud_frac = df['cloud_area_fraction']
# Change the DPI of the resulting figure. Higher DPI drastically improves the
# look of the text rendering.
# plt.rcParams['savefig.dpi'] = 100
def makeStationPlot(plotTitle, plotFileName, maxDataAge, maxLat, minLat,
maxLon, minLon, stationDensity, textSize, figX, figY, dpi,
showCountryBorders, showStateBorders, showCountyBorders):
#
# Data Polling
# Get data from AWC TDS
dataURL = "https://www.aviationweather.gov/adds/dataserver_current/httpparam?dataSource=metars&requestType=retrieve&format=csv&minLat=" + str(
minLat) + "&minLon=" + str(minLon) + "&maxLat=" + str(
maxLat) + "&maxLon=" + str(maxLon) + "&hoursBeforeNow=" + str(
maxDataAge)
# First read in the data. We use pandas because it simplifies a lot of tasks, like dealing
# with strings
data = pd.read_csv(dataURL,
header=5,
usecols=(1, 3, 4, 5, 6, 7, 8, 12, 21, 22),
names=[
'stid', 'lat', 'lon', 'air_temperature',
'dew_point_temperature', 'wind_dir', 'wind_speed',
'slp', 'weather', 'cloud_fraction'
],
na_values=-99999)
#
# Data Handling
# convert T and Td from °C to °F
data['air_temperature'] = (data['air_temperature'] * (9 / 5.0)) + 32
data['dew_point_temperature'] = (data['dew_point_temperature'] *
(9 / 5.0)) + 32
# change sky category to %
data['cloud_fraction'] = data['cloud_fraction'].replace(
'SKC', 0.0).replace('CLR', 0.0).replace('CAVOK', 0.0).replace(
'FEW', 0.1875).replace('SCT',
0.4375).replace('BKN', 0.750).replace(
'OVC', 1.000).replace('OVX', 1.000)
# Drop rows with missing winds
data = data.dropna(how='any', subset=['wind_dir', 'wind_speed'])
# Set up the map projection
proj = ccrs.LambertConformal(
central_longitude=(minLon + (maxLon - minLon) / 2),
central_latitude=(minLat + (maxLat - minLat) / 2))
# Set station density, in x meter radius
point_locs = proj.transform_points(ccrs.PlateCarree(), data['lon'].values,
data['lat'].values)
data = data[reduce_point_density(point_locs, stationDensity * 1000)]
# Get the wind components, converting from m/s to knots as will be appropriate
u, v = wind_components(
(data['wind_speed'].values * units('m/s')).to('knots'),
data['wind_dir'].values * units.degree)
# Convert the fraction value into a code of 0-8 and compensate for NaN values
cloud_frac = (8 * data['cloud_fraction'])
cloud_frac[np.isnan(cloud_frac)] = 10
cloud_frac = cloud_frac.astype(int)
# Map weather strings to WMO codes. Only use the first symbol if there are multiple
wx = [
wx_code_map[s.split()[0] if ' ' in s else s]
for s in data['weather'].fillna('')
]
#
# Plot Setup
# Set DPI of the resulting figure
plt.rcParams['savefig.dpi'] = dpi
# Create the figure and an axes set to the projection.
fig = plt.figure(figsize=(figX, figY))
ax = fig.add_subplot(1, 1, 1, projection=proj)
# Set plot bounds
ax.set_extent((minLon, maxLon, minLat, maxLat))
# Add geographic features
if showCountyBorders:
ax.add_feature(USCOUNTIES.with_scale('500k'),
edgecolor='gray',
linewidth=0.25)
if showStateBorders:
state_borders = cfeature.NaturalEarthFeature(
category='cultural',
name='admin_1_states_provinces_lakes',
scale='50m',
facecolor='none')
ax.add_feature(state_borders, edgecolor='gray', linewidth=0.5)
if showCountryBorders:
country_borders = cfeature.NaturalEarthFeature(
category='cultural',
name='admin_0_countries',
scale='50m',
facecolor='none')
ax.add_feature(country_borders, edgecolor='black', linewidth=0.7)
#
# Create Station Plots
# Set station location, setup plot
stationplot = StationPlot(ax,
data['lon'].values,
data['lat'].values,
clip_on=True,
transform=ccrs.PlateCarree(),
fontsize=textSize)
# Plot the temperature and dew point
stationplot.plot_parameter('NW', data['air_temperature'], color='red')
stationplot.plot_parameter('SW',
data['dew_point_temperature'],
color='darkgreen')
# Plot pressure data
stationplot.plot_parameter('NE',
data['slp'],
formatter=lambda v: format(10 * v, '.0f')[-3:])
# Plot cloud cover
stationplot.plot_symbol('C', cloud_frac, sky_cover)
# Plot current weather
stationplot.plot_symbol('W', wx, current_weather)
# Add wind barbs
stationplot.plot_barb(u, v)
# Plot station id
stationplot.plot_text((2, 0), data['stid'])
# Set a title and show the plot
ax.set_title(plotTitle)
# Export fig
fig.savefig('/home/CarterHumphreys/bin/send2web/' +
datetime.utcnow().strftime("%Y%m%d-%H00") + '_' +
plotFileName + '.png',
bbox_inches='tight')
fig.savefig('/home/CarterHumphreys/bin/send2web/' + plotFileName + '.png',
bbox_inches='tight')
