def tds_connect(tds):
"""
:return: datasets list after opening connection to Thredds catalogue
"""
cat = TDSCatalog(tds + 'aggregated' + xml_suf)
return list(cat.datasets.values())
Use Siphon to query the NetCDF Subset Service (NCSS).
"""
from datetime import datetime
import matplotlib.pyplot as plt
import numpy as np
from siphon.catalog import TDSCatalog
###########################################
# First we construct a TDSCatalog instance pointing to our dataset of interest, in
# this case TDS' "Best" virtual dataset for the GFS global 0.5 degree collection of
# GRIB files. We see this catalog contains a single dataset.
best_gfs = TDSCatalog(
'http://thredds.ucar.edu/thredds/catalog/grib/NCEP/GFS/'
'Global_0p5deg/catalog.xml?dataset=grib/NCEP/GFS/Global_0p5deg/Best')
print(best_gfs.datasets)
###########################################
# We pull out this dataset and get the NCSS access point
best_ds = best_gfs.datasets[0]
ncss = best_ds.subset()
###########################################
# We can then use the `ncss` object to create a new query object, which
# facilitates asking for data from the server.
query = ncss.query()
###########################################
# We construct a query asking for data corresponding to latitude 40N and longitude 105W,
def test_dataset_no_handler(nids_url):
"""Test requesting a service that has no handler gives a ValueError."""
cat = TDSCatalog(nids_url)
with pytest.raises(ValueError) as err:
cat.datasets[0].access_with_service('UDDC')
assert 'is not an access method supported' in str(err.value)
print('Generating %s...' % pname)
# ----------------------------------------------------------------------------------------------------------------------
# Figure out needed GFS file based on command-line input and define some helpful Cartopy variables
# ----------------------------------------------------------------------------------------------------------------------
DesiredTime = datetime.strptime(dt,'%Y%m%d%H') # This is a 'datetime' variable for the desired map date and time
ModelInit = DesiredTime - timedelta(hours=int(ForecastHour)) # Get the model init time for DesiredTime and ForecastHour
pc = ccrs.PlateCarree()
states = NaturalEarthFeature(category='cultural', scale='50m', facecolor='none', name='admin_1_states_provinces_shp')
catUrl = ModelInit.strftime('https://rda.ucar.edu/thredds/catalog/files/g/ds084.1/%Y/%Y%m%d/catalog.xml')
datasetName = ModelInit.strftime('gfs.0p25.%Y%m%d%H.f') + ForecastHour + '.grib2'
# ----------------------------------------------------------------------------------------------------------------------
# Access file on RDA server and define CONUS region (to avoid downloading the global GFS field)
# ----------------------------------------------------------------------------------------------------------------------
catalog = TDSCatalog(catUrl) # We're using 'TDSCatalog' from the 'siphon' Python package to access the above URL
ds = catalog.datasets[datasetName] # Let's get the 24-hour forecast file
dataset = ds.remote_access() # Calls remote_access() -- this gives us access to the file
latitude = dataset.variables['lat'][:] # Get 1-D array with latitude values
rows = np.where((latitude>=20.)&(latitude<=55.))[0] # List of locations within 20-55N
r0, r1 = [min(rows), max(rows)+1]
longitude = dataset.variables['lon'][:] # Get 1-D array with longitude values
cols = np.where((longitude>=220.)&(longitude<=300.))[0] # List of locations within 120-60W (0-360 longitude format)
c0, c1 = [min(cols), max(cols)+1]
glon, glat = np.meshgrid(longitude[c0:c1], latitude[r0:r1]) # Create 2-D latitude and longitude grids for cartopy use
# ----------------------------------------------------------------------------------------------------------------------
# Request fields from remote file depending on the selected map of interest
# ----------------------------------------------------------------------------------------------------------------------
var_list = list(dataset.variables) # list of all variable names in 'dataset' (helpful when developing or debugging)
bcolor = 'blue' # wind barb color
import numpy as np
#sleep 30s; shutdown -h now
baseurl = 'http://thredds-test.unidata.ucar.edu/thredds/catalog/satellite/goes16/GOES16/'
channel = '13' #01-13
date = '20180624' #20180615 or current
sector = 'Mesoscale-1' #Mesoscale-1, Mesoscale-2, CONUS, FullDisk
#sector = 'CONUS'
savelocation = '/home/scarani/Desktop/output/goes/' + sector + '/'
baseurl = 'http://thredds-test.unidata.ucar.edu/thredds/catalog/satellite/goes16/GOES16/'
cat = TDSCatalog(baseurl + str(sector) + '/Channel' + str(channel) + '/' + str(date) +
'/catalog.xml')
data = cat.datasets
ds = cat.datasets[-120]
data = cat.datasets
# 'Mercator'
# 'Fixed Grid'
ds = ds.remote_access(service='OPENDAP')
ds = NetCDF4DataStore(ds)
ds = xr.open_dataset(ds)
print(ds.projection)
timestamp = datetime.strptime(ds.start_date_time, '%Y%j%H%M%S')
data_var = ds.metpy.parse_cf('Sectorized_CMI')
x = ds['x']
import matplotlib.patheffects as PathEffects
import metpy.calc as mpcalc
from metpy.units import units
from mpl_toolkits.axes_grid1 import make_axes_locatable, axes_size
from netCDF4 import num2date
import numpy as np
import scipy.ndimage as ndimage
import shapefile
from siphon.catalog import TDSCatalog
import xarray as xr
# Set current time.
now = datetime.utcnow()
# Import latest RAP dataset.
imprt = TDSCatalog('https://thredds-test.unidata.ucar.edu/thredds/catalog/'
'grib/NCEP/RAP/CONUS_13km/latest.xml')
dataset = imprt.datasets[0]
ncss = dataset.subset()
# Query the imported datasets wind data.
wind = ncss.query()
wind.variables('u-component_of_wind_isobaric',
'v-component_of_wind_isobaric')
wind.add_lonlat().vertical_level(500 * 100)
wind.time(now)
wind.lonlat_box(north=55, south=20, east=281, west=230)
wind_data = ncss.get_data(wind)
# Query the imported datasets height data.
hght = ncss.query()
hght.variables('Geopotential_height_isobaric')
elif(model=='NAM'):
data_link = 'http://thredds.ucar.edu/thredds/catalog/grib/NCEP/NAM/' + \
'CONUS_12km/catalog.html?dataset=grib/NCEP/NAM/CONUS_12km/Best'
elif(model=='RAP'):
data_link = 'https://thredds.ucar.edu/thredds/catalog/grib/NCEP/RAP/CONUS_13km/' + \
'catalog.html?dataset=grib/NCEP/RAP/CONUS_13km/Best'
elif(model=='HRRR'):
data_link = 'https://thredds.ucar.edu/thredds/catalog/grib/NCEP/HRRR/' + \
'CONUS_2p5km/catalog.html?dataset=grib/NCEP/HRRR/CONUS_2p5km/Best'
###########################################
# First we construct a TDSCatalog instance pointing to our dataset of interest, in
# this case TDS' "Best" virtual dataset for the GFS global 0.5 degree collection of
# GRIB files. We see this catalog contains a single dataset.
best_gfs = TDSCatalog(data_link)
print(best_gfs.datasets)
# Grab the NAM data
###########################################
# We pull out this dataset and get the NCSS access point
best_ds = best_gfs.datasets[0]
ncss = best_ds.subset()
###########################################
# We can then use the `ncss` object to create a new query object, which
# facilitates asking for data from the server.
query = ncss.query()
###########################################
def test_tds50_catalogref_follow():
"""Test following a catalog ref url on TDS 5."""
cat = TDSCatalog(
'http://thredds-test.unidata.ucar.edu/thredds/catalog.xml')
assert len(cat.catalog_refs[0].follow().catalog_refs) == 59
def test_catalog_ref_str():
"""Test that catalog references are properly represented as strings."""
url = 'http://thredds.ucar.edu/thredds/catalog.xml'
cat = TDSCatalog(url)
assert str(cat.catalog_refs[0]) == 'Forecast Model Data'
def test_ramadda_catalog():
"""Test parsing a catalog from RAMADDA."""
url = 'http://weather.rsmas.miami.edu/repository?output=thredds.catalog'
cat = TDSCatalog(url)
assert len(cat.catalog_refs) == 12
def test_catalog_representation():
"""Test string representation of the catalog object."""
url = 'http://thredds-test.unidata.ucar.edu/thredds/catalog.xml'
cat = TDSCatalog(url)
assert str(cat) == 'Unidata THREDDS Data Server'
def test_html_link(recwarn):
"""Test that we fall-back when given an HTML catalog page."""
url = ('http://thredds-test.unidata.ucar.edu/thredds/catalog/'
'grib/NCEP/RAP/CONUS_13km/catalog.html')
TDSCatalog(url)
assert 'Changing' in str(recwarn.pop(UserWarning).message)
cax = fig.add_axes([left, bottom, width, height])
cbar = plt.colorbar(im, cax=cax, ticks=clevs, orientation='vertical')
cbar.ax.tick_params(labelsize=10)
cbar.set_label('500-hPa temperature [K]', size=11) # MODIFY THIS for other fields!!
"""
Code begins below; modify 'ymdh' as needed for experimenting (since Unidata THREDDS only keeps a few days of forecasts)
"""
ymdh = '2021020812' # YYYYMMDDHH; user can change to a command-line input with the line ymdh = sys.argv[1]
#### Access GFS forecast initialized at the time given by 'ymdh'
init_time = datetime.strptime(ymdh, '%Y%m%d%H')
catalogURL = 'https://tds.scigw.unidata.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p25deg/' + \
init_time.strftime('GFS_Global_0p25deg_%Y%m%d_%H00.grib2/catalog.xml')
catalog = TDSCatalog(catalogURL)
dataset = catalog.datasets[init_time.strftime('GFS_Global_0p25deg_%Y%m%d_%H00.grib2')].remote_access()
fieldList = list(dataset.variables) # list of all variable names in 'dataset' (helpful when developing or debugging)
#### Get attribute info for temperature on pressure levels
temperature = dataset.variables['Temperature_isobaric']
for dim in temperature.dimensions:
print(dim, dataset.variables[dim].units) # print the dimension name and its units
hoursName, levName, latName, lonName = temperature.dimensions
hours = dataset.variables[hoursName][:] # NumPy array of forecast hours (hours since 'init_time')
plevs = dataset.variables[levName][:] # NumPy array of pressure levels (units of Pa)
lat = dataset.variables[latName][:] # NumPy array of latitudes
lon = dataset.variables[lonName][:] # NumPy array of longitudes (0-360 format)
#### Get data for desired pressure level range and geographic region (below approach ensures dim order doesn't matter)
hour = 24 # 24-hour forecast
def main():
# Parse the arguments from the command line
parser = argparse.ArgumentParser()
parser.add_argument('-m',
'--map',
help='Which type of map to be generated.')
parser.add_argument('-t',
'--time',
nargs='+',
help='Access and plot weather data X hours from now.',
type=int,
default=0)
args = parser.parse_args()
if args.map == 'verywide':
map_ = map_utils.VeryWide()
elif args.map == 'regional':
map_ = map_utils.Regional()
elif args.map == 'local':
map_ = map_utils.Local()
elif args.map == 'tropical':
map_ = map_utils.Tropical()
else:
print("Invalid Map Type Requested.")
return
# Create maps for each time declared on command line
for t in args.time:
# Acquire the datasets from the GFS Global Catalog
GFS_data = TDSCatalog(
'http://thredds.ucar.edu/thredds/catalog/grib/NCEP/GFS/'
'Global_0p25deg/catalog.xml?dataset=grib/NCEP/GFS/Global_0p25deg/Best'
)
# Pull out our dataset and get the NCSS access point used to query data from the dataset
dataset = GFS_data.datasets[0]
ncss = dataset.subset()
# Use the `ncss` object to create a new query object
query = ncss.query()
time = (datetime.utcnow() + timedelta(hours=t)
) # Time of data requested
query.time(time)
query.accept('netcdf4')
query.variables('Temperature_surface')
# Set the lat lon box for which specific area of the dataset to query
if map_.map_type is not 'tropical':
query.lonlat_box(north=map_.NorthSouthEastWest[0],
south=map_.NorthSouthEastWest[1],
east=map_.NorthSouthEastWest[2],
west=map_.NorthSouthEastWest[3])
else:
query.lonlat_box(north=map_.NorthSouthEastWest[0] + 10,
south=map_.NorthSouthEastWest[1],
east=map_.NorthSouthEastWest[2],
west=map_.NorthSouthEastWest[3])
data = ncss.get_data(query)
# Grab the keys from the data we want
temperatures = data.variables['Temperature_surface']
latitudes = data.variables['lat']
longitudes = data.variables['lon']
# Remove 1d arrays from data for plotting
temperatures = temperatures[:].squeeze()
latitudes = latitudes[:].squeeze()
longitudes = longitudes[:].squeeze()
# Convert temps to Fahrenheit from Kelvin
temperatures = convert_temperature_to_fahrenheit(temperatures)
# Combine 1D latitude and longitudes into a 2D grid of locations
lon_2d, lat_2d = np.meshgrid(longitudes, latitudes)
# Create figure for plotting
fig = plt.figure(figsize=(15, 9))
ax = fig.add_subplot(1, 1, 1, projection=ccrs.Mercator())
ax.set_extent(map_.NorthSouthEastWest[::-1], crs=ccrs.Geodetic())
# Add map features depending on map type
ax.add_feature(cfeature.STATES.with_scale('50m'), linewidth=0.5)
if map_.map_type == 'regional' or map_.map_type == 'local':
reader = shpreader.Reader('../county_data/countyl010g.shp')
counties = list(reader.geometries())
COUNTIES = cfeature.ShapelyFeature(counties, ccrs.PlateCarree())
ax.add_feature(COUNTIES,
facecolor='none',
edgecolor='black',
linewidth=0.3)
elif map_.map_type == 'tropical':
countries = cfeature.NaturalEarthFeature(category='cultural',
name='admin_0_countries',
scale='50m',
facecolor='none')
ax.add_feature(cfeature.LAND)
ax.add_feature(countries, edgecolor='black', linewidth=0.5)
# Contour temperature value at each lat/lon
cf = ax.contourf(lon_2d,
lat_2d,
temperatures,
40,
extend='both',
transform=ccrs.PlateCarree(),
cmap='coolwarm')
# Plot a colorbar to show temperature values
colorbar = plt.colorbar(cf, ax=ax, fraction=0.032)
colorbar.set_label('Temperature (\u00b0F)')
# Plot all the cities
if map_.map_type is not 'tropical':
for city in map_.cities:
for lat in range(len(latitudes)):
for lon in range(len(longitudes)):
if round_temps(
city.lat) == latitudes[lat] and round_temps(
city.lon) == (longitudes[lon] - 360):
cityTemp_latlon = Utils.plot_latlon_cityTemp_by_maptype(
lat=city.lat,
lon=city.lon,
map_type=map_.map_type)
ax.text(cityTemp_latlon[1],
cityTemp_latlon[0],
int(round(temperatures[lat][lon])),
fontsize='10',
fontweight='bold',
transform=ccrs.PlateCarree())
ax.plot(city.lon,
city.lat,
'ro',
zorder=9,
markersize=2.00,
transform=ccrs.Geodetic())
cityName_latlon = Utils.plot_latlon_cityName_by_maptype(
lat=city.lat, lon=city.lon, map_type=map_.map_type)
ax.text(cityName_latlon[1],
cityName_latlon[0],
city.city_name,
fontsize='small',
fontweight='bold',
transform=ccrs.PlateCarree())
# Create a title with the time value
ax.set_title('Temperature forecast (\u00b0F) for {} UTC'.format(
str(time)[:-7]),
fontsize=12,
loc='left')
# Company copyright
text = AnchoredText(r'$\mathcircled{{c}}$ NickelBlock Forecasting',
loc=4,
prop={'size': 9},
frameon=True)
ax.add_artist(text)
# Data model
data_model = AnchoredText('GFS 12z model',
loc=3,
prop={'size': 9},
frameon=True)
ax.add_artist(data_model)
# Add logo
logo = Utils.get_logo()
if map_.map_type is not 'tropical':
ax.figure.figimage(logo, 1105, 137, zorder=1)
else:
ax.figure.figimage(logo, 1105, 181, zorder=1)
plt.savefig('{}_Temperature_Hour_{}.png'.format(map_.map_type, t))
from matplotlib.dates import AutoDateLocator, DateFormatter
import matplotlib.pyplot as plt
import metpy.calc as mpcalc
from metpy.units import units
from siphon.catalog import TDSCatalog
from siphon.ncss import NCSS
from metpy.calc import get_wind_components
from metpy.calc import reduce_point_density
from metpy.cbook import get_test_data
from metpy.plots import add_metpy_logo, StationPlot
from metpy.plots.wx_symbols import current_weather, sky_cover, wx_code_map
from metpy.units import units
# Request METAR data from TDS
metar = TDSCatalog('http://thredds.ucar.edu/thredds/catalog/nws/metar/'
'ncdecoded/catalog.xml')
dataset = list(metar.datasets.values())[0]
print(list(dataset.access_urls))
# Access netcdf subset and use siphon to request data
ncss_url = dataset.access_urls['NetcdfSubset']
ncss = NCSS(ncss_url)
print(ncss.variables)
# get current date and time
now = datetime.utcnow()
now = datetime(now.year, now.month, now.day, now.hour)
# define time range you want the data for
start = now - timedelta(days=1)
end = now
def test_catalog_follow():
"""Test catalog reference following."""
url = 'http://thredds.ucar.edu/thredds/catalog.xml'
ref_name = 'Forecast Model Data'
cat = TDSCatalog(url).catalog_refs[ref_name].follow()
assert cat
from datetime import datetime
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import matplotlib.pyplot as plt
from matplotlib import patheffects
from netCDF4 import Dataset
import numpy as np
from scipy import interpolate
from siphon.catalog import TDSCatalog
date = datetime.utcnow()
channel = 13
idx = -2
region = 'Mesoscale-1'
"""
Open and return a netCDF Dataset object for a given date, channel, and image index
of GOES-16 data from THREDDS test server.
"""
cat = TDSCatalog(
'http://thredds-test.unidata.ucar.edu/thredds/catalog/satellite/goes16/GOES16/'
'{}/Channel{:02d}/{:%Y%m%d}/catalog.xml'.format(region, channel, date))
ds = cat.datasets[idx].remote_access(service='OPENDAP')
def test_basic():
"""Test of parsing a basic catalog."""
url = 'http://thredds-test.unidata.ucar.edu/thredds/catalog.xml'
cat = TDSCatalog(url)
assert 'Forecast Model Data' in cat.catalog_refs
def Map_Jets(i, im_save_path):
from siphon.catalog import TDSCatalog
top_cat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog.xml')
ref = top_cat.catalog_refs['Forecast Model Data']
new_cat = ref.follow()
model = new_cat.catalog_refs[4]
gfs_cat = model.follow()
ds = gfs_cat.datasets[1]
subset = ds.subset()
query_data = subset.query()
query_data.lonlat_box(west=-130, east=-50, south=10, north=60)
# Allow for NetCDF files
query_data.accept('netcdf4')
query_data.time(i)
data = query_data.variables('Geopotential_height_isobaric',
'Pressure_reduced_to_MSL_msl',
'u-component_of_wind_isobaric',
'v-component_of_wind_isobaric')
# Finally attempt to access the data
data = subset.get_data(query_data)
lat = data.variables['lat'][:].squeeze()
lon = data.variables['lon'][:].squeeze()
lev_250 = np.where(data.variables['isobaric4'][:] == 25000)[0][0]
hght_250 = data.variables['Geopotential_height_isobaric'][0, lev_250, :, :]
u_250 = data.variables['u-component_of_wind_isobaric'][0, lev_250, :, :]
v_250 = data.variables['v-component_of_wind_isobaric'][0, lev_250, :, :]
# Create a figure object, title it, and do the plots.
fig = plt.figure(figsize=(17., 11.))
add_metpy_logo(fig, 30, 940, size='small')
# Add the map and set the extent
ax6 = plt.subplot(1, 1, 1, projection=plotcrs)
# Add state boundaries to plot
ax6.add_feature(states_provinces, edgecolor='b', linewidth=1)
# Add country borders to plot
ax6.add_feature(country_borders, edgecolor='k', linewidth=1)
# Convert number of hours since the reference time into an actual date
time_var = data.variables[find_time_var(
data.variables['v-component_of_wind_isobaric'])]
time_final = num2date(time_var[:].squeeze(), time_var.units)
print(
str(time_final)[:4] + "_" + str(time_final)[5:7] + "_" +
str(time_final)[8:10] + "_" + str(time_final)[11:13] + "Z")
file_time = str(time_final)[:4] + "_" + str(time_final)[5:7] + "_" + str(
time_final)[8:10] + "_" + str(time_final)[11:13] + "Z"
# Plot Title
plt.title('GFS: 250mb Heights and Jet Streaks (m/s)',
loc='left',
fontsize=16)
plt.title(' {0:%d %B %Y %H:%MZ}'.format(time_final),
loc='right',
fontsize=16)
# Heights
#---------------------------------------------------------------------------------------------------
MIN = hght_250.min()
MAX = hght_250.max()
#print hght_250.min(),hght_250.max()
hght_250 = ndimage.gaussian_filter(hght_250, sigma=3,
order=0) * units.meter
clev250 = np.arange(MIN, MAX, 80)
cs = ax6.contour(lon,
lat,
hght_250.m,
clev250,
colors='black',
linewidths=2.0,
linestyles='solid',
transform=ccrs.PlateCarree())
#plt.clabel(cs, fontsize=10, inline=1, inline_spacing=10, fmt='%i',
# rightside_up=True, use_clabeltext=True)
# Winds
#---------------------------------------------------------------------------------------------------
#lon_slice = slice(None, None, 7)
#lat_slice = slice(None, None, 7)
#ax4.barbs(lon[lon_slice], lat[lat_slice],
# u_250[lon_slice, lat_slice].magnitude,
# v_250[lon_slice, lat_slice].magnitude,
# transform=ccrs.PlateCarree(), zorder=2)
wspd250 = mpcalc.get_wind_speed(u_250, v_250)
clevsped250 = np.arange(50, 100, 1)
cf = ax6.contourf(lon,
lat,
wspd250,
clevsped250,
cmap="gist_ncar",
transform=ccrs.PlateCarree())
#cbar = plt.colorbar(cf, cax=cax, orientation='horizontal', extend='max', extendrect=True,pad=0.2)
cbaxes = fig.add_axes(colorbar_axis)
cbar = plt.colorbar(cf, orientation='horizontal', cax=cbaxes)
ax6.set_extent(extent, datacrs)
plt.close(fig)
#---------------------------------------------------------------------------------------------------
#---------------------------------------------------------------------------------------------------
GFS_Jet = im_save_path + "GFS/Jets/"
if not os.path.isdir(GFS_Jet):
os.makedirs(GFS_Jet)
fig.savefig(GFS_Jet + "250mb_Heights_Winds_" + file_time + ".png",
bbox_inches='tight',
dpi=120)
print('done.')
def test_access_elements():
"""Test parsing access elements in TDS client catalog."""
url = 'http://oceandata.sci.gsfc.nasa.gov/opendap/SeaWiFS/L3SMI/2001/001/catalog.xml'
cat = TDSCatalog(url)
assert len(list(cat.datasets)) != 0
# Copyright (c) 2013-2015 University Corporation for Atmospheric Research/Unidata.
# Distributed under the terms of the MIT License.
# SPDX-License-Identifier: MIT
"""
===============
THREDDS Catalog
===============
Read a catalog from THREDDS Data Server.
This example grabs a remote catalog and prints out the catalog references
contained within.
"""
# This is currently a placeholder for a better example
from __future__ import print_function
from siphon.catalog import TDSCatalog
###########################################
cat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog.xml')
print(list(cat.catalog_refs.keys()))
def test_latest_attribute():
"""Test using the catalog latest attribute."""
url = ('http://thredds-test.unidata.ucar.edu/thredds/catalog/'
'grib/NCEP/RAP/CONUS_13km/catalog.xml')
cat = TDSCatalog(url)
assert cat.latest.name == 'RR_CONUS_13km_20150527_0100.grib2'
import matplotlib.pyplot as plt
import metpy.calc as mpcalc
from netCDF4 import num2date
import numpy as np
from metpy.units import units
import scipy.ndimage as ndimage
from siphon.catalog import TDSCatalog
from siphon.ncss import NCSS
# =============================================================================
# RETRIEVE RAP AND HRRR DATA
# =============================================================================
RAP = 'http://thredds-jetstream.unidata.ucar.edu/thredds/catalog/grib/NCEP/RAP/CONUS_20km/latest.xml'
HRRR= 'http://thredds-jetstream.unidata.ucar.edu/thredds/catalog/grib/NCEP/HRRR/CONUS_2p5km/latest.xml'
GFS = 'http://thredds-jetstream.unidata.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p5deg/latest.xml'
DATA = TDSCatalog(GFS)
BEST_DATA = list(DATA.datasets.values())[0]
NCSS_DATA = NCSS(BEST_DATA.access_urls['NetcdfSubset'])
NOW = datetime.utcnow()
LATEST_DATA = NCSS_DATA.query().time(NOW).accept('netcdf4')
# =============================================================================
# UPPER-AIR VARIABLES
# =============================================================================
# 250: JET STREAM, GEOPOTENTIAL HEIGHT, POTENTIAL VORTICITY, IRROTATIONAL WIND
def 250hPa_GFS_jet_stream_SLP(lon_west, lon_east, lat_south, lat_north):
def 250hPa_GFS_jet_stream_jet_dyn(lon_west, lon_east, lat_south, lat_north):
# 500: VORTICITY, GEOPOTENTIAL HEIGHT, VORTICITY ADVECTION
def 500hPa_GFS_vorticity(lon_west, lon_east, lat_south, lat_north):
LATEST_DATA.variables('Geopotential_height_isobaric', 'u-component_of_wind_isobaric', 'v-component_of_wind_isobaric').add_lonlat()
def test_tds_top_catalog():
"""Test parsing top-level catalog."""
url = 'http://thredds.ucar.edu/thredds/catalog.xml'
cat = TDSCatalog(url)
assert cat
def get_data_in_point_between_dates(self, point, dates, variables):
"""Request variables for a specific point and a set period from Thredds"""
try:
catalog = TDSCatalog(self.url_catalog)
best_dataset = catalog.datasets[0]
ncss = best_dataset.subset()
query = ncss.query()
query.lonlat_point(point['lon'], point['lat']).time_range(
dates['start'], dates['end'])
query.variables(",".join(variables))
# Set extra params for performance #
#query.vertical_level(0.49402499198913574)
query.accept("netcdf")
self.logger.debug(
"thredds query lon: {lon}, lat: {lat}, start: {start}, end: {end}, vars: {vars}"
.format(lon=point["lon"],
lat=point["lat"],
start=dates["start"],
end=dates["end"],
vars=",".join(variables)))
data = ncss.get_data(query)
scale_factor = {}
offset = {}
for variable_name in variables:
scale_factor[variable_name] = ncss.metadata.variables[
variable_name]['attributes']['scale_factor'][0]
offset[variable_name] = ncss.metadata.variables[variable_name][
'attributes']['add_offset'][0]
values = {}
for i in range(data.variables.get('time').size):
dictionary_date_variables = {}
for variable_name in variables:
value = float(
np.ma.getdata(data[variable_name][0][i], subok=True))
if float(ncss.metadata.variables[variables[0]]
['attributes']['missing_value'][0]) == value:
raise LandException(
"This point is located on land: lon: {lon}, lat: {lat}"
.format(lon=point["lon"], lat=point["lat"]))
dictionary_date_variables[
variable_name] = self.get_real_value(
value, scale_factor[variable_name],
offset[variable_name])
date = num2date(data.variables['time'][0][i],
data.variables['time'].units,
data.variables['time'].calendar)
if date.tzinfo is None:
date = date.replace(tzinfo=pytz.utc)
values[date] = dictionary_date_variables
except LandException as ex:
self.logger.warn(
"This point is located on land: lon: {lon}, lat: {lat}".format(
lon=point["lon"], lat=point["lat"]))
raise
except Exception as err:
self.logger.error("Fatal error in thredds.", exc_info=True)
raise err
return values
def test_simple_radar_cat():
"""Test parsing of radar server catalog."""
url = 'http://thredds.ucar.edu/thredds/radarServer/nexrad/level2/IDD/dataset.xml'
cat = TDSCatalog(url)
assert cat
def test_dataset_unavailable_service(nids_url):
"""Test requesting a service that isn't present gives a ValueError."""
cat = TDSCatalog(nids_url)
with pytest.raises(ValueError) as err:
cat.datasets[0].access_with_service('NetcdfSubset')
assert 'not available' in str(err.value)
import cartopy.feature as cfeature
from matplotlib import patheffects
import matplotlib.pyplot as plt
from metpy.io import GiniFile
from metpy.plots.ctables import registry
from metpy.units import units
from netCDF4 import num2date
import scipy.ndimage as ndimage
from siphon.catalog import TDSCatalog
import xarray as xr
##############################################
# Get satellite data and set projection based on that data.
# Scan the catalog and download the data
satcat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog/satellite/'
'WV/WEST-CONUS_4km/current/catalog.xml')
dataset = satcat.datasets[0]
f = GiniFile(dataset.remote_open())
gini_ds = xr.open_dataset(f)
# Pull parts out of the data file
dat = gini_ds.metpy.parse_cf('WV')
data_var = gini_ds.variables['WV']
x = gini_ds.variables['x'][:]
y = gini_ds.variables['y'][:]
timestamp = f.prod_desc.datetime
##############################################
# Use Siphon to obtain data that is close to the time of the satellite file
gfscat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog/grib/'
def test_dataset_remote_open(nids_url):
"""Test using the remote_open method to request HTTP access."""
cat = TDSCatalog(nids_url)
fobj = cat.datasets[0].remote_open()
assert fobj.read(8) == b'\x01\r\r\n562 '
import json
import ssl
from datetime import datetime
from StringIO import StringIO
from bs4 import BeautifulSoup
from lxml import etree
from lxml.html.soupparser import fromstring
from thredds_crawler.crawl import Crawl
from django.conf import settings
import os
#url = "http://*****:*****@'), "catalogRefs/CatalogTELEDEM.html")
url = "http://localhost:8080/thredds/CatalogTELEDM.html"
cat = TDSCatalog('http://localhost:8080/thredds/CatalogTELEDM.html')
#context = ssl._create_unverified_context()
#catT = TDSCatalog(url)
cat = Crawl(url)
datasets = [i.id for i in cat.datasets]
root = "/home/mers/Bureau/teledm/donnees/"
catalog = {}
for item in datasets:
p = catalog
for x in item.split('/'):
p = p.setdefault(x, {})
ds = {}
for d in datasets:
dp = d.split('/')
