def _read_gsod_file(gsod_tar, station, year):
tar_station_filename = station + '-' + str(year) + '.op.gz'
try:
gsod_tar.getmember('./' + tar_station_filename)
except KeyError:
return None
ncdc_temp_dir = os.path.join(NCDC_GSOD_DIR, 'temp')
util.mkdir_if_doesnt_exist(ncdc_temp_dir)
temp_path = os.path.join(ncdc_temp_dir, tar_station_filename)
gsod_tar.extract('./' + tar_station_filename, ncdc_temp_dir)
with gzip.open(temp_path, 'rb') as gunzip_f:
columns = [
# name, length, # of spaces separating previous column, dtype
('USAF', 6, 0, 'U6'),
('WBAN', 5, 1, 'U5'),
('date', 8, 2, object),
('mean_temp', 6, 2, float),
('mean_temp_count', 2, 1, int),
('dew_point', 6, 2, float),
('dew_point_count', 2, 1, int),
('sea_level_pressure', 6, 2, float),
('sea_level_pressure_count', 2, 1, int),
('station_pressure', 6, 2, float),
('station_pressure_count', 2, 1, int),
('visibility', 5, 2, float),
('visibility_count', 2, 1, int),
('mean_wind_speed', 5, 2, float),
('mean_wind_speed_count', 2, 1, int),
('max_wind_speed', 5, 2, float),
('max_gust', 5, 2, float),
('max_temp', 6, 2, float),
('max_temp_flag', 1, 0, 'U1'),
('min_temp', 6, 1, float),
('min_temp_flag', 1, 0, 'U1'),
('precip', 5, 1, float),
('precip_flag', 1, 0, 'U1'),
('snow_depth', 5, 1, float),
('FRSHTT', 6, 2, 'U6'),
]
dtype = np.dtype([
(column[0], column[3])
for column in columns])
# note: ignore initial 0
delimiter = itertools.chain(*[column[1:3][::-1] for column in columns])
usecols = list(range(1, len(columns) * 2, 2))
data = np.genfromtxt(gunzip_f, skip_header=1, delimiter=delimiter,
usecols=usecols, dtype=dtype, converters={5: _convert_date_string})
os.remove(temp_path)
# somehow we can end up with single-element arrays that are 0-dimensional??
# (occurs on tyler's machine but is hard to reproduce)
if data.ndim == 0:
data = data.flatten()
return data
def get_raster(layer,
bbox,
path=None,
update_cache=False,
check_modified=False,
mosaic=False):
"""downloads National Elevation Dataset raster tiles that cover the given bounding box
for the specified data layer.
Parameters
----------
layer : str
dataset layer name. (see get_available_layers for list)
bbox : (sequence of float|str)
bounding box of in geographic coordinates of area to download tiles
in the format (min longitude, min latitude, max longitude, max latitude)
path : ``None`` or path
if ``None`` default path will be used
update_cache: ``True`` or ``False`` (default)
if ``False`` and output file already exists use it.
check_modified: ``True`` or ``False`` (default)
if tile exists in path, check if newer file exists online and download if available.
mosaic: ``True`` or ``False`` (default)
if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires
rasterio package and GDAL.
Returns
-------
raster_tiles : geojson FeatureCollection
metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file']
"""
_check_layer(layer)
raster_tiles = _download_tiles(get_raster_availability(layer, bbox),
path=path,
check_modified=check_modified)
if mosaic:
if path is None:
path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH)
util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox'))
xmin, ymin, xmax, ymax = [float(n) for n in bbox]
uid = util.generate_raster_uid(layer, xmin, ymin, xmax, ymax)
output_path = os.path.join(path, 'by_boundingbox', uid + '.tif')
if os.path.isfile(output_path) and not update_cache:
return output_path
raster_files = [
tile['properties']['file'] for tile in raster_tiles['features']
]
util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path)
return [output_path]
return raster_tiles
def get_raster(product_key,
bbox,
fmt=None,
path=None,
check_modified=False,
mosaic=False):
"""downloads National Elevation Dataset raster tiles that cover the given bounding box
for the specified data layer.
Parameters
----------
product_key : str
dataset name. (see get_available_datasets for list)
bbox : (sequence of float|str)
bounding box of in geographic coordinates of area to download tiles
in the format (min longitude, min latitude, max longitude, max latitude)
fmt : ``None`` or str
available formats vary in different datasets. If ``None``, preference will be given
to geotiff and then img, followed by whatever fmt is available
path : ``None`` or path
if ``None`` default path will be used
update_cache: ``True`` or ``False`` (default)
if ``False`` then tiles will not be re-downloaded if they exist in the path
check_modified: ``True`` or ``False`` (default)
if tile exists in path, check if newer file exists online and download if available.
mosaic: ``True`` or ``False`` (default)
if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires
rasterio package and GDAL.
Returns
-------
raster_tiles : geojson FeatureCollection
metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file']
"""
raster_tiles = _download_tiles(
get_raster_availability(product_key, bbox, fmt), path, check_modified)
if mosaic:
if path is None:
path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH)
util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox'))
uid = util.generate_raster_uid(product_key, xmin, ymin, xmax, ymax)
output_path = os.path.join(path, 'by_boundingbox', uid + '.tif')
if os.path.isfile(output_path) and not update_cache:
return output_path
raster_files = [
tile['properties']['file'] for tile in raster_tiles['features']
]
util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path)
return [output_path]
return raster_tiles
def _get_client(wsdl_url, suds_cache=("default",), suds_timeout=None, user_cache=False):
"""
Open and re-use (persist) a suds.client.Client instance _suds_client throughout
the session, to minimize WOF server impact and improve performance. _suds_client
is global in scope.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
suds_cache : ``None`` or tuple
suds client local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the suds default (1 day) will be used.
Use ``None`` to turn off caching.
suds_timeout : int or float
suds SOAP URL open timeout (seconds).
If unspecified, the suds default (90 seconds) will be used.
user_cache : bool
If False (default), use the system temp location to store cache WSDL and
other files. Use the default user ulmo directory if True.
Returns
-------
_suds_client : suds Client
Newly or previously instantiated (reused) suds Client object.
"""
global _suds_client
# Handle new or changed client request (create new client)
if _suds_client is None or _suds_client.wsdl.url != wsdl_url or not suds_timeout is None:
if user_cache:
cache_dir = os.path.join(util.get_ulmo_dir(), 'suds')
util.mkdir_if_doesnt_exist(cache_dir)
_suds_client = suds.client.Client(wsdl_url, cache=ObjectCache(location=cache_dir))
else:
_suds_client = suds.client.Client(wsdl_url)
if suds_cache is None:
_suds_client.set_options(cache=None)
else:
cache = _suds_client.options.cache
# could add some error catching ...
if suds_cache[0] == "default":
cache.setduration(days=1)
else:
cache.setduration(**dict([suds_cache]))
if not suds_timeout is None:
_suds_client.set_options(timeout=suds_timeout)
return _suds_client
def get_raster(layer, bbox, path=None, update_cache=False,
check_modified=False, mosaic=False):
"""downloads National Elevation Dataset raster tiles that cover the given bounding box
for the specified data layer.
Parameters
----------
layer : str
dataset layer name. (see get_available_layers for list)
bbox : (sequence of float|str)
bounding box of in geographic coordinates of area to download tiles
in the format (min longitude, min latitude, max longitude, max latitude)
path : ``None`` or path
if ``None`` default path will be used
update_cache: ``True`` or ``False`` (default)
if ``False`` and output file already exists use it.
check_modified: ``True`` or ``False`` (default)
if tile exists in path, check if newer file exists online and download if available.
mosaic: ``True`` or ``False`` (default)
if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires
rasterio package and GDAL.
Returns
-------
raster_tiles : geojson FeatureCollection
metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file']
"""
_check_layer(layer)
raster_tiles = _download_tiles(get_raster_availability(layer, bbox), path=path,
check_modified=check_modified)
if mosaic:
if path is None:
path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH)
util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox'))
xmin, ymin, xmax, ymax = [float(n) for n in bbox]
uid = util.generate_raster_uid(layer, xmin, ymin, xmax, ymax)
output_path = os.path.join(path, 'by_boundingbox', uid + '.tif')
if os.path.isfile(output_path) and not update_cache:
return output_path
raster_files = [tile['properties']['file'] for tile in raster_tiles['features']]
util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path)
return [output_path]
return raster_tiles
def get_raster(product_key, bbox, fmt=None, path=None, check_modified=False, mosaic=False):
"""downloads National Elevation Dataset raster tiles that cover the given bounding box
for the specified data layer.
Parameters
----------
product_key : str
dataset name. (see get_available_datasets for list)
bbox : (sequence of float|str)
bounding box of in geographic coordinates of area to download tiles
in the format (min longitude, min latitude, max longitude, max latitude)
fmt : ``None`` or str
available formats vary in different datasets. If ``None``, preference will be given
to geotiff and then img, followed by whatever fmt is available
path : ``None`` or path
if ``None`` default path will be used
update_cache: ``True`` or ``False`` (default)
if ``False`` then tiles will not be re-downloaded if they exist in the path
check_modified: ``True`` or ``False`` (default)
if tile exists in path, check if newer file exists online and download if available.
mosaic: ``True`` or ``False`` (default)
if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires
rasterio package and GDAL.
Returns
-------
raster_tiles : geojson FeatureCollection
metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file']
"""
raster_tiles = _download_tiles(get_raster_availability(product_key, bbox, fmt),
path, check_modified)
if mosaic:
if path is None:
path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH)
util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox'))
uid = util.generate_raster_uid(product_key, xmin, ymin, xmax, ymax)
output_path = os.path.join(path, 'by_boundingbox', uid + '.tif')
if os.path.isfile(output_path) and not update_cache:
return output_path
raster_files = [tile['properties']['file'] for tile in raster_tiles['features']]
util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path)
return [output_path]
return raster_tiles
def _read_gsod_file(gsod_tar, station, year):
tar_station_filename = station + '-' + str(year) + '.op.gz'
try:
gsod_tar.getmember('./' + tar_station_filename)
except KeyError:
return None
ncdc_extract_dir = os.path.join(NCDC_GSOD_DIR, 'extract')
util.mkdir_if_doesnt_exist(ncdc_extract_dir)
temp_path = os.path.join(ncdc_extract_dir, tar_station_filename)
gsod_tar.extract('./' + tar_station_filename, ncdc_extract_dir)
with _open_gzip(temp_path, 'rb') as gunzip_f:
out_file_name = temp_path.replace('op.gz', 'txt')
print (out_file_name)
outF = open(out_file_name, 'wb')
outF.write( gunzip_f.read() )
outF.close()
os.remove(temp_path)
def get_services(bbox=None, user_cache=False):
"""Retrieves a list of services.
Parameters
----------
bbox : ``None`` or 4-tuple
Optional argument for a bounding box that covers the area you want to
look for services in. This should be a tuple containing (min_longitude,
min_latitude, max_longitude, and max_latitude) with these values in
decimal degrees. If not provided then the full set of services will be
queried from HIS Central.
user_cache : bool
If False (default), use the system temp location to store cache WSDL and
other files. Use the default user ulmo directory if True.
Returns
-------
services_dicts : list
A list of dicts that each contain information on an individual service.
"""
if user_cache:
cache_dir = os.path.join(util.get_ulmo_dir(), 'suds')
util.mkdir_if_doesnt_exist(cache_dir)
suds_client = suds.client.Client(HIS_CENTRAL_WSDL_URL,
cache=ObjectCache(location=cache_dir))
else:
suds_client = suds.client.Client(HIS_CENTRAL_WSDL_URL)
if bbox is None:
services = suds_client.service.GetWaterOneFlowServiceInfo()
else:
x_min, y_min, x_max, y_max = bbox
services = suds_client.service.GetServicesInBox2(
xmin=x_min, ymin=y_min, xmax=x_max, ymax=y_max)
services = [
_service_dict(service_info)
for service_info in services.ServiceInfo
]
return services
