def _read_gsod_file(gsod_tar, station, year): tar_station_filename = station + '-' + str(year) + '.op.gz' try: gsod_tar.getmember('./' + tar_station_filename) except KeyError: return None ncdc_temp_dir = os.path.join(NCDC_GSOD_DIR, 'temp') util.mkdir_if_doesnt_exist(ncdc_temp_dir) temp_path = os.path.join(ncdc_temp_dir, tar_station_filename) gsod_tar.extract('./' + tar_station_filename, ncdc_temp_dir) with gzip.open(temp_path, 'rb') as gunzip_f: columns = [ # name, length, # of spaces separating previous column, dtype ('USAF', 6, 0, 'U6'), ('WBAN', 5, 1, 'U5'), ('date', 8, 2, object), ('mean_temp', 6, 2, float), ('mean_temp_count', 2, 1, int), ('dew_point', 6, 2, float), ('dew_point_count', 2, 1, int), ('sea_level_pressure', 6, 2, float), ('sea_level_pressure_count', 2, 1, int), ('station_pressure', 6, 2, float), ('station_pressure_count', 2, 1, int), ('visibility', 5, 2, float), ('visibility_count', 2, 1, int), ('mean_wind_speed', 5, 2, float), ('mean_wind_speed_count', 2, 1, int), ('max_wind_speed', 5, 2, float), ('max_gust', 5, 2, float), ('max_temp', 6, 2, float), ('max_temp_flag', 1, 0, 'U1'), ('min_temp', 6, 1, float), ('min_temp_flag', 1, 0, 'U1'), ('precip', 5, 1, float), ('precip_flag', 1, 0, 'U1'), ('snow_depth', 5, 1, float), ('FRSHTT', 6, 2, 'U6'), ] dtype = np.dtype([ (column[0], column[3]) for column in columns]) # note: ignore initial 0 delimiter = itertools.chain(*[column[1:3][::-1] for column in columns]) usecols = list(range(1, len(columns) * 2, 2)) data = np.genfromtxt(gunzip_f, skip_header=1, delimiter=delimiter, usecols=usecols, dtype=dtype, converters={5: _convert_date_string}) os.remove(temp_path) # somehow we can end up with single-element arrays that are 0-dimensional?? # (occurs on tyler's machine but is hard to reproduce) if data.ndim == 0: data = data.flatten() return data
def get_raster(layer, bbox, path=None, update_cache=False, check_modified=False, mosaic=False): """downloads National Elevation Dataset raster tiles that cover the given bounding box for the specified data layer. Parameters ---------- layer : str dataset layer name. (see get_available_layers for list) bbox : (sequence of float|str) bounding box of in geographic coordinates of area to download tiles in the format (min longitude, min latitude, max longitude, max latitude) path : ``None`` or path if ``None`` default path will be used update_cache: ``True`` or ``False`` (default) if ``False`` and output file already exists use it. check_modified: ``True`` or ``False`` (default) if tile exists in path, check if newer file exists online and download if available. mosaic: ``True`` or ``False`` (default) if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires rasterio package and GDAL. Returns ------- raster_tiles : geojson FeatureCollection metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file'] """ _check_layer(layer) raster_tiles = _download_tiles(get_raster_availability(layer, bbox), path=path, check_modified=check_modified) if mosaic: if path is None: path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH) util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox')) xmin, ymin, xmax, ymax = [float(n) for n in bbox] uid = util.generate_raster_uid(layer, xmin, ymin, xmax, ymax) output_path = os.path.join(path, 'by_boundingbox', uid + '.tif') if os.path.isfile(output_path) and not update_cache: return output_path raster_files = [ tile['properties']['file'] for tile in raster_tiles['features'] ] util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path) return [output_path] return raster_tiles
def get_raster(product_key, bbox, fmt=None, path=None, check_modified=False, mosaic=False): """downloads National Elevation Dataset raster tiles that cover the given bounding box for the specified data layer. Parameters ---------- product_key : str dataset name. (see get_available_datasets for list) bbox : (sequence of float|str) bounding box of in geographic coordinates of area to download tiles in the format (min longitude, min latitude, max longitude, max latitude) fmt : ``None`` or str available formats vary in different datasets. If ``None``, preference will be given to geotiff and then img, followed by whatever fmt is available path : ``None`` or path if ``None`` default path will be used update_cache: ``True`` or ``False`` (default) if ``False`` then tiles will not be re-downloaded if they exist in the path check_modified: ``True`` or ``False`` (default) if tile exists in path, check if newer file exists online and download if available. mosaic: ``True`` or ``False`` (default) if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires rasterio package and GDAL. Returns ------- raster_tiles : geojson FeatureCollection metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file'] """ raster_tiles = _download_tiles( get_raster_availability(product_key, bbox, fmt), path, check_modified) if mosaic: if path is None: path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH) util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox')) uid = util.generate_raster_uid(product_key, xmin, ymin, xmax, ymax) output_path = os.path.join(path, 'by_boundingbox', uid + '.tif') if os.path.isfile(output_path) and not update_cache: return output_path raster_files = [ tile['properties']['file'] for tile in raster_tiles['features'] ] util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path) return [output_path] return raster_tiles
def _get_client(wsdl_url, suds_cache=("default",), suds_timeout=None, user_cache=False): """ Open and re-use (persist) a suds.client.Client instance _suds_client throughout the session, to minimize WOF server impact and improve performance. _suds_client is global in scope. Parameters ---------- wsdl_url : str URL of a service's web service definition language (WSDL) description. All WaterOneFlow services publish a WSDL description and this url is the entry point to the service. suds_cache : ``None`` or tuple suds client local cache duration for WSDL description and client object. Pass a cache duration tuple like ('days', 3) to set a custom duration. Duration may be in months, weeks, days, hours, or seconds. If unspecified, the suds default (1 day) will be used. Use ``None`` to turn off caching. suds_timeout : int or float suds SOAP URL open timeout (seconds). If unspecified, the suds default (90 seconds) will be used. user_cache : bool If False (default), use the system temp location to store cache WSDL and other files. Use the default user ulmo directory if True. Returns ------- _suds_client : suds Client Newly or previously instantiated (reused) suds Client object. """ global _suds_client # Handle new or changed client request (create new client) if _suds_client is None or _suds_client.wsdl.url != wsdl_url or not suds_timeout is None: if user_cache: cache_dir = os.path.join(util.get_ulmo_dir(), 'suds') util.mkdir_if_doesnt_exist(cache_dir) _suds_client = suds.client.Client(wsdl_url, cache=ObjectCache(location=cache_dir)) else: _suds_client = suds.client.Client(wsdl_url) if suds_cache is None: _suds_client.set_options(cache=None) else: cache = _suds_client.options.cache # could add some error catching ... if suds_cache[0] == "default": cache.setduration(days=1) else: cache.setduration(**dict([suds_cache])) if not suds_timeout is None: _suds_client.set_options(timeout=suds_timeout) return _suds_client
def get_raster(layer, bbox, path=None, update_cache=False, check_modified=False, mosaic=False): """downloads National Elevation Dataset raster tiles that cover the given bounding box for the specified data layer. Parameters ---------- layer : str dataset layer name. (see get_available_layers for list) bbox : (sequence of float|str) bounding box of in geographic coordinates of area to download tiles in the format (min longitude, min latitude, max longitude, max latitude) path : ``None`` or path if ``None`` default path will be used update_cache: ``True`` or ``False`` (default) if ``False`` and output file already exists use it. check_modified: ``True`` or ``False`` (default) if tile exists in path, check if newer file exists online and download if available. mosaic: ``True`` or ``False`` (default) if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires rasterio package and GDAL. Returns ------- raster_tiles : geojson FeatureCollection metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file'] """ _check_layer(layer) raster_tiles = _download_tiles(get_raster_availability(layer, bbox), path=path, check_modified=check_modified) if mosaic: if path is None: path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH) util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox')) xmin, ymin, xmax, ymax = [float(n) for n in bbox] uid = util.generate_raster_uid(layer, xmin, ymin, xmax, ymax) output_path = os.path.join(path, 'by_boundingbox', uid + '.tif') if os.path.isfile(output_path) and not update_cache: return output_path raster_files = [tile['properties']['file'] for tile in raster_tiles['features']] util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path) return [output_path] return raster_tiles
def get_raster(product_key, bbox, fmt=None, path=None, check_modified=False, mosaic=False): """downloads National Elevation Dataset raster tiles that cover the given bounding box for the specified data layer. Parameters ---------- product_key : str dataset name. (see get_available_datasets for list) bbox : (sequence of float|str) bounding box of in geographic coordinates of area to download tiles in the format (min longitude, min latitude, max longitude, max latitude) fmt : ``None`` or str available formats vary in different datasets. If ``None``, preference will be given to geotiff and then img, followed by whatever fmt is available path : ``None`` or path if ``None`` default path will be used update_cache: ``True`` or ``False`` (default) if ``False`` then tiles will not be re-downloaded if they exist in the path check_modified: ``True`` or ``False`` (default) if tile exists in path, check if newer file exists online and download if available. mosaic: ``True`` or ``False`` (default) if ``True``, mosaic and clip downloaded tiles to the extents of the bbox provided. Requires rasterio package and GDAL. Returns ------- raster_tiles : geojson FeatureCollection metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file'] """ raster_tiles = _download_tiles(get_raster_availability(product_key, bbox, fmt), path, check_modified) if mosaic: if path is None: path = os.path.join(util.get_ulmo_dir(), DEFAULT_FILE_PATH) util.mkdir_if_doesnt_exist(os.path.join(path, 'by_boundingbox')) uid = util.generate_raster_uid(product_key, xmin, ymin, xmax, ymax) output_path = os.path.join(path, 'by_boundingbox', uid + '.tif') if os.path.isfile(output_path) and not update_cache: return output_path raster_files = [tile['properties']['file'] for tile in raster_tiles['features']] util.mosaic_and_clip(raster_files, xmin, ymin, xmax, ymax, output_path) return [output_path] return raster_tiles
def _read_gsod_file(gsod_tar, station, year): tar_station_filename = station + '-' + str(year) + '.op.gz' try: gsod_tar.getmember('./' + tar_station_filename) except KeyError: return None ncdc_extract_dir = os.path.join(NCDC_GSOD_DIR, 'extract') util.mkdir_if_doesnt_exist(ncdc_extract_dir) temp_path = os.path.join(ncdc_extract_dir, tar_station_filename) gsod_tar.extract('./' + tar_station_filename, ncdc_extract_dir) with _open_gzip(temp_path, 'rb') as gunzip_f: out_file_name = temp_path.replace('op.gz', 'txt') print (out_file_name) outF = open(out_file_name, 'wb') outF.write( gunzip_f.read() ) outF.close() os.remove(temp_path)
def get_services(bbox=None, user_cache=False): """Retrieves a list of services. Parameters ---------- bbox : ``None`` or 4-tuple Optional argument for a bounding box that covers the area you want to look for services in. This should be a tuple containing (min_longitude, min_latitude, max_longitude, and max_latitude) with these values in decimal degrees. If not provided then the full set of services will be queried from HIS Central. user_cache : bool If False (default), use the system temp location to store cache WSDL and other files. Use the default user ulmo directory if True. Returns ------- services_dicts : list A list of dicts that each contain information on an individual service. """ if user_cache: cache_dir = os.path.join(util.get_ulmo_dir(), 'suds') util.mkdir_if_doesnt_exist(cache_dir) suds_client = suds.client.Client(HIS_CENTRAL_WSDL_URL, cache=ObjectCache(location=cache_dir)) else: suds_client = suds.client.Client(HIS_CENTRAL_WSDL_URL) if bbox is None: services = suds_client.service.GetWaterOneFlowServiceInfo() else: x_min, y_min, x_max, y_max = bbox services = suds_client.service.GetServicesInBox2( xmin=x_min, ymin=y_min, xmax=x_max, ymax=y_max) services = [ _service_dict(service_info) for service_info in services.ServiceInfo ] return services