def test_SpatialCoordinateVariables_add_to_dataset():
lat = SpatialCoordinateVariable(numpy.arange(19, -1, -1))
lon = SpatialCoordinateVariable(numpy.arange(10))
coords = SpatialCoordinateVariables(lon, lat, Proj(init='EPSG:4326'))
lat_varname = 'lat'
lon_varname = 'lon'
outfilename = 'test.nc'
try:
with Dataset(outfilename, 'w') as target_ds:
coords.add_to_dataset(target_ds, lon_varname, lat_varname)
assert lat_varname in target_ds.dimensions
assert lat_varname in target_ds.variables
assert len(target_ds.dimensions[lat_varname]) == lat.values.size
assert numpy.array_equal(lat.values,
target_ds.variables[lat_varname][:])
assert lon_varname in target_ds.dimensions
assert lon_varname in target_ds.variables
assert len(target_ds.dimensions[lon_varname]) == lon.values.size
assert numpy.array_equal(lon.values,
target_ds.variables[lon_varname][:])
finally:
if os.path.exists(outfilename):
os.remove(outfilename)
def test_SpatialCoordinateVariables_slice_by_bbox():
lat = SpatialCoordinateVariable(numpy.arange(19, -1, -1))
lon = SpatialCoordinateVariable(numpy.arange(10))
proj = Proj(init='EPSG:4326')
coords = SpatialCoordinateVariables(lon, lat, proj)
subset = coords.slice_by_bbox(BBox((1.75, 3.7, 6.2, 16.7), proj))
assert numpy.array_equal(subset.x.values, numpy.arange(2, 6))
assert subset.x.values[0] == 2
assert subset.x.values[-1] == 5
assert subset.y.values[0] == 16
assert subset.y.values[-1] == 4
def test_window_for_bbox():
coords = SpatialCoordinateVariables.from_bbox(
BBox([-124, 82, -122, 90], Proj(init='epsg:4326')), 20, 20)
window = coords.get_window_for_bbox(BBox([-123.9, 82.4, -122.1, 89.6]))
assert window.x_slice == slice(1, 19)
assert window.y_slice == slice(1, 19)
def _get_subsets(self, elevation, data, coords: SpatialCoordinateVariables, bbox):
""" Returns subsets of elevation, data, and coords, clipped to the given bounds """
x_slice = slice(*coords.x.indices_for_range(bbox.xmin, bbox.xmax))
y_slice = slice(*coords.y.indices_for_range(bbox.ymin, bbox.ymax))
return elevation[y_slice, x_slice], data[y_slice, x_slice], coords.slice_by_bbox(bbox)
def load_data(self):
self.data_name = self.path.split(os.sep)[-1].split('.')[0]
with Dataset(self.path, 'r') as ds:
# This should be: self.variables = clover.netcdf.utilities.data_variables(ds)
# but unfortunately clover's data_variables() method is broken under python3
# currently this may cause problems with grid_mapping or *_bnds variables
dimensions = list(ds.dimensions.keys())
self.variables = [var for var in ds.variables if var not in dimensions]
recognized_dims = {'x': 'y', 'lon': 'lat', 'longitude': 'latitude'}
for x, y in recognized_dims.items():
if x in dimensions:
self.x_dim, self.y_dim = x, y
break
if self.x_dim is None:
raise KeyError("NetCDF file doesn't have recognizable dimension names (x, lat, latitude, etc.)")
# we need a time_info attribute even if there's no time info
self.time_info = TemporalInfo()
recognized_time_dims = ['time', 'year', 'month', 'date']
for timedim in recognized_time_dims:
if timedim in dimensions:
self.t_dim = timedim
if timedim in ds.variables:
timevar = DateVariable(ds.variables[timedim])
self.time_info.timestamps = timevar.datetimes.tolist()
# we don't want timezones, but sometimes clover adds them
if self.time_info.timestamps[0].tzinfo is not None:
self.time_info.timestamps = [d.replace(tzinfo=None) for d in self.time_info.timestamps]
else: # no time variable
wx.MessageDialog(App.get().app_controller.main_window,
caption='Missing Time Data',
message=('{} has a time dimension "{}" but no corresponding coordinate ' +
'variable. Using only final timestep.').format(self.data_name, timedim), style=wx.OK).ShowModal()
break
# Determine spatial extent; if multiple vars, they must all have the same extent
self.x = SpatialCoordinateVariable(ds.variables[self.x_dim])
self.y = SpatialCoordinateVariable(ds.variables[self.y_dim])
self.y_increasing = self.y.values[0] < self.y.values[-1]
grid = clover.netcdf.describe.describe(ds)['variables'][self.variables[0]]['spatial_grid']
ext = grid['extent']
self.extent = Extent(*[ext[d] for d in ['xmin', 'ymin', 'xmax', 'ymax']])
self._resolution = grid['x_resolution']
self.affine = SpatialCoordinateVariables(self.x, self.y, None).affine
self.var_shape = ds.variables[self.variables[0]].shape
# if a var has a temporal dimension but no temporal data, we treat it as non-temporal:
if not self.time_info.is_temporal and len(self.var_shape) == 3:
self.var_shape = self.var_shape[1:]
def describe(path_or_dataset):
if isinstance(path_or_dataset, string_types):
dataset = Dataset(path_or_dataset)
else:
dataset = path_or_dataset
description = {
'dimensions': {},
'variables': {},
'attributes': get_ncattrs(dataset)
}
for dimension_name in dataset.dimensions:
dimension = dataset.dimensions[dimension_name]
description['dimensions'][dimension_name] = {'length': len(dimension)}
for variable_name in dataset.variables:
variable = dataset.variables[variable_name]
if not variable.dimensions:
# Do not collect info about dimensionless variables (e.g., CRS variable)
continue
dtype = str(variable.dtype)
if "'" in dtype:
dtype = dtype.split("'")[1]
attributes = get_ncattrs(variable)
variable_info = {
'attributes':
attributes,
'dimensions':
variable.dimensions,
'data_type':
dtype,
'name':
attributes.get('long_name') or attributes.get('standard_name')
or variable_name
}
if dtype not in ('str', ):
if len(variable.shape) > 2:
# Avoid loading the entire array into memory by iterating along the first index (usually time)
variable_info.update({
'min':
min(variable[i, :].min().item()
for i in range(variable.shape[0])),
'max':
max(variable[i, :].max().item()
for i in range(variable.shape[0]))
})
else:
data = variable[:]
variable_info.update({
'min': data.min().item(),
'max': data.max().item()
})
if variable_name in dataset.dimensions and dtype not in ('str', ):
dimension_variable = dataset.variables[variable_name]
if len(dimension_variable.dimensions
) == 1: # range dimensions don't make sense for interval
interval = get_interval(dimension_variable)
if interval:
variable_info['interval'] = interval
else:
# Data variable
proj4 = get_crs(dataset, variable_name)
#extent
if len(variable.dimensions) >= 2:
x_variable_name = None
y_variable_name = None
time_variable_name = None
for dimension_name in (x for x in variable.dimensions
if x in dataset.variables):
attributes = get_ncattrs(dataset.variables[dimension_name])
standard_name = attributes.get('standard_name', None)
if standard_name in X_DIMENSION_STANDARD_NAMES or dimension_name in X_DIMENSION_COMMON_NAMES:
x_variable_name = dimension_name
elif standard_name in Y_DIMENSION_STANDARD_NAMES or dimension_name in Y_DIMENSION_COMMON_NAMES:
y_variable_name = dimension_name
elif standard_name in TIME_DIMENSION_STANDARD_NAMES or dimension_name in TIME_DIMENSION_COMMON_NAMES:
if len(dataset.dimensions[dimension_name]) > 1:
time_variable_name = dimension_name
if x_variable_name and y_variable_name:
if proj4 is None and is_geographic(dataset, variable_name):
# Assume WGS84
proj4 = PROJ4_GEOGRAPHIC
coordinates = SpatialCoordinateVariables(
SpatialCoordinateVariable(
dataset.variables[x_variable_name]),
SpatialCoordinateVariable(
dataset.variables[y_variable_name]),
Proj(str(proj4)) if proj4 else None)
variable_info['spatial_grid'] = {
'extent': coordinates.bbox.as_dict(),
'x_dimension': x_variable_name,
'x_resolution': coordinates.x.pixel_size,
'y_dimension': y_variable_name,
'y_resolution': coordinates.y.pixel_size
}
if time_variable_name:
time_variable = dataset.variables[time_variable_name]
time_info = {
'dimension': time_variable_name,
}
try:
date_variable = DateVariable(time_variable)
values = date_variable.datetimes
time_info['extent'] = [
values.min().isoformat(),
values.max().isoformat()
]
time_info['interval_unit'] = date_variable.unit
interval = get_interval(time_variable)
if interval is not None:
time_info['interval'] = interval
except ValueError:
pass
variable_info['time'] = time_info
if proj4:
variable_info['proj4'] = proj4
description['variables'][variable_name] = variable_info
return description
def warp_like(ds,
ds_projection,
variables,
out_ds,
template_ds,
template_varname,
resampling=RESAMPLING.nearest):
"""
Warp one or more variables in a NetCDF file based on the coordinate reference system and
spatial domain of a template NetCDF file.
:param ds: source dataset
:param ds_projection: source dataset coordiante reference system, proj4 string or EPSG:NNNN code
:param variables: list of variable names in source dataset to warp
:param out_ds: output dataset. Must be opened in write or append mode.
:param template_ds: template dataset
:param template_varname: variable name for template data variable in template dataset
:param resampling: resampling method. See rasterio.warp.RESAMPLING for options
"""
template_variable = template_ds.variables[template_varname]
template_prj = Proj(get_crs(template_ds, template_varname))
template_mask = template_variable[:].mask
template_y_name, template_x_name = template_variable.dimensions[-2:]
template_coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=template_prj)
# template_geo_bbox = template_coords.bbox.project(ds_prj, edge_points=21) # TODO: add when needing to subset
ds_y_name, ds_x_name = ds.variables[variables[0]].dimensions[-2:]
proj = Proj(
init=ds_projection) if 'EPSG:' in ds_projection.upper() else Proj(
str(ds_projection))
ds_coords = SpatialCoordinateVariables.from_dataset(ds,
x_name=ds_x_name,
y_name=ds_y_name,
projection=proj)
with rasterio.drivers():
# Copy dimensions for variable across to output
for dim_name in template_variable.dimensions:
if not dim_name in out_ds.dimensions:
if dim_name in template_ds.variables and not dim_name in out_ds.variables:
copy_variable(template_ds, out_ds, dim_name)
else:
copy_dimension(template_ds, out_ds, dim_name)
for variable_name in variables:
click.echo('Processing: {0}'.format(variable_name))
variable = ds.variables[variable_name]
fill_value = getattr(variable, '_FillValue',
variable[0, 0].fill_value)
for dim_name in variable.dimensions[:-2]:
if not dim_name in out_ds.dimensions:
if dim_name in ds.variables:
copy_variable(ds, out_ds, dim_name)
else:
copy_dimension(ds, out_ds, dim_name)
out_var = out_ds.createVariable(
variable_name,
variable.dtype,
dimensions=variable.dimensions[:-2] +
template_variable.dimensions,
fill_value=fill_value)
reproject_kwargs = {
'src_transform': ds_coords.affine,
'src_crs': CRS.from_string(ds_projection),
'dst_transform': template_coords.affine,
'dst_crs': template_prj.srs,
'resampling': resampling,
'src_nodata': fill_value,
'dst_nodata': fill_value,
'threads': 4
}
# TODO: may only need to select out what is in window
if len(variable.shape) == 3:
idxs = range(variable.shape[0])
with click.progressbar(idxs) as bar:
for i in bar:
# print('processing slice: {0}'.format(i))
data = variable[i, :]
out = numpy.ma.empty(template_coords.shape,
dtype=data.dtype)
out.mask = template_mask
out.fill(fill_value)
reproject(data, out, **reproject_kwargs)
out_var[i, :] = out
else:
data = variable[:]
out = numpy.ma.empty(template_coords.shape, dtype=data.dtype)
out.mask = template_mask
out.fill(fill_value)
reproject(data, out, **reproject_kwargs)
out_var[:] = out
def map_eems(
eems_file,
# output_directory,
scale,
format,
src_crs,
resampling):
"""
Render a NetCDF EEMS model to a web map.
"""
from EEMSBasePackage import EEMSCmd, EEMSProgram
model = EEMSProgram(eems_file)
# For each data producing command, store the netcdf file that contains it
file_vars = dict()
raw_variables = set()
for cmd in model.orderedCmds: # This is bottom up, may want to invert
filename = None
variable = None
if cmd.HasResultName():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetResultName()
elif cmd.IsReadCmd():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetParam('NewFieldName')
raw_variables.add(variable)
if filename and variable:
if not filename in file_vars:
file_vars[filename] = []
file_vars[filename].append(variable)
filenames = file_vars.keys()
for filename in filenames:
if not os.path.exists(filename):
raise click.ClickException(
'Could not find data file from EEMS model: {0}'.format(
filename))
dst_crs = 'EPSG:3857'
output_directory = tempfile.mkdtemp()
click.echo('Using temp directory: {0}'.format(output_directory))
# if not os.path.exists(output_directory):
# os.makedirs(output_directory)
# Since fuzzy renderer is hardcoded, we can output it now
fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'],
'1,-1')
fuzzy_renderer.get_legend(image_height=150)[0].to_image().save(
os.path.join(output_directory, 'fuzzy_legend.png'))
template_filename = filenames[0]
template_var = file_vars[template_filename][0]
with Dataset(template_filename) as ds:
var_obj = ds.variables[template_var]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions != 2:
raise click.ClickException(
'Only 2 dimensions are allowed on data variables for now')
ds_crs = get_crs(ds, template_var)
if not ds_crs and is_geographic(ds, template_var):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None)
#
# if mask is not None and not mask.shape == shape[-2:]:
# # Will likely break before this if collecting statistics
# raise click.BadParameter(
# 'mask variable shape does not match shape of input spatial dimensions',
# param='--mask', param_hint='--mask'
# )
#
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list())
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(RESAMPLING, resampling),
'dst_shape': (dst_height, dst_width)
}
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide valid src_crs to get interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing dataset {0}'.format(filename))
for variable in file_vars[filename]:
click.echo('Processing variable {0}'.format(variable))
if not variable in ds.variables:
raise click.ClickException(
'variable {0} was not found in file: {1}'.format(
variable, filename))
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
data = var_obj[:]
# if mask is not None:
# data = numpy.ma.masked_array(data, mask=mask)
if variable in raw_variables:
palette = DEFAULT_PALETTES['raw']
palette_stretch = '{0},{1}'.format(data.max(), data.min())
renderer = palette_to_stretched_renderer(
palette, palette_stretch)
renderer.get_legend(
image_height=150, max_precision=2)[0].to_image().save(
os.path.join(output_directory,
'{0}_legend.png'.format(variable)))
else:
renderer = fuzzy_renderer
image_filename = os.path.join(
output_directory, '{0}.{1}'.format(variable, format))
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale=scale,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[variable] = local_filename
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('clover.cli')).get_template('eems_map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
tree=[[cmd, depth]
for (cmd, depth) in model.GetCmdTree()],
raw_variables=list(raw_variables)))
webbrowser.open(index_html)
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None):
outputs = results.format_args()
for k, v in six.iteritems(outputs):
if is_raster(v) and publish_raster_results:
service_name = '{0}/{1}'.format(job.uuid, k)
rel_path = '{}.nc'.format(service_name)
abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path)
os.makedirs(os.path.dirname(abs_path))
with Dataset(abs_path, 'w', format='NETCDF4') as ds:
if v.extent.projection.is_latlong():
x_var = 'longitude'
y_var = 'latitude'
else:
x_var = 'x'
y_var = 'y'
coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape))
coord_vars.add_to_dataset(ds, x_var, y_var)
fill_value = v.fill_value if is_masked(v) else None
data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value)
data_var[:] = v
set_crs(ds, 'data', v.extent.projection)
if callable(renderer_or_fn):
renderer = renderer_or_fn(v)
elif renderer_or_fn is None:
renderer = StretchedRenderer(
[(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))]
)
else:
renderer = renderer_or_fn
with transaction.atomic():
service = Service.objects.create(
name=service_name,
description='This service has been automatically generated from the result of a geoprocessing job.',
data_path=rel_path,
projection=v.extent.projection.srs,
full_extent=v.extent,
initial_extent=v.extent,
)
Variable.objects.create(
service=service,
index=0,
variable='data',
projection=v.extent.projection.srs,
x_dimension=x_var,
y_dimension=y_var,
name='data',
renderer=renderer,
full_extent=v.extent
)
ProcessingResultService.objects.create(job=job, service=service)
outputs[k] = service_name
elif is_ndarray(v):
if v.size < numpy.get_printoptions()['threshold']:
outputs[k] = v.tolist()
else:
outputs[k] = str(v)
return outputs
def render_netcdf(filename_pattern, variable, output_directory, renderer_file,
save_file, renderer_type, colormap, fill, colorspace,
palette, palette_stretch, scale, id_variable, lh,
legend_breaks, legend_ticks, legend_precision, format,
src_crs, dst_crs, res, resampling, anchors, interactive_map,
mask_path):
"""
Render netcdf files to images.
colormap is ignored if renderer_file is provided
--dst-crs is ignored if using --map option (always uses EPSG:3857
If no colormap or palette is provided, a default palette may be chosen based on the name of the variable.
If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It
must be in the same CRS as the input datasets, and have the same spatial
dimensions.
"""
# Parameter overrides
if interactive_map:
dst_crs = 'EPSG:3857'
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='filename_pattern',
param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
mask = get_mask(mask_path) if mask_path is not None else None
if renderer_file is not None and not save_file:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist',
param='renderer_file',
param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
if variable in renderer_dict and not 'colors' in renderer_dict:
renderer_dict = renderer_dict[variable]
renderer_type = renderer_dict['type']
if renderer_type == 'stretched':
colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
if 'min' in colors or 'max' in colors or 'mean' in colors:
statistics = collect_statistics(filenames, (variable, ),
mask=mask)[variable]
for entry in renderer_dict['colors']:
if isinstance(entry[0], basestring):
if entry[0] in ('min', 'max', 'mean'):
entry[0] = statistics[entry[0]]
elif '*' in entry[0]:
rel_value, statistic = entry[0].split('*')
entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
if palette is not None:
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
elif colormap is None and variable in DEFAULT_PALETTES:
palette, palette_stretch = DEFAULT_PALETTES[variable]
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
else:
if colormap is None:
colormap = 'min:#000000,max:#FFFFFF'
renderer = colormap_to_stretched_renderer(colormap,
colorspace,
filenames,
variable,
fill_value=fill,
mask=mask)
elif renderer_type == 'classified':
if not palette:
raise click.BadParameter(
'palette required for classified (for now)',
param='--palette',
param_hint='--palette')
renderer = palette_to_classified_renderer(
palette,
filenames,
variable,
method='equal',
fill_value=fill,
mask=mask) # TODO: other methods
if save_file:
if os.path.exists(save_file):
with open(save_file, 'r+') as output_file:
data = json.loads(output_file.read())
output_file.seek(0)
output_file.truncate()
data[variable] = renderer.serialize()
output_file.write(json.dumps(data, indent=4))
else:
with open(save_file, 'w') as output_file:
output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'stretched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(
image_height=lh,
breaks=legend_breaks,
ticks=legend_ticks,
max_precision=legend_precision)[0].to_image()
elif renderer_type == 'classified':
legend = composite_elements(renderer.get_legend())
legend.save(
os.path.join(output_directory, '{0}_legend.png'.format(variable)))
with Dataset(filenames[0]) as ds:
var_obj = ds.variables[variable]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions == 3:
if id_variable:
if shape[0] != ds.variables[id_variable][:].shape[0]:
raise click.BadParameter(
'must be same dimensionality as 3rd dimension of {0}'.
format(variable),
param='--id_variable',
param_hint='--id_variable')
else:
# Guess from the 3rd dimension
guess = dimensions[0]
if guess in ds.variables and ds.variables[guess][:].shape[
0] == shape[0]:
id_variable = guess
ds_crs = get_crs(ds, variable)
if not ds_crs and is_geographic(ds, variable):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds,
x_dim,
y_dim,
projection=Proj(src_crs.to_dict()) if src_crs else None)
if mask is not None and not mask.shape == shape[-2:]:
# Will likely break before this if collecting statistics
raise click.BadParameter(
'mask variable shape does not match shape of input spatial dimensions',
param='--mask',
param_hint='--mask')
flip_y = False
reproject_kwargs = None
if dst_crs is not None:
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs,
dst_crs,
src_width,
src_height,
*coords.bbox.as_list(),
resolution=res)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(RESAMPLING, resampling),
'dst_shape': (dst_height, dst_width)
}
else:
dst_transform = coords.affine
dst_height, dst_width = coords.shape
dst_crs = src_crs
if coords.y.is_ascending_order():
# Only needed if we are not already reprojecting the data, since that will flip it automatically
flip_y = True
if anchors or interactive_map:
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide at least src_crs to get Leaflet anchors or interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(
dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
if anchors:
click.echo('Anchors: {0}'.format(leaflet_anchors))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing {0}'.format(filename))
filename_root = os.path.split(filename)[1].replace('.nc', '')
if not variable in ds.variables:
raise click.BadParameter(
'variable {0} was not found in file: {1}'.format(
variable, filename),
param='variable',
param_hint='VARIABLE')
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
if num_dimensions == 2:
data = var_obj[:]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
image_filename = os.path.join(
output_directory,
'{0}_{1}.{2}'.format(filename_root, variable, format))
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
elif num_dimensions == 3:
for index in range(shape[0]):
id = ds.variables[id_variable][
index] if id_variable is not None else index
image_filename = os.path.join(
output_directory,
'{0}_{1}__{2}.{3}'.format(filename_root, variable, id,
format))
data = var_obj[index]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)
[0]] = local_filename
# TODO: not tested recently. Make sure still correct
# else:
# # Assume last 2 components of shape are lat & lon, rest are iterated over
# id_variables = None
# if id_variable is not None:
# id_variables = id_variable.split(',')
# for index, name in enumerate(id_variables):
# if name:
# assert data.shape[index] == ds.variables[name][:].shape[0]
#
# ranges = []
# for dim in data.shape[:-2]:
# ranges.append(range(0, dim))
# for combined_index in product(*ranges):
# id_parts = []
# for index, dim_index in enumerate(combined_index):
# if id_variables is not None and index < len(id_variables) and id_variables[index]:
# id = ds.variables[id_variables[index]][dim_index]
#
# if not isinstance(id, basestring):
# if isinstance(id, Iterable):
# id = '_'.join((str(i) for i in id))
# else:
# id = str(id)
#
# id_parts.append(id)
#
# else:
# id_parts.append(str(dim_index))
#
# combined_id = '_'.join(id_parts)
# image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format))
# if reproject_kwargs:
# data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this
# render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format)
#
# local_filename = os.path.split(image_filename)[1]
# layers[os.path.splitext(local_filename)[0]] = local_filename
if interactive_map:
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('clover.cli')).get_template('map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
variable=variable))
webbrowser.open(index_html)
def handle(self, *args, **options):
message = (
"WARNING: This will update all service data, casting each to it's smallest possible data type. Do you want "
"to continue? [y/n]"
)
if input(message).lower() not in {'y', 'yes'}:
return
for service in Service.objects.all():
if service.variable_set.all().count() > 1:
print("Skipping service '{}' with more than one variable...".format(service.name))
continue
variable = service.variable_set.all().get()
path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
tmp_dir = mkdtemp()
tmp_path = os.path.join(tmp_dir, os.path.basename(service.data_path))
try:
with Dataset(path, 'r') as ds:
data = ds.variables[variable.variable][:]
coords = SpatialCoordinateVariables.from_bbox(service.full_extent, *reversed(data.shape))
if data.dtype.kind != 'i':
print("Ignoring service '{}' with non-int type".format(service.name))
continue
# The fill value will be the minimum value of the chosen type, so we want to make sure it's not
# included in the actual data range
min_value = data.min() - 1
max_value = data.max()
# Determine the most suitable data type by finding the minimum type for the min/max values and then
# using the type that will accurately represent both
min_type = str(numpy.min_scalar_type(min_value))
max_type = str(numpy.min_scalar_type(max_value))
min_unsigned, min_size = min_type.split('int')
max_unsigned, max_size = max_type.split('int')
dtype = '{}int{}'.format(min_unsigned and max_unsigned, max(int(min_size), int(max_size)))
if data.dtype == dtype:
print("Service '{}' already has the smallest possible type: {}".format(service.name, dtype))
continue
print("Converting service '{}' to type: {}".format(service.name, dtype))
with Dataset(tmp_path, 'w', format='NETCDF4') as ds:
coords.add_to_dataset(ds, variable.x_dimension, variable.y_dimension)
data = data.astype(dtype)
fill_value = numpy.ma.maximum_fill_value(numpy.dtype(dtype))
numpy.ma.set_fill_value(data, fill_value)
data_var = ds.createVariable(
variable.variable, dtype, dimensions=(variable.y_dimension, variable.x_dimension),
fill_value=fill_value
)
data_var[:] = data
set_crs(ds, variable.variable, service.full_extent.projection)
os.unlink(path)
shutil.copy2(tmp_path, path)
finally:
try:
shutil.rmtree(tmp_dir)
except OSError:
pass
def test_SpatialCoordinateVariables_bbox():
proj = Proj(init='EPSG:4326')
bbox = BBox((10.5, 5, 110.5, 55), projection=proj)
coords = SpatialCoordinateVariables.from_bbox(bbox, 10, 5)
assert coords.bbox.as_list() == bbox.as_list()
def netcdf_to_raster(path_or_dataset,
variable_name,
outfilename,
index=0,
projection=None):
"""
Exports a 2D slice from a netcdf file to a raster file.
Only GeoTiffs are supported at this time.
Parameters
----------
path_or_dataset: path to NetCDF file or open Dataset
variable_name: name of data variable to export from dataset
outfilename: output filename
index: index within 3rd dimension (in first position) or 0
projection: pyproj.Proj object. Automatically determined from file if possible
"""
if isinstance(path_or_dataset, string_types):
dataset = Dataset(path_or_dataset)
else:
dataset = path_or_dataset
projection = projection or get_crs(dataset, variable_name)
if not projection:
raise ValueError(
'Projection must be provided; '
'no projection information can be determined from file')
# TODO figure out cleaner way to get affine or coords
y_name, x_name = dataset.variables[variable_name].dimensions[:2]
coords = SpatialCoordinateVariables.from_dataset(dataset,
x_name,
y_name,
projection=projection)
affine = coords.affine
if outfilename.lower().endswith('.tif'):
format = 'GTiff'
else:
raise ValueError(
'Only GeoTiff outputs supported, filename must have .tif extension'
)
variable = dataset.variables[variable_name]
ndims = len(variable.shape)
if ndims == 2:
if index != 0:
raise ValueError('Index out of range, must be 0')
data = variable[:]
elif ndims == 3:
# Assumes that time dimension is first
if index < 0 or index >= variable.shape[0]:
raise ValueError('Index out of range, '
'must be between 0 and {0}'.variable.shape[0])
data = variable[index]
else:
raise ValueError(
'Unsupported number of dimensions {0} for variable {1}, '
'must be 2 or 3'.format(ndims, variable_name))
array_to_raster(data,
outfilename,
format=format,
projection=projection,
affine=affine)
def raster_to_netcdf(filename_or_raster,
outfilename=None,
variable_name='data',
format='NETCDF4',
**kwargs):
"""
Parameters
----------
filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset
outfilename: name of output file. If blank, will be same name as input with *.nc extension added
variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4
format
kwargs: arguments passed to variable creation: zlib
Note: only rasters with descending y coordinates are currently supported
"""
start = time.time()
if isinstance(filename_or_raster, string_types):
if not os.path.exists(filename_or_raster):
raise ValueError(
'File does not exist: {0}'.format(filename_or_raster))
src = rasterio.open(filename_or_raster)
managed_raster = True
else:
src = filename_or_raster
managed_raster = False
if not src.count == 1:
raise NotImplementedError(
'ERROR: multi-band rasters not yet supported for this operation')
prj = pyproj.Proj(**src.crs)
outfilename = outfilename or src.name + '.nc'
with Dataset(outfilename, 'w', format=format) as target:
if prj.is_latlong():
x_varname = 'longitude'
y_varname = 'latitude'
else:
x_varname = 'x'
y_varname = 'y'
# TODO: may need to do this in blocks if source is big
data = src.read(1, masked=True)
coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj),
src.width, src.height)
coords.add_to_dataset(target, x_varname, y_varname, **kwargs)
out_var = target.createVariable(variable_name,
data.dtype,
dimensions=(y_varname, x_varname),
**kwargs)
out_var[:] = data
set_crs(target, variable_name, prj, set_proj4_att=False)
if managed_raster:
src.close()
print('Elapsed {0:.3f} seconds'.format(time.time() - start))
def handle(self, *args, **options):
elevation_service = Service.objects.get(name='west2_dem')
with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, elevation_service.data_path)) as ds:
coords = SpatialCoordinateVariables.from_dataset(
ds, x_name='lon', y_name='lat', projection=Proj(elevation_service.projection)
)
elevation = ds.variables['elevation'][:]
message = 'WARNING: This will replace all your transfer limits. Do you want to continue? [y/n]'
if input(message).lower() not in {'y', 'yes'}:
return
self.transfers_by_source = {}
with transaction.atomic():
TransferLimit.objects.all().delete()
for time_period in ('1961_1990', '1981_2010'):
for variable in VARIABLES:
print('Processing {} for {}...'.format(variable, time_period))
variable_service = Service.objects.get(name='west2_{}Y_{}'.format(time_period, variable))
with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, variable_service.data_path)) as ds:
data = ds.variables[variable][:]
for zone in SeedZone.objects.all():
clipped_elevation, clipped_data, clipped_coords = self._get_subsets(
elevation, data, coords, BBox(zone.polygon.extent)
)
zone_mask = rasterize(
((json.loads(zone.polygon.geojson), 1),), out_shape=clipped_elevation.shape,
transform=clipped_coords.affine, fill=0, dtype=numpy.dtype('uint8')
)
masked_dem = numpy.ma.masked_where(zone_mask == 0, clipped_elevation)
min_elevation = max(math.floor(numpy.nanmin(masked_dem) / 0.3048), 0)
max_elevation = math.ceil(numpy.nanmax(masked_dem) / 0.3048)
bands = list(self._get_bands_fn(zone.source)(zone.zone_id, min_elevation, max_elevation))
if not bands:
print('WARNING: No elevation bands found for {}, zone {}'.format(
zone.source, zone.zone_id
))
continue
for band in bands:
low, high = band
# Bands are exclusive of the low number, so the first band is a special case, since we
# want to include 0. So we work around it by making the actual low -1
if low == 0:
low = -1
# Elevation bands are represented in feet
masked_data = numpy.ma.masked_where(
(zone_mask == 0) | (clipped_elevation <= low * 0.3048) |
(clipped_elevation > high * 0.3048),
clipped_data
)
self._write_limit(variable, time_period, zone, masked_data, low, high)
for source, transfers_by_variable in self.transfers_by_source.items():
for variable, transfers in transfers_by_variable.items():
TransferLimit.objects.filter(
variable=variable, zone__source=source
).update(
avg_transfer=mean(transfers)
)
def mask(
input,
output,
variable,
like,
netcdf3,
all_touched,
invert,
zip):
"""
Create a NetCDF mask from a shapefile.
Values are equivalent to a numpy mask: 0 for unmasked areas, and 1 for masked areas.
Template NetCDF dataset must have a valid projection defined or be inferred from dimensions (e.g., lat / long)
"""
with Dataset(like) as template_ds:
template_varname = data_variables(template_ds).keys()[0]
template_variable = template_ds.variables[template_varname]
template_crs = get_crs(template_ds, template_varname)
if template_crs:
template_crs = CRS.from_string(template_crs)
elif is_geographic(template_ds, template_varname):
template_crs = CRS({'init': 'EPSG:4326'})
else:
raise click.UsageError('template dataset must have a valid projection defined')
spatial_dimensions = template_variable.dimensions[-2:]
mask_shape = template_variable.shape[-2:]
template_y_name, template_x_name = spatial_dimensions
coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=Proj(**template_crs.to_dict())
)
with fiona.open(input, 'r') as shp:
transform_required = CRS(shp.crs) != template_crs
# Project bbox for filtering
bbox = coords.bbox
if transform_required:
bbox = bbox.project(Proj(**shp.crs), edge_points=21)
geometries = []
for f in shp.filter(bbox=bbox.as_list()):
geom = f['geometry']
if transform_required:
geom = transform_geom(shp.crs, template_crs, geom)
geometries.append(geom)
click.echo('Converting {0} features to mask'.format(len(geometries)))
if invert:
fill_value = 0
default_value = 1
else:
fill_value = 1
default_value = 0
with rasterio.drivers():
# Rasterize features to 0, leaving background as 1
mask = rasterize(
geometries,
out_shape=mask_shape,
transform=coords.affine,
all_touched=all_touched,
fill=fill_value,
default_value=default_value,
dtype=numpy.uint8
)
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
dtype = 'int8' if netcdf3 else 'uint8'
with Dataset(output, 'w', format=format) as out:
coords.add_to_dataset(out, template_x_name, template_y_name)
out_var = out.createVariable(variable, dtype, dimensions=spatial_dimensions, zlib=zip,
fill_value=get_fill_value(dtype))
out_var[:] = mask