def test_get_crs(tmpdir):
""" Test reading proj4 string from CF convention parameters """
ds = Dataset(str(tmpdir.join('test.nc')), 'w')
data_var = ds.createVariable('data', 'S1')
data_var.setncattr('grid_mapping', 'crs_Lambert')
crs_var = ds.createVariable('crs_Lambert', 'S1')
in_proj4 = '+proj=lcc +units=m +lat_1=30 +lat_2=60 +lat_0=47.5 +lon_0=-97 +x_0=3825000 +y_0=3200000'
# These parameters match the above proj4 string
ncatts = dict()
ncatts['grid_mapping_name'] = 'lambert_conformal_conic'
ncatts['latitude_of_projection_origin'] = 47.5
ncatts['longitude_of_central_meridian'] = -97
ncatts['standard_parallel'] = [30, 60]
ncatts['false_northing'] = 3200000
ncatts['false_easting'] = 3825000
set_ncattrs(crs_var, ncatts)
out_proj4 = get_crs(ds, 'data')
assert out_proj4 is not None
out_data = CRS.from_string(out_proj4).to_dict()
assert len(out_data) == 8 # There should be 8 parameters
assert CRS.from_string(in_proj4).to_dict() == out_data
# Test WGS84 lat/long
data_var = ds.createVariable('data2', 'S1')
data_var.setncattr('grid_mapping', 'crs_latlong')
crs_var = ds.createVariable('crs_latlong', 'S1')
in_proj4 = '+proj=latlong +a={0} +rf={1}'.format(pj_ellps['WGS84']['a'],
pj_ellps['WGS84']['rf'])
# These parameters match the above proj4 string
ncatts = dict()
ncatts['grid_mapping_name'] = 'latitude_longitude'
ncatts['semi_major_axis'] = 6378137.0
ncatts['inverse_flattening'] = 298.257223563
set_ncattrs(crs_var, ncatts)
out_proj4 = get_crs(ds, 'data2')
assert out_proj4 is not None
out_data = CRS.from_string(out_proj4).to_dict()
assert len(out_data) == 4 # There should be 4 parameters
# Note: pyproj adds units=m even for latlong, which is incorrect but not our problem
assert CRS.from_string(in_proj4 + ' +units=m').to_dict() == out_data
def test_get_crs(tmpdir):
""" Test reading proj4 string from CF convention parameters """
ds = Dataset(str(tmpdir.join('test.nc')), 'w')
data_var = ds.createVariable('data', 'S1')
data_var.setncattr('grid_mapping', 'crs_Lambert')
crs_var = ds.createVariable('crs_Lambert', 'S1')
in_proj4 = '+proj=lcc +units=m +lat_1=30 +lat_2=60 +lat_0=47.5 +lon_0=-97 +x_0=3825000 +y_0=3200000'
# These parameters match the above proj4 string
ncatts = dict()
ncatts['grid_mapping_name'] = 'lambert_conformal_conic'
ncatts['latitude_of_projection_origin'] = 47.5
ncatts['longitude_of_central_meridian'] = -97
ncatts['standard_parallel'] = [30, 60]
ncatts['false_northing'] = 3200000
ncatts['false_easting'] = 3825000
set_ncattrs(crs_var, ncatts)
out_proj4 = get_crs(ds, 'data')
assert out_proj4 is not None
out_data = CRS.from_string(out_proj4).to_dict()
assert len(out_data) == 8 # There should be 8 parameters
assert CRS.from_string(in_proj4).to_dict() == out_data
# Test WGS84 lat/long
data_var = ds.createVariable('data2', 'S1')
data_var.setncattr('grid_mapping', 'crs_latlong')
crs_var = ds.createVariable('crs_latlong', 'S1')
in_proj4 = '+proj=latlong +a={0} +rf={1}'.format(pj_ellps['WGS84']['a'], pj_ellps['WGS84']['rf'])
# These parameters match the above proj4 string
ncatts = dict()
ncatts['grid_mapping_name'] = 'latitude_longitude'
ncatts['semi_major_axis'] = 6378137.0
ncatts['inverse_flattening'] = 298.257223563
set_ncattrs(crs_var, ncatts)
out_proj4 = get_crs(ds, 'data2')
assert out_proj4 is not None
out_data = CRS.from_string(out_proj4).to_dict()
assert len(out_data) == 4 # There should be 4 parameters
# Note: pyproj adds units=m even for latlong, which is incorrect but not our problem
assert CRS.from_string(in_proj4 + ' +units=m').to_dict() == out_data
def test_symmetric_proj4(tmpdir):
""" Test writing and reading proj4 string as attribute of variable """
ds = Dataset(str(tmpdir.join('test.nc')), 'w')
proj4 = '+proj=stere +units=m +datum=WGS84 +lat_ts=60 +lat_0=90 +lon_0=263 +lat_1=60 +x_0=3475000 +y_0=7475000'
ds.createVariable('data', 'S1')
set_crs(ds, 'data', Proj(proj4), set_proj4_att=True)
out_proj4 = get_crs(ds, 'data')
out_data = CRS.from_string(out_proj4).to_dict()
assert len(out_data) == 9 # There should be 9 parameters
assert CRS.from_string(proj4).to_dict() == out_data
def test_symmetric_proj4(tmpdir):
""" Test writing and reading proj4 string as attribute of variable """
ds = Dataset(str(tmpdir.join('test.nc')), 'w')
proj4 = '+proj=stere +units=m +datum=WGS84 +lat_ts=60 +lat_0=90 +lon_0=263 +lat_1=60 +x_0=3475000 +y_0=7475000'
ds.createVariable('data', 'S1')
set_crs(ds, 'data', Proj(proj4), set_proj4_att=True)
out_proj4 = get_crs(ds, 'data')
out_data = CRS.from_string(out_proj4).to_dict()
assert len(out_data) == 9 # There should be 9 parameters
assert CRS.from_string(proj4).to_dict() == out_data
def warp(
filename_pattern,
output_directory,
variables,
src_crs,
like,
resampling):
if variables:
variables = variables.strip().split(',')
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern', param='filename_pattern', param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
# For now, template dataset is required
template_ds = Dataset(like)
template_varname = data_variables(template_ds).keys()[0]
for filename in filenames:
with Dataset(filename) as ds:
if not variables:
ds_variables = data_variables(ds).keys()
else:
# filter to only variables present in this dataset
ds_variables = [v for v in variables if v in ds.variables]
ds_crs = get_crs(ds, ds_variables[0]) or src_crs
with Dataset(os.path.join(output_directory, os.path.split(filename)[1]), 'w') as out_ds:
click.echo('Processing: {0}'.format(filename))
warp_like(
ds,
ds_projection=ds_crs,
variables=ds_variables,
out_ds=out_ds,
template_ds=template_ds,
template_varname=template_varname,
resampling=getattr(Resampling, resampling)
)
def warp(filename_pattern, output_directory, variables, src_crs, like,
resampling):
if variables:
variables = variables.strip().split(',')
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='filename_pattern',
param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
# For now, template dataset is required
template_ds = Dataset(like)
template_varname = data_variables(template_ds).keys()[0]
for filename in filenames:
with Dataset(filename) as ds:
if not variables:
ds_variables = data_variables(ds).keys()
else:
# filter to only variables present in this dataset
ds_variables = [v for v in variables if v in ds.variables]
ds_crs = get_crs(ds, ds_variables[0]) or src_crs
with Dataset(
os.path.join(output_directory,
os.path.split(filename)[1]), 'w') as out_ds:
click.echo('Processing: {0}'.format(filename))
warp_like(ds,
ds_projection=ds_crs,
variables=ds_variables,
out_ds=out_ds,
template_ds=template_ds,
template_varname=template_varname,
resampling=getattr(Resampling, resampling))
def test_utm(tmpdir):
ds = Dataset(str(tmpdir.join('test.nc')), 'w')
proj4 = '+init=epsg:3157' # UTM Zone 10
ds.createVariable('data', 'S1')
set_crs(ds, 'data', Proj(proj4), set_proj4_att=True)
out_proj4 = get_crs(ds, 'data')
out_data = CRS.from_string(out_proj4).to_dict()
# ESPG will have been converted to long form
assert len(out_data) == 6
expected = {
u'zone': 10,
u'ellps': u'GRS80',
u'no_defs': True,
u'proj': u'utm',
u'units': u'm',
u'towgs84': u'0,0,0,0,0,0,0'
}
assert expected == out_data
def test_utm(tmpdir):
ds = Dataset(str(tmpdir.join('test.nc')), 'w')
proj4 = '+init=epsg:3157' # UTM Zone 10
ds.createVariable('data', 'S1')
set_crs(ds, 'data', Proj(proj4), set_proj4_att=True)
out_proj4 = get_crs(ds, 'data')
out_data = CRS.from_string(out_proj4).to_dict()
# ESPG will have been converted to long form
assert len(out_data) == 6
expected = {
u'zone': 10,
u'ellps': u'GRS80',
u'no_defs': True,
u'proj': u'utm',
u'units': u'm',
u'towgs84': u'0,0,0,0,0,0,0'
}
assert expected == out_data
def map_eems(
eems_file,
# output_directory,
scale,
format,
src_crs,
resampling):
"""
Render a NetCDF EEMS model to a web map.
"""
from EEMSBasePackage import EEMSCmd, EEMSProgram
model = EEMSProgram(eems_file)
# For each data producing command, store the netcdf file that contains it
file_vars = dict()
raw_variables = set()
for cmd in model.orderedCmds: # This is bottom up, may want to invert
filename = None
variable = None
if cmd.HasResultName():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetResultName()
elif cmd.IsReadCmd():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetParam('NewFieldName')
raw_variables.add(variable)
if filename and variable:
if not filename in file_vars:
file_vars[filename] = []
file_vars[filename].append(variable)
filenames = file_vars.keys()
for filename in filenames:
if not os.path.exists(filename):
raise click.ClickException(
'Could not find data file from EEMS model: {0}'.format(
filename))
dst_crs = 'EPSG:3857'
output_directory = tempfile.mkdtemp()
click.echo('Using temp directory: {0}'.format(output_directory))
# if not os.path.exists(output_directory):
# os.makedirs(output_directory)
# Since fuzzy renderer is hardcoded, we can output it now
fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'],
'1,-1')
fuzzy_renderer.get_legend(image_height=150)[0].to_image().save(
os.path.join(output_directory, 'fuzzy_legend.png'))
template_filename = filenames[0]
template_var = file_vars[template_filename][0]
with Dataset(template_filename) as ds:
var_obj = ds.variables[template_var]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions != 2:
raise click.ClickException(
'Only 2 dimensions are allowed on data variables for now')
ds_crs = get_crs(ds, template_var)
if not ds_crs and is_geographic(ds, template_var):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None)
#
# if mask is not None and not mask.shape == shape[-2:]:
# # Will likely break before this if collecting statistics
# raise click.BadParameter(
# 'mask variable shape does not match shape of input spatial dimensions',
# param='--mask', param_hint='--mask'
# )
#
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list())
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide valid src_crs to get interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing dataset {0}'.format(filename))
for variable in file_vars[filename]:
click.echo('Processing variable {0}'.format(variable))
if not variable in ds.variables:
raise click.ClickException(
'variable {0} was not found in file: {1}'.format(
variable, filename))
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
data = var_obj[:]
# if mask is not None:
# data = numpy.ma.masked_array(data, mask=mask)
if variable in raw_variables:
palette = DEFAULT_PALETTES['raw']
palette_stretch = '{0},{1}'.format(data.max(), data.min())
renderer = palette_to_stretched_renderer(
palette, palette_stretch)
renderer.get_legend(
image_height=150, max_precision=2)[0].to_image().save(
os.path.join(output_directory,
'{0}_legend.png'.format(variable)))
else:
renderer = fuzzy_renderer
image_filename = os.path.join(
output_directory, '{0}.{1}'.format(variable, format))
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale=scale,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[variable] = local_filename
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('trefoil.cli')).get_template('eems_map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
tree=[[cmd, depth]
for (cmd, depth) in model.GetCmdTree()],
raw_variables=list(raw_variables)))
webbrowser.open(index_html)
def describe(path_or_dataset):
if isinstance(path_or_dataset, string_types):
dataset = Dataset(path_or_dataset)
else:
dataset = path_or_dataset
description = {
'dimensions': {},
'variables': {},
'attributes': get_ncattrs(dataset)
}
for dimension_name in dataset.dimensions:
dimension = dataset.dimensions[dimension_name]
description['dimensions'][dimension_name] = {'length': len(dimension)}
for variable_name in dataset.variables:
variable = dataset.variables[variable_name]
if not variable.dimensions:
# Do not collect info about dimensionless variables (e.g., CRS variable)
continue
dtype = str(variable.dtype)
if "'" in dtype:
dtype = dtype.split("'")[1]
attributes = get_ncattrs(variable)
variable_info = {
'attributes':
attributes,
'dimensions':
variable.dimensions,
'data_type':
dtype,
'name':
attributes.get('long_name') or attributes.get('standard_name')
or variable_name
}
if dtype not in ('str', ):
if len(variable.shape) > 2:
# Avoid loading the entire array into memory by iterating along the first index (usually time)
variable_info.update({
'min':
min(variable[i, :].min().item()
for i in range(variable.shape[0])),
'max':
max(variable[i, :].max().item()
for i in range(variable.shape[0]))
})
else:
data = variable[:]
variable_info.update({
'min': data.min().item(),
'max': data.max().item()
})
if variable_name in dataset.dimensions and dtype not in ('str', ):
dimension_variable = dataset.variables[variable_name]
if len(dimension_variable.dimensions
) == 1: # range dimensions don't make sense for interval
interval = get_interval(dimension_variable)
if interval:
variable_info['interval'] = interval
else:
# Data variable
proj4 = get_crs(dataset, variable_name)
#extent
if len(variable.dimensions) >= 2:
x_variable_name = None
y_variable_name = None
time_variable_name = None
for dimension_name in (x for x in variable.dimensions
if x in dataset.variables):
attributes = get_ncattrs(dataset.variables[dimension_name])
standard_name = attributes.get('standard_name', None)
if standard_name in X_DIMENSION_STANDARD_NAMES or dimension_name in X_DIMENSION_COMMON_NAMES:
x_variable_name = dimension_name
elif standard_name in Y_DIMENSION_STANDARD_NAMES or dimension_name in Y_DIMENSION_COMMON_NAMES:
y_variable_name = dimension_name
elif standard_name in TIME_DIMENSION_STANDARD_NAMES or dimension_name in TIME_DIMENSION_COMMON_NAMES:
if len(dataset.dimensions[dimension_name]) > 1:
time_variable_name = dimension_name
if x_variable_name and y_variable_name:
if proj4 is None and is_geographic(dataset, variable_name):
# Assume WGS84
proj4 = PROJ4_GEOGRAPHIC
coordinates = SpatialCoordinateVariables(
SpatialCoordinateVariable(
dataset.variables[x_variable_name]),
SpatialCoordinateVariable(
dataset.variables[y_variable_name]),
Proj(str(proj4)) if proj4 else None)
variable_info['spatial_grid'] = {
'extent': coordinates.bbox.as_dict(),
'x_dimension': x_variable_name,
'x_resolution': coordinates.x.pixel_size,
'y_dimension': y_variable_name,
'y_resolution': coordinates.y.pixel_size
}
if time_variable_name:
time_variable = dataset.variables[time_variable_name]
time_info = {
'dimension': time_variable_name,
}
try:
date_variable = DateVariable(time_variable)
values = date_variable.datetimes
time_info['extent'] = [
values.min().isoformat(),
values.max().isoformat()
]
time_info['interval_unit'] = date_variable.unit
interval = get_interval(time_variable)
if interval is not None:
time_info['interval'] = interval
except ValueError:
pass
variable_info['time'] = time_info
if proj4:
variable_info['proj4'] = proj4
description['variables'][variable_name] = variable_info
return description
def warp_like(ds,
ds_projection,
variables,
out_ds,
template_ds,
template_varname,
resampling=Resampling.nearest):
"""
Warp one or more variables in a NetCDF file based on the coordinate reference system and
spatial domain of a template NetCDF file.
:param ds: source dataset
:param ds_projection: source dataset coordiante reference system, proj4 string or EPSG:NNNN code
:param variables: list of variable names in source dataset to warp
:param out_ds: output dataset. Must be opened in write or append mode.
:param template_ds: template dataset
:param template_varname: variable name for template data variable in template dataset
:param resampling: resampling method. See rasterio.enums.Resampling for options
"""
template_variable = template_ds.variables[template_varname]
template_prj = Proj(get_crs(template_ds, template_varname))
template_mask = template_variable[:].mask
template_y_name, template_x_name = template_variable.dimensions[-2:]
template_coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=template_prj)
# template_geo_bbox = template_coords.bbox.project(ds_prj, edge_points=21) # TODO: add when needing to subset
ds_y_name, ds_x_name = ds.variables[variables[0]].dimensions[-2:]
proj = Proj(
init=ds_projection) if 'EPSG:' in ds_projection.upper() else Proj(
str(ds_projection))
ds_coords = SpatialCoordinateVariables.from_dataset(ds,
x_name=ds_x_name,
y_name=ds_y_name,
projection=proj)
with rasterio.Env():
# Copy dimensions for variable across to output
for dim_name in template_variable.dimensions:
if not dim_name in out_ds.dimensions:
if dim_name in template_ds.variables and not dim_name in out_ds.variables:
copy_variable(template_ds, out_ds, dim_name)
else:
copy_dimension(template_ds, out_ds, dim_name)
for variable_name in variables:
click.echo('Processing: {0}'.format(variable_name))
variable = ds.variables[variable_name]
fill_value = getattr(variable, '_FillValue',
variable[0, 0].fill_value)
for dim_name in variable.dimensions[:-2]:
if not dim_name in out_ds.dimensions:
if dim_name in ds.variables:
copy_variable(ds, out_ds, dim_name)
else:
copy_dimension(ds, out_ds, dim_name)
out_var = out_ds.createVariable(
variable_name,
variable.dtype,
dimensions=variable.dimensions[:-2] +
template_variable.dimensions,
fill_value=fill_value)
reproject_kwargs = {
'src_transform': ds_coords.affine,
'src_crs': CRS.from_string(ds_projection),
'dst_transform': template_coords.affine,
'dst_crs': template_prj.srs,
'resampling': resampling,
'src_nodata': fill_value,
'dst_nodata': fill_value,
'threads': 4
}
# TODO: may only need to select out what is in window
if len(variable.shape) == 3:
idxs = range(variable.shape[0])
with click.progressbar(idxs) as bar:
for i in bar:
# print('processing slice: {0}'.format(i))
data = variable[i, :]
out = numpy.ma.empty(template_coords.shape,
dtype=data.dtype)
out.mask = template_mask
out.fill(fill_value)
reproject(data, out, **reproject_kwargs)
out_var[i, :] = out
else:
data = variable[:]
out = numpy.ma.empty(template_coords.shape, dtype=data.dtype)
out.mask = template_mask
out.fill(fill_value)
reproject(data, out, **reproject_kwargs)
out_var[:] = out
def mask(
input,
output,
variable,
like,
netcdf3,
all_touched,
invert,
zip):
"""
Create a NetCDF mask from a shapefile.
Values are equivalent to a numpy mask: 0 for unmasked areas, and 1 for masked areas.
Template NetCDF dataset must have a valid projection defined or be inferred from dimensions (e.g., lat / long)
"""
with Dataset(like) as template_ds:
template_varname = data_variables(template_ds).keys()[0]
template_variable = template_ds.variables[template_varname]
template_crs = get_crs(template_ds, template_varname)
if template_crs:
template_crs = CRS.from_string(template_crs)
elif is_geographic(template_ds, template_varname):
template_crs = CRS({'init': 'EPSG:4326'})
else:
raise click.UsageError('template dataset must have a valid projection defined')
spatial_dimensions = template_variable.dimensions[-2:]
mask_shape = template_variable.shape[-2:]
template_y_name, template_x_name = spatial_dimensions
coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=Proj(**template_crs.to_dict())
)
with fiona.open(input, 'r') as shp:
transform_required = CRS(shp.crs) != template_crs
# Project bbox for filtering
bbox = coords.bbox
if transform_required:
bbox = bbox.project(Proj(**shp.crs), edge_points=21)
geometries = []
for f in shp.filter(bbox=bbox.as_list()):
geom = f['geometry']
if transform_required:
geom = transform_geom(shp.crs, template_crs, geom)
geometries.append(geom)
click.echo('Converting {0} features to mask'.format(len(geometries)))
if invert:
fill_value = 0
default_value = 1
else:
fill_value = 1
default_value = 0
with rasterio.Env():
# Rasterize features to 0, leaving background as 1
mask = rasterize(
geometries,
out_shape=mask_shape,
transform=coords.affine,
all_touched=all_touched,
fill=fill_value,
default_value=default_value,
dtype=numpy.uint8
)
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
dtype = 'int8' if netcdf3 else 'uint8'
with Dataset(output, 'w', format=format) as out:
coords.add_to_dataset(out, template_x_name, template_y_name)
out_var = out.createVariable(variable, dtype, dimensions=spatial_dimensions, zlib=zip,
fill_value=get_fill_value(dtype))
out_var[:] = mask
def zones(
input,
output,
variable,
attribute,
like,
netcdf3,
zip):
"""
Create zones in a NetCDF from features in a shapefile. This is intended
to be used as input to zonal statistics functions; it is not intended
as a direct replacement for rasterizing geometries into NetCDF.
Only handles < 65,535 features for now.
If --attribute is provided, any features that do not have this will not be
assigned to zones.
A values lookup will be used to store values. The zones are indices of
the unique values encountered when extracting features.
The original values are stored in an additional variable with the name of
the zones variable plus '_values'.
Template NetCDF dataset must have a valid projection defined or be inferred
from dimensions (e.g., lat / long).
"""
with Dataset(like) as template_ds:
template_varname = list(data_variables(template_ds).keys())[0]
template_variable = template_ds.variables[template_varname]
template_crs = get_crs(template_ds, template_varname)
if template_crs:
template_crs = CRS.from_string(template_crs)
elif is_geographic(template_ds, template_varname):
template_crs = CRS({'init': 'EPSG:4326'})
else:
raise click.UsageError('template dataset must have a valid projection defined')
spatial_dimensions = template_variable.dimensions[-2:]
out_shape = template_variable.shape[-2:]
template_y_name, template_x_name = spatial_dimensions
coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=Proj(**template_crs.to_dict())
)
with fiona.open(input, 'r') as shp:
if attribute:
if not attribute in shp.meta['schema']['properties']:
raise click.BadParameter('{0} not found in dataset'.format(attribute),
param='--attribute', param_hint='--attribute')
att_dtype = shp.meta['schema']['properties'][attribute].split(':')[0]
if not att_dtype in ('int', 'str'):
raise click.BadParameter('integer or string attribute required'.format(attribute),
param='--attribute', param_hint='--attribute')
transform_required = CRS(shp.crs) != template_crs
geometries = []
values = set()
values_lookup = {}
# Project bbox for filtering
bbox = coords.bbox
if transform_required:
bbox = bbox.project(Proj(**shp.crs), edge_points=21)
index = 0
for f in shp.filter(bbox=bbox.as_list()):
value = f['properties'].get(attribute) if attribute else int(f['id'])
if value is not None:
geom = f['geometry']
if transform_required:
geom = transform_geom(shp.crs, template_crs, geom)
geometries.append((geom, index))
if not value in values:
values.add(value)
values_lookup[index] = value
index += 1
# Otherwise, these will not be rasterized
num_geometries = len(geometries)
# Save a slot at the end for nodata
if num_geometries < 255:
dtype = numpy.dtype('uint8')
elif num_geometries < 65535:
dtype = numpy.dtype('uint16')
else:
raise click.UsageError('Too many features to rasterize: {0}, Exceptioning...'.format(num_geometries))
fill_value = get_fill_value(dtype)
click.echo('Rasterizing {0} features into zones'.format(num_geometries))
with rasterio.Env():
zones = rasterize(
geometries,
out_shape=out_shape,
transform=coords.affine,
all_touched=False, # True produces undesirable results for adjacent polygons
fill=fill_value,
dtype=dtype
)
format = 'NETCDF4'
out_dtype = dtype
if netcdf3:
format = 'NETCDF3_CLASSIC'
if dtype == numpy.uint8:
out_dtype = numpy.dtype('int16')
elif dtype == numpy.uint16:
out_dtype = numpy.dtype('int32')
# Have to convert fill_value to mask since we changed data type
zones = numpy.ma.masked_array(zones, mask=(zones == fill_value))
with Dataset(output, 'w', format=format) as out:
values_varname = '{0}_values'.format(variable)
coords.add_to_dataset(out, template_x_name, template_y_name)
out_var = out.createVariable(variable, out_dtype,
dimensions=spatial_dimensions,
zlib=zip,
fill_value=get_fill_value(out_dtype))
out_var.setncattr('values', values_varname)
out_var[:] = zones
out_values = numpy.array([values_lookup[k] for k in range(0, len(values_lookup))])
if netcdf3 and out_values.dtype == numpy.int64:
out_values = out_values.astype('int32')
out.createDimension(values_varname, len(out_values))
values_var = out.createVariable(values_varname, out_values.dtype,
dimensions=(values_varname, ),
zlib=zip)
values_var[:] = out_values
def netcdf_to_raster(
path_or_dataset,
variable_name,
outfilename,
index=0,
projection=None):
"""
Exports a 2D slice from a netcdf file to a raster file.
Only GeoTiffs are supported at this time.
Parameters
----------
path_or_dataset: path to NetCDF file or open Dataset
variable_name: name of data variable to export from dataset
outfilename: output filename
index: index within 3rd dimension (in first position) or 0
projection: pyproj.Proj object. Automatically determined from file if possible
"""
if isinstance(path_or_dataset, string_types):
dataset = Dataset(path_or_dataset)
else:
dataset = path_or_dataset
projection = projection or get_crs(dataset, variable_name)
if not projection:
raise ValueError('Projection must be provided; '
'no projection information can be determined from file')
# TODO figure out cleaner way to get affine or coords
y_name, x_name = dataset.variables[variable_name].dimensions[:2]
coords = SpatialCoordinateVariables.from_dataset(
dataset, x_name, y_name, projection=projection)
affine = coords.affine
if outfilename.lower().endswith('.tif'):
format = 'GTiff'
else:
raise ValueError('Only GeoTiff outputs supported, filename must have .tif extension')
variable = dataset.variables[variable_name]
ndims = len(variable.shape)
if ndims == 2:
if index != 0:
raise ValueError('Index out of range, must be 0')
data = variable[:]
elif ndims == 3:
# Assumes that time dimension is first
if index < 0 or index >= variable.shape[0]:
raise ValueError('Index out of range, '
'must be between 0 and {0}'.variable.shape[0])
data = variable[index]
else:
raise ValueError(
'Unsupported number of dimensions {0} for variable {1}, '
'must be 2 or 3'.format(ndims, variable_name))
array_to_raster(
data,
outfilename,
format=format,
projection=projection,
affine=affine)
def render_netcdf(
filename_pattern,
variable,
output_directory,
renderer_file,
save_file,
renderer_type,
colormap,
fill,
colorspace,
palette,
palette_stretch,
scale,
id_variable,
lh,
legend_breaks,
legend_ticks,
legend_precision,
format,
src_crs,
dst_crs,
res,
resampling,
anchors,
interactive_map,
mask_path):
"""
Render netcdf files to images.
colormap is ignored if renderer_file is provided
--dst-crs is ignored if using --map option (always uses EPSG:3857
If no colormap or palette is provided, a default palette may be chosen based on the name of the variable.
If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It
must be in the same CRS as the input datasets, and have the same spatial
dimensions.
"""
# Parameter overrides
if interactive_map:
dst_crs = 'EPSG:3857'
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern', param='filename_pattern', param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
mask = get_mask(mask_path) if mask_path is not None else None
if renderer_file is not None and not save_file:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist', param='renderer_file', param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
if variable in renderer_dict and not 'colors' in renderer_dict:
renderer_dict = renderer_dict[variable]
renderer_type = renderer_dict['type']
if renderer_type == 'stretched':
colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
if 'min' in colors or 'max' in colors or 'mean' in colors:
statistics = collect_statistics(filenames, (variable,), mask=mask)[variable]
for entry in renderer_dict['colors']:
if isinstance(entry[0], basestring):
if entry[0] in ('min', 'max', 'mean'):
entry[0] = statistics[entry[0]]
elif '*' in entry[0]:
rel_value, statistic = entry[0].split('*')
entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
if palette is not None:
renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask)
elif colormap is None and variable in DEFAULT_PALETTES:
palette, palette_stretch = DEFAULT_PALETTES[variable]
renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask)
else:
if colormap is None:
colormap = 'min:#000000,max:#FFFFFF'
renderer = colormap_to_stretched_renderer(colormap, colorspace, filenames, variable, fill_value=fill, mask=mask)
elif renderer_type == 'classified':
if not palette:
raise click.BadParameter('palette required for classified (for now)',
param='--palette', param_hint='--palette')
renderer = palette_to_classified_renderer(palette, filenames, variable, method='equal', fill_value=fill, mask=mask) # TODO: other methods
if save_file:
if os.path.exists(save_file):
with open(save_file, 'r+') as output_file:
data = json.loads(output_file.read())
output_file.seek(0)
output_file.truncate()
data[variable] = renderer.serialize()
output_file.write(json.dumps(data, indent=4))
else:
with open(save_file, 'w') as output_file:
output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'stretched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(image_height=lh, breaks=legend_breaks, ticks=legend_ticks, max_precision=legend_precision)[0].to_image()
elif renderer_type == 'classified':
legend = composite_elements(renderer.get_legend())
legend.save(os.path.join(output_directory, '{0}_legend.png'.format(variable)))
with Dataset(filenames[0]) as ds:
var_obj = ds.variables[variable]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions == 3:
if id_variable:
if shape[0] != ds.variables[id_variable][:].shape[0]:
raise click.BadParameter('must be same dimensionality as 3rd dimension of {0}'.format(variable),
param='--id_variable', param_hint='--id_variable')
else:
# Guess from the 3rd dimension
guess = dimensions[0]
if guess in ds.variables and ds.variables[guess][:].shape[0] == shape[0]:
id_variable = guess
ds_crs = get_crs(ds, variable)
if not ds_crs and is_geographic(ds, variable):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS({'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs.to_dict()) if src_crs else None
)
if mask is not None and not mask.shape == shape[-2:]:
# Will likely break before this if collecting statistics
raise click.BadParameter(
'mask variable shape does not match shape of input spatial dimensions',
param='--mask', param_hint='--mask'
)
flip_y = False
reproject_kwargs = None
if dst_crs is not None:
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height,
*coords.bbox.as_list(), resolution=res
)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
else:
dst_transform = coords.affine
dst_height, dst_width = coords.shape
dst_crs = src_crs
if coords.y.is_ascending_order():
# Only needed if we are not already reprojecting the data, since that will flip it automatically
flip_y = True
if anchors or interactive_map:
if not (dst_crs or src_crs):
raise click.BadParameter('must provide at least src_crs to get Leaflet anchors or interactive map',
param='--src-crs', param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(BBox.from_affine(dst_transform, dst_width, dst_height,
projection=Proj(dst_crs) if dst_crs else None))
if anchors:
click.echo('Anchors: {0}'.format(leaflet_anchors))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing {0}'.format(filename))
filename_root = os.path.split(filename)[1].replace('.nc', '')
if not variable in ds.variables:
raise click.BadParameter('variable {0} was not found in file: {1}'.format(variable, filename),
param='variable', param_hint='VARIABLE')
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException('All datasets must have the same dimensions for {0}'.format(variable))
if num_dimensions == 2:
data = var_obj[:]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
image_filename = os.path.join(output_directory, '{0}_{1}.{2}'.format(filename_root, variable, format))
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
elif num_dimensions == 3:
for index in range(shape[0]):
id = ds.variables[id_variable][index] if id_variable is not None else index
image_filename = os.path.join(output_directory, '{0}_{1}__{2}.{3}'.format(filename_root, variable, id, format))
data = var_obj[index]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
# TODO: not tested recently. Make sure still correct
# else:
# # Assume last 2 components of shape are lat & lon, rest are iterated over
# id_variables = None
# if id_variable is not None:
# id_variables = id_variable.split(',')
# for index, name in enumerate(id_variables):
# if name:
# assert data.shape[index] == ds.variables[name][:].shape[0]
#
# ranges = []
# for dim in data.shape[:-2]:
# ranges.append(range(0, dim))
# for combined_index in product(*ranges):
# id_parts = []
# for index, dim_index in enumerate(combined_index):
# if id_variables is not None and index < len(id_variables) and id_variables[index]:
# id = ds.variables[id_variables[index]][dim_index]
#
# if not isinstance(id, basestring):
# if isinstance(id, Iterable):
# id = '_'.join((str(i) for i in id))
# else:
# id = str(id)
#
# id_parts.append(id)
#
# else:
# id_parts.append(str(dim_index))
#
# combined_id = '_'.join(id_parts)
# image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format))
# if reproject_kwargs:
# data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this
# render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format)
#
# local_filename = os.path.split(image_filename)[1]
# layers[os.path.splitext(local_filename)[0]] = local_filename
if interactive_map:
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(loader=PackageLoader('trefoil.cli')).get_template('map.html')
out.write(
template.render(
layers=json.dumps(layers),
bounds=str(leaflet_anchors),
variable=variable
)
)
webbrowser.open(index_html)
def map_eems(
eems_file,
# output_directory,
scale,
format,
src_crs,
resampling):
"""
Render a NetCDF EEMS model to a web map.
"""
from EEMSBasePackage import EEMSCmd, EEMSProgram
model = EEMSProgram(eems_file)
# For each data producing command, store the netcdf file that contains it
file_vars = dict()
raw_variables = set()
for cmd in model.orderedCmds: # This is bottom up, may want to invert
filename = None
variable = None
if cmd.HasResultName():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetResultName()
elif cmd.IsReadCmd():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetParam('NewFieldName')
raw_variables.add(variable)
if filename and variable:
if not filename in file_vars:
file_vars[filename] = []
file_vars[filename].append(variable)
filenames =file_vars.keys()
for filename in filenames:
if not os.path.exists(filename):
raise click.ClickException('Could not find data file from EEMS model: {0}'.format(filename))
dst_crs = 'EPSG:3857'
output_directory = tempfile.mkdtemp()
click.echo('Using temp directory: {0}'.format(output_directory))
# if not os.path.exists(output_directory):
# os.makedirs(output_directory)
# Since fuzzy renderer is hardcoded, we can output it now
fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'], '1,-1')
fuzzy_renderer.get_legend(image_height=150)[0].to_image().save(os.path.join(output_directory, 'fuzzy_legend.png'))
template_filename = filenames[0]
template_var = file_vars[template_filename][0]
with Dataset(template_filename) as ds:
var_obj = ds.variables[template_var]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions != 2:
raise click.ClickException('Only 2 dimensions are allowed on data variables for now')
ds_crs = get_crs(ds, template_var)
if not ds_crs and is_geographic(ds, template_var):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS({'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None
)
#
# if mask is not None and not mask.shape == shape[-2:]:
# # Will likely break before this if collecting statistics
# raise click.BadParameter(
# 'mask variable shape does not match shape of input spatial dimensions',
# param='--mask', param_hint='--mask'
# )
#
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height,
*coords.bbox.as_list()
)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
if not (dst_crs or src_crs):
raise click.BadParameter('must provide valid src_crs to get interactive map',
param='--src-crs', param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(BBox.from_affine(dst_transform, dst_width, dst_height,
projection=Proj(dst_crs) if dst_crs else None))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing dataset {0}'.format(filename))
for variable in file_vars[filename]:
click.echo('Processing variable {0}'.format(variable))
if not variable in ds.variables:
raise click.ClickException('variable {0} was not found in file: {1}'.format(variable, filename))
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException('All datasets must have the same dimensions for {0}'.format(variable))
data = var_obj[:]
# if mask is not None:
# data = numpy.ma.masked_array(data, mask=mask)
if variable in raw_variables:
palette = DEFAULT_PALETTES['raw']
palette_stretch = '{0},{1}'.format(data.max(), data.min())
renderer = palette_to_stretched_renderer(palette, palette_stretch)
renderer.get_legend(image_height=150, max_precision=2)[0].to_image().save(os.path.join(output_directory, '{0}_legend.png'.format(variable)))
else:
renderer = fuzzy_renderer
image_filename = os.path.join(output_directory, '{0}.{1}'.format(variable, format))
data = warp_array(data, **reproject_kwargs)
render_image(renderer, data, image_filename, scale=scale, format=format)
local_filename = os.path.split(image_filename)[1]
layers[variable] = local_filename
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(loader=PackageLoader('trefoil.cli')).get_template('eems_map.html')
out.write(
template.render(
layers=json.dumps(layers),
bounds=str(leaflet_anchors),
tree=[[cmd, depth] for (cmd, depth) in model.GetCmdTree()],
raw_variables=list(raw_variables)
)
)
webbrowser.open(index_html)
def render_netcdf(filename_pattern, variable, output_directory, renderer_file,
save_file, renderer_type, colormap, fill, colorspace,
palette, palette_stretch, scale, id_variable, lh,
legend_breaks, legend_ticks, legend_precision, format,
src_crs, dst_crs, res, resampling, anchors, interactive_map,
mask_path):
"""
Render netcdf files to images.
colormap is ignored if renderer_file is provided
--dst-crs is ignored if using --map option (always uses EPSG:3857
If no colormap or palette is provided, a default palette may be chosen based on the name of the variable.
If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It
must be in the same CRS as the input datasets, and have the same spatial
dimensions.
"""
# Parameter overrides
if interactive_map:
dst_crs = 'EPSG:3857'
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='filename_pattern',
param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
mask = get_mask(mask_path) if mask_path is not None else None
if renderer_file is not None and not save_file:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist',
param='renderer_file',
param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
if variable in renderer_dict and not 'colors' in renderer_dict:
renderer_dict = renderer_dict[variable]
renderer_type = renderer_dict['type']
if renderer_type == 'stretched':
colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
if 'min' in colors or 'max' in colors or 'mean' in colors:
statistics = collect_statistics(filenames, (variable, ),
mask=mask)[variable]
for entry in renderer_dict['colors']:
if isinstance(entry[0], basestring):
if entry[0] in ('min', 'max', 'mean'):
entry[0] = statistics[entry[0]]
elif '*' in entry[0]:
rel_value, statistic = entry[0].split('*')
entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
if palette is not None:
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
elif colormap is None and variable in DEFAULT_PALETTES:
palette, palette_stretch = DEFAULT_PALETTES[variable]
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
else:
if colormap is None:
colormap = 'min:#000000,max:#FFFFFF'
renderer = colormap_to_stretched_renderer(colormap,
colorspace,
filenames,
variable,
fill_value=fill,
mask=mask)
elif renderer_type == 'classified':
if not palette:
raise click.BadParameter(
'palette required for classified (for now)',
param='--palette',
param_hint='--palette')
renderer = palette_to_classified_renderer(
palette,
filenames,
variable,
method='equal',
fill_value=fill,
mask=mask) # TODO: other methods
if save_file:
if os.path.exists(save_file):
with open(save_file, 'r+') as output_file:
data = json.loads(output_file.read())
output_file.seek(0)
output_file.truncate()
data[variable] = renderer.serialize()
output_file.write(json.dumps(data, indent=4))
else:
with open(save_file, 'w') as output_file:
output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'stretched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(
image_height=lh,
breaks=legend_breaks,
ticks=legend_ticks,
max_precision=legend_precision)[0].to_image()
elif renderer_type == 'classified':
legend = composite_elements(renderer.get_legend())
legend.save(
os.path.join(output_directory, '{0}_legend.png'.format(variable)))
with Dataset(filenames[0]) as ds:
var_obj = ds.variables[variable]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions == 3:
if id_variable:
if shape[0] != ds.variables[id_variable][:].shape[0]:
raise click.BadParameter(
'must be same dimensionality as 3rd dimension of {0}'.
format(variable),
param='--id_variable',
param_hint='--id_variable')
else:
# Guess from the 3rd dimension
guess = dimensions[0]
if guess in ds.variables and ds.variables[guess][:].shape[
0] == shape[0]:
id_variable = guess
ds_crs = get_crs(ds, variable)
if not ds_crs and is_geographic(ds, variable):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds,
x_dim,
y_dim,
projection=Proj(src_crs.to_dict()) if src_crs else None)
if mask is not None and not mask.shape == shape[-2:]:
# Will likely break before this if collecting statistics
raise click.BadParameter(
'mask variable shape does not match shape of input spatial dimensions',
param='--mask',
param_hint='--mask')
flip_y = False
reproject_kwargs = None
if dst_crs is not None:
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs,
dst_crs,
src_width,
src_height,
*coords.bbox.as_list(),
resolution=res)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
else:
dst_transform = coords.affine
dst_height, dst_width = coords.shape
dst_crs = src_crs
if coords.y.is_ascending_order():
# Only needed if we are not already reprojecting the data, since that will flip it automatically
flip_y = True
if anchors or interactive_map:
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide at least src_crs to get Leaflet anchors or interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(
dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
if anchors:
click.echo('Anchors: {0}'.format(leaflet_anchors))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing {0}'.format(filename))
filename_root = os.path.split(filename)[1].replace('.nc', '')
if not variable in ds.variables:
raise click.BadParameter(
'variable {0} was not found in file: {1}'.format(
variable, filename),
param='variable',
param_hint='VARIABLE')
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
if num_dimensions == 2:
data = var_obj[:]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
image_filename = os.path.join(
output_directory,
'{0}_{1}.{2}'.format(filename_root, variable, format))
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
elif num_dimensions == 3:
for index in range(shape[0]):
id = ds.variables[id_variable][
index] if id_variable is not None else index
image_filename = os.path.join(
output_directory,
'{0}_{1}__{2}.{3}'.format(filename_root, variable, id,
format))
data = var_obj[index]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)
[0]] = local_filename
# TODO: not tested recently. Make sure still correct
# else:
# # Assume last 2 components of shape are lat & lon, rest are iterated over
# id_variables = None
# if id_variable is not None:
# id_variables = id_variable.split(',')
# for index, name in enumerate(id_variables):
# if name:
# assert data.shape[index] == ds.variables[name][:].shape[0]
#
# ranges = []
# for dim in data.shape[:-2]:
# ranges.append(range(0, dim))
# for combined_index in product(*ranges):
# id_parts = []
# for index, dim_index in enumerate(combined_index):
# if id_variables is not None and index < len(id_variables) and id_variables[index]:
# id = ds.variables[id_variables[index]][dim_index]
#
# if not isinstance(id, basestring):
# if isinstance(id, Iterable):
# id = '_'.join((str(i) for i in id))
# else:
# id = str(id)
#
# id_parts.append(id)
#
# else:
# id_parts.append(str(dim_index))
#
# combined_id = '_'.join(id_parts)
# image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format))
# if reproject_kwargs:
# data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this
# render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format)
#
# local_filename = os.path.split(image_filename)[1]
# layers[os.path.splitext(local_filename)[0]] = local_filename
if interactive_map:
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('trefoil.cli')).get_template('map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
variable=variable))
webbrowser.open(index_html)
def mask(input, output, variable, like, netcdf3, all_touched, invert, zip):
"""
Create a NetCDF mask from a shapefile.
Values are equivalent to a numpy mask: 0 for unmasked areas, and 1 for masked areas.
Template NetCDF dataset must have a valid projection defined or be inferred from dimensions (e.g., lat / long)
"""
with Dataset(like) as template_ds:
template_varname = data_variables(template_ds).keys()[0]
template_variable = template_ds.variables[template_varname]
template_crs = get_crs(template_ds, template_varname)
if template_crs:
template_crs = CRS.from_string(template_crs)
elif is_geographic(template_ds, template_varname):
template_crs = CRS({'init': 'EPSG:4326'})
else:
raise click.UsageError(
'template dataset must have a valid projection defined')
spatial_dimensions = template_variable.dimensions[-2:]
mask_shape = template_variable.shape[-2:]
template_y_name, template_x_name = spatial_dimensions
coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=Proj(**template_crs.to_dict()))
with fiona.open(input, 'r') as shp:
transform_required = CRS(shp.crs) != template_crs
# Project bbox for filtering
bbox = coords.bbox
if transform_required:
bbox = bbox.project(Proj(**shp.crs), edge_points=21)
geometries = []
for f in shp.filter(bbox=bbox.as_list()):
geom = f['geometry']
if transform_required:
geom = transform_geom(shp.crs, template_crs, geom)
geometries.append(geom)
click.echo('Converting {0} features to mask'.format(len(geometries)))
if invert:
fill_value = 0
default_value = 1
else:
fill_value = 1
default_value = 0
with rasterio.Env():
# Rasterize features to 0, leaving background as 1
mask = rasterize(geometries,
out_shape=mask_shape,
transform=coords.affine,
all_touched=all_touched,
fill=fill_value,
default_value=default_value,
dtype=numpy.uint8)
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
dtype = 'int8' if netcdf3 else 'uint8'
with Dataset(output, 'w', format=format) as out:
coords.add_to_dataset(out, template_x_name, template_y_name)
out_var = out.createVariable(variable,
dtype,
dimensions=spatial_dimensions,
zlib=zip,
fill_value=get_fill_value(dtype))
out_var[:] = mask
def netcdf_to_raster(path_or_dataset,
variable_name,
outfilename,
index=0,
projection=None):
"""
Exports a 2D slice from a netcdf file to a raster file.
Only GeoTiffs are supported at this time.
Parameters
----------
path_or_dataset: path to NetCDF file or open Dataset
variable_name: name of data variable to export from dataset
outfilename: output filename
index: index within 3rd dimension (in first position) or 0
projection: pyproj.Proj object. Automatically determined from file if possible
"""
if isinstance(path_or_dataset, string_types):
dataset = Dataset(path_or_dataset)
else:
dataset = path_or_dataset
projection = projection or get_crs(dataset, variable_name)
if not projection:
raise ValueError(
'Projection must be provided; '
'no projection information can be determined from file')
# TODO figure out cleaner way to get affine or coords
y_name, x_name = dataset.variables[variable_name].dimensions[:2]
coords = SpatialCoordinateVariables.from_dataset(dataset,
x_name,
y_name,
projection=projection)
affine = coords.affine
if outfilename.lower().endswith('.tif'):
format = 'GTiff'
else:
raise ValueError(
'Only GeoTiff outputs supported, filename must have .tif extension'
)
variable = dataset.variables[variable_name]
ndims = len(variable.shape)
if ndims == 2:
if index != 0:
raise ValueError('Index out of range, must be 0')
data = variable[:]
elif ndims == 3:
# Assumes that time dimension is first
if index < 0 or index >= variable.shape[0]:
raise ValueError('Index out of range, '
'must be between 0 and {0}'.variable.shape[0])
data = variable[index]
else:
raise ValueError(
'Unsupported number of dimensions {0} for variable {1}, '
'must be 2 or 3'.format(ndims, variable_name))
array_to_raster(data,
outfilename,
format=format,
projection=projection,
affine=affine)