def load_region(self, region_name):
"""Loads data for a region if not already loaded, closing previous
region as needed.
Parameters
----------
region_name : str
"""
if region_name == self.region:
return
self.close()
elevation_service = Service.objects.get(
name="{}_dem".format(region_name))
elevation_record = elevation_service.variable_set.first()
self.dataset = Dataset(DATA_PATH / elevation_service.data_path)
self.coords = SpatialCoordinateVariables.from_dataset(
self.dataset,
x_name=elevation_record.x_dimension,
y_name=elevation_record.y_dimension,
projection=Proj(elevation_service.projection),
)
self.data = self.dataset.variables[elevation_record.name]
self.data.set_auto_mask(False)
self.nodata_value = self.data._FillValue
self.region = region_name
def load_variable_data(self, variable, region, year, model=None):
if variable == 'LAT':
with Dataset(
os.path.join(NC_SERVICE_DIR, 'regions', region,
'{}_dem.nc'.format(region))) as ds:
return create_latitude_data(
SpatialCoordinateVariables.from_dataset(ds))
if model is not None:
year = '{model}_{year}'.format(model=model, year=year)
service = Service.objects.get(
name='{region}_{year}Y_{variable}'.format(
region=region, year=year, variable=variable))
variable = service.variable_set.first()
self.service = variable.service
self.dataset = None
data = self.get_grid_for_variable(variable)
return Raster(data, variable.full_extent, 1, 0, Y_INCREASING)
def warp_like(ds,
ds_projection,
variables,
out_ds,
template_ds,
template_varname,
resampling=Resampling.nearest):
"""
Warp one or more variables in a NetCDF file based on the coordinate reference system and
spatial domain of a template NetCDF file.
:param ds: source dataset
:param ds_projection: source dataset coordiante reference system, proj4 string or EPSG:NNNN code
:param variables: list of variable names in source dataset to warp
:param out_ds: output dataset. Must be opened in write or append mode.
:param template_ds: template dataset
:param template_varname: variable name for template data variable in template dataset
:param resampling: resampling method. See rasterio.enums.Resampling for options
"""
template_variable = template_ds.variables[template_varname]
template_prj = Proj(get_crs(template_ds, template_varname))
template_mask = template_variable[:].mask
template_y_name, template_x_name = template_variable.dimensions[-2:]
template_coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=template_prj)
# template_geo_bbox = template_coords.bbox.project(ds_prj, edge_points=21) # TODO: add when needing to subset
ds_y_name, ds_x_name = ds.variables[variables[0]].dimensions[-2:]
proj = Proj(
init=ds_projection) if 'EPSG:' in ds_projection.upper() else Proj(
str(ds_projection))
ds_coords = SpatialCoordinateVariables.from_dataset(ds,
x_name=ds_x_name,
y_name=ds_y_name,
projection=proj)
with rasterio.Env():
# Copy dimensions for variable across to output
for dim_name in template_variable.dimensions:
if not dim_name in out_ds.dimensions:
if dim_name in template_ds.variables and not dim_name in out_ds.variables:
copy_variable(template_ds, out_ds, dim_name)
else:
copy_dimension(template_ds, out_ds, dim_name)
for variable_name in variables:
click.echo('Processing: {0}'.format(variable_name))
variable = ds.variables[variable_name]
fill_value = getattr(variable, '_FillValue',
variable[0, 0].fill_value)
for dim_name in variable.dimensions[:-2]:
if not dim_name in out_ds.dimensions:
if dim_name in ds.variables:
copy_variable(ds, out_ds, dim_name)
else:
copy_dimension(ds, out_ds, dim_name)
out_var = out_ds.createVariable(
variable_name,
variable.dtype,
dimensions=variable.dimensions[:-2] +
template_variable.dimensions,
fill_value=fill_value)
reproject_kwargs = {
'src_transform': ds_coords.affine,
'src_crs': CRS.from_string(ds_projection),
'dst_transform': template_coords.affine,
'dst_crs': template_prj.srs,
'resampling': resampling,
'src_nodata': fill_value,
'dst_nodata': fill_value,
'threads': 4
}
# TODO: may only need to select out what is in window
if len(variable.shape) == 3:
idxs = range(variable.shape[0])
with click.progressbar(idxs) as bar:
for i in bar:
# print('processing slice: {0}'.format(i))
data = variable[i, :]
out = numpy.ma.empty(template_coords.shape,
dtype=data.dtype)
out.mask = template_mask
out.fill(fill_value)
reproject(data, out, **reproject_kwargs)
out_var[i, :] = out
else:
data = variable[:]
out = numpy.ma.empty(template_coords.shape, dtype=data.dtype)
out.mask = template_mask
out.fill(fill_value)
reproject(data, out, **reproject_kwargs)
out_var[:] = out
def netcdf_to_raster(
path_or_dataset,
variable_name,
outfilename,
index=0,
projection=None):
"""
Exports a 2D slice from a netcdf file to a raster file.
Only GeoTiffs are supported at this time.
Parameters
----------
path_or_dataset: path to NetCDF file or open Dataset
variable_name: name of data variable to export from dataset
outfilename: output filename
index: index within 3rd dimension (in first position) or 0
projection: pyproj.Proj object. Automatically determined from file if possible
"""
if isinstance(path_or_dataset, string_types):
dataset = Dataset(path_or_dataset)
else:
dataset = path_or_dataset
projection = projection or get_crs(dataset, variable_name)
if not projection:
raise ValueError('Projection must be provided; '
'no projection information can be determined from file')
# TODO figure out cleaner way to get affine or coords
y_name, x_name = dataset.variables[variable_name].dimensions[:2]
coords = SpatialCoordinateVariables.from_dataset(
dataset, x_name, y_name, projection=projection)
affine = coords.affine
if outfilename.lower().endswith('.tif'):
format = 'GTiff'
else:
raise ValueError('Only GeoTiff outputs supported, filename must have .tif extension')
variable = dataset.variables[variable_name]
ndims = len(variable.shape)
if ndims == 2:
if index != 0:
raise ValueError('Index out of range, must be 0')
data = variable[:]
elif ndims == 3:
# Assumes that time dimension is first
if index < 0 or index >= variable.shape[0]:
raise ValueError('Index out of range, '
'must be between 0 and {0}'.variable.shape[0])
data = variable[index]
else:
raise ValueError(
'Unsupported number of dimensions {0} for variable {1}, '
'must be 2 or 3'.format(ndims, variable_name))
array_to_raster(
data,
outfilename,
format=format,
projection=projection,
affine=affine)
def handle(self, datasets, directory, overwrite, *args, **options):
old_files = []
for dataset in datasets:
filename = os.path.basename(dataset)
name = os.path.splitext(filename)[0]
if directory is not None:
filename = '{}/{}'.format(directory.strip('/'), filename)
name = '{}/{}'.format(directory.strip('/'), name)
with transaction.atomic():
existing = Service.objects.filter(name__iexact=name)
if existing.exists():
if overwrite:
old_files.append(
os.path.join(SERVICE_DIR,
existing.get().data_path))
existing.delete()
else:
raise CommandError(
"A service named '{}' already exists".format(name))
with Dataset(dataset, 'r') as ds:
variables = []
x_dimension = None
y_dimension = None
projection = None
desc = describe(ds)
for variable, variable_info in desc['variables'].items():
if 'spatial_grid' in variable_info:
variables.append(variable)
spatial_grid = variable_info['spatial_grid']
x_dimension = spatial_grid['x_dimension']
y_dimension = spatial_grid['y_dimension']
projection = Proj(variable_info['proj4'])
if not variables:
raise CommandError('No usable variables found')
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dimension, y_dimension, projection=projection)
service = Service.objects.create(name=name,
data_path=filename,
projection=coords.projection,
full_extent=coords.bbox,
initial_extent=coords.bbox)
for variable in variables:
Variable.objects.create(service=service,
index=0,
variable=variable,
projection=projection,
x_dimension=x_dimension,
y_dimension=y_dimension,
name=variable,
renderer=StretchedRenderer([
(variable_info['min'],
Color(0, 0, 0)),
(variable_info['max'],
Color(255, 255, 255))
]),
full_extent=coords.bbox)
print('Added {}...'.format(name))
for path in old_files:
if os.path.exists(path):
os.remove(path)
for dataset in datasets:
target_dir = Path(SERVICE_DIR) / (directory or '')
if not os.path.exists(target_dir):
os.makedirs(target_dir)
shutil.copy(dataset, target_dir)
def zones(
input,
output,
variable,
attribute,
like,
netcdf3,
zip):
"""
Create zones in a NetCDF from features in a shapefile. This is intended
to be used as input to zonal statistics functions; it is not intended
as a direct replacement for rasterizing geometries into NetCDF.
Only handles < 65,535 features for now.
If --attribute is provided, any features that do not have this will not be
assigned to zones.
A values lookup will be used to store values. The zones are indices of
the unique values encountered when extracting features.
The original values are stored in an additional variable with the name of
the zones variable plus '_values'.
Template NetCDF dataset must have a valid projection defined or be inferred
from dimensions (e.g., lat / long).
"""
with Dataset(like) as template_ds:
template_varname = list(data_variables(template_ds).keys())[0]
template_variable = template_ds.variables[template_varname]
template_crs = get_crs(template_ds, template_varname)
if template_crs:
template_crs = CRS.from_string(template_crs)
elif is_geographic(template_ds, template_varname):
template_crs = CRS({'init': 'EPSG:4326'})
else:
raise click.UsageError('template dataset must have a valid projection defined')
spatial_dimensions = template_variable.dimensions[-2:]
out_shape = template_variable.shape[-2:]
template_y_name, template_x_name = spatial_dimensions
coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=Proj(**template_crs.to_dict())
)
with fiona.open(input, 'r') as shp:
if attribute:
if not attribute in shp.meta['schema']['properties']:
raise click.BadParameter('{0} not found in dataset'.format(attribute),
param='--attribute', param_hint='--attribute')
att_dtype = shp.meta['schema']['properties'][attribute].split(':')[0]
if not att_dtype in ('int', 'str'):
raise click.BadParameter('integer or string attribute required'.format(attribute),
param='--attribute', param_hint='--attribute')
transform_required = CRS(shp.crs) != template_crs
geometries = []
values = set()
values_lookup = {}
# Project bbox for filtering
bbox = coords.bbox
if transform_required:
bbox = bbox.project(Proj(**shp.crs), edge_points=21)
index = 0
for f in shp.filter(bbox=bbox.as_list()):
value = f['properties'].get(attribute) if attribute else int(f['id'])
if value is not None:
geom = f['geometry']
if transform_required:
geom = transform_geom(shp.crs, template_crs, geom)
geometries.append((geom, index))
if not value in values:
values.add(value)
values_lookup[index] = value
index += 1
# Otherwise, these will not be rasterized
num_geometries = len(geometries)
# Save a slot at the end for nodata
if num_geometries < 255:
dtype = numpy.dtype('uint8')
elif num_geometries < 65535:
dtype = numpy.dtype('uint16')
else:
raise click.UsageError('Too many features to rasterize: {0}, Exceptioning...'.format(num_geometries))
fill_value = get_fill_value(dtype)
click.echo('Rasterizing {0} features into zones'.format(num_geometries))
with rasterio.Env():
zones = rasterize(
geometries,
out_shape=out_shape,
transform=coords.affine,
all_touched=False, # True produces undesirable results for adjacent polygons
fill=fill_value,
dtype=dtype
)
format = 'NETCDF4'
out_dtype = dtype
if netcdf3:
format = 'NETCDF3_CLASSIC'
if dtype == numpy.uint8:
out_dtype = numpy.dtype('int16')
elif dtype == numpy.uint16:
out_dtype = numpy.dtype('int32')
# Have to convert fill_value to mask since we changed data type
zones = numpy.ma.masked_array(zones, mask=(zones == fill_value))
with Dataset(output, 'w', format=format) as out:
values_varname = '{0}_values'.format(variable)
coords.add_to_dataset(out, template_x_name, template_y_name)
out_var = out.createVariable(variable, out_dtype,
dimensions=spatial_dimensions,
zlib=zip,
fill_value=get_fill_value(out_dtype))
out_var.setncattr('values', values_varname)
out_var[:] = zones
out_values = numpy.array([values_lookup[k] for k in range(0, len(values_lookup))])
if netcdf3 and out_values.dtype == numpy.int64:
out_values = out_values.astype('int32')
out.createDimension(values_varname, len(out_values))
values_var = out.createVariable(values_varname, out_values.dtype,
dimensions=(values_varname, ),
zlib=zip)
values_var[:] = out_values
def render_netcdf(
filename_pattern,
variable,
output_directory,
renderer_file,
save_file,
renderer_type,
colormap,
fill,
colorspace,
palette,
palette_stretch,
scale,
id_variable,
lh,
legend_breaks,
legend_ticks,
legend_precision,
format,
src_crs,
dst_crs,
res,
resampling,
anchors,
interactive_map,
mask_path):
"""
Render netcdf files to images.
colormap is ignored if renderer_file is provided
--dst-crs is ignored if using --map option (always uses EPSG:3857
If no colormap or palette is provided, a default palette may be chosen based on the name of the variable.
If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It
must be in the same CRS as the input datasets, and have the same spatial
dimensions.
"""
# Parameter overrides
if interactive_map:
dst_crs = 'EPSG:3857'
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern', param='filename_pattern', param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
mask = get_mask(mask_path) if mask_path is not None else None
if renderer_file is not None and not save_file:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist', param='renderer_file', param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
if variable in renderer_dict and not 'colors' in renderer_dict:
renderer_dict = renderer_dict[variable]
renderer_type = renderer_dict['type']
if renderer_type == 'stretched':
colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
if 'min' in colors or 'max' in colors or 'mean' in colors:
statistics = collect_statistics(filenames, (variable,), mask=mask)[variable]
for entry in renderer_dict['colors']:
if isinstance(entry[0], basestring):
if entry[0] in ('min', 'max', 'mean'):
entry[0] = statistics[entry[0]]
elif '*' in entry[0]:
rel_value, statistic = entry[0].split('*')
entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
if palette is not None:
renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask)
elif colormap is None and variable in DEFAULT_PALETTES:
palette, palette_stretch = DEFAULT_PALETTES[variable]
renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask)
else:
if colormap is None:
colormap = 'min:#000000,max:#FFFFFF'
renderer = colormap_to_stretched_renderer(colormap, colorspace, filenames, variable, fill_value=fill, mask=mask)
elif renderer_type == 'classified':
if not palette:
raise click.BadParameter('palette required for classified (for now)',
param='--palette', param_hint='--palette')
renderer = palette_to_classified_renderer(palette, filenames, variable, method='equal', fill_value=fill, mask=mask) # TODO: other methods
if save_file:
if os.path.exists(save_file):
with open(save_file, 'r+') as output_file:
data = json.loads(output_file.read())
output_file.seek(0)
output_file.truncate()
data[variable] = renderer.serialize()
output_file.write(json.dumps(data, indent=4))
else:
with open(save_file, 'w') as output_file:
output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'stretched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(image_height=lh, breaks=legend_breaks, ticks=legend_ticks, max_precision=legend_precision)[0].to_image()
elif renderer_type == 'classified':
legend = composite_elements(renderer.get_legend())
legend.save(os.path.join(output_directory, '{0}_legend.png'.format(variable)))
with Dataset(filenames[0]) as ds:
var_obj = ds.variables[variable]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions == 3:
if id_variable:
if shape[0] != ds.variables[id_variable][:].shape[0]:
raise click.BadParameter('must be same dimensionality as 3rd dimension of {0}'.format(variable),
param='--id_variable', param_hint='--id_variable')
else:
# Guess from the 3rd dimension
guess = dimensions[0]
if guess in ds.variables and ds.variables[guess][:].shape[0] == shape[0]:
id_variable = guess
ds_crs = get_crs(ds, variable)
if not ds_crs and is_geographic(ds, variable):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS({'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs.to_dict()) if src_crs else None
)
if mask is not None and not mask.shape == shape[-2:]:
# Will likely break before this if collecting statistics
raise click.BadParameter(
'mask variable shape does not match shape of input spatial dimensions',
param='--mask', param_hint='--mask'
)
flip_y = False
reproject_kwargs = None
if dst_crs is not None:
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height,
*coords.bbox.as_list(), resolution=res
)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
else:
dst_transform = coords.affine
dst_height, dst_width = coords.shape
dst_crs = src_crs
if coords.y.is_ascending_order():
# Only needed if we are not already reprojecting the data, since that will flip it automatically
flip_y = True
if anchors or interactive_map:
if not (dst_crs or src_crs):
raise click.BadParameter('must provide at least src_crs to get Leaflet anchors or interactive map',
param='--src-crs', param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(BBox.from_affine(dst_transform, dst_width, dst_height,
projection=Proj(dst_crs) if dst_crs else None))
if anchors:
click.echo('Anchors: {0}'.format(leaflet_anchors))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing {0}'.format(filename))
filename_root = os.path.split(filename)[1].replace('.nc', '')
if not variable in ds.variables:
raise click.BadParameter('variable {0} was not found in file: {1}'.format(variable, filename),
param='variable', param_hint='VARIABLE')
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException('All datasets must have the same dimensions for {0}'.format(variable))
if num_dimensions == 2:
data = var_obj[:]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
image_filename = os.path.join(output_directory, '{0}_{1}.{2}'.format(filename_root, variable, format))
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
elif num_dimensions == 3:
for index in range(shape[0]):
id = ds.variables[id_variable][index] if id_variable is not None else index
image_filename = os.path.join(output_directory, '{0}_{1}__{2}.{3}'.format(filename_root, variable, id, format))
data = var_obj[index]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
# TODO: not tested recently. Make sure still correct
# else:
# # Assume last 2 components of shape are lat & lon, rest are iterated over
# id_variables = None
# if id_variable is not None:
# id_variables = id_variable.split(',')
# for index, name in enumerate(id_variables):
# if name:
# assert data.shape[index] == ds.variables[name][:].shape[0]
#
# ranges = []
# for dim in data.shape[:-2]:
# ranges.append(range(0, dim))
# for combined_index in product(*ranges):
# id_parts = []
# for index, dim_index in enumerate(combined_index):
# if id_variables is not None and index < len(id_variables) and id_variables[index]:
# id = ds.variables[id_variables[index]][dim_index]
#
# if not isinstance(id, basestring):
# if isinstance(id, Iterable):
# id = '_'.join((str(i) for i in id))
# else:
# id = str(id)
#
# id_parts.append(id)
#
# else:
# id_parts.append(str(dim_index))
#
# combined_id = '_'.join(id_parts)
# image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format))
# if reproject_kwargs:
# data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this
# render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format)
#
# local_filename = os.path.split(image_filename)[1]
# layers[os.path.splitext(local_filename)[0]] = local_filename
if interactive_map:
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(loader=PackageLoader('trefoil.cli')).get_template('map.html')
out.write(
template.render(
layers=json.dumps(layers),
bounds=str(leaflet_anchors),
variable=variable
)
)
webbrowser.open(index_html)
def netcdf_to_raster(path_or_dataset,
variable_name,
outfilename,
index=0,
projection=None):
"""
Exports a 2D slice from a netcdf file to a raster file.
Only GeoTiffs are supported at this time.
Parameters
----------
path_or_dataset: path to NetCDF file or open Dataset
variable_name: name of data variable to export from dataset
outfilename: output filename
index: index within 3rd dimension (in first position) or 0
projection: pyproj.Proj object. Automatically determined from file if possible
"""
if isinstance(path_or_dataset, string_types):
dataset = Dataset(path_or_dataset)
else:
dataset = path_or_dataset
projection = projection or get_crs(dataset, variable_name)
if not projection:
raise ValueError(
'Projection must be provided; '
'no projection information can be determined from file')
# TODO figure out cleaner way to get affine or coords
y_name, x_name = dataset.variables[variable_name].dimensions[:2]
coords = SpatialCoordinateVariables.from_dataset(dataset,
x_name,
y_name,
projection=projection)
affine = coords.affine
if outfilename.lower().endswith('.tif'):
format = 'GTiff'
else:
raise ValueError(
'Only GeoTiff outputs supported, filename must have .tif extension'
)
variable = dataset.variables[variable_name]
ndims = len(variable.shape)
if ndims == 2:
if index != 0:
raise ValueError('Index out of range, must be 0')
data = variable[:]
elif ndims == 3:
# Assumes that time dimension is first
if index < 0 or index >= variable.shape[0]:
raise ValueError('Index out of range, '
'must be between 0 and {0}'.variable.shape[0])
data = variable[index]
else:
raise ValueError(
'Unsupported number of dimensions {0} for variable {1}, '
'must be 2 or 3'.format(ndims, variable_name))
array_to_raster(data,
outfilename,
format=format,
projection=projection,
affine=affine)
def map_eems(
eems_file,
# output_directory,
scale,
format,
src_crs,
resampling):
"""
Render a NetCDF EEMS model to a web map.
"""
from EEMSBasePackage import EEMSCmd, EEMSProgram
model = EEMSProgram(eems_file)
# For each data producing command, store the netcdf file that contains it
file_vars = dict()
raw_variables = set()
for cmd in model.orderedCmds: # This is bottom up, may want to invert
filename = None
variable = None
if cmd.HasResultName():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetResultName()
elif cmd.IsReadCmd():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetParam('NewFieldName')
raw_variables.add(variable)
if filename and variable:
if not filename in file_vars:
file_vars[filename] = []
file_vars[filename].append(variable)
filenames =file_vars.keys()
for filename in filenames:
if not os.path.exists(filename):
raise click.ClickException('Could not find data file from EEMS model: {0}'.format(filename))
dst_crs = 'EPSG:3857'
output_directory = tempfile.mkdtemp()
click.echo('Using temp directory: {0}'.format(output_directory))
# if not os.path.exists(output_directory):
# os.makedirs(output_directory)
# Since fuzzy renderer is hardcoded, we can output it now
fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'], '1,-1')
fuzzy_renderer.get_legend(image_height=150)[0].to_image().save(os.path.join(output_directory, 'fuzzy_legend.png'))
template_filename = filenames[0]
template_var = file_vars[template_filename][0]
with Dataset(template_filename) as ds:
var_obj = ds.variables[template_var]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions != 2:
raise click.ClickException('Only 2 dimensions are allowed on data variables for now')
ds_crs = get_crs(ds, template_var)
if not ds_crs and is_geographic(ds, template_var):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS({'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None
)
#
# if mask is not None and not mask.shape == shape[-2:]:
# # Will likely break before this if collecting statistics
# raise click.BadParameter(
# 'mask variable shape does not match shape of input spatial dimensions',
# param='--mask', param_hint='--mask'
# )
#
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height,
*coords.bbox.as_list()
)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
if not (dst_crs or src_crs):
raise click.BadParameter('must provide valid src_crs to get interactive map',
param='--src-crs', param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(BBox.from_affine(dst_transform, dst_width, dst_height,
projection=Proj(dst_crs) if dst_crs else None))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing dataset {0}'.format(filename))
for variable in file_vars[filename]:
click.echo('Processing variable {0}'.format(variable))
if not variable in ds.variables:
raise click.ClickException('variable {0} was not found in file: {1}'.format(variable, filename))
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException('All datasets must have the same dimensions for {0}'.format(variable))
data = var_obj[:]
# if mask is not None:
# data = numpy.ma.masked_array(data, mask=mask)
if variable in raw_variables:
palette = DEFAULT_PALETTES['raw']
palette_stretch = '{0},{1}'.format(data.max(), data.min())
renderer = palette_to_stretched_renderer(palette, palette_stretch)
renderer.get_legend(image_height=150, max_precision=2)[0].to_image().save(os.path.join(output_directory, '{0}_legend.png'.format(variable)))
else:
renderer = fuzzy_renderer
image_filename = os.path.join(output_directory, '{0}.{1}'.format(variable, format))
data = warp_array(data, **reproject_kwargs)
render_image(renderer, data, image_filename, scale=scale, format=format)
local_filename = os.path.split(image_filename)[1]
layers[variable] = local_filename
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(loader=PackageLoader('trefoil.cli')).get_template('eems_map.html')
out.write(
template.render(
layers=json.dumps(layers),
bounds=str(leaflet_anchors),
tree=[[cmd, depth] for (cmd, depth) in model.GetCmdTree()],
raw_variables=list(raw_variables)
)
)
webbrowser.open(index_html)
def mask(input, output, variable, like, netcdf3, all_touched, invert, zip):
"""
Create a NetCDF mask from a shapefile.
Values are equivalent to a numpy mask: 0 for unmasked areas, and 1 for masked areas.
Template NetCDF dataset must have a valid projection defined or be inferred from dimensions (e.g., lat / long)
"""
with Dataset(like) as template_ds:
template_varname = data_variables(template_ds).keys()[0]
template_variable = template_ds.variables[template_varname]
template_crs = get_crs(template_ds, template_varname)
if template_crs:
template_crs = CRS.from_string(template_crs)
elif is_geographic(template_ds, template_varname):
template_crs = CRS({'init': 'EPSG:4326'})
else:
raise click.UsageError(
'template dataset must have a valid projection defined')
spatial_dimensions = template_variable.dimensions[-2:]
mask_shape = template_variable.shape[-2:]
template_y_name, template_x_name = spatial_dimensions
coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=Proj(**template_crs.to_dict()))
with fiona.open(input, 'r') as shp:
transform_required = CRS(shp.crs) != template_crs
# Project bbox for filtering
bbox = coords.bbox
if transform_required:
bbox = bbox.project(Proj(**shp.crs), edge_points=21)
geometries = []
for f in shp.filter(bbox=bbox.as_list()):
geom = f['geometry']
if transform_required:
geom = transform_geom(shp.crs, template_crs, geom)
geometries.append(geom)
click.echo('Converting {0} features to mask'.format(len(geometries)))
if invert:
fill_value = 0
default_value = 1
else:
fill_value = 1
default_value = 0
with rasterio.Env():
# Rasterize features to 0, leaving background as 1
mask = rasterize(geometries,
out_shape=mask_shape,
transform=coords.affine,
all_touched=all_touched,
fill=fill_value,
default_value=default_value,
dtype=numpy.uint8)
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
dtype = 'int8' if netcdf3 else 'uint8'
with Dataset(output, 'w', format=format) as out:
coords.add_to_dataset(out, template_x_name, template_y_name)
out_var = out.createVariable(variable,
dtype,
dimensions=spatial_dimensions,
zlib=zip,
fill_value=get_fill_value(dtype))
out_var[:] = mask
def mask(
input,
output,
variable,
like,
netcdf3,
all_touched,
invert,
zip):
"""
Create a NetCDF mask from a shapefile.
Values are equivalent to a numpy mask: 0 for unmasked areas, and 1 for masked areas.
Template NetCDF dataset must have a valid projection defined or be inferred from dimensions (e.g., lat / long)
"""
with Dataset(like) as template_ds:
template_varname = data_variables(template_ds).keys()[0]
template_variable = template_ds.variables[template_varname]
template_crs = get_crs(template_ds, template_varname)
if template_crs:
template_crs = CRS.from_string(template_crs)
elif is_geographic(template_ds, template_varname):
template_crs = CRS({'init': 'EPSG:4326'})
else:
raise click.UsageError('template dataset must have a valid projection defined')
spatial_dimensions = template_variable.dimensions[-2:]
mask_shape = template_variable.shape[-2:]
template_y_name, template_x_name = spatial_dimensions
coords = SpatialCoordinateVariables.from_dataset(
template_ds,
x_name=template_x_name,
y_name=template_y_name,
projection=Proj(**template_crs.to_dict())
)
with fiona.open(input, 'r') as shp:
transform_required = CRS(shp.crs) != template_crs
# Project bbox for filtering
bbox = coords.bbox
if transform_required:
bbox = bbox.project(Proj(**shp.crs), edge_points=21)
geometries = []
for f in shp.filter(bbox=bbox.as_list()):
geom = f['geometry']
if transform_required:
geom = transform_geom(shp.crs, template_crs, geom)
geometries.append(geom)
click.echo('Converting {0} features to mask'.format(len(geometries)))
if invert:
fill_value = 0
default_value = 1
else:
fill_value = 1
default_value = 0
with rasterio.Env():
# Rasterize features to 0, leaving background as 1
mask = rasterize(
geometries,
out_shape=mask_shape,
transform=coords.affine,
all_touched=all_touched,
fill=fill_value,
default_value=default_value,
dtype=numpy.uint8
)
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
dtype = 'int8' if netcdf3 else 'uint8'
with Dataset(output, 'w', format=format) as out:
coords.add_to_dataset(out, template_x_name, template_y_name)
out_var = out.createVariable(variable, dtype, dimensions=spatial_dimensions, zlib=zip,
fill_value=get_fill_value(dtype))
out_var[:] = mask
def main(in_pattern, out_pattern, boundary, single, varname):
"""
Clips and masks large NetCDF datasets to regional datasets based on the boundary. The in_pattern and out_pattern
arguments should be filename patterns (can include path) with the pattern: /path/to/in_netcdf_{variable}.nc.
Example usage: python cut_to_region.py NorthAmerica/NA_{variable}.nc USWest/west_{variable}.nc west.shp
"""
if single and not varname:
print('--varname is required when --single is used')
sys.exit(-1)
if single:
if not os.path.exists(in_pattern):
print('Input file {} does not exist.'.format(in_pattern))
sys.exit(-1)
input_paths = [(in_pattern, varname)]
else:
input_paths = [(in_pattern.format(variable=x), x) for x in VARIABLES]
for path, _ in input_paths:
if not os.path.exists(path):
print('Input file {} does not exist.'.format(path))
sys.exit(-1)
with fiona.open(boundary, 'r') as shp:
features = []
wgs84 = Proj('+init=EPSG:4326')
shp_projection = Proj(shp.crs)
bounds = shp.bounds
ll = transform(shp_projection, wgs84, bounds[0], bounds[1])
ur = transform(shp_projection, wgs84, bounds[2], bounds[3])
bbox = BBox([*ll, *ur], projection=wgs84)
for feature in shp.items():
geometry = transform_geom(shp.crs, {'init': 'EPSG: 4326'},
feature[1]['geometry'])
features.append(geometry)
for in_path, variable in input_paths:
if single:
out_path = out_pattern
else:
out_path = out_pattern.format(variable=variable)
if os.path.exists(out_path):
confirm = input(
"The output file '{}' already exists? Do you with to replace it? [y/n] "
.format(out_path))
if confirm.lower().strip() not in ['y', 'yes']:
print('Exiting...')
sys.exit()
with Dataset(in_path, 'r') as ds:
coords = SpatialCoordinateVariables.from_dataset(
ds, x_name='longitude', y_name='latitude')
x_start, x_stop = coords.x.indices_for_range(bbox.xmin, bbox.xmax)
y_start, y_stop = coords.y.indices_for_range(bbox.ymin, bbox.ymax)
x_slice = slice(x_start, x_stop)
y_slice = slice(y_start, y_stop)
clipped_coords = coords.slice_by_bbox(bbox)
grid = ds.variables[variable][y_slice, x_slice]
if is_masked(grid):
mask = grid.mask.astype('uint8')
else:
mask = numpy.zeros(grid.shape, dtype='uint8')
mask |= rasterize(((x, 0) for x in features),
out_shape=mask.shape,
transform=clipped_coords.affine,
fill=1,
default_value=0)
grid = numpy.ma.masked_where(mask == 1, grid.data)
print('Writing {}...'.format(out_path))
with Dataset(out_path, 'w', format='NETCDF4') as ds:
clipped_coords.add_to_dataset(ds, 'longitude', 'latitude')
data_var = ds.createVariable(variable,
grid.dtype,
dimensions=('latitude', 'longitude'),
fill_value=grid.fill_value)
if data_var.shape != grid.shape:
grid = grid[:data_var.shape[0], :data_var.shape[1]]
data_var[:] = grid
set_crs(ds, variable, Proj('+init=EPSG:4326'))
def render_netcdf(filename_pattern, variable, output_directory, renderer_file,
save_file, renderer_type, colormap, fill, colorspace,
palette, palette_stretch, scale, id_variable, lh,
legend_breaks, legend_ticks, legend_precision, format,
src_crs, dst_crs, res, resampling, anchors, interactive_map,
mask_path):
"""
Render netcdf files to images.
colormap is ignored if renderer_file is provided
--dst-crs is ignored if using --map option (always uses EPSG:3857
If no colormap or palette is provided, a default palette may be chosen based on the name of the variable.
If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It
must be in the same CRS as the input datasets, and have the same spatial
dimensions.
"""
# Parameter overrides
if interactive_map:
dst_crs = 'EPSG:3857'
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='filename_pattern',
param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
mask = get_mask(mask_path) if mask_path is not None else None
if renderer_file is not None and not save_file:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist',
param='renderer_file',
param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
if variable in renderer_dict and not 'colors' in renderer_dict:
renderer_dict = renderer_dict[variable]
renderer_type = renderer_dict['type']
if renderer_type == 'stretched':
colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
if 'min' in colors or 'max' in colors or 'mean' in colors:
statistics = collect_statistics(filenames, (variable, ),
mask=mask)[variable]
for entry in renderer_dict['colors']:
if isinstance(entry[0], basestring):
if entry[0] in ('min', 'max', 'mean'):
entry[0] = statistics[entry[0]]
elif '*' in entry[0]:
rel_value, statistic = entry[0].split('*')
entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
if palette is not None:
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
elif colormap is None and variable in DEFAULT_PALETTES:
palette, palette_stretch = DEFAULT_PALETTES[variable]
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
else:
if colormap is None:
colormap = 'min:#000000,max:#FFFFFF'
renderer = colormap_to_stretched_renderer(colormap,
colorspace,
filenames,
variable,
fill_value=fill,
mask=mask)
elif renderer_type == 'classified':
if not palette:
raise click.BadParameter(
'palette required for classified (for now)',
param='--palette',
param_hint='--palette')
renderer = palette_to_classified_renderer(
palette,
filenames,
variable,
method='equal',
fill_value=fill,
mask=mask) # TODO: other methods
if save_file:
if os.path.exists(save_file):
with open(save_file, 'r+') as output_file:
data = json.loads(output_file.read())
output_file.seek(0)
output_file.truncate()
data[variable] = renderer.serialize()
output_file.write(json.dumps(data, indent=4))
else:
with open(save_file, 'w') as output_file:
output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'stretched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(
image_height=lh,
breaks=legend_breaks,
ticks=legend_ticks,
max_precision=legend_precision)[0].to_image()
elif renderer_type == 'classified':
legend = composite_elements(renderer.get_legend())
legend.save(
os.path.join(output_directory, '{0}_legend.png'.format(variable)))
with Dataset(filenames[0]) as ds:
var_obj = ds.variables[variable]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions == 3:
if id_variable:
if shape[0] != ds.variables[id_variable][:].shape[0]:
raise click.BadParameter(
'must be same dimensionality as 3rd dimension of {0}'.
format(variable),
param='--id_variable',
param_hint='--id_variable')
else:
# Guess from the 3rd dimension
guess = dimensions[0]
if guess in ds.variables and ds.variables[guess][:].shape[
0] == shape[0]:
id_variable = guess
ds_crs = get_crs(ds, variable)
if not ds_crs and is_geographic(ds, variable):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds,
x_dim,
y_dim,
projection=Proj(src_crs.to_dict()) if src_crs else None)
if mask is not None and not mask.shape == shape[-2:]:
# Will likely break before this if collecting statistics
raise click.BadParameter(
'mask variable shape does not match shape of input spatial dimensions',
param='--mask',
param_hint='--mask')
flip_y = False
reproject_kwargs = None
if dst_crs is not None:
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs,
dst_crs,
src_width,
src_height,
*coords.bbox.as_list(),
resolution=res)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
else:
dst_transform = coords.affine
dst_height, dst_width = coords.shape
dst_crs = src_crs
if coords.y.is_ascending_order():
# Only needed if we are not already reprojecting the data, since that will flip it automatically
flip_y = True
if anchors or interactive_map:
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide at least src_crs to get Leaflet anchors or interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(
dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
if anchors:
click.echo('Anchors: {0}'.format(leaflet_anchors))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing {0}'.format(filename))
filename_root = os.path.split(filename)[1].replace('.nc', '')
if not variable in ds.variables:
raise click.BadParameter(
'variable {0} was not found in file: {1}'.format(
variable, filename),
param='variable',
param_hint='VARIABLE')
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
if num_dimensions == 2:
data = var_obj[:]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
image_filename = os.path.join(
output_directory,
'{0}_{1}.{2}'.format(filename_root, variable, format))
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
elif num_dimensions == 3:
for index in range(shape[0]):
id = ds.variables[id_variable][
index] if id_variable is not None else index
image_filename = os.path.join(
output_directory,
'{0}_{1}__{2}.{3}'.format(filename_root, variable, id,
format))
data = var_obj[index]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)
[0]] = local_filename
# TODO: not tested recently. Make sure still correct
# else:
# # Assume last 2 components of shape are lat & lon, rest are iterated over
# id_variables = None
# if id_variable is not None:
# id_variables = id_variable.split(',')
# for index, name in enumerate(id_variables):
# if name:
# assert data.shape[index] == ds.variables[name][:].shape[0]
#
# ranges = []
# for dim in data.shape[:-2]:
# ranges.append(range(0, dim))
# for combined_index in product(*ranges):
# id_parts = []
# for index, dim_index in enumerate(combined_index):
# if id_variables is not None and index < len(id_variables) and id_variables[index]:
# id = ds.variables[id_variables[index]][dim_index]
#
# if not isinstance(id, basestring):
# if isinstance(id, Iterable):
# id = '_'.join((str(i) for i in id))
# else:
# id = str(id)
#
# id_parts.append(id)
#
# else:
# id_parts.append(str(dim_index))
#
# combined_id = '_'.join(id_parts)
# image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format))
# if reproject_kwargs:
# data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this
# render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format)
#
# local_filename = os.path.split(image_filename)[1]
# layers[os.path.splitext(local_filename)[0]] = local_filename
if interactive_map:
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('trefoil.cli')).get_template('map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
variable=variable))
webbrowser.open(index_html)
def map_eems(
eems_file,
# output_directory,
scale,
format,
src_crs,
resampling):
"""
Render a NetCDF EEMS model to a web map.
"""
from EEMSBasePackage import EEMSCmd, EEMSProgram
model = EEMSProgram(eems_file)
# For each data producing command, store the netcdf file that contains it
file_vars = dict()
raw_variables = set()
for cmd in model.orderedCmds: # This is bottom up, may want to invert
filename = None
variable = None
if cmd.HasResultName():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetResultName()
elif cmd.IsReadCmd():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetParam('NewFieldName')
raw_variables.add(variable)
if filename and variable:
if not filename in file_vars:
file_vars[filename] = []
file_vars[filename].append(variable)
filenames = file_vars.keys()
for filename in filenames:
if not os.path.exists(filename):
raise click.ClickException(
'Could not find data file from EEMS model: {0}'.format(
filename))
dst_crs = 'EPSG:3857'
output_directory = tempfile.mkdtemp()
click.echo('Using temp directory: {0}'.format(output_directory))
# if not os.path.exists(output_directory):
# os.makedirs(output_directory)
# Since fuzzy renderer is hardcoded, we can output it now
fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'],
'1,-1')
fuzzy_renderer.get_legend(image_height=150)[0].to_image().save(
os.path.join(output_directory, 'fuzzy_legend.png'))
template_filename = filenames[0]
template_var = file_vars[template_filename][0]
with Dataset(template_filename) as ds:
var_obj = ds.variables[template_var]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions != 2:
raise click.ClickException(
'Only 2 dimensions are allowed on data variables for now')
ds_crs = get_crs(ds, template_var)
if not ds_crs and is_geographic(ds, template_var):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None)
#
# if mask is not None and not mask.shape == shape[-2:]:
# # Will likely break before this if collecting statistics
# raise click.BadParameter(
# 'mask variable shape does not match shape of input spatial dimensions',
# param='--mask', param_hint='--mask'
# )
#
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list())
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(Resampling, resampling),
'dst_shape': (dst_height, dst_width)
}
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide valid src_crs to get interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing dataset {0}'.format(filename))
for variable in file_vars[filename]:
click.echo('Processing variable {0}'.format(variable))
if not variable in ds.variables:
raise click.ClickException(
'variable {0} was not found in file: {1}'.format(
variable, filename))
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
data = var_obj[:]
# if mask is not None:
# data = numpy.ma.masked_array(data, mask=mask)
if variable in raw_variables:
palette = DEFAULT_PALETTES['raw']
palette_stretch = '{0},{1}'.format(data.max(), data.min())
renderer = palette_to_stretched_renderer(
palette, palette_stretch)
renderer.get_legend(
image_height=150, max_precision=2)[0].to_image().save(
os.path.join(output_directory,
'{0}_legend.png'.format(variable)))
else:
renderer = fuzzy_renderer
image_filename = os.path.join(
output_directory, '{0}.{1}'.format(variable, format))
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale=scale,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[variable] = local_filename
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('trefoil.cli')).get_template('eems_map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
tree=[[cmd, depth]
for (cmd, depth) in model.GetCmdTree()],
raw_variables=list(raw_variables)))
webbrowser.open(index_html)