def warp_like(ds, ds_projection, variables, out_ds, template_ds, template_varname, resampling=RESAMPLING.nearest): """ Warp one or more variables in a NetCDF file based on the coordinate reference system and spatial domain of a template NetCDF file. :param ds: source dataset :param ds_projection: source dataset coordiante reference system, proj4 string or EPSG:NNNN code :param variables: list of variable names in source dataset to warp :param out_ds: output dataset. Must be opened in write or append mode. :param template_ds: template dataset :param template_varname: variable name for template data variable in template dataset :param resampling: resampling method. See rasterio.warp.RESAMPLING for options """ template_variable = template_ds.variables[template_varname] template_prj = Proj(get_crs(template_ds, template_varname)) template_mask = template_variable[:].mask template_y_name, template_x_name = template_variable.dimensions[-2:] template_coords = SpatialCoordinateVariables.from_dataset( template_ds, x_name=template_x_name, y_name=template_y_name, projection=template_prj) # template_geo_bbox = template_coords.bbox.project(ds_prj, edge_points=21) # TODO: add when needing to subset ds_y_name, ds_x_name = ds.variables[variables[0]].dimensions[-2:] proj = Proj( init=ds_projection) if 'EPSG:' in ds_projection.upper() else Proj( str(ds_projection)) ds_coords = SpatialCoordinateVariables.from_dataset(ds, x_name=ds_x_name, y_name=ds_y_name, projection=proj) with rasterio.drivers(): # Copy dimensions for variable across to output for dim_name in template_variable.dimensions: if not dim_name in out_ds.dimensions: if dim_name in template_ds.variables and not dim_name in out_ds.variables: copy_variable(template_ds, out_ds, dim_name) else: copy_dimension(template_ds, out_ds, dim_name) for variable_name in variables: click.echo('Processing: {0}'.format(variable_name)) variable = ds.variables[variable_name] fill_value = getattr(variable, '_FillValue', variable[0, 0].fill_value) for dim_name in variable.dimensions[:-2]: if not dim_name in out_ds.dimensions: if dim_name in ds.variables: copy_variable(ds, out_ds, dim_name) else: copy_dimension(ds, out_ds, dim_name) out_var = out_ds.createVariable( variable_name, variable.dtype, dimensions=variable.dimensions[:-2] + template_variable.dimensions, fill_value=fill_value) reproject_kwargs = { 'src_transform': ds_coords.affine, 'src_crs': CRS.from_string(ds_projection), 'dst_transform': template_coords.affine, 'dst_crs': template_prj.srs, 'resampling': resampling, 'src_nodata': fill_value, 'dst_nodata': fill_value, 'threads': 4 } # TODO: may only need to select out what is in window if len(variable.shape) == 3: idxs = range(variable.shape[0]) with click.progressbar(idxs) as bar: for i in bar: # print('processing slice: {0}'.format(i)) data = variable[i, :] out = numpy.ma.empty(template_coords.shape, dtype=data.dtype) out.mask = template_mask out.fill(fill_value) reproject(data, out, **reproject_kwargs) out_var[i, :] = out else: data = variable[:] out = numpy.ma.empty(template_coords.shape, dtype=data.dtype) out.mask = template_mask out.fill(fill_value) reproject(data, out, **reproject_kwargs) out_var[:] = out
def map_eems( eems_file, # output_directory, scale, format, src_crs, resampling): """ Render a NetCDF EEMS model to a web map. """ from EEMSBasePackage import EEMSCmd, EEMSProgram model = EEMSProgram(eems_file) # For each data producing command, store the netcdf file that contains it file_vars = dict() raw_variables = set() for cmd in model.orderedCmds: # This is bottom up, may want to invert filename = None variable = None if cmd.HasResultName(): filename = cmd.GetParam('OutFileName') variable = cmd.GetResultName() elif cmd.IsReadCmd(): filename = cmd.GetParam('OutFileName') variable = cmd.GetParam('NewFieldName') raw_variables.add(variable) if filename and variable: if not filename in file_vars: file_vars[filename] = [] file_vars[filename].append(variable) filenames = file_vars.keys() for filename in filenames: if not os.path.exists(filename): raise click.ClickException( 'Could not find data file from EEMS model: {0}'.format( filename)) dst_crs = 'EPSG:3857' output_directory = tempfile.mkdtemp() click.echo('Using temp directory: {0}'.format(output_directory)) # if not os.path.exists(output_directory): # os.makedirs(output_directory) # Since fuzzy renderer is hardcoded, we can output it now fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'], '1,-1') fuzzy_renderer.get_legend(image_height=150)[0].to_image().save( os.path.join(output_directory, 'fuzzy_legend.png')) template_filename = filenames[0] template_var = file_vars[template_filename][0] with Dataset(template_filename) as ds: var_obj = ds.variables[template_var] dimensions = var_obj.dimensions shape = var_obj.shape num_dimensions = len(shape) if num_dimensions != 2: raise click.ClickException( 'Only 2 dimensions are allowed on data variables for now') ds_crs = get_crs(ds, template_var) if not ds_crs and is_geographic(ds, template_var): ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84 src_crs = CRS.from_string(ds_crs) if ds_crs else CRS( {'init': src_crs}) if src_crs else None # get transforms, assume last 2 dimensions on variable are spatial in row, col order y_dim, x_dim = dimensions[-2:] coords = SpatialCoordinateVariables.from_dataset( ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None) # # if mask is not None and not mask.shape == shape[-2:]: # # Will likely break before this if collecting statistics # raise click.BadParameter( # 'mask variable shape does not match shape of input spatial dimensions', # param='--mask', param_hint='--mask' # ) # if not src_crs: raise click.BadParameter('must provide src_crs to reproject', param='--src-crs', param_hint='--src-crs') dst_crs = CRS.from_string(dst_crs) src_height, src_width = coords.shape dst_transform, dst_width, dst_height = calculate_default_transform( src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list()) reproject_kwargs = { 'src_crs': src_crs, 'src_transform': coords.affine, 'dst_crs': dst_crs, 'dst_transform': dst_transform, 'resampling': getattr(RESAMPLING, resampling), 'dst_shape': (dst_height, dst_width) } if not (dst_crs or src_crs): raise click.BadParameter( 'must provide valid src_crs to get interactive map', param='--src-crs', param_hint='--src-crs') leaflet_anchors = get_leaflet_anchors( BBox.from_affine(dst_transform, dst_width, dst_height, projection=Proj(dst_crs) if dst_crs else None)) layers = {} for filename in filenames: with Dataset(filename) as ds: click.echo('Processing dataset {0}'.format(filename)) for variable in file_vars[filename]: click.echo('Processing variable {0}'.format(variable)) if not variable in ds.variables: raise click.ClickException( 'variable {0} was not found in file: {1}'.format( variable, filename)) var_obj = ds.variables[variable] if not var_obj.dimensions == dimensions: raise click.ClickException( 'All datasets must have the same dimensions for {0}'. format(variable)) data = var_obj[:] # if mask is not None: # data = numpy.ma.masked_array(data, mask=mask) if variable in raw_variables: palette = DEFAULT_PALETTES['raw'] palette_stretch = '{0},{1}'.format(data.max(), data.min()) renderer = palette_to_stretched_renderer( palette, palette_stretch) renderer.get_legend( image_height=150, max_precision=2)[0].to_image().save( os.path.join(output_directory, '{0}_legend.png'.format(variable))) else: renderer = fuzzy_renderer image_filename = os.path.join( output_directory, '{0}.{1}'.format(variable, format)) data = warp_array(data, **reproject_kwargs) render_image(renderer, data, image_filename, scale=scale, format=format) local_filename = os.path.split(image_filename)[1] layers[variable] = local_filename index_html = os.path.join(output_directory, 'index.html') with open(index_html, 'w') as out: template = Environment( loader=PackageLoader('clover.cli')).get_template('eems_map.html') out.write( template.render(layers=json.dumps(layers), bounds=str(leaflet_anchors), tree=[[cmd, depth] for (cmd, depth) in model.GetCmdTree()], raw_variables=list(raw_variables))) webbrowser.open(index_html)
def render_netcdf(filename_pattern, variable, output_directory, renderer_file, save_file, renderer_type, colormap, fill, colorspace, palette, palette_stretch, scale, id_variable, lh, legend_breaks, legend_ticks, legend_precision, format, src_crs, dst_crs, res, resampling, anchors, interactive_map, mask_path): """ Render netcdf files to images. colormap is ignored if renderer_file is provided --dst-crs is ignored if using --map option (always uses EPSG:3857 If no colormap or palette is provided, a default palette may be chosen based on the name of the variable. If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It must be in the same CRS as the input datasets, and have the same spatial dimensions. """ # Parameter overrides if interactive_map: dst_crs = 'EPSG:3857' filenames = glob.glob(filename_pattern) if not filenames: raise click.BadParameter('No files found matching that pattern', param='filename_pattern', param_hint='FILENAME_PATTERN') if not os.path.exists(output_directory): os.makedirs(output_directory) mask = get_mask(mask_path) if mask_path is not None else None if renderer_file is not None and not save_file: if not os.path.exists(renderer_file): raise click.BadParameter('does not exist', param='renderer_file', param_hint='renderer_file') # see https://bitbucket.org/databasin/ncdjango/wiki/Home for format renderer_dict = json.loads(open(renderer_file).read()) if variable in renderer_dict and not 'colors' in renderer_dict: renderer_dict = renderer_dict[variable] renderer_type = renderer_dict['type'] if renderer_type == 'stretched': colors = ','.join([str(c[0]) for c in renderer_dict['colors']]) if 'min' in colors or 'max' in colors or 'mean' in colors: statistics = collect_statistics(filenames, (variable, ), mask=mask)[variable] for entry in renderer_dict['colors']: if isinstance(entry[0], basestring): if entry[0] in ('min', 'max', 'mean'): entry[0] = statistics[entry[0]] elif '*' in entry[0]: rel_value, statistic = entry[0].split('*') entry[0] = float(rel_value) * statistics[statistic] renderer = renderer_from_dict(renderer_dict) else: if renderer_type == 'stretched': if palette is not None: renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask) elif colormap is None and variable in DEFAULT_PALETTES: palette, palette_stretch = DEFAULT_PALETTES[variable] renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask) else: if colormap is None: colormap = 'min:#000000,max:#FFFFFF' renderer = colormap_to_stretched_renderer(colormap, colorspace, filenames, variable, fill_value=fill, mask=mask) elif renderer_type == 'classified': if not palette: raise click.BadParameter( 'palette required for classified (for now)', param='--palette', param_hint='--palette') renderer = palette_to_classified_renderer( palette, filenames, variable, method='equal', fill_value=fill, mask=mask) # TODO: other methods if save_file: if os.path.exists(save_file): with open(save_file, 'r+') as output_file: data = json.loads(output_file.read()) output_file.seek(0) output_file.truncate() data[variable] = renderer.serialize() output_file.write(json.dumps(data, indent=4)) else: with open(save_file, 'w') as output_file: output_file.write(json.dumps({variable: renderer.serialize()})) if renderer_type == 'stretched': if legend_ticks is not None and not legend_breaks: legend_ticks = [float(v) for v in legend_ticks.split(',')] legend = renderer.get_legend( image_height=lh, breaks=legend_breaks, ticks=legend_ticks, max_precision=legend_precision)[0].to_image() elif renderer_type == 'classified': legend = composite_elements(renderer.get_legend()) legend.save( os.path.join(output_directory, '{0}_legend.png'.format(variable))) with Dataset(filenames[0]) as ds: var_obj = ds.variables[variable] dimensions = var_obj.dimensions shape = var_obj.shape num_dimensions = len(shape) if num_dimensions == 3: if id_variable: if shape[0] != ds.variables[id_variable][:].shape[0]: raise click.BadParameter( 'must be same dimensionality as 3rd dimension of {0}'. format(variable), param='--id_variable', param_hint='--id_variable') else: # Guess from the 3rd dimension guess = dimensions[0] if guess in ds.variables and ds.variables[guess][:].shape[ 0] == shape[0]: id_variable = guess ds_crs = get_crs(ds, variable) if not ds_crs and is_geographic(ds, variable): ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84 src_crs = CRS.from_string(ds_crs) if ds_crs else CRS( {'init': src_crs}) if src_crs else None # get transforms, assume last 2 dimensions on variable are spatial in row, col order y_dim, x_dim = dimensions[-2:] coords = SpatialCoordinateVariables.from_dataset( ds, x_dim, y_dim, projection=Proj(src_crs.to_dict()) if src_crs else None) if mask is not None and not mask.shape == shape[-2:]: # Will likely break before this if collecting statistics raise click.BadParameter( 'mask variable shape does not match shape of input spatial dimensions', param='--mask', param_hint='--mask') flip_y = False reproject_kwargs = None if dst_crs is not None: if not src_crs: raise click.BadParameter('must provide src_crs to reproject', param='--src-crs', param_hint='--src-crs') dst_crs = CRS.from_string(dst_crs) src_height, src_width = coords.shape dst_transform, dst_width, dst_height = calculate_default_transform( src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list(), resolution=res) reproject_kwargs = { 'src_crs': src_crs, 'src_transform': coords.affine, 'dst_crs': dst_crs, 'dst_transform': dst_transform, 'resampling': getattr(RESAMPLING, resampling), 'dst_shape': (dst_height, dst_width) } else: dst_transform = coords.affine dst_height, dst_width = coords.shape dst_crs = src_crs if coords.y.is_ascending_order(): # Only needed if we are not already reprojecting the data, since that will flip it automatically flip_y = True if anchors or interactive_map: if not (dst_crs or src_crs): raise click.BadParameter( 'must provide at least src_crs to get Leaflet anchors or interactive map', param='--src-crs', param_hint='--src-crs') leaflet_anchors = get_leaflet_anchors( BBox.from_affine( dst_transform, dst_width, dst_height, projection=Proj(dst_crs) if dst_crs else None)) if anchors: click.echo('Anchors: {0}'.format(leaflet_anchors)) layers = {} for filename in filenames: with Dataset(filename) as ds: click.echo('Processing {0}'.format(filename)) filename_root = os.path.split(filename)[1].replace('.nc', '') if not variable in ds.variables: raise click.BadParameter( 'variable {0} was not found in file: {1}'.format( variable, filename), param='variable', param_hint='VARIABLE') var_obj = ds.variables[variable] if not var_obj.dimensions == dimensions: raise click.ClickException( 'All datasets must have the same dimensions for {0}'. format(variable)) if num_dimensions == 2: data = var_obj[:] if mask is not None: data = numpy.ma.masked_array(data, mask=mask) image_filename = os.path.join( output_directory, '{0}_{1}.{2}'.format(filename_root, variable, format)) if reproject_kwargs: data = warp_array(data, **reproject_kwargs) render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format) local_filename = os.path.split(image_filename)[1] layers[os.path.splitext(local_filename)[0]] = local_filename elif num_dimensions == 3: for index in range(shape[0]): id = ds.variables[id_variable][ index] if id_variable is not None else index image_filename = os.path.join( output_directory, '{0}_{1}__{2}.{3}'.format(filename_root, variable, id, format)) data = var_obj[index] if mask is not None: data = numpy.ma.masked_array(data, mask=mask) if reproject_kwargs: data = warp_array(data, **reproject_kwargs) render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format) local_filename = os.path.split(image_filename)[1] layers[os.path.splitext(local_filename) [0]] = local_filename # TODO: not tested recently. Make sure still correct # else: # # Assume last 2 components of shape are lat & lon, rest are iterated over # id_variables = None # if id_variable is not None: # id_variables = id_variable.split(',') # for index, name in enumerate(id_variables): # if name: # assert data.shape[index] == ds.variables[name][:].shape[0] # # ranges = [] # for dim in data.shape[:-2]: # ranges.append(range(0, dim)) # for combined_index in product(*ranges): # id_parts = [] # for index, dim_index in enumerate(combined_index): # if id_variables is not None and index < len(id_variables) and id_variables[index]: # id = ds.variables[id_variables[index]][dim_index] # # if not isinstance(id, basestring): # if isinstance(id, Iterable): # id = '_'.join((str(i) for i in id)) # else: # id = str(id) # # id_parts.append(id) # # else: # id_parts.append(str(dim_index)) # # combined_id = '_'.join(id_parts) # image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format)) # if reproject_kwargs: # data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this # render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format) # # local_filename = os.path.split(image_filename)[1] # layers[os.path.splitext(local_filename)[0]] = local_filename if interactive_map: index_html = os.path.join(output_directory, 'index.html') with open(index_html, 'w') as out: template = Environment( loader=PackageLoader('clover.cli')).get_template('map.html') out.write( template.render(layers=json.dumps(layers), bounds=str(leaflet_anchors), variable=variable)) webbrowser.open(index_html)
def netcdf_to_raster(path_or_dataset, variable_name, outfilename, index=0, projection=None): """ Exports a 2D slice from a netcdf file to a raster file. Only GeoTiffs are supported at this time. Parameters ---------- path_or_dataset: path to NetCDF file or open Dataset variable_name: name of data variable to export from dataset outfilename: output filename index: index within 3rd dimension (in first position) or 0 projection: pyproj.Proj object. Automatically determined from file if possible """ if isinstance(path_or_dataset, string_types): dataset = Dataset(path_or_dataset) else: dataset = path_or_dataset projection = projection or get_crs(dataset, variable_name) if not projection: raise ValueError( 'Projection must be provided; ' 'no projection information can be determined from file') # TODO figure out cleaner way to get affine or coords y_name, x_name = dataset.variables[variable_name].dimensions[:2] coords = SpatialCoordinateVariables.from_dataset(dataset, x_name, y_name, projection=projection) affine = coords.affine if outfilename.lower().endswith('.tif'): format = 'GTiff' else: raise ValueError( 'Only GeoTiff outputs supported, filename must have .tif extension' ) variable = dataset.variables[variable_name] ndims = len(variable.shape) if ndims == 2: if index != 0: raise ValueError('Index out of range, must be 0') data = variable[:] elif ndims == 3: # Assumes that time dimension is first if index < 0 or index >= variable.shape[0]: raise ValueError('Index out of range, ' 'must be between 0 and {0}'.variable.shape[0]) data = variable[index] else: raise ValueError( 'Unsupported number of dimensions {0} for variable {1}, ' 'must be 2 or 3'.format(ndims, variable_name)) array_to_raster(data, outfilename, format=format, projection=projection, affine=affine)
def handle(self, *args, **options): elevation_service = Service.objects.get(name='west2_dem') with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, elevation_service.data_path)) as ds: coords = SpatialCoordinateVariables.from_dataset( ds, x_name='lon', y_name='lat', projection=Proj(elevation_service.projection) ) elevation = ds.variables['elevation'][:] message = 'WARNING: This will replace all your transfer limits. Do you want to continue? [y/n]' if input(message).lower() not in {'y', 'yes'}: return self.transfers_by_source = {} with transaction.atomic(): TransferLimit.objects.all().delete() for time_period in ('1961_1990', '1981_2010'): for variable in VARIABLES: print('Processing {} for {}...'.format(variable, time_period)) variable_service = Service.objects.get(name='west2_{}Y_{}'.format(time_period, variable)) with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, variable_service.data_path)) as ds: data = ds.variables[variable][:] for zone in SeedZone.objects.all(): clipped_elevation, clipped_data, clipped_coords = self._get_subsets( elevation, data, coords, BBox(zone.polygon.extent) ) zone_mask = rasterize( ((json.loads(zone.polygon.geojson), 1),), out_shape=clipped_elevation.shape, transform=clipped_coords.affine, fill=0, dtype=numpy.dtype('uint8') ) masked_dem = numpy.ma.masked_where(zone_mask == 0, clipped_elevation) min_elevation = max(math.floor(numpy.nanmin(masked_dem) / 0.3048), 0) max_elevation = math.ceil(numpy.nanmax(masked_dem) / 0.3048) bands = list(self._get_bands_fn(zone.source)(zone.zone_id, min_elevation, max_elevation)) if not bands: print('WARNING: No elevation bands found for {}, zone {}'.format( zone.source, zone.zone_id )) continue for band in bands: low, high = band # Bands are exclusive of the low number, so the first band is a special case, since we # want to include 0. So we work around it by making the actual low -1 if low == 0: low = -1 # Elevation bands are represented in feet masked_data = numpy.ma.masked_where( (zone_mask == 0) | (clipped_elevation <= low * 0.3048) | (clipped_elevation > high * 0.3048), clipped_data ) self._write_limit(variable, time_period, zone, masked_data, low, high) for source, transfers_by_variable in self.transfers_by_source.items(): for variable, transfers in transfers_by_variable.items(): TransferLimit.objects.filter( variable=variable, zone__source=source ).update( avg_transfer=mean(transfers) )
def mask( input, output, variable, like, netcdf3, all_touched, invert, zip): """ Create a NetCDF mask from a shapefile. Values are equivalent to a numpy mask: 0 for unmasked areas, and 1 for masked areas. Template NetCDF dataset must have a valid projection defined or be inferred from dimensions (e.g., lat / long) """ with Dataset(like) as template_ds: template_varname = data_variables(template_ds).keys()[0] template_variable = template_ds.variables[template_varname] template_crs = get_crs(template_ds, template_varname) if template_crs: template_crs = CRS.from_string(template_crs) elif is_geographic(template_ds, template_varname): template_crs = CRS({'init': 'EPSG:4326'}) else: raise click.UsageError('template dataset must have a valid projection defined') spatial_dimensions = template_variable.dimensions[-2:] mask_shape = template_variable.shape[-2:] template_y_name, template_x_name = spatial_dimensions coords = SpatialCoordinateVariables.from_dataset( template_ds, x_name=template_x_name, y_name=template_y_name, projection=Proj(**template_crs.to_dict()) ) with fiona.open(input, 'r') as shp: transform_required = CRS(shp.crs) != template_crs # Project bbox for filtering bbox = coords.bbox if transform_required: bbox = bbox.project(Proj(**shp.crs), edge_points=21) geometries = [] for f in shp.filter(bbox=bbox.as_list()): geom = f['geometry'] if transform_required: geom = transform_geom(shp.crs, template_crs, geom) geometries.append(geom) click.echo('Converting {0} features to mask'.format(len(geometries))) if invert: fill_value = 0 default_value = 1 else: fill_value = 1 default_value = 0 with rasterio.drivers(): # Rasterize features to 0, leaving background as 1 mask = rasterize( geometries, out_shape=mask_shape, transform=coords.affine, all_touched=all_touched, fill=fill_value, default_value=default_value, dtype=numpy.uint8 ) format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4' dtype = 'int8' if netcdf3 else 'uint8' with Dataset(output, 'w', format=format) as out: coords.add_to_dataset(out, template_x_name, template_y_name) out_var = out.createVariable(variable, dtype, dimensions=spatial_dimensions, zlib=zip, fill_value=get_fill_value(dtype)) out_var[:] = mask