def test_SpatialCoordinateVariables_add_to_dataset(): lat = SpatialCoordinateVariable(numpy.arange(19, -1, -1)) lon = SpatialCoordinateVariable(numpy.arange(10)) coords = SpatialCoordinateVariables(lon, lat, Proj(init='EPSG:4326')) lat_varname = 'lat' lon_varname = 'lon' outfilename = 'test.nc' try: with Dataset(outfilename, 'w') as target_ds: coords.add_to_dataset(target_ds, lon_varname, lat_varname) assert lat_varname in target_ds.dimensions assert lat_varname in target_ds.variables assert len(target_ds.dimensions[lat_varname]) == lat.values.size assert numpy.array_equal(lat.values, target_ds.variables[lat_varname][:]) assert lon_varname in target_ds.dimensions assert lon_varname in target_ds.variables assert len(target_ds.dimensions[lon_varname]) == lon.values.size assert numpy.array_equal(lon.values, target_ds.variables[lon_varname][:]) finally: if os.path.exists(outfilename): os.remove(outfilename)
def test_SpatialCoordinateVariables_slice_by_bbox(): lat = SpatialCoordinateVariable(numpy.arange(19, -1, -1)) lon = SpatialCoordinateVariable(numpy.arange(10)) proj = Proj(init='EPSG:4326') coords = SpatialCoordinateVariables(lon, lat, proj) subset = coords.slice_by_bbox(BBox((1.75, 3.7, 6.2, 16.7), proj)) assert numpy.array_equal(subset.x.values, numpy.arange(2, 6)) assert subset.x.values[0] == 2 assert subset.x.values[-1] == 5 assert subset.y.values[0] == 16 assert subset.y.values[-1] == 4
def test_window_for_bbox(): coords = SpatialCoordinateVariables.from_bbox( BBox([-124, 82, -122, 90], Proj(init='epsg:4326')), 20, 20) window = coords.get_window_for_bbox(BBox([-123.9, 82.4, -122.1, 89.6])) assert window.x_slice == slice(1, 19) assert window.y_slice == slice(1, 19)
def _get_subsets(self, elevation, data, coords: SpatialCoordinateVariables, bbox): """ Returns subsets of elevation, data, and coords, clipped to the given bounds """ x_slice = slice(*coords.x.indices_for_range(bbox.xmin, bbox.xmax)) y_slice = slice(*coords.y.indices_for_range(bbox.ymin, bbox.ymax)) return elevation[y_slice, x_slice], data[y_slice, x_slice], coords.slice_by_bbox(bbox)
def load_data(self): self.data_name = self.path.split(os.sep)[-1].split('.')[0] with Dataset(self.path, 'r') as ds: # This should be: self.variables = clover.netcdf.utilities.data_variables(ds) # but unfortunately clover's data_variables() method is broken under python3 # currently this may cause problems with grid_mapping or *_bnds variables dimensions = list(ds.dimensions.keys()) self.variables = [var for var in ds.variables if var not in dimensions] recognized_dims = {'x': 'y', 'lon': 'lat', 'longitude': 'latitude'} for x, y in recognized_dims.items(): if x in dimensions: self.x_dim, self.y_dim = x, y break if self.x_dim is None: raise KeyError("NetCDF file doesn't have recognizable dimension names (x, lat, latitude, etc.)") # we need a time_info attribute even if there's no time info self.time_info = TemporalInfo() recognized_time_dims = ['time', 'year', 'month', 'date'] for timedim in recognized_time_dims: if timedim in dimensions: self.t_dim = timedim if timedim in ds.variables: timevar = DateVariable(ds.variables[timedim]) self.time_info.timestamps = timevar.datetimes.tolist() # we don't want timezones, but sometimes clover adds them if self.time_info.timestamps[0].tzinfo is not None: self.time_info.timestamps = [d.replace(tzinfo=None) for d in self.time_info.timestamps] else: # no time variable wx.MessageDialog(App.get().app_controller.main_window, caption='Missing Time Data', message=('{} has a time dimension "{}" but no corresponding coordinate ' + 'variable. Using only final timestep.').format(self.data_name, timedim), style=wx.OK).ShowModal() break # Determine spatial extent; if multiple vars, they must all have the same extent self.x = SpatialCoordinateVariable(ds.variables[self.x_dim]) self.y = SpatialCoordinateVariable(ds.variables[self.y_dim]) self.y_increasing = self.y.values[0] < self.y.values[-1] grid = clover.netcdf.describe.describe(ds)['variables'][self.variables[0]]['spatial_grid'] ext = grid['extent'] self.extent = Extent(*[ext[d] for d in ['xmin', 'ymin', 'xmax', 'ymax']]) self._resolution = grid['x_resolution'] self.affine = SpatialCoordinateVariables(self.x, self.y, None).affine self.var_shape = ds.variables[self.variables[0]].shape # if a var has a temporal dimension but no temporal data, we treat it as non-temporal: if not self.time_info.is_temporal and len(self.var_shape) == 3: self.var_shape = self.var_shape[1:]
def describe(path_or_dataset): if isinstance(path_or_dataset, string_types): dataset = Dataset(path_or_dataset) else: dataset = path_or_dataset description = { 'dimensions': {}, 'variables': {}, 'attributes': get_ncattrs(dataset) } for dimension_name in dataset.dimensions: dimension = dataset.dimensions[dimension_name] description['dimensions'][dimension_name] = {'length': len(dimension)} for variable_name in dataset.variables: variable = dataset.variables[variable_name] if not variable.dimensions: # Do not collect info about dimensionless variables (e.g., CRS variable) continue dtype = str(variable.dtype) if "'" in dtype: dtype = dtype.split("'")[1] attributes = get_ncattrs(variable) variable_info = { 'attributes': attributes, 'dimensions': variable.dimensions, 'data_type': dtype, 'name': attributes.get('long_name') or attributes.get('standard_name') or variable_name } if dtype not in ('str', ): if len(variable.shape) > 2: # Avoid loading the entire array into memory by iterating along the first index (usually time) variable_info.update({ 'min': min(variable[i, :].min().item() for i in range(variable.shape[0])), 'max': max(variable[i, :].max().item() for i in range(variable.shape[0])) }) else: data = variable[:] variable_info.update({ 'min': data.min().item(), 'max': data.max().item() }) if variable_name in dataset.dimensions and dtype not in ('str', ): dimension_variable = dataset.variables[variable_name] if len(dimension_variable.dimensions ) == 1: # range dimensions don't make sense for interval interval = get_interval(dimension_variable) if interval: variable_info['interval'] = interval else: # Data variable proj4 = get_crs(dataset, variable_name) #extent if len(variable.dimensions) >= 2: x_variable_name = None y_variable_name = None time_variable_name = None for dimension_name in (x for x in variable.dimensions if x in dataset.variables): attributes = get_ncattrs(dataset.variables[dimension_name]) standard_name = attributes.get('standard_name', None) if standard_name in X_DIMENSION_STANDARD_NAMES or dimension_name in X_DIMENSION_COMMON_NAMES: x_variable_name = dimension_name elif standard_name in Y_DIMENSION_STANDARD_NAMES or dimension_name in Y_DIMENSION_COMMON_NAMES: y_variable_name = dimension_name elif standard_name in TIME_DIMENSION_STANDARD_NAMES or dimension_name in TIME_DIMENSION_COMMON_NAMES: if len(dataset.dimensions[dimension_name]) > 1: time_variable_name = dimension_name if x_variable_name and y_variable_name: if proj4 is None and is_geographic(dataset, variable_name): # Assume WGS84 proj4 = PROJ4_GEOGRAPHIC coordinates = SpatialCoordinateVariables( SpatialCoordinateVariable( dataset.variables[x_variable_name]), SpatialCoordinateVariable( dataset.variables[y_variable_name]), Proj(str(proj4)) if proj4 else None) variable_info['spatial_grid'] = { 'extent': coordinates.bbox.as_dict(), 'x_dimension': x_variable_name, 'x_resolution': coordinates.x.pixel_size, 'y_dimension': y_variable_name, 'y_resolution': coordinates.y.pixel_size } if time_variable_name: time_variable = dataset.variables[time_variable_name] time_info = { 'dimension': time_variable_name, } try: date_variable = DateVariable(time_variable) values = date_variable.datetimes time_info['extent'] = [ values.min().isoformat(), values.max().isoformat() ] time_info['interval_unit'] = date_variable.unit interval = get_interval(time_variable) if interval is not None: time_info['interval'] = interval except ValueError: pass variable_info['time'] = time_info if proj4: variable_info['proj4'] = proj4 description['variables'][variable_name] = variable_info return description
def warp_like(ds, ds_projection, variables, out_ds, template_ds, template_varname, resampling=RESAMPLING.nearest): """ Warp one or more variables in a NetCDF file based on the coordinate reference system and spatial domain of a template NetCDF file. :param ds: source dataset :param ds_projection: source dataset coordiante reference system, proj4 string or EPSG:NNNN code :param variables: list of variable names in source dataset to warp :param out_ds: output dataset. Must be opened in write or append mode. :param template_ds: template dataset :param template_varname: variable name for template data variable in template dataset :param resampling: resampling method. See rasterio.warp.RESAMPLING for options """ template_variable = template_ds.variables[template_varname] template_prj = Proj(get_crs(template_ds, template_varname)) template_mask = template_variable[:].mask template_y_name, template_x_name = template_variable.dimensions[-2:] template_coords = SpatialCoordinateVariables.from_dataset( template_ds, x_name=template_x_name, y_name=template_y_name, projection=template_prj) # template_geo_bbox = template_coords.bbox.project(ds_prj, edge_points=21) # TODO: add when needing to subset ds_y_name, ds_x_name = ds.variables[variables[0]].dimensions[-2:] proj = Proj( init=ds_projection) if 'EPSG:' in ds_projection.upper() else Proj( str(ds_projection)) ds_coords = SpatialCoordinateVariables.from_dataset(ds, x_name=ds_x_name, y_name=ds_y_name, projection=proj) with rasterio.drivers(): # Copy dimensions for variable across to output for dim_name in template_variable.dimensions: if not dim_name in out_ds.dimensions: if dim_name in template_ds.variables and not dim_name in out_ds.variables: copy_variable(template_ds, out_ds, dim_name) else: copy_dimension(template_ds, out_ds, dim_name) for variable_name in variables: click.echo('Processing: {0}'.format(variable_name)) variable = ds.variables[variable_name] fill_value = getattr(variable, '_FillValue', variable[0, 0].fill_value) for dim_name in variable.dimensions[:-2]: if not dim_name in out_ds.dimensions: if dim_name in ds.variables: copy_variable(ds, out_ds, dim_name) else: copy_dimension(ds, out_ds, dim_name) out_var = out_ds.createVariable( variable_name, variable.dtype, dimensions=variable.dimensions[:-2] + template_variable.dimensions, fill_value=fill_value) reproject_kwargs = { 'src_transform': ds_coords.affine, 'src_crs': CRS.from_string(ds_projection), 'dst_transform': template_coords.affine, 'dst_crs': template_prj.srs, 'resampling': resampling, 'src_nodata': fill_value, 'dst_nodata': fill_value, 'threads': 4 } # TODO: may only need to select out what is in window if len(variable.shape) == 3: idxs = range(variable.shape[0]) with click.progressbar(idxs) as bar: for i in bar: # print('processing slice: {0}'.format(i)) data = variable[i, :] out = numpy.ma.empty(template_coords.shape, dtype=data.dtype) out.mask = template_mask out.fill(fill_value) reproject(data, out, **reproject_kwargs) out_var[i, :] = out else: data = variable[:] out = numpy.ma.empty(template_coords.shape, dtype=data.dtype) out.mask = template_mask out.fill(fill_value) reproject(data, out, **reproject_kwargs) out_var[:] = out
def map_eems( eems_file, # output_directory, scale, format, src_crs, resampling): """ Render a NetCDF EEMS model to a web map. """ from EEMSBasePackage import EEMSCmd, EEMSProgram model = EEMSProgram(eems_file) # For each data producing command, store the netcdf file that contains it file_vars = dict() raw_variables = set() for cmd in model.orderedCmds: # This is bottom up, may want to invert filename = None variable = None if cmd.HasResultName(): filename = cmd.GetParam('OutFileName') variable = cmd.GetResultName() elif cmd.IsReadCmd(): filename = cmd.GetParam('OutFileName') variable = cmd.GetParam('NewFieldName') raw_variables.add(variable) if filename and variable: if not filename in file_vars: file_vars[filename] = [] file_vars[filename].append(variable) filenames = file_vars.keys() for filename in filenames: if not os.path.exists(filename): raise click.ClickException( 'Could not find data file from EEMS model: {0}'.format( filename)) dst_crs = 'EPSG:3857' output_directory = tempfile.mkdtemp() click.echo('Using temp directory: {0}'.format(output_directory)) # if not os.path.exists(output_directory): # os.makedirs(output_directory) # Since fuzzy renderer is hardcoded, we can output it now fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'], '1,-1') fuzzy_renderer.get_legend(image_height=150)[0].to_image().save( os.path.join(output_directory, 'fuzzy_legend.png')) template_filename = filenames[0] template_var = file_vars[template_filename][0] with Dataset(template_filename) as ds: var_obj = ds.variables[template_var] dimensions = var_obj.dimensions shape = var_obj.shape num_dimensions = len(shape) if num_dimensions != 2: raise click.ClickException( 'Only 2 dimensions are allowed on data variables for now') ds_crs = get_crs(ds, template_var) if not ds_crs and is_geographic(ds, template_var): ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84 src_crs = CRS.from_string(ds_crs) if ds_crs else CRS( {'init': src_crs}) if src_crs else None # get transforms, assume last 2 dimensions on variable are spatial in row, col order y_dim, x_dim = dimensions[-2:] coords = SpatialCoordinateVariables.from_dataset( ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None) # # if mask is not None and not mask.shape == shape[-2:]: # # Will likely break before this if collecting statistics # raise click.BadParameter( # 'mask variable shape does not match shape of input spatial dimensions', # param='--mask', param_hint='--mask' # ) # if not src_crs: raise click.BadParameter('must provide src_crs to reproject', param='--src-crs', param_hint='--src-crs') dst_crs = CRS.from_string(dst_crs) src_height, src_width = coords.shape dst_transform, dst_width, dst_height = calculate_default_transform( src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list()) reproject_kwargs = { 'src_crs': src_crs, 'src_transform': coords.affine, 'dst_crs': dst_crs, 'dst_transform': dst_transform, 'resampling': getattr(RESAMPLING, resampling), 'dst_shape': (dst_height, dst_width) } if not (dst_crs or src_crs): raise click.BadParameter( 'must provide valid src_crs to get interactive map', param='--src-crs', param_hint='--src-crs') leaflet_anchors = get_leaflet_anchors( BBox.from_affine(dst_transform, dst_width, dst_height, projection=Proj(dst_crs) if dst_crs else None)) layers = {} for filename in filenames: with Dataset(filename) as ds: click.echo('Processing dataset {0}'.format(filename)) for variable in file_vars[filename]: click.echo('Processing variable {0}'.format(variable)) if not variable in ds.variables: raise click.ClickException( 'variable {0} was not found in file: {1}'.format( variable, filename)) var_obj = ds.variables[variable] if not var_obj.dimensions == dimensions: raise click.ClickException( 'All datasets must have the same dimensions for {0}'. format(variable)) data = var_obj[:] # if mask is not None: # data = numpy.ma.masked_array(data, mask=mask) if variable in raw_variables: palette = DEFAULT_PALETTES['raw'] palette_stretch = '{0},{1}'.format(data.max(), data.min()) renderer = palette_to_stretched_renderer( palette, palette_stretch) renderer.get_legend( image_height=150, max_precision=2)[0].to_image().save( os.path.join(output_directory, '{0}_legend.png'.format(variable))) else: renderer = fuzzy_renderer image_filename = os.path.join( output_directory, '{0}.{1}'.format(variable, format)) data = warp_array(data, **reproject_kwargs) render_image(renderer, data, image_filename, scale=scale, format=format) local_filename = os.path.split(image_filename)[1] layers[variable] = local_filename index_html = os.path.join(output_directory, 'index.html') with open(index_html, 'w') as out: template = Environment( loader=PackageLoader('clover.cli')).get_template('eems_map.html') out.write( template.render(layers=json.dumps(layers), bounds=str(leaflet_anchors), tree=[[cmd, depth] for (cmd, depth) in model.GetCmdTree()], raw_variables=list(raw_variables))) webbrowser.open(index_html)
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None): outputs = results.format_args() for k, v in six.iteritems(outputs): if is_raster(v) and publish_raster_results: service_name = '{0}/{1}'.format(job.uuid, k) rel_path = '{}.nc'.format(service_name) abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path) os.makedirs(os.path.dirname(abs_path)) with Dataset(abs_path, 'w', format='NETCDF4') as ds: if v.extent.projection.is_latlong(): x_var = 'longitude' y_var = 'latitude' else: x_var = 'x' y_var = 'y' coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape)) coord_vars.add_to_dataset(ds, x_var, y_var) fill_value = v.fill_value if is_masked(v) else None data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value) data_var[:] = v set_crs(ds, 'data', v.extent.projection) if callable(renderer_or_fn): renderer = renderer_or_fn(v) elif renderer_or_fn is None: renderer = StretchedRenderer( [(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))] ) else: renderer = renderer_or_fn with transaction.atomic(): service = Service.objects.create( name=service_name, description='This service has been automatically generated from the result of a geoprocessing job.', data_path=rel_path, projection=v.extent.projection.srs, full_extent=v.extent, initial_extent=v.extent, ) Variable.objects.create( service=service, index=0, variable='data', projection=v.extent.projection.srs, x_dimension=x_var, y_dimension=y_var, name='data', renderer=renderer, full_extent=v.extent ) ProcessingResultService.objects.create(job=job, service=service) outputs[k] = service_name elif is_ndarray(v): if v.size < numpy.get_printoptions()['threshold']: outputs[k] = v.tolist() else: outputs[k] = str(v) return outputs
def render_netcdf(filename_pattern, variable, output_directory, renderer_file, save_file, renderer_type, colormap, fill, colorspace, palette, palette_stretch, scale, id_variable, lh, legend_breaks, legend_ticks, legend_precision, format, src_crs, dst_crs, res, resampling, anchors, interactive_map, mask_path): """ Render netcdf files to images. colormap is ignored if renderer_file is provided --dst-crs is ignored if using --map option (always uses EPSG:3857 If no colormap or palette is provided, a default palette may be chosen based on the name of the variable. If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It must be in the same CRS as the input datasets, and have the same spatial dimensions. """ # Parameter overrides if interactive_map: dst_crs = 'EPSG:3857' filenames = glob.glob(filename_pattern) if not filenames: raise click.BadParameter('No files found matching that pattern', param='filename_pattern', param_hint='FILENAME_PATTERN') if not os.path.exists(output_directory): os.makedirs(output_directory) mask = get_mask(mask_path) if mask_path is not None else None if renderer_file is not None and not save_file: if not os.path.exists(renderer_file): raise click.BadParameter('does not exist', param='renderer_file', param_hint='renderer_file') # see https://bitbucket.org/databasin/ncdjango/wiki/Home for format renderer_dict = json.loads(open(renderer_file).read()) if variable in renderer_dict and not 'colors' in renderer_dict: renderer_dict = renderer_dict[variable] renderer_type = renderer_dict['type'] if renderer_type == 'stretched': colors = ','.join([str(c[0]) for c in renderer_dict['colors']]) if 'min' in colors or 'max' in colors or 'mean' in colors: statistics = collect_statistics(filenames, (variable, ), mask=mask)[variable] for entry in renderer_dict['colors']: if isinstance(entry[0], basestring): if entry[0] in ('min', 'max', 'mean'): entry[0] = statistics[entry[0]] elif '*' in entry[0]: rel_value, statistic = entry[0].split('*') entry[0] = float(rel_value) * statistics[statistic] renderer = renderer_from_dict(renderer_dict) else: if renderer_type == 'stretched': if palette is not None: renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask) elif colormap is None and variable in DEFAULT_PALETTES: palette, palette_stretch = DEFAULT_PALETTES[variable] renderer = palette_to_stretched_renderer(palette, palette_stretch, filenames, variable, fill_value=fill, mask=mask) else: if colormap is None: colormap = 'min:#000000,max:#FFFFFF' renderer = colormap_to_stretched_renderer(colormap, colorspace, filenames, variable, fill_value=fill, mask=mask) elif renderer_type == 'classified': if not palette: raise click.BadParameter( 'palette required for classified (for now)', param='--palette', param_hint='--palette') renderer = palette_to_classified_renderer( palette, filenames, variable, method='equal', fill_value=fill, mask=mask) # TODO: other methods if save_file: if os.path.exists(save_file): with open(save_file, 'r+') as output_file: data = json.loads(output_file.read()) output_file.seek(0) output_file.truncate() data[variable] = renderer.serialize() output_file.write(json.dumps(data, indent=4)) else: with open(save_file, 'w') as output_file: output_file.write(json.dumps({variable: renderer.serialize()})) if renderer_type == 'stretched': if legend_ticks is not None and not legend_breaks: legend_ticks = [float(v) for v in legend_ticks.split(',')] legend = renderer.get_legend( image_height=lh, breaks=legend_breaks, ticks=legend_ticks, max_precision=legend_precision)[0].to_image() elif renderer_type == 'classified': legend = composite_elements(renderer.get_legend()) legend.save( os.path.join(output_directory, '{0}_legend.png'.format(variable))) with Dataset(filenames[0]) as ds: var_obj = ds.variables[variable] dimensions = var_obj.dimensions shape = var_obj.shape num_dimensions = len(shape) if num_dimensions == 3: if id_variable: if shape[0] != ds.variables[id_variable][:].shape[0]: raise click.BadParameter( 'must be same dimensionality as 3rd dimension of {0}'. format(variable), param='--id_variable', param_hint='--id_variable') else: # Guess from the 3rd dimension guess = dimensions[0] if guess in ds.variables and ds.variables[guess][:].shape[ 0] == shape[0]: id_variable = guess ds_crs = get_crs(ds, variable) if not ds_crs and is_geographic(ds, variable): ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84 src_crs = CRS.from_string(ds_crs) if ds_crs else CRS( {'init': src_crs}) if src_crs else None # get transforms, assume last 2 dimensions on variable are spatial in row, col order y_dim, x_dim = dimensions[-2:] coords = SpatialCoordinateVariables.from_dataset( ds, x_dim, y_dim, projection=Proj(src_crs.to_dict()) if src_crs else None) if mask is not None and not mask.shape == shape[-2:]: # Will likely break before this if collecting statistics raise click.BadParameter( 'mask variable shape does not match shape of input spatial dimensions', param='--mask', param_hint='--mask') flip_y = False reproject_kwargs = None if dst_crs is not None: if not src_crs: raise click.BadParameter('must provide src_crs to reproject', param='--src-crs', param_hint='--src-crs') dst_crs = CRS.from_string(dst_crs) src_height, src_width = coords.shape dst_transform, dst_width, dst_height = calculate_default_transform( src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list(), resolution=res) reproject_kwargs = { 'src_crs': src_crs, 'src_transform': coords.affine, 'dst_crs': dst_crs, 'dst_transform': dst_transform, 'resampling': getattr(RESAMPLING, resampling), 'dst_shape': (dst_height, dst_width) } else: dst_transform = coords.affine dst_height, dst_width = coords.shape dst_crs = src_crs if coords.y.is_ascending_order(): # Only needed if we are not already reprojecting the data, since that will flip it automatically flip_y = True if anchors or interactive_map: if not (dst_crs or src_crs): raise click.BadParameter( 'must provide at least src_crs to get Leaflet anchors or interactive map', param='--src-crs', param_hint='--src-crs') leaflet_anchors = get_leaflet_anchors( BBox.from_affine( dst_transform, dst_width, dst_height, projection=Proj(dst_crs) if dst_crs else None)) if anchors: click.echo('Anchors: {0}'.format(leaflet_anchors)) layers = {} for filename in filenames: with Dataset(filename) as ds: click.echo('Processing {0}'.format(filename)) filename_root = os.path.split(filename)[1].replace('.nc', '') if not variable in ds.variables: raise click.BadParameter( 'variable {0} was not found in file: {1}'.format( variable, filename), param='variable', param_hint='VARIABLE') var_obj = ds.variables[variable] if not var_obj.dimensions == dimensions: raise click.ClickException( 'All datasets must have the same dimensions for {0}'. format(variable)) if num_dimensions == 2: data = var_obj[:] if mask is not None: data = numpy.ma.masked_array(data, mask=mask) image_filename = os.path.join( output_directory, '{0}_{1}.{2}'.format(filename_root, variable, format)) if reproject_kwargs: data = warp_array(data, **reproject_kwargs) render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format) local_filename = os.path.split(image_filename)[1] layers[os.path.splitext(local_filename)[0]] = local_filename elif num_dimensions == 3: for index in range(shape[0]): id = ds.variables[id_variable][ index] if id_variable is not None else index image_filename = os.path.join( output_directory, '{0}_{1}__{2}.{3}'.format(filename_root, variable, id, format)) data = var_obj[index] if mask is not None: data = numpy.ma.masked_array(data, mask=mask) if reproject_kwargs: data = warp_array(data, **reproject_kwargs) render_image(renderer, data, image_filename, scale, flip_y=flip_y, format=format) local_filename = os.path.split(image_filename)[1] layers[os.path.splitext(local_filename) [0]] = local_filename # TODO: not tested recently. Make sure still correct # else: # # Assume last 2 components of shape are lat & lon, rest are iterated over # id_variables = None # if id_variable is not None: # id_variables = id_variable.split(',') # for index, name in enumerate(id_variables): # if name: # assert data.shape[index] == ds.variables[name][:].shape[0] # # ranges = [] # for dim in data.shape[:-2]: # ranges.append(range(0, dim)) # for combined_index in product(*ranges): # id_parts = [] # for index, dim_index in enumerate(combined_index): # if id_variables is not None and index < len(id_variables) and id_variables[index]: # id = ds.variables[id_variables[index]][dim_index] # # if not isinstance(id, basestring): # if isinstance(id, Iterable): # id = '_'.join((str(i) for i in id)) # else: # id = str(id) # # id_parts.append(id) # # else: # id_parts.append(str(dim_index)) # # combined_id = '_'.join(id_parts) # image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format)) # if reproject_kwargs: # data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this # render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format) # # local_filename = os.path.split(image_filename)[1] # layers[os.path.splitext(local_filename)[0]] = local_filename if interactive_map: index_html = os.path.join(output_directory, 'index.html') with open(index_html, 'w') as out: template = Environment( loader=PackageLoader('clover.cli')).get_template('map.html') out.write( template.render(layers=json.dumps(layers), bounds=str(leaflet_anchors), variable=variable)) webbrowser.open(index_html)
def handle(self, *args, **options): message = ( "WARNING: This will update all service data, casting each to it's smallest possible data type. Do you want " "to continue? [y/n]" ) if input(message).lower() not in {'y', 'yes'}: return for service in Service.objects.all(): if service.variable_set.all().count() > 1: print("Skipping service '{}' with more than one variable...".format(service.name)) continue variable = service.variable_set.all().get() path = os.path.join(SERVICE_DATA_ROOT, service.data_path) tmp_dir = mkdtemp() tmp_path = os.path.join(tmp_dir, os.path.basename(service.data_path)) try: with Dataset(path, 'r') as ds: data = ds.variables[variable.variable][:] coords = SpatialCoordinateVariables.from_bbox(service.full_extent, *reversed(data.shape)) if data.dtype.kind != 'i': print("Ignoring service '{}' with non-int type".format(service.name)) continue # The fill value will be the minimum value of the chosen type, so we want to make sure it's not # included in the actual data range min_value = data.min() - 1 max_value = data.max() # Determine the most suitable data type by finding the minimum type for the min/max values and then # using the type that will accurately represent both min_type = str(numpy.min_scalar_type(min_value)) max_type = str(numpy.min_scalar_type(max_value)) min_unsigned, min_size = min_type.split('int') max_unsigned, max_size = max_type.split('int') dtype = '{}int{}'.format(min_unsigned and max_unsigned, max(int(min_size), int(max_size))) if data.dtype == dtype: print("Service '{}' already has the smallest possible type: {}".format(service.name, dtype)) continue print("Converting service '{}' to type: {}".format(service.name, dtype)) with Dataset(tmp_path, 'w', format='NETCDF4') as ds: coords.add_to_dataset(ds, variable.x_dimension, variable.y_dimension) data = data.astype(dtype) fill_value = numpy.ma.maximum_fill_value(numpy.dtype(dtype)) numpy.ma.set_fill_value(data, fill_value) data_var = ds.createVariable( variable.variable, dtype, dimensions=(variable.y_dimension, variable.x_dimension), fill_value=fill_value ) data_var[:] = data set_crs(ds, variable.variable, service.full_extent.projection) os.unlink(path) shutil.copy2(tmp_path, path) finally: try: shutil.rmtree(tmp_dir) except OSError: pass
def test_SpatialCoordinateVariables_bbox(): proj = Proj(init='EPSG:4326') bbox = BBox((10.5, 5, 110.5, 55), projection=proj) coords = SpatialCoordinateVariables.from_bbox(bbox, 10, 5) assert coords.bbox.as_list() == bbox.as_list()
def netcdf_to_raster(path_or_dataset, variable_name, outfilename, index=0, projection=None): """ Exports a 2D slice from a netcdf file to a raster file. Only GeoTiffs are supported at this time. Parameters ---------- path_or_dataset: path to NetCDF file or open Dataset variable_name: name of data variable to export from dataset outfilename: output filename index: index within 3rd dimension (in first position) or 0 projection: pyproj.Proj object. Automatically determined from file if possible """ if isinstance(path_or_dataset, string_types): dataset = Dataset(path_or_dataset) else: dataset = path_or_dataset projection = projection or get_crs(dataset, variable_name) if not projection: raise ValueError( 'Projection must be provided; ' 'no projection information can be determined from file') # TODO figure out cleaner way to get affine or coords y_name, x_name = dataset.variables[variable_name].dimensions[:2] coords = SpatialCoordinateVariables.from_dataset(dataset, x_name, y_name, projection=projection) affine = coords.affine if outfilename.lower().endswith('.tif'): format = 'GTiff' else: raise ValueError( 'Only GeoTiff outputs supported, filename must have .tif extension' ) variable = dataset.variables[variable_name] ndims = len(variable.shape) if ndims == 2: if index != 0: raise ValueError('Index out of range, must be 0') data = variable[:] elif ndims == 3: # Assumes that time dimension is first if index < 0 or index >= variable.shape[0]: raise ValueError('Index out of range, ' 'must be between 0 and {0}'.variable.shape[0]) data = variable[index] else: raise ValueError( 'Unsupported number of dimensions {0} for variable {1}, ' 'must be 2 or 3'.format(ndims, variable_name)) array_to_raster(data, outfilename, format=format, projection=projection, affine=affine)
def raster_to_netcdf(filename_or_raster, outfilename=None, variable_name='data', format='NETCDF4', **kwargs): """ Parameters ---------- filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset outfilename: name of output file. If blank, will be same name as input with *.nc extension added variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4 format kwargs: arguments passed to variable creation: zlib Note: only rasters with descending y coordinates are currently supported """ start = time.time() if isinstance(filename_or_raster, string_types): if not os.path.exists(filename_or_raster): raise ValueError( 'File does not exist: {0}'.format(filename_or_raster)) src = rasterio.open(filename_or_raster) managed_raster = True else: src = filename_or_raster managed_raster = False if not src.count == 1: raise NotImplementedError( 'ERROR: multi-band rasters not yet supported for this operation') prj = pyproj.Proj(**src.crs) outfilename = outfilename or src.name + '.nc' with Dataset(outfilename, 'w', format=format) as target: if prj.is_latlong(): x_varname = 'longitude' y_varname = 'latitude' else: x_varname = 'x' y_varname = 'y' # TODO: may need to do this in blocks if source is big data = src.read(1, masked=True) coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj), src.width, src.height) coords.add_to_dataset(target, x_varname, y_varname, **kwargs) out_var = target.createVariable(variable_name, data.dtype, dimensions=(y_varname, x_varname), **kwargs) out_var[:] = data set_crs(target, variable_name, prj, set_proj4_att=False) if managed_raster: src.close() print('Elapsed {0:.3f} seconds'.format(time.time() - start))
def handle(self, *args, **options): elevation_service = Service.objects.get(name='west2_dem') with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, elevation_service.data_path)) as ds: coords = SpatialCoordinateVariables.from_dataset( ds, x_name='lon', y_name='lat', projection=Proj(elevation_service.projection) ) elevation = ds.variables['elevation'][:] message = 'WARNING: This will replace all your transfer limits. Do you want to continue? [y/n]' if input(message).lower() not in {'y', 'yes'}: return self.transfers_by_source = {} with transaction.atomic(): TransferLimit.objects.all().delete() for time_period in ('1961_1990', '1981_2010'): for variable in VARIABLES: print('Processing {} for {}...'.format(variable, time_period)) variable_service = Service.objects.get(name='west2_{}Y_{}'.format(time_period, variable)) with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, variable_service.data_path)) as ds: data = ds.variables[variable][:] for zone in SeedZone.objects.all(): clipped_elevation, clipped_data, clipped_coords = self._get_subsets( elevation, data, coords, BBox(zone.polygon.extent) ) zone_mask = rasterize( ((json.loads(zone.polygon.geojson), 1),), out_shape=clipped_elevation.shape, transform=clipped_coords.affine, fill=0, dtype=numpy.dtype('uint8') ) masked_dem = numpy.ma.masked_where(zone_mask == 0, clipped_elevation) min_elevation = max(math.floor(numpy.nanmin(masked_dem) / 0.3048), 0) max_elevation = math.ceil(numpy.nanmax(masked_dem) / 0.3048) bands = list(self._get_bands_fn(zone.source)(zone.zone_id, min_elevation, max_elevation)) if not bands: print('WARNING: No elevation bands found for {}, zone {}'.format( zone.source, zone.zone_id )) continue for band in bands: low, high = band # Bands are exclusive of the low number, so the first band is a special case, since we # want to include 0. So we work around it by making the actual low -1 if low == 0: low = -1 # Elevation bands are represented in feet masked_data = numpy.ma.masked_where( (zone_mask == 0) | (clipped_elevation <= low * 0.3048) | (clipped_elevation > high * 0.3048), clipped_data ) self._write_limit(variable, time_period, zone, masked_data, low, high) for source, transfers_by_variable in self.transfers_by_source.items(): for variable, transfers in transfers_by_variable.items(): TransferLimit.objects.filter( variable=variable, zone__source=source ).update( avg_transfer=mean(transfers) )
def mask( input, output, variable, like, netcdf3, all_touched, invert, zip): """ Create a NetCDF mask from a shapefile. Values are equivalent to a numpy mask: 0 for unmasked areas, and 1 for masked areas. Template NetCDF dataset must have a valid projection defined or be inferred from dimensions (e.g., lat / long) """ with Dataset(like) as template_ds: template_varname = data_variables(template_ds).keys()[0] template_variable = template_ds.variables[template_varname] template_crs = get_crs(template_ds, template_varname) if template_crs: template_crs = CRS.from_string(template_crs) elif is_geographic(template_ds, template_varname): template_crs = CRS({'init': 'EPSG:4326'}) else: raise click.UsageError('template dataset must have a valid projection defined') spatial_dimensions = template_variable.dimensions[-2:] mask_shape = template_variable.shape[-2:] template_y_name, template_x_name = spatial_dimensions coords = SpatialCoordinateVariables.from_dataset( template_ds, x_name=template_x_name, y_name=template_y_name, projection=Proj(**template_crs.to_dict()) ) with fiona.open(input, 'r') as shp: transform_required = CRS(shp.crs) != template_crs # Project bbox for filtering bbox = coords.bbox if transform_required: bbox = bbox.project(Proj(**shp.crs), edge_points=21) geometries = [] for f in shp.filter(bbox=bbox.as_list()): geom = f['geometry'] if transform_required: geom = transform_geom(shp.crs, template_crs, geom) geometries.append(geom) click.echo('Converting {0} features to mask'.format(len(geometries))) if invert: fill_value = 0 default_value = 1 else: fill_value = 1 default_value = 0 with rasterio.drivers(): # Rasterize features to 0, leaving background as 1 mask = rasterize( geometries, out_shape=mask_shape, transform=coords.affine, all_touched=all_touched, fill=fill_value, default_value=default_value, dtype=numpy.uint8 ) format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4' dtype = 'int8' if netcdf3 else 'uint8' with Dataset(output, 'w', format=format) as out: coords.add_to_dataset(out, template_x_name, template_y_name) out_var = out.createVariable(variable, dtype, dimensions=spatial_dimensions, zlib=zip, fill_value=get_fill_value(dtype)) out_var[:] = mask