def test_window_for_bbox(): coords = SpatialCoordinateVariables.from_bbox( BBox([-124, 82, -122, 90], Proj(init='epsg:4326')), 20, 20) window = coords.get_window_for_bbox(BBox([-123.9, 82.4, -122.1, 89.6])) assert window.x_slice == slice(1, 19) assert window.y_slice == slice(1, 19)
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None): outputs = results.format_args() for k, v in six.iteritems(outputs): if is_raster(v) and publish_raster_results: service_name = '{0}/{1}'.format(job.uuid, k) rel_path = '{}.nc'.format(service_name) abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path) os.makedirs(os.path.dirname(abs_path)) with Dataset(abs_path, 'w', format='NETCDF4') as ds: if v.extent.projection.is_latlong(): x_var = 'longitude' y_var = 'latitude' else: x_var = 'x' y_var = 'y' coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape)) coord_vars.add_to_dataset(ds, x_var, y_var) fill_value = v.fill_value if is_masked(v) else None data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value) data_var[:] = v set_crs(ds, 'data', v.extent.projection) if callable(renderer_or_fn): renderer = renderer_or_fn(v) elif renderer_or_fn is None: renderer = StretchedRenderer( [(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))] ) else: renderer = renderer_or_fn with transaction.atomic(): service = Service.objects.create( name=service_name, description='This service has been automatically generated from the result of a geoprocessing job.', data_path=rel_path, projection=v.extent.projection.srs, full_extent=v.extent, initial_extent=v.extent, ) Variable.objects.create( service=service, index=0, variable='data', projection=v.extent.projection.srs, x_dimension=x_var, y_dimension=y_var, name='data', renderer=renderer, full_extent=v.extent ) ProcessingResultService.objects.create(job=job, service=service) outputs[k] = service_name elif is_ndarray(v): if v.size < numpy.get_printoptions()['threshold']: outputs[k] = v.tolist() else: outputs[k] = str(v) return outputs
def handle(self, *args, **options): message = ( "WARNING: This will update all service data, casting each to it's smallest possible data type. Do you want " "to continue? [y/n]" ) if input(message).lower() not in {'y', 'yes'}: return for service in Service.objects.all(): if service.variable_set.all().count() > 1: print("Skipping service '{}' with more than one variable...".format(service.name)) continue variable = service.variable_set.all().get() path = os.path.join(SERVICE_DATA_ROOT, service.data_path) tmp_dir = mkdtemp() tmp_path = os.path.join(tmp_dir, os.path.basename(service.data_path)) try: with Dataset(path, 'r') as ds: data = ds.variables[variable.variable][:] coords = SpatialCoordinateVariables.from_bbox(service.full_extent, *reversed(data.shape)) if data.dtype.kind != 'i': print("Ignoring service '{}' with non-int type".format(service.name)) continue # The fill value will be the minimum value of the chosen type, so we want to make sure it's not # included in the actual data range min_value = data.min() - 1 max_value = data.max() # Determine the most suitable data type by finding the minimum type for the min/max values and then # using the type that will accurately represent both min_type = str(numpy.min_scalar_type(min_value)) max_type = str(numpy.min_scalar_type(max_value)) min_unsigned, min_size = min_type.split('int') max_unsigned, max_size = max_type.split('int') dtype = '{}int{}'.format(min_unsigned and max_unsigned, max(int(min_size), int(max_size))) if data.dtype == dtype: print("Service '{}' already has the smallest possible type: {}".format(service.name, dtype)) continue print("Converting service '{}' to type: {}".format(service.name, dtype)) with Dataset(tmp_path, 'w', format='NETCDF4') as ds: coords.add_to_dataset(ds, variable.x_dimension, variable.y_dimension) data = data.astype(dtype) fill_value = numpy.ma.maximum_fill_value(numpy.dtype(dtype)) numpy.ma.set_fill_value(data, fill_value) data_var = ds.createVariable( variable.variable, dtype, dimensions=(variable.y_dimension, variable.x_dimension), fill_value=fill_value ) data_var[:] = data set_crs(ds, variable.variable, service.full_extent.projection) os.unlink(path) shutil.copy2(tmp_path, path) finally: try: shutil.rmtree(tmp_dir) except OSError: pass
def test_SpatialCoordinateVariables_bbox(): proj = Proj(init='EPSG:4326') bbox = BBox((10.5, 5, 110.5, 55), projection=proj) coords = SpatialCoordinateVariables.from_bbox(bbox, 10, 5) assert coords.bbox.as_list() == bbox.as_list()
def raster_to_netcdf(filename_or_raster, outfilename=None, variable_name='data', format='NETCDF4', **kwargs): """ Parameters ---------- filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset outfilename: name of output file. If blank, will be same name as input with *.nc extension added variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4 format kwargs: arguments passed to variable creation: zlib Note: only rasters with descending y coordinates are currently supported """ start = time.time() if isinstance(filename_or_raster, string_types): if not os.path.exists(filename_or_raster): raise ValueError( 'File does not exist: {0}'.format(filename_or_raster)) src = rasterio.open(filename_or_raster) managed_raster = True else: src = filename_or_raster managed_raster = False if not src.count == 1: raise NotImplementedError( 'ERROR: multi-band rasters not yet supported for this operation') prj = pyproj.Proj(**src.crs) outfilename = outfilename or src.name + '.nc' with Dataset(outfilename, 'w', format=format) as target: if prj.is_latlong(): x_varname = 'longitude' y_varname = 'latitude' else: x_varname = 'x' y_varname = 'y' # TODO: may need to do this in blocks if source is big data = src.read(1, masked=True) coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj), src.width, src.height) coords.add_to_dataset(target, x_varname, y_varname, **kwargs) out_var = target.createVariable(variable_name, data.dtype, dimensions=(y_varname, x_varname), **kwargs) out_var[:] = data set_crs(target, variable_name, prj, set_proj4_att=False) if managed_raster: src.close() print('Elapsed {0:.3f} seconds'.format(time.time() - start))