def test_window_for_bbox():
coords = SpatialCoordinateVariables.from_bbox(
BBox([-124, 82, -122, 90], Proj(init='epsg:4326')), 20, 20)
window = coords.get_window_for_bbox(BBox([-123.9, 82.4, -122.1, 89.6]))
assert window.x_slice == slice(1, 19)
assert window.y_slice == slice(1, 19)
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None):
outputs = results.format_args()
for k, v in six.iteritems(outputs):
if is_raster(v) and publish_raster_results:
service_name = '{0}/{1}'.format(job.uuid, k)
rel_path = '{}.nc'.format(service_name)
abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path)
os.makedirs(os.path.dirname(abs_path))
with Dataset(abs_path, 'w', format='NETCDF4') as ds:
if v.extent.projection.is_latlong():
x_var = 'longitude'
y_var = 'latitude'
else:
x_var = 'x'
y_var = 'y'
coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape))
coord_vars.add_to_dataset(ds, x_var, y_var)
fill_value = v.fill_value if is_masked(v) else None
data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value)
data_var[:] = v
set_crs(ds, 'data', v.extent.projection)
if callable(renderer_or_fn):
renderer = renderer_or_fn(v)
elif renderer_or_fn is None:
renderer = StretchedRenderer(
[(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))]
)
else:
renderer = renderer_or_fn
with transaction.atomic():
service = Service.objects.create(
name=service_name,
description='This service has been automatically generated from the result of a geoprocessing job.',
data_path=rel_path,
projection=v.extent.projection.srs,
full_extent=v.extent,
initial_extent=v.extent,
)
Variable.objects.create(
service=service,
index=0,
variable='data',
projection=v.extent.projection.srs,
x_dimension=x_var,
y_dimension=y_var,
name='data',
renderer=renderer,
full_extent=v.extent
)
ProcessingResultService.objects.create(job=job, service=service)
outputs[k] = service_name
elif is_ndarray(v):
if v.size < numpy.get_printoptions()['threshold']:
outputs[k] = v.tolist()
else:
outputs[k] = str(v)
return outputs
def handle(self, *args, **options):
message = (
"WARNING: This will update all service data, casting each to it's smallest possible data type. Do you want "
"to continue? [y/n]"
)
if input(message).lower() not in {'y', 'yes'}:
return
for service in Service.objects.all():
if service.variable_set.all().count() > 1:
print("Skipping service '{}' with more than one variable...".format(service.name))
continue
variable = service.variable_set.all().get()
path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
tmp_dir = mkdtemp()
tmp_path = os.path.join(tmp_dir, os.path.basename(service.data_path))
try:
with Dataset(path, 'r') as ds:
data = ds.variables[variable.variable][:]
coords = SpatialCoordinateVariables.from_bbox(service.full_extent, *reversed(data.shape))
if data.dtype.kind != 'i':
print("Ignoring service '{}' with non-int type".format(service.name))
continue
# The fill value will be the minimum value of the chosen type, so we want to make sure it's not
# included in the actual data range
min_value = data.min() - 1
max_value = data.max()
# Determine the most suitable data type by finding the minimum type for the min/max values and then
# using the type that will accurately represent both
min_type = str(numpy.min_scalar_type(min_value))
max_type = str(numpy.min_scalar_type(max_value))
min_unsigned, min_size = min_type.split('int')
max_unsigned, max_size = max_type.split('int')
dtype = '{}int{}'.format(min_unsigned and max_unsigned, max(int(min_size), int(max_size)))
if data.dtype == dtype:
print("Service '{}' already has the smallest possible type: {}".format(service.name, dtype))
continue
print("Converting service '{}' to type: {}".format(service.name, dtype))
with Dataset(tmp_path, 'w', format='NETCDF4') as ds:
coords.add_to_dataset(ds, variable.x_dimension, variable.y_dimension)
data = data.astype(dtype)
fill_value = numpy.ma.maximum_fill_value(numpy.dtype(dtype))
numpy.ma.set_fill_value(data, fill_value)
data_var = ds.createVariable(
variable.variable, dtype, dimensions=(variable.y_dimension, variable.x_dimension),
fill_value=fill_value
)
data_var[:] = data
set_crs(ds, variable.variable, service.full_extent.projection)
os.unlink(path)
shutil.copy2(tmp_path, path)
finally:
try:
shutil.rmtree(tmp_dir)
except OSError:
pass
def test_SpatialCoordinateVariables_bbox():
proj = Proj(init='EPSG:4326')
bbox = BBox((10.5, 5, 110.5, 55), projection=proj)
coords = SpatialCoordinateVariables.from_bbox(bbox, 10, 5)
assert coords.bbox.as_list() == bbox.as_list()
def raster_to_netcdf(filename_or_raster,
outfilename=None,
variable_name='data',
format='NETCDF4',
**kwargs):
"""
Parameters
----------
filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset
outfilename: name of output file. If blank, will be same name as input with *.nc extension added
variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4
format
kwargs: arguments passed to variable creation: zlib
Note: only rasters with descending y coordinates are currently supported
"""
start = time.time()
if isinstance(filename_or_raster, string_types):
if not os.path.exists(filename_or_raster):
raise ValueError(
'File does not exist: {0}'.format(filename_or_raster))
src = rasterio.open(filename_or_raster)
managed_raster = True
else:
src = filename_or_raster
managed_raster = False
if not src.count == 1:
raise NotImplementedError(
'ERROR: multi-band rasters not yet supported for this operation')
prj = pyproj.Proj(**src.crs)
outfilename = outfilename or src.name + '.nc'
with Dataset(outfilename, 'w', format=format) as target:
if prj.is_latlong():
x_varname = 'longitude'
y_varname = 'latitude'
else:
x_varname = 'x'
y_varname = 'y'
# TODO: may need to do this in blocks if source is big
data = src.read(1, masked=True)
coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj),
src.width, src.height)
coords.add_to_dataset(target, x_varname, y_varname, **kwargs)
out_var = target.createVariable(variable_name,
data.dtype,
dimensions=(y_varname, x_varname),
**kwargs)
out_var[:] = data
set_crs(target, variable_name, prj, set_proj4_att=False)
if managed_raster:
src.close()
print('Elapsed {0:.3f} seconds'.format(time.time() - start))
