def test_generate_scores_workflow_sanity():
# Simple
raster_1 = Raster(numpy.reshape(numpy.arange(100), (10, 10)),
BBox((0, 0, 10, 10)), 1, 0)
raster_2 = Raster(numpy.reshape(numpy.arange(100, 200), (10, 10)),
BBox((0, 0, 10, 10)), 1, 0)
limits = [{'min': 30, 'max': 70}, {'min': 140, 'max': 160}]
GenerateScores()(variables=[raster_1, raster_2],
limits=limits,
region='test')
def get_render_configurations(self, request, **kwargs):
tile_bounds = list(
mercantile.bounds(int(self.kwargs['x']), int(self.kwargs['y']),
int(self.kwargs['z'])))
extent = BBox(
tile_bounds,
projection=Proj('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs')
).project(
Proj(
'+proj=merc +lon_0=0 +k=1 +x_0=0 +y_0=0 +a=6378137 +b=6378137 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs'
))
base_config = ImageConfiguration(
extent=extent,
size=TILE_SIZE,
image_format='png',
background_color=TRANSPARENT_BACKGROUND_COLOR)
return base_config, [
RenderConfiguration(variable=x,
extent=extent,
size=TILE_SIZE,
image_format='png')
for x in self.service.variable_set.all()
]
def test_bbox():
bbox = BBox(TEST_COORDS, TEST_COORDS_PRJ)
assert bbox.xmin == TEST_COORDS[0]
assert bbox.ymin == TEST_COORDS[1]
assert bbox.xmax == TEST_COORDS[2]
assert bbox.ymax == TEST_COORDS[3]
assert bbox.projection.srs == TEST_COORDS_PRJ.srs
def get_read_window(self, extent):
"""Calculates a read window and transform to read data within the extent.
Example
-------
window, transform = instance.get_read_window(extent)
data = instance.data[window]
Parameters
----------
extent : (xmin, ymin, xmax, ymax)
Returns
-------
(y_slice, x_slice), SpatialCoordinateVariables
"""
# Calculate indexes to slice based on extent
bbox = BBox(extent)
x_slice = slice(*self.coords.x.indices_for_range(bbox.xmin, bbox.xmax))
y_slice = slice(*self.coords.y.indices_for_range(bbox.ymin, bbox.ymax))
# expand by 1px in all directions to make sure something isn't getting cut off
# if bounds fall very close to pixel edges
x_slice = slice(max(x_slice.start - 1, 0),
min(x_slice.stop + 1, len(self.coords.x)),
x_slice.step)
y_slice = slice(max(y_slice.start - 1, 0),
min(y_slice.stop + 1, len(self.coords.y)),
y_slice.step)
# get updated coords for the slices
coords = self.coords.slice_by_window(Window(y_slice, x_slice))
return (y_slice, x_slice), coords
def __getitem__(self, items):
arr = super(Raster, self).__getitem__(items)
if not isinstance(arr, numpy.ndarray) or isinstance(
arr, MaskedConstant):
return arr
Raster.__array_finalize__(arr, self)
if self.extent is None or self.x_dim is None or self.y_dim is None:
return arr.view(numpy.ndarray) # Cast back to regular numpy array
if isinstance(items, slice):
items = (items, )
if isinstance(items, tuple):
xmin, ymin, xmax, ymax = self.extent.as_list()
cell_size = (self.extent.width / self.shape[self.x_dim],
self.extent.height / self.shape[self.y_dim])
has_valid_extent = True
for i, item in enumerate(items):
if not isinstance(item, slice) and i in {
self.x_dim, self.y_dim
}:
# Not a slice, so we can't reliably preserve extent information
has_valid_extent = False
break
if i == self.x_dim:
xmin += cell_size[0] * (item.start or 0)
end = min(item.stop or self.shape[self.x_dim],
self.shape[self.x_dim])
if end < 0:
end = self.shape[self.x_dim] + end + 1
xmax -= cell_size[0] * (self.shape[self.x_dim] - end)
if i == self.y_dim:
start = item.start or 0
end = min(item.stop or self.shape[self.y_dim],
self.shape[self.y_dim])
if end < 0:
end = self.shape[self.y_dim] + end
end = self.shape[self.y_dim] - end
if not self.y_increasing:
start, end = end, start
ymin += cell_size[1] * start
ymax -= cell_size[1] * end
if has_valid_extent:
arr.extent = BBox((xmin, ymin, xmax, ymax),
projection=self.extent.projection)
return arr
else:
return arr.view(numpy.ndarray)
return arr.view(numpy.ndarray)
def test_bbox_local_projection():
bbox = BBox(TEST_COORDS, TEST_COORDS_PRJ)
out = CRS.from_string(bbox.get_local_albers_projection().srs)
expected = CRS.from_string(
"+lon_0=-124.5625 +ellps=WGS84 +datum=WGS84 +y_0=0 +no_defs=True +proj=aea +x_0=0 +units=m +lat_2=48.9375 +lat_1=48.6875 +lat_0=0 "
)
assert expected == out
def test_projection():
bbox = BBox(TEST_COORDS, TEST_COORDS_PRJ)
proj_bbox = bbox.project(Proj(init="EPSG:3857"))
# Calculated by running this previously under controlled conditions. No validation against truth of projection values.
assert numpy.allclose(proj_bbox.as_list(), [
-13887106.476460878, 6211469.632719522, -13845361.6674134,
6274861.394006577
])
def _normalize_bbox(self, bbox, size):
"""Returns this bbox normalized to match the ratio of the given size."""
bbox_ratio = float(bbox.width) / float(bbox.height)
size_ratio = float(size[0]) / float(size[1])
if round(size_ratio, 4) == round(bbox_ratio, 4):
return bbox
else:
if bbox.height * size_ratio >= bbox.width:
diff = bbox.height * size_ratio - bbox.width
return BBox((bbox.xmin - diff / 2, bbox.ymin,
bbox.xmax + diff / 2, bbox.ymax), bbox.projection)
else:
diff = abs(bbox.width / size_ratio - bbox.height)
return BBox((bbox.xmin, bbox.ymin - diff / 2, bbox.xmax,
bbox.ymax + diff / 2), bbox.projection)
def bbox(self):
half_x_pixel_size = self.x.pixel_size / 2.0
half_y_pixel_size = self.y.pixel_size / 2.0
return BBox(
(self.x.min - half_x_pixel_size, self.y.min - half_y_pixel_size,
self.x.max + half_x_pixel_size, self.y.max + half_y_pixel_size),
self.projection)
def crop_image(self, im):
im_ul = (self.center_point_px[0] - self.target_size[0] // 2,
self.center_point_px[1] - self.target_size[1] // 2)
box = (*im_ul, im_ul[0] + self.target_size[0],
im_ul[1] + self.target_size[1])
return im.crop(box), BBox(
(self.to_world(box[0], box[3])) + self.to_world(box[2], box[1]),
projection=Proj(init='epsg:3857'))
def test_generate_scores_workflow_validity():
"""Test simple 2x2 grid against pre-calculated values"""
ahm = Raster(numpy.array([[284, 274], [307, 298]]), BBox((0, 0, 10, 10)),
1, 0)
cmd = Raster(numpy.array([[292, 305], [300, 291]]), BBox((0, 0, 10, 10)),
1, 0)
limits = [{'min': 264, 'max': 304}, {'min': 271, 'max': 311}]
expected_mask = numpy.array([[False, False], [True, False]])
expected_results = numpy.ma.masked_array([[95, 14], [None, 30]],
mask=expected_mask)
results = GenerateScores()(variables=[ahm, cmd],
limits=limits,
region='test')
assert (results['raster_out'].mask == expected_mask).all()
assert (results['raster_out'] == expected_results).all()
def test_raster(self):
arr = numpy.reshape(numpy.arange(100), (10, 10))
extent = BBox((0, 0, 10, 10))
raster = Raster(arr, extent, 1, 0)
assert isinstance(raster, Raster)
raster = raster[:]
assert isinstance(raster, Raster)
assert isinstance(raster.extent, BBox)
clipped = raster[3:, 3:-1]
assert isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert clipped.extent.as_list() == [3, 0, 10, 7]
raster.y_increasing = True
clipped = raster[3:, 3:-1]
assert isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert clipped.extent.as_list() == [3, 3, 10, 10]
clipped = raster[0]
assert not isinstance(clipped, Raster)
assert isinstance(raster.extent, BBox)
assert is_ndarray(clipped)
assert (clipped == numpy.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])).all()
# Make sure binops don't lose extent info
assert isinstance((raster == raster).extent, BBox)
assert isinstance((raster != raster).extent, BBox)
assert isinstance((raster > 2).extent, BBox)
assert isinstance((2 > raster).extent, BBox)
assert isinstance((raster + 2).extent, BBox)
assert isinstance((2 + raster).extent, BBox)
assert isinstance((raster - 2).extent, BBox)
assert isinstance((2 - raster).extent, BBox)
assert isinstance((raster * 2).extent, BBox)
assert isinstance((2 * raster).extent, BBox)
assert isinstance((raster / 2).extent, BBox)
assert isinstance((2 / raster).extent, BBox)
assert isinstance((raster // 2).extent, BBox)
assert isinstance((2 // raster).extent, BBox)
assert isinstance((raster**2).extent, BBox)
assert isinstance((2**raster).extent, BBox)
assert isinstance((raster % 2).extent, BBox)
assert isinstance((2 % raster).extent, BBox)
assert isinstance((raster << 2).extent, BBox)
assert isinstance((2 << raster).extent, BBox)
assert isinstance((raster >> 2).extent, BBox)
assert isinstance((2 >> raster).extent, BBox)
assert isinstance((raster | 2).extent, BBox)
assert isinstance((2 | raster).extent, BBox)
assert isinstance((raster & 2).extent, BBox)
assert isinstance((2 & raster).extent, BBox)
assert isinstance((raster ^ 2).extent, BBox)
assert isinstance((2 ^ raster).extent, BBox)
def get_results_image(self, bounds, size, single_color, kept_colors, gained_colors, species, historic, futures):
kept_colors = self.get_colors(kept_colors, len(futures)+1)
gained_colors = self.get_colors(gained_colors, len(futures)+1)
extent = BBox(bounds, projection=WGS84)
self.service = Service.objects.get(name='{}_p{}_800m_pa'.format(species, historic))
variable = self.service.variable_set.all().first()
native_extent = extent.project(Proj(str(variable.projection)))
coords = SpatialCoordinateVariables.from_bbox(variable.full_extent, *self.get_grid_spatial_dimensions(variable))
x_slice = coords.x.indices_for_range(native_extent.xmin, native_extent.xmax)
y_slice = coords.y.indices_for_range(native_extent.ymin, native_extent.ymax)
historic_data = self.get_grid_for_variable(variable, x_slice=x_slice, y_slice=y_slice)
self.close_dataset()
if not futures:
data = historic_data
renderer = UniqueValuesRenderer([(1, Color.from_hex(single_color))], fill_value=0)
else:
future_grids = []
for future in futures:
self.service = Service.objects.get(name='{}_15gcm_{}_pa'.format(species, future))
variable = self.service.variable_set.all().first()
future_grids.append(self.get_grid_for_variable(variable, x_slice=x_slice, y_slice=y_slice))
self.close_dataset()
future_data = sum(future_grids)
del future_grids
data = numpy.zeros_like(historic_data, numpy.uint8)
data[historic_data == 1] = 1
kept_idx = (historic_data == 1) & (future_data > 0)
data[kept_idx] = future_data[kept_idx] + 1
gained_idx = (historic_data == 0) & (future_data > 0)
data[gained_idx] = future_data[gained_idx] + len(kept_colors) + 1
data[data.mask == 1] = 0
values = numpy.unique(data)
renderer = UniqueValuesRenderer(
[
(i+1, Color.from_hex(c))
for (i, c) in enumerate(kept_colors)
if i+1 in values
] +
[
(i+len(kept_colors)+1, Color.from_hex(c))
for (i, c) in enumerate(gained_colors)
if i+len(kept_colors)+1 in values
],
fill_value=0
)
image = renderer.render_image(data.data).convert('RGBA')
return GeoImage(image, native_extent).warp(extent, size).image
def test_SpatialCoordinateVariables_slice_by_bbox():
lat = SpatialCoordinateVariable(numpy.arange(19, -1, -1))
lon = SpatialCoordinateVariable(numpy.arange(10))
proj = Proj(init='EPSG:4326')
coords = SpatialCoordinateVariables(lon, lat, proj)
subset = coords.slice_by_bbox(BBox((1.75, 3.7, 6.2, 16.7), proj))
assert numpy.array_equal(subset.x.values, numpy.arange(2, 6))
assert subset.x.values[0] == 2
assert subset.x.values[-1] == 5
assert subset.y.values[0] == 16
assert subset.y.values[-1] == 4
def to_python(self, value):
if not value or isinstance(value, BBox):
return value
try:
data = json.loads(value)
projection = pyproj.Proj(str(data.get('proj4'))) if data.get('proj4') else None
return BBox(
(data['xmin'], data['ymin'], data['xmax'], data['ymax']), projection=projection
)
except (ValueError, KeyError):
raise ValidationError("")
def to_python(self, value):
if not value or isinstance(value, BBox):
return value
try:
projection = None
if 'xmin' in value:
data = json.loads(value)
values = [data[k] for k in ('xmin', 'ymin', 'xmax', 'ymax')]
if 'spatialReference' in data:
projection = SrField().to_python(
json.dumps(data.get('spatialReference')))
else:
values = [float(x.strip()) for x in value.split(',')]
return BBox(values, projection=projection)
except ValueError:
raise ValidationError('Invalid bbox')
def get_basemap_image(self, basemap, bounds, size, zoom):
tiles = list(mercantile.tiles(*bounds, zoom, truncate=True))
bounds_merc = [*mercantile.xy(*bounds[:2]), *mercantile.xy(*bounds[2:])]
image = Image.new('RGBA', size)
to_image = world_to_image(BBox(bounds_merc, projection=WEB_MERCATOR), size)
async def fetch_tile(client, url, tile):
tile_bounds = mercantile.xy_bounds(tile)
if url.startswith('//'):
url = 'http:' + url
async with client.get(url.format(x=tile.x, y=tile.y, z=tile.z, s='server')) as r:
tile_im = Image.open(BytesIO(await r.read()))
image.paste(tile_im, [int(round(x)) for x in to_image(*tile_bounds[0:4:3])]) # [0:4:3] = [xmin, ymax] (upper-left)
async def fetch_tiles():
async with aiohttp.ClientSession() as client:
futures = [ensure_future(fetch_tile(client, basemap, tile)) for tile in tiles]
await asyncio.wait(futures, return_when=asyncio.ALL_COMPLETED)
asyncio.get_event_loop().run_until_complete(fetch_tiles())
return image
def raster_to_netcdf(filename_or_raster,
outfilename=None,
variable_name='data',
format='NETCDF4',
**kwargs):
"""
Parameters
----------
filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset
outfilename: name of output file. If blank, will be same name as input with *.nc extension added
variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4
format
kwargs: arguments passed to variable creation: zlib
Note: only rasters with descending y coordinates are currently supported
"""
start = time.time()
if isinstance(filename_or_raster, string_types):
if not os.path.exists(filename_or_raster):
raise ValueError(
'File does not exist: {0}'.format(filename_or_raster))
src = rasterio.open(filename_or_raster)
managed_raster = True
else:
src = filename_or_raster
managed_raster = False
if not src.count == 1:
raise NotImplementedError(
'ERROR: multi-band rasters not yet supported for this operation')
prj = pyproj.Proj(**src.crs)
outfilename = outfilename or src.name + '.nc'
with Dataset(outfilename, 'w', format=format) as target:
if is_latlong(prj):
x_varname = 'longitude'
y_varname = 'latitude'
else:
x_varname = 'x'
y_varname = 'y'
# TODO: may need to do this in blocks if source is big
data = src.read(1, masked=True)
coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj),
src.width, src.height)
coords.add_to_dataset(target, x_varname, y_varname, **kwargs)
out_var = target.createVariable(variable_name,
data.dtype,
dimensions=(y_varname, x_varname),
**kwargs)
out_var[:] = data
set_crs(target, variable_name, prj, set_proj4_att=False)
if managed_raster:
src.close()
print('Elapsed {0:.3f} seconds'.format(time.time() - start))
def handle(self, data_files, overwrite, *args, **options):
with transaction.atomic():
for data_file in data_files:
if not os.path.isdir(SERVICE_DATA_ROOT):
raise CommandError('Directory %s does not exist.' % SERVICE_DATA_ROOT)
# Check for existence of file or service. There's a possible race
# condition here, but this is a management command, not a user command.
file_exists = False
svc_exists = False
target = os.path.join(SERVICE_DATA_ROOT, os.path.basename(data_file))
if os.path.exists(target):
if not overwrite:
self.stderr.write('File %s already exists.\n' % target)
file_exists = True
svc_name = os.path.basename(data_file).split('.')[0]
if Service.objects.filter(name=svc_name).exists():
if overwrite:
Service.objects.filter(name=svc_name).delete()
else:
self.stderr.write('Service %s already exists.\n' % svc_name)
svc_exists = True
if (file_exists or svc_exists) and not overwrite:
raise CommandError('No changes made.')
desc = describe(data_file)
grid = next(v['spatial_grid'] for k, v in desc['variables'].items() if v.get('spatial_grid'))
extent = grid['extent']
proj = extent['proj4']
bbox = BBox(
[extent[c] for c in ['xmin', 'ymin', 'xmax', 'ymax']], pyproj.Proj(proj)
)
renderer = UniqueValuesRenderer([(1, Color(0, 0, 0, 255))], fill_value=0)
if file_exists:
os.remove(os.path.join(SERVICE_DATA_ROOT, os.path.basename(data_file)))
shutil.copy(data_file, SERVICE_DATA_ROOT)
service = Service.objects.create(
name=svc_name,
projection=proj,
full_extent=bbox,
initial_extent=bbox,
data_path=os.path.basename(data_file)
)
for i, (variable_name, variable) in enumerate(desc['variables'].items()):
grid = variable.get('spatial_grid')
if grid is None:
continue
extent = grid['extent']
bbox = BBox(
[extent[c] for c in ['xmin', 'ymin', 'xmax', 'ymax']], pyproj.Proj(extent['proj4'])
)
Variable.objects.create(
service=service,
index=i,
variable=variable_name,
projection=proj,
x_dimension=grid['x_dimension'],
y_dimension=grid['y_dimension'],
name=variable_name,
renderer=renderer,
full_extent=bbox
)
def main(in_pattern, out_pattern, boundary, single, varname):
"""
Clips and masks large NetCDF datasets to regional datasets based on the boundary. The in_pattern and out_pattern
arguments should be filename patterns (can include path) with the pattern: /path/to/in_netcdf_{variable}.nc.
Example usage: python cut_to_region.py NorthAmerica/NA_{variable}.nc USWest/west_{variable}.nc west.shp
"""
if single and not varname:
print('--varname is required when --single is used')
sys.exit(-1)
if single:
if not os.path.exists(in_pattern):
print('Input file {} does not exist.'.format(in_pattern))
sys.exit(-1)
input_paths = [(in_pattern, varname)]
else:
input_paths = [(in_pattern.format(variable=x), x) for x in VARIABLES]
for path, _ in input_paths:
if not os.path.exists(path):
print('Input file {} does not exist.'.format(path))
sys.exit(-1)
with fiona.open(boundary, 'r') as shp:
features = []
wgs84 = Proj('+init=EPSG:4326')
shp_projection = Proj(shp.crs)
bounds = shp.bounds
ll = transform(shp_projection, wgs84, bounds[0], bounds[1])
ur = transform(shp_projection, wgs84, bounds[2], bounds[3])
bbox = BBox([*ll, *ur], projection=wgs84)
for feature in shp.items():
geometry = transform_geom(shp.crs, {'init': 'EPSG: 4326'},
feature[1]['geometry'])
features.append(geometry)
for in_path, variable in input_paths:
if single:
out_path = out_pattern
else:
out_path = out_pattern.format(variable=variable)
if os.path.exists(out_path):
confirm = input(
"The output file '{}' already exists? Do you with to replace it? [y/n] "
.format(out_path))
if confirm.lower().strip() not in ['y', 'yes']:
print('Exiting...')
sys.exit()
with Dataset(in_path, 'r') as ds:
coords = SpatialCoordinateVariables.from_dataset(
ds, x_name='longitude', y_name='latitude')
x_start, x_stop = coords.x.indices_for_range(bbox.xmin, bbox.xmax)
y_start, y_stop = coords.y.indices_for_range(bbox.ymin, bbox.ymax)
x_slice = slice(x_start, x_stop)
y_slice = slice(y_start, y_stop)
clipped_coords = coords.slice_by_bbox(bbox)
grid = ds.variables[variable][y_slice, x_slice]
if is_masked(grid):
mask = grid.mask.astype('uint8')
else:
mask = numpy.zeros(grid.shape, dtype='uint8')
mask |= rasterize(((x, 0) for x in features),
out_shape=mask.shape,
transform=clipped_coords.affine,
fill=1,
default_value=0)
grid = numpy.ma.masked_where(mask == 1, grid.data)
print('Writing {}...'.format(out_path))
with Dataset(out_path, 'w', format='NETCDF4') as ds:
clipped_coords.add_to_dataset(ds, 'longitude', 'latitude')
data_var = ds.createVariable(variable,
grid.dtype,
dimensions=('latitude', 'longitude'),
fill_value=grid.fill_value)
if data_var.shape != grid.shape:
grid = grid[:data_var.shape[0], :data_var.shape[1]]
data_var[:] = grid
set_crs(ds, variable, Proj('+init=EPSG:4326'))
def get(self, request, *args, **kwargs):
tile_bounds = list(
mercantile.bounds(int(self.kwargs['x']), int(self.kwargs['y']),
int(self.kwargs['z'])))
extent = BBox(tile_bounds, projection=WGS84).project(WEB_MERCATOR)
try:
service_names = request.GET['services'].split(',')
except KeyError:
return HttpResponse(status=400)
grids = []
for service_name in service_names:
self.service = Service.objects.get(name=service_name)
variable = self.service.variable_set.all()[:1].get()
native_extent = extent.project(
pyproj.Proj(str(variable.projection)))
dimensions = self.get_grid_spatial_dimensions(variable)
cell_size = (float(variable.full_extent.width) / dimensions[0],
float(variable.full_extent.height) / dimensions[1])
grid_bounds = [
int(
math.floor(
float(native_extent.xmin - variable.full_extent.xmin) /
cell_size[0])) - 1,
int(
math.floor(
float(native_extent.ymin - variable.full_extent.ymin) /
cell_size[1])) - 1,
int(
math.ceil(
float(native_extent.xmax - variable.full_extent.xmin) /
cell_size[0])) + 1,
int(
math.ceil(
float(native_extent.ymax - variable.full_extent.ymin) /
cell_size[1])) + 1
]
grid_bounds = [
min(max(grid_bounds[0], 0), dimensions[0]),
min(max(grid_bounds[1], 0), dimensions[1]),
min(max(grid_bounds[2], 0), dimensions[0]),
min(max(grid_bounds[3], 0), dimensions[1])
]
if not (grid_bounds[2] - grid_bounds[0]
and grid_bounds[3] - grid_bounds[1]):
continue
grid_extent = BBox(
(variable.full_extent.xmin + grid_bounds[0] * cell_size[0],
variable.full_extent.ymin + grid_bounds[1] * cell_size[1],
variable.full_extent.xmin + grid_bounds[2] * cell_size[0],
variable.full_extent.ymin + grid_bounds[3] * cell_size[1]),
native_extent.projection)
if not self.is_y_increasing(variable):
y_max = dimensions[1] - grid_bounds[1]
y_min = dimensions[1] - grid_bounds[3]
grid_bounds[1] = y_min
grid_bounds[3] = y_max
grids.append(
self.get_grid_for_variable(variable,
x_slice=(grid_bounds[0],
grid_bounds[2]),
y_slice=(grid_bounds[1],
grid_bounds[3])))
self.close_dataset()
intersection = reduce(lambda x, y: x & y, grids)
image = RENDERER.render_image(
intersection,
row_major_order=self.is_row_major(variable)).convert('RGBA')
# If y values are increasing, the rendered image needs to be flipped vertically
if self.is_y_increasing(variable):
image = image.transpose(Image.FLIP_TOP_BOTTOM)
image = GeoImage(image, grid_extent).warp(extent, TILE_SIZE).image
image, content_type = self.format_image(image, 'png')
return self.create_response(request, image, content_type)
def render_tif(filename_pattern, output_directory, renderer_file, save,
renderer_type, colormap, colorspace, palette, scale,
id_variable, lh, legend_breaks, legend_ticks, src_crs, dst_crs,
res, resampling, anchors):
"""
Render single-band GeoTIFF files to images.
colormap is ignored if renderer_file is provided
"""
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='filename_pattern',
param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
if renderer_file is not None and not save:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist',
param='renderer_file',
param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
# if renderer_dict['type'] == 'stretched':
# colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
# if 'min' in colors or 'max' in colors or 'mean' in colors:
# statistics = collect_statistics(filenames, (variable,))[variable]
# for entry in renderer_dict['colors']:
# if isinstance(entry[0], basestring):
# if entry[0] in ('min', 'max', 'mean'):
# entry[0] = statistics[entry[0]]
# elif '*' in entry[0]:
# rel_value, statistic = entry[0].split('*')
# entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
# if palette is not None:
# renderer = _palette_to_stretched_renderer(palette, 'min,max', filenames, variable)
#
# else:
renderer = _colormap_to_stretched_renderer(colormap, colorspace,
filenames)
elif renderer_type == 'unique':
renderer = UniqueValuesRenderer(_parse_colormap(colormap),
colorspace)
else:
raise NotImplementedError('other renderers not yet built')
# if save:
# if not renderer_file:
# raise click.BadParameter('must be provided to save', param='renderer_file', param_hint='renderer_file')
#
# if os.path.exists(renderer_file):
# with open(renderer_file, 'r+') as output_file:
# data = json.loads(output_file.read())
# output_file.seek(0)
# output_file.truncate()
# data[variable] = renderer.serialize()
# output_file.write(json.dumps(data, indent=4))
# else:
# with open(renderer_file, 'w') as output_file:
# output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'streteched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(image_height=lh,
breaks=legend_breaks,
ticks=legend_ticks,
max_precision=2)[0].to_image()
elif renderer_type == 'unique':
legend = renderer.get_legend(image_height=lh)[0].to_image()
legend.save(os.path.join(output_directory, 'legend.png'))
for filename in filenames:
with rasterio.open(filename) as ds:
print('Processing', filename)
filename_root = os.path.split(filename)[1].replace('.nc', '')
data = ds.read(1, masked=True)
# # get transforms, assume last 2 dimensions on variable are spatial in row, col order
# y_dim, x_dim = ds.variables[variable].dimensions[-2:]
# y_len, x_len = data.shape[-2:]
# coords = SpatialCoordinateVariables.from_dataset(ds, x_dim, y_dim)#, projection=Proj(src_crs))
#
# if coords.y.is_ascending_order():
# data = data[::-1]
#
reproject_kwargs = None
if dst_crs is not None:
# TODO: extract this out into a general trefoil reprojection function
ds_crs = ds.crs
if not (src_crs or ds_crs):
raise click.BadParameter(
'must provide src_crs to reproject',
param='src_crs',
param_hint='src_crs')
dst_crs = {'init': dst_crs}
src_crs = ds_crs if ds_crs else {'init': src_crs}
left, bottom, top, right = ds.bounds
dst_affine, dst_width, dst_height = calculate_default_transform(
left, bottom, right, top, ds.width, ds.height, src_crs,
dst_crs)
dst_shape = (dst_height, dst_width)
# proj_bbox = coords.bbox.project(Proj(dst_crs))
#
# x_dif = proj_bbox.xmax - proj_bbox.xmin
# y_dif = proj_bbox.ymax - proj_bbox.ymin
#
# total_len = float(x_len + y_len)
# # Cellsize is dimension weighted average of x and y dimensions per projected pixel, unless otherwise provided
# avg_cellsize = ((x_dif / float(x_len)) * (float(x_len) / total_len)) + ((y_dif / float(y_len)) * (float(y_len) / total_len))
#
# cellsize = res or avg_cellsize
# dst_affine = Affine(cellsize, 0, proj_bbox.xmin, 0, -cellsize, proj_bbox.ymax)
# dst_shape = (
# max(int(ceil((y_dif) / cellsize)), 1), # height
# max(int(ceil(x_dif / cellsize)), 1) # width
# )
# TODO: replace with method in rasterio
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': ds.affine,
'dst_crs': dst_crs,
'dst_transform': dst_affine,
'resampling': getattr(Resampling, resampling),
'dst_shape': dst_shape
}
if anchors:
# Reproject the bbox of the output to WGS84
full_bbox = BBox(
(dst_affine.c, dst_affine.f +
dst_affine.e * dst_shape[0], dst_affine.c +
dst_affine.a * dst_shape[1], dst_affine.f),
projection=Proj(dst_crs))
wgs84_bbox = full_bbox.project(Proj(init='EPSG:4326'))
print('WGS84 Anchors: {0}'.format(
[[wgs84_bbox.ymin, wgs84_bbox.xmin],
[wgs84_bbox.ymax, wgs84_bbox.xmax]]))
elif anchors:
# Reproject the bbox of the output to WGS84
full_bbox = BBox(ds.bounds, projection=Proj(ds.crs))
wgs84_bbox = full_bbox.project(Proj(init='EPSG:4326'))
print('WGS84 Anchors: {0}'.format(
[[wgs84_bbox.ymin, wgs84_bbox.xmin],
[wgs84_bbox.ymax, wgs84_bbox.xmax]]))
image_filename = os.path.join(output_directory,
'{0}.png'.format(filename_root))
render_image(renderer,
data,
image_filename,
scale,
reproject_kwargs=reproject_kwargs)
def execute(self, bounds, zoom, basemap, single_color, kept_colors, gained_colors, species, **kwargs):
historic = kwargs['historic']
futures = kwargs.get('futures', [])
opacity = kwargs.get('opacity', 1.)
map_image = self.get_basemap_image(basemap, bounds, IMAGE_SIZE, zoom)
results_image = self.get_results_image(
bounds, IMAGE_SIZE, single_color, kept_colors, gained_colors, species, historic, futures
)
map_image.paste(Image.blend(map_image, results_image, opacity), (0, 0), results_image)
image_data = BytesIO()
map_image.save(image_data, 'png')
def format_x_coord(x):
return '{}° W'.format(round(abs(x), 2)) if x < 0 else '{}° E'.format(round(x, 2))
def format_y_coord(y):
return '{}° S'.format(round(abs(y), 2)) if y < 0 else '{}° N'.format(round(y, 2))
to_world = image_to_world(BBox(bounds, projection=WGS84).project(WEB_MERCATOR), IMAGE_SIZE)
images_dir = self.get_images_dir()
with open(os.path.join(images_dir, 'scale.png'), 'rb') as f:
scale_data = b64encode(f.read())
scale_bar_x = 38
scale_bar_y = IMAGE_SIZE[1] - 15
scale_bar_start = transform(WEB_MERCATOR, WGS84, *to_world(scale_bar_x, scale_bar_y))
scale_bar_end = transform(WEB_MERCATOR, WGS84, *to_world(scale_bar_x + 96, scale_bar_y))
scale = '{} mi'.format(round(vincenty(reversed(scale_bar_start), reversed(scale_bar_end)).miles, 1))
context = {
'today': now(),
'image_data': b64encode(image_data.getvalue()),
'east': format_x_coord(bounds[0]),
'south': format_y_coord(bounds[1]),
'west': format_x_coord(bounds[2]),
'north': format_y_coord(bounds[3]),
'scale_image_data': scale_data,
'scale': scale,
'single_color': single_color,
'kept_colors': self.get_colors(kept_colors, len(futures)+1),
'gained_colors': self.get_colors(gained_colors, len(futures)+1),
'historic': historic.replace('_', '-'),
'futures': [
{'rcp': RCP_LABELS[rcp], 'year': YEAR_LABELS[year]}
for rcp, year in [x.split('_', 1) for x in futures]
],
'species': SPECIES_LABELS[species],
}
uuid = str(uuid4())
report_dir = os.path.join(settings.MEDIA_ROOT, uuid)
if not os.path.exists(report_dir):
os.makedirs(report_dir)
path = os.path.join(report_dir, 'SPHT Report.pdf')
with open(path, 'wb') as f:
HTML(BytesIO(render_to_string('spht/report.html', context).encode())).write_pdf(f)
return '{}{}/SPHT Report.pdf'.format(settings.MEDIA_URL, uuid)
def test_SpatialCoordinateVariables_bbox():
proj = Proj(init='EPSG:4326')
bbox = BBox((10.5, 5, 110.5, 55), projection=proj)
coords = SpatialCoordinateVariables.from_bbox(bbox, 10, 5)
assert coords.bbox.as_list() == bbox.as_list()
def hydrate_initial_extent(self, bundle):
if bundle.data.get('initial_extent'):
bundle.data['initial_extent'] = BBox(bundle.data['initial_extent'])
return bundle
def test_window_for_bbox():
coords = SpatialCoordinateVariables.from_bbox(BBox([-124, 82, -122, 90], Proj(init='epsg:4326')), 20, 20)
window = coords.get_window_for_bbox(BBox([-123.9, 82.4, -122.1, 89.6]))
assert window.x_slice == slice(1, 19)
assert window.y_slice == slice(1, 19)
def hydrate_full_extent(self, bundle):
if bundle.data.get('full_extent'):
bundle.data['full_extent'] = BBox(bundle.data['full_extent'])
return bundle
def to_netcdf(
files,
output,
variable,
dtype,
src_crs,
x_name,
y_name,
z_name,
datetime_pattern,
netcdf3,
compress,
packed,
xy_dtype,
# z_dtype,
calendar,
autocrop):
"""
Convert rasters to NetCDF and stack them according to a dimension.
X and Y dimension names will be named according to the source projection (lon, lat if geographic projection, x, y
otherwise) unless specified.
Will overwrite an existing NetCDF file.
Only the first band of the input will be turned into a NetCDF file.
"""
# TODO: add format string template to this to parse out components
filenames = list(glob.glob(files))
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='files',
param_hint='FILES')
z_values = []
if datetime_pattern is not None:
datetimes = (datetime.strptime(x, datetime_pattern) for x in filenames)
# Sort both datimes and filenames by datetimes
z_values, filenames = [
list(x)
for x in zip(*sorted(zip(datetimes, filenames), key=itemgetter(0)))
]
items = tuple(enumerate(filenames))
has_z = len(filenames) > 1
if has_z and not z_name:
raise click.BadParameter('Required when > 1 input file',
param='--z',
param_hint='--z')
if src_crs:
src_crs = CRS.from_string(src_crs)
template_ds = rasterio.open(filenames[0])
src_crs = template_ds.crs or src_crs
if not src_crs:
raise click.BadParameter(
'Required when no CRS information available in source files',
param='--src-crs',
param_hint='--src-crs')
prj = Proj(**src_crs.to_dict())
bounds = template_ds.bounds
width = template_ds.width
height = template_ds.height
window = None
src_dtype = numpy.dtype(template_ds.dtypes[0])
dtype = numpy.dtype(dtype) if dtype else src_dtype
if dtype == src_dtype:
fill_value = template_ds.nodata
if src_dtype.kind in ('u', 'i'):
# nodata always comes from rasterio as floating point
fill_value = int(fill_value)
else:
fill_value = get_fill_value(dtype)
x_name = x_name or ('lon' if src_crs.is_geographic else 'x')
y_name = y_name or ('lat' if src_crs.is_geographic else 'y')
var_kwargs = {'fill_value': fill_value}
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
with Dataset(output, 'w', format=format) as out:
if packed or autocrop:
mins = []
maxs = []
windows = []
click.echo('Inspecting input datasets...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True)
if packed:
mins.append(data.min())
maxs.append(data.max())
if autocrop:
data_window = get_data_window(data)
if data_window != ((0, height), (0, width)):
windows.append(data_window)
if packed:
min_value = min(mins)
max_value = max(maxs)
scale, offset = get_pack_atts(dtype, min_value, max_value)
if autocrop and windows:
window = union(windows)
bounds = template_ds.window_bounds(window)
height = window[0][1] - window[0][0]
width = window[1][1] - window[1][0]
coords = SpatialCoordinateVariables.from_bbox(BBox(bounds, prj), width,
height, xy_dtype)
coords.add_to_dataset(out, x_name, y_name, zlib=compress)
var_dimensions = [y_name, x_name]
shape = list(coords.shape)
if has_z:
shape.insert(0, len(filenames))
out.createDimension(z_name, shape[0])
var_dimensions.insert(0, z_name)
if z_values:
dates = DateVariable(numpy.array(z_values),
units_start_date=z_values[0],
calendar=calendar)
dates.add_to_dataset(out, z_name)
click.echo('Creating {0}:{1} with shape {2}'.format(
output, variable, shape))
out_var = out.createVariable(variable,
dtype,
dimensions=var_dimensions,
zlib=compress,
**var_kwargs)
set_crs(out, variable, prj, set_proj4_att=True)
if packed:
out_var.setncattr('scale_factor', scale)
out_var.setncattr('add_offset', offset)
click.echo('Copying data from input files...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True, window=window)
if has_z:
out_var[index, :] = data
else:
out_var[:] = data
out.sync()
def __init__(self,
size,
point,
zoom,
center,
tile_layers,
region,
zone_id,
opacity,
constraint_geometry=None):
self._configure_event_loop()
self.num_tiles = [
math.ceil(size[x] / TILE_SIZE[x]) + 1 for x in (0, 1)
]
center_tile = mercantile.tile(center[1], center[0], zoom)
mercator = Proj(init='epsg:3857')
wgs84 = Proj(init='epsg:4326')
center_tile_bbox = BBox(mercantile.bounds(*center_tile),
projection=wgs84).project(mercator,
edge_points=0)
center_to_image = world_to_image(center_tile_bbox, TILE_SIZE)
center_to_world = image_to_world(center_tile_bbox, TILE_SIZE)
center_point_px = center_to_image(*mercantile.xy(center[1], center[0]))
self.ul_tile = mercantile.tile(*transform(
mercator, wgs84,
*center_to_world(center_point_px[0] -
math.ceil(IMAGE_SIZE[0] / 2), center_point_px[1] -
math.ceil(IMAGE_SIZE[1] / 2)), zoom))
lr_tile = mercantile.Tile(x=min(2**zoom,
self.ul_tile.x + self.num_tiles[0]),
y=min(2**zoom,
self.ul_tile.y + self.num_tiles[1]),
z=zoom)
ul = mercantile.xy(*mercantile.ul(*self.ul_tile))
lr = mercantile.xy(*mercantile.ul(*lr_tile))
self.image_bbox = BBox((ul[0], lr[1], lr[0], ul[1]))
self.image_size = (TILE_SIZE[0] * self.num_tiles[0],
TILE_SIZE[1] * self.num_tiles[1])
self.to_image = world_to_image(self.image_bbox, self.image_size)
self.to_world = image_to_world(self.image_bbox, self.image_size)
self.point_px = [
round(x) for x in self.to_image(*mercantile.xy(*point))
]
self.center_point_px = self.to_image(
*mercantile.xy(center[1], center[0]))
self.target_size = size
self.point = point
self.zoom = zoom
self.center = center
self.tile_layers = tile_layers
self.region = region
self.zone_id = zone_id
self.opacity = opacity
self.constraint_geometry = constraint_geometry
def handle(self, region_name, *args, **options):
name = region_name[0]
from django.conf import settings
BASE_DIR = settings.NC_SERVICE_DATA_ROOT
# determine extent and lat/lon variable names from DEM
dem_path = os.path.join(BASE_DIR, "regions", name, "{}_dem.nc".format(name))
with Dataset(dem_path, "r") as ds:
dims = ds.dimensions.keys()
lat = "lat" if "lat" in dims else "latitude"
lon = "lon" if "lon" in dims else "longitude"
l = float(ds.variables[lon][:].min())
b = float(ds.variables[lat][:].min())
r = float(ds.variables[lon][:].max())
t = float(ds.variables[lat][:].max())
extent = BBox((l, b, r, t), projection=pyproj.Proj(WGS84))
# Generate DEM service
with transaction.atomic():
print("Adding {}".format(name))
print("---")
print("elevation")
service_name = "{}_dem".format(name)
if Service.objects.filter(name__iexact=service_name).exists():
print("{} already exists, skipping.".format(service_name))
else:
dem_service = Service.objects.create(
name=service_name,
data_path="regions/{name}/{name}_dem.nc".format(name=name),
projection=WGS84,
full_extent=extent,
initial_extent=extent,
)
with Dataset(dem_path, "r") as ds:
v_min = numpy.nanmin(ds.variables["elevation"][:]).item()
v_max = numpy.nanmax(ds.variables["elevation"][:]).item()
renderer = StretchedRenderer([(v_min, Color(0, 0, 0)), (v_max, Color(255, 255, 255))])
Variable.objects.create(
service=dem_service,
index=0,
variable="elevation",
projection=WGS84,
x_dimension=lon,
y_dimension=lat,
name="elevation",
renderer=renderer,
full_extent=extent,
)
# Generate ClimateNA services
with transaction.atomic():
for year in PERIODS:
print("")
print(year)
print("---")
for var in VARS:
print(var)
service_name = "{}_{}Y_{}".format(name, year, var)
if not Service.objects.filter(name__iexact=service_name).exists():
data_path = "regions/{name}/{year}Y/{name}_{year}Y_{var}.nc".format(
name=name, year=year, var=var
)
if not os.path.exists(os.path.join(BASE_DIR, data_path)):
print("{} does not exist, skipping.".format(service_name))
continue
service = Service.objects.create(
name=service_name,
data_path=data_path,
projection=WGS84,
full_extent=extent,
initial_extent=extent,
)
with Dataset(os.path.join(BASE_DIR, service.data_path), "r") as ds:
dims = ds.dimensions.keys()
lat = "lat" if "lat" in dims else "latitude"
lon = "lon" if "lon" in dims else "longitude"
v_min = numpy.nanmin(ds.variables[var][:]).item()
v_max = numpy.nanmax(ds.variables[var][:]).item()
renderer = StretchedRenderer([(v_min, Color(0, 0, 0)), (v_max, Color(255, 255, 255))])
variable = Variable.objects.create(
service=service,
index=0,
variable=var,
projection=WGS84,
x_dimension=lon,
y_dimension=lat,
name=var,
renderer=renderer,
full_extent=extent,
)
else:
print("{} already exists, skipping.".format(service_name))