def construct_raster(self, z, xmin, xmax, ymin, ymax):
"""
Create an empty tif raster file on disk using the input tile range. The
new raster aligns with the xyz tile scheme and can be filled
sequentially with raster algebra results.
"""
# Compute bounds and scale to construct raster.
bounds = []
for x in range(xmin, xmax + 1):
for y in range(ymin, ymax + 1):
bounds.append(tile_bounds(x, y, z))
bounds = [
min([bnd[0] for bnd in bounds]),
min([bnd[1] for bnd in bounds]),
max([bnd[2] for bnd in bounds]),
max([bnd[3] for bnd in bounds]),
]
scale = tile_scale(z)
# Create tempfile.
raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
self.exportfile = NamedTemporaryFile(dir=raster_workdir, suffix='.tif')
# Instantiate raster using the tempfile path.
return GDALRaster({
'srid': WEB_MERCATOR_SRID,
'width': (xmax - xmin + 1) * WEB_MERCATOR_TILESIZE,
'height': (ymax - ymin + 1) * WEB_MERCATOR_TILESIZE,
'scale': (scale, -scale),
'origin': (bounds[0], bounds[3]),
'driver': 'tif',
'bands': [{'data': [0], 'nodata_value': 0}],
'name': self.exportfile.name,
'datatype': ALGEBRA_PIXEL_TYPE_GDAL,
})
def test_vector_tile_endpoint_json(self):
# Get url for a tile.
self.url = reverse('vectortiles-list',
kwargs={
'aggregationlayer': self.agglayer.id,
'z': 11,
'x': 552,
'y': 859,
'frmt': 'json'
})
# Setup request with fromula that will multiply the rasterlayer by itself
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
bounds = tile_bounds(552, 859, 11)
bounds = OGRGeometry.from_bbox(bounds)
bounds.srid = WEB_MERCATOR_SRID
result = json.loads(response.content.decode())
self.assertEqual(
'St Petersburg',
result['features'][0]['properties']['name'],
)
self.assertEqual(
'Coverall',
result['features'][1]['properties']['name'],
)
self.assertEqual(
[-9220429.22801057, 3228174.60948196],
result['features'][0]['geometry']['coordinates'][0][0][0],
)
def construct_raster(self, z, xmin, xmax, ymin, ymax):
"""
Create an empty tif raster file on disk using the input tile range. The
new raster aligns with the xyz tile scheme and can be filled
sequentially with raster algebra results.
"""
# Compute bounds and scale to construct raster.
bounds = []
for x in range(xmin, xmax + 1):
for y in range(ymin, ymax + 1):
bounds.append(tile_bounds(x, y, z))
bounds = [
min([bnd[0] for bnd in bounds]),
min([bnd[1] for bnd in bounds]),
max([bnd[2] for bnd in bounds]),
max([bnd[3] for bnd in bounds]),
]
scale = tile_scale(z)
# Create tempfile.
raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
self.exportfile = NamedTemporaryFile(dir=raster_workdir, suffix='.tif')
# Instantiate raster using the tempfile path.
return GDALRaster({
'srid': WEB_MERCATOR_SRID,
'width': (xmax - xmin + 1) * WEB_MERCATOR_TILESIZE,
'height': (ymax - ymin + 1) * WEB_MERCATOR_TILESIZE,
'scale': (scale, -scale),
'origin': (bounds[0], bounds[3]),
'driver': 'tif',
'bands': [{'data': [0], 'nodata_value': 0}],
'name': self.exportfile.name,
'datatype': ALGEBRA_PIXEL_TYPE_GDAL,
})
def test_tile_index_range(self):
bounds = tile_bounds(43, 67, 8)
geom = OGRGeometry.from_bbox(bounds)
# With the default tolerance 0, the edging tiles are
# included.
idx = tile_index_range(geom.extent, 11)
self.assertEqual(idx[2] - idx[0], 2**3)
self.assertEqual(idx[3] - idx[1], 2**3)
# With a larger tolerance, the strictly overlaping tiles are included.
idx = tile_index_range(geom.extent, 11, tolerance=1e-3)
self.assertEqual(idx[2] - idx[0], 2**3 - 1)
self.assertEqual(idx[3] - idx[1], 2**3 - 1)
def test_tile_index_range(self):
bounds = tile_bounds(43, 67, 8)
geom = OGRGeometry.from_bbox(bounds)
# With the default tolerance 0, the edging tiles are
# included.
idx = tile_index_range(geom.extent, 11)
self.assertEqual(idx[2] - idx[0], 2 ** 3)
self.assertEqual(idx[3] - idx[1], 2 ** 3)
# With a larger tolerance, the strictly overlaping tiles are included.
idx = tile_index_range(geom.extent, 11, tolerance=1e-3)
self.assertEqual(idx[2] - idx[0], 2 ** 3 - 1)
self.assertEqual(idx[3] - idx[1], 2 ** 3 - 1)
def list(self, request, aggregationlayer, x, y, z, frmt, *args, **kwargs):
# Select which agglayer to use for this tile.
lyr = get_object_or_404(AggregationLayer, pk=aggregationlayer)
# Compute tile boundary coorner coordinates.
bounds_coords = tile_bounds(int(x), int(y), int(z))
# Create a geometry with a 1% buffer around the tile. This buffered
# tile boundary will be used for clipping the geometry. The overflow
# will visually dissolve the polygons on the frontend visualization.
bounds = OGRGeometry.from_bbox(bounds_coords)
bounds.srid = WEB_MERCATOR_SRID
bounds = bounds.geos
bounds_buffer = bounds.buffer(
(bounds_coords[2] - bounds_coords[0]) / 100)
# Get the intersection of the aggregation areas and the tile boundary.
# use buffer to clip the aggregation area.
result = AggregationArea.objects.filter(
aggregationlayer=lyr,
geom__intersects=bounds,
).annotate(intersection=Intersection('geom', bounds_buffer)).only(
'id', 'name')
# Render intersection as vector tile in two different available formats.
if frmt == 'json':
result = [
'{{"geometry": {0}, "properties": {{"id": {1}, "name": "{2}"}}}}'
.format(dat.intersection.geojson, dat.id, dat.name)
for dat in result
]
result = ','.join(result)
result = '{"type": "FeatureCollection","features":[' + result + ']}'
return HttpResponse(result, content_type="application/json")
elif frmt == 'pbf':
features = [{
"geometry": bytes(dat.intersection.wkb),
"properties": {
"id": dat.id,
"name": dat.name,
"attributes": dat.attributes,
},
} for dat in result]
data = [
{
"name": lyr.name,
"features": features,
},
]
vtile = mapbox_vector_tile.encode(data,
quantize_bounds=bounds_coords)
return HttpResponse(vtile, content_type='application/x-protobuf')
def list(self, request, aggregationlayer, x, y, z, frmt, *args, **kwargs):
# Select which agglayer to use for this tile.
lyr = get_object_or_404(AggregationLayer, pk=aggregationlayer)
# Compute tile boundary coorner coordinates.
bounds_coords = tile_bounds(int(x), int(y), int(z))
# Create a geometry with a 1% buffer around the tile. This buffered
# tile boundary will be used for clipping the geometry. The overflow
# will visually dissolve the polygons on the frontend visualization.
bounds = OGRGeometry.from_bbox(bounds_coords)
bounds.srid = WEB_MERCATOR_SRID
bounds = bounds.geos
bounds_buffer = bounds.buffer((bounds_coords[2] - bounds_coords[0]) / 100)
# Get the intersection of the aggregation areas and the tile boundary.
# use buffer to clip the aggregation area.
result = AggregationArea.objects.filter(
aggregationlayer=lyr,
geom__intersects=bounds,
).annotate(
intersection=Intersection('geom', bounds_buffer)
).only('id', 'name')
# Render intersection as vector tile in two different available formats.
if frmt == 'json':
result = ['{{"geometry": {0}, "properties": {{"id": {1}, "name": "{2}"}}}}'.format(dat.intersection.geojson, dat.id, dat.name) for dat in result]
result = ','.join(result)
result = '{"type": "FeatureCollection","features":[' + result + ']}'
return HttpResponse(result, content_type="application/json")
elif frmt == 'pbf':
features = [
{
"geometry": bytes(dat.intersection.wkb),
"properties": {
"id": dat.id,
"name": dat.name,
"attributes": dat.attributes,
},
} for dat in result
]
data = [
{
"name": lyr.name,
"features": features,
},
]
vtile = mapbox_vector_tile.encode(data, quantize_bounds=bounds_coords)
return HttpResponse(vtile, content_type='application/x-protobuf')
def get_raster_tile(layer_id, tilez, tilex, tiley):
"""
Get the raster from a tile for further processing. If the requested tile
does not exists in the database, higher level tiles are searched. If a
higher level tile is found, it is warped to the requested zoom level. This
ensures that a tile can be requested at any zoom level.
"""
# Loop through zoom levels to search for a tile
for zoom in range(tilez, -1, -1):
# Compute multiplier to find parent raster
multiplier = 2**(tilez - zoom)
# Fetch tile
tile = RasterTile.objects.filter(
tilex=tilex / multiplier,
tiley=tiley / multiplier,
tilez=zoom,
rasterlayer_id=layer_id,
)
if tile.exists():
# Extract raster from tile model
result = tile[0].rast
# If the tile is a parent of the original, warp it to the
# original request tile.
if zoom < tilez:
# Compute bounds, scale and size of child tile
bounds = tile_bounds(tilex, tiley, tilez)
tilesize = int(
getattr(settings, 'RASTER_TILESIZE',
WEB_MERCATOR_TILESIZE))
tilescale = tile_scale(tilez)
# Warp parent tile to child tile in memory.
result = result.warp({
'driver': 'MEM',
'width': tilesize,
'height': tilesize,
'scale': [tilescale, -tilescale],
'origin': [bounds[0], bounds[3]],
})
return result
def get_raster_tile(layer_id, tilez, tilex, tiley):
"""
Get the raster from a tile for further processing. If the requested tile
does not exists in the database, higher level tiles are searched. If a
higher level tile is found, it is warped to the requested zoom level. This
ensures that a tile can be requested at any zoom level.
"""
# Loop through zoom levels to search for a tile
for zoom in range(tilez, -1, -1):
# Compute multiplier to find parent raster
multiplier = 2 ** (tilez - zoom)
# Fetch tile
tile = RasterTile.objects.filter(
tilex=tilex / multiplier,
tiley=tiley / multiplier,
tilez=zoom,
rasterlayer_id=layer_id,
)
if tile.exists():
# Extract raster from tile model
result = tile[0].rast
# If the tile is a parent of the original, warp it to the
# original request tile.
if zoom < tilez:
# Compute bounds, scale and size of child tile
bounds = tile_bounds(tilex, tiley, tilez)
tilesize = int(getattr(settings, 'RASTER_TILESIZE', WEB_MERCATOR_TILESIZE))
tilescale = tile_scale(tilez)
# Warp parent tile to child tile in memory.
result = result.warp({
'driver': 'MEM',
'width': tilesize,
'height': tilesize,
'scale': [tilescale, -tilescale],
'origin': [bounds[0], bounds[3]],
})
return result
def test_vector_tile_endpoint_json(self):
# Get url for a tile.
self.url = reverse('vectortiles-list', kwargs={'aggregationlayer': self.agglayer.id, 'z': 11, 'x': 552, 'y': 859, 'frmt': 'json'})
# Setup request with fromula that will multiply the rasterlayer by itself
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
bounds = tile_bounds(552, 859, 11)
bounds = OGRGeometry.from_bbox(bounds)
bounds.srid = WEB_MERCATOR_SRID
result = json.loads(response.content.decode())
self.assertEqual(
'St Petersburg',
result['features'][0]['properties']['name'],
)
self.assertEqual(
'Coverall',
result['features'][1]['properties']['name'],
)
self.assertEqual(
[-9220429.22801057, 3228174.60948196],
result['features'][0]['geometry']['coordinates'][0][0][0],
)
def process_quadrant(self, indexrange, zoom):
"""
Create raster tiles for a quadrant of tiles defined by a x-y-z index
range and a zoom level.
"""
# TODO Use a standalone celery task for this method in order to
# gain speedup from parallelism.
self._quadrant_count += 1
self.log(
'Starting tile creation for quadrant {0} at zoom level {1}'.format(
self._quadrant_count, zoom),
status=self.rasterlayer.parsestatus.CREATING_TILES)
# Compute scale of tiles for this zoomlevel
tilescale = utils.tile_scale(zoom)
# Compute quadrant bounds and create destination file
bounds = utils.tile_bounds(indexrange[0], indexrange[1], zoom)
dest_file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.tif')
# Snap dataset to the quadrant
snapped_dataset = self.dataset.warp({
'name':
dest_file.name,
'origin': [bounds[0], bounds[3]],
'scale': [tilescale, -tilescale],
'width': (indexrange[2] - indexrange[0] + 1) * self.tilesize,
'height': (indexrange[3] - indexrange[1] + 1) * self.tilesize,
})
# Create all tiles in this quadrant in batches
batch = []
for tilex in range(indexrange[0], indexrange[2] + 1):
for tiley in range(indexrange[1], indexrange[3] + 1):
# Calculate raster tile origin
bounds = utils.tile_bounds(tilex, tiley, zoom)
# Construct band data arrays
pixeloffset = ((tilex - indexrange[0]) * self.tilesize,
(tiley - indexrange[1]) * self.tilesize)
band_data = [{
'data':
band.data(offset=pixeloffset,
size=(self.tilesize, self.tilesize)),
'nodata_value':
band.nodata_value
} for band in snapped_dataset.bands]
# Ignore tile if its only nodata.
if all([
numpy.all(dat['data'] == dat['nodata_value'])
for dat in band_data
]):
continue
# Add tile data to histogram
if zoom == self.max_zoom:
self.push_histogram(band_data)
# Warp source raster into this tile (in memory)
dest = GDALRaster({
'width':
self.tilesize,
'height':
self.tilesize,
'origin': [bounds[0], bounds[3]],
'scale': [tilescale, -tilescale],
'srid':
WEB_MERCATOR_SRID,
'datatype':
snapped_dataset.bands[0].datatype(),
'bands':
band_data,
})
# Store tile in batch array
batch.append(
RasterTile(rast=dest,
rasterlayer_id=self.rasterlayer.id,
tilex=tilex,
tiley=tiley,
tilez=zoom))
# Commit batch to database and reset it
if len(batch) == self.batch_step_size:
RasterTile.objects.bulk_create(batch)
batch = []
# Commit remaining objects
if len(batch):
RasterTile.objects.bulk_create(batch)
def process_quadrant(self, indexrange, zoom):
"""
Create raster tiles for a quadrant of tiles defined by a x-y-z index
range and a zoom level.
"""
self._quadrant_count += 1
self.log(
'Starting tile creation for quadrant {0} at zoom level {1}'.format(self._quadrant_count, zoom),
status=self.rasterlayer.parsestatus.CREATING_TILES
)
# Compute scale of tiles for this zoomlevel
tilescale = utils.tile_scale(zoom)
# Compute quadrant bounds and create destination file
bounds = utils.tile_bounds(indexrange[0], indexrange[1], zoom)
dest_file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.tif')
# Snap dataset to the quadrant
snapped_dataset = self.dataset.warp({
'name': dest_file.name,
'origin': [bounds[0], bounds[3]],
'scale': [tilescale, -tilescale],
'width': (indexrange[2] - indexrange[0] + 1) * self.tilesize,
'height': (indexrange[3] - indexrange[1] + 1) * self.tilesize,
})
# Create all tiles in this quadrant in batches
batch = []
for tilex in range(indexrange[0], indexrange[2] + 1):
for tiley in range(indexrange[1], indexrange[3] + 1):
# Calculate raster tile origin
bounds = utils.tile_bounds(tilex, tiley, zoom)
# Construct band data arrays
pixeloffset = (
(tilex - indexrange[0]) * self.tilesize,
(tiley - indexrange[1]) * self.tilesize
)
band_data = [
{
'data': band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)),
'nodata_value': band.nodata_value
} for band in snapped_dataset.bands
]
# Ignore tile if its only nodata.
if all([numpy.all(dat['data'] == dat['nodata_value']) for dat in band_data]):
continue
# Add tile data to histogram
if zoom == self.max_zoom:
self.push_histogram(band_data)
# Warp source raster into this tile (in memory)
dest = GDALRaster({
'width': self.tilesize,
'height': self.tilesize,
'origin': [bounds[0], bounds[3]],
'scale': [tilescale, -tilescale],
'srid': WEB_MERCATOR_SRID,
'datatype': snapped_dataset.bands[0].datatype(),
'bands': band_data,
})
# Store tile in batch array
batch.append(
RasterTile(
rast=dest,
rasterlayer_id=self.rasterlayer.id,
tilex=tilex,
tiley=tiley,
tilez=zoom
)
)
# Commit batch to database and reset it
if len(batch) == BATCH_STEP_SIZE:
RasterTile.objects.bulk_create(batch)
batch = []
# Commit remaining objects
if len(batch):
RasterTile.objects.bulk_create(batch)
