def union(inputtiles, parsenames):
tiles = sutils.tile_parser(inputtiles, parsenames)
xmin, xmax, ymin, ymax = sutils.get_range(tiles)
zoom = sutils.get_zoom(tiles)
# make an array of shape (xrange + 3, yrange + 3)
burn = sutils.burnXYZs(tiles, xmin, xmax, ymin, ymax, 0)
nw = mercantile.xy(*mercantile.ul(xmin, ymin, zoom))
se = mercantile.xy(*mercantile.ul(xmax + 1, ymax + 1, zoom))
aff = Affine(((se[0] - nw[0]) / float(xmax - xmin + 1)), 0.0, nw[0],
0.0, -((nw[1] - se[1]) / float(ymax - ymin + 1)), nw[1])
unprojecter = sutils.Unprojecter()
unionedTiles = [
{
'geometry': unprojecter.unproject(feature),
'properties': {},
'type': 'Feature'
} for feature, shapes in features.shapes(np.asarray(np.flipud(np.rot90(burn)).astype(np.uint8), order='C'), transform=aff) if shapes == 1
]
return unionedTiles
def get_tile(tx, ty, zoom):
print tx, ty, zoom
bounds = mercantile.bounds(tx, ty, zoom)
print bounds
lat1 = int(bounds.north)
lat2 = int(bounds.south)
lon1 = int(bounds.west)
lon2 = int(bounds.east)
tile = np.zeros((256, 256))
tile_base = mercantile.xy(*mercantile.ul(tx, ty, zoom))
print tile_base
for lat in range(lat2, lat1 + 1):
for lon in range(lon1, lon2 + 1):
key = "meshmap:block:%d:%d" % (lat, lon)
block = np.frombuffer(redis.get(key), dtype=np.uint8)
block = block.reshape((120, 80))
blat1 = min(lat + 1, bounds.north)
blat2 = max(lat, bounds.south)
blon1 = max(lon, bounds.west)
blon2 = min(lon + 1, bounds.east)
print blat1, blat2, blon1, blon2
p1 = mercantile.xy(blon1, blat1)
p2 = mercantile.xy(blon2, blat2)
print p1, p2
print p1[0] - tile_base[0], p1[1] - tile_base[1]
def make_transform(tilerange, zoom):
ulx, uly = mercantile.xy(*mercantile.ul(tilerange['x']['min'], tilerange['y']['min'], zoom))
lrx, lry = mercantile.xy(*mercantile.ul(tilerange['x']['max'], tilerange['y']['max'], zoom))
xcell = (lrx - ulx) / float(tilerange['x']['max'] - tilerange['x']['min'])
ycell = (uly - lry) / float(tilerange['y']['max'] - tilerange['y']['min'])
return Affine(xcell, 0, ulx,
0, -ycell, uly)
def project_geom(geom):
coordinates = []
# zhangqi 新增代码 替换
# return {
# 'type': geom['type'],
# 'coordinates': [
# [mercantile.xy(*coords) for coords in part]
# for part in geom['coordinates']
# ]
# }
if geom['type'] == 'MultiPolygon':
coordinates = []
for part in geom['coordinates']:
tmp = []
for p in part:
tmp.append([mercantile.xy(*coords) for coords in p])
coordinates.append(tmp)
return {'type': geom['type'], 'coordinates': coordinates}
else:
return {
'type':
geom['type'],
'coordinates': [[mercantile.xy(*coords) for coords in part]
for part in geom['coordinates']]
}
def make_src_meta(bounds, size, creation_opts={}):
"""
Create metadata for output tiles
"""
ul = merc.xy(bounds.west, bounds.north)
lr = merc.xy(bounds.east, bounds.south)
aff = make_affine(size, size, ul, lr)
## default values
src_meta = {
'driver': 'GTiff',
'height': size,
'width': size,
'count': 4,
'dtype': np.uint8,
'affine': aff,
"crs": 'EPSG:3857',
'compress': 'JPEG',
'tiled': True,
'blockxsize': 256,
'blockysize': 256
}
for c in creation_opts.keys():
src_meta[c] = creation_opts[c]
return src_meta
def make_src_meta(bounds, size, creation_opts={}):
"""
Create metadata for output tiles
"""
ul = merc.xy(bounds.west, bounds.north)
lr = merc.xy(bounds.east, bounds.south)
aff = make_affine(size, size, ul, lr)
## default values
src_meta = {
'driver': 'GTiff',
'height': size,
'width': size,
'count': 4,
'dtype': np.uint8,
'affine': aff,
"crs": 'EPSG:3857',
'compress': 'JPEG',
'tiled': True,
'blockxsize': 256,
'blockysize': 256
}
for c in creation_opts.keys():
src_meta[c] = creation_opts[c]
return src_meta
def make_transform(tilerange, zoom):
ulx, uly = mercantile.xy(*mercantile.ul(tilerange['x']['min'], tilerange['y']['min'], zoom))
lrx, lry = mercantile.xy(*mercantile.ul(tilerange['x']['max'], tilerange['y']['max'], zoom))
xcell = (lrx - ulx) / float(tilerange['x']['max'] - tilerange['x']['min'])
ycell = (uly - lry) / float(tilerange['y']['max'] - tilerange['y']['min'])
return Affine(xcell, 0, ulx,
0, -ycell, uly)
async def serve(cls, request):
# TODO: develop and test some 'request' plugin hooks
endpoint = request.path.split('.')[-1]
path_args = request.match_info
request_hook_response = Configs.plugins.hook('request', endpoint=endpoint, request=request, **path_args)
# if a request_hook_response is received, return and skip the regular processing:
if request_hook_response:
return request_hook_response
# fetch query parameters from request info
x = int(request.match_info['x'])
y = int(request.match_info['y'])
zoom = int(request.match_info['z'])
layers = request.match_info['layers']
try:
recipe_name = request.match_info['recipe']
except KeyError:
recipe_name = 'default_recipe'
# check that recipe exists
if recipe_name not in Configs.recipes.keys():
logger.error('Recipe {0} not found in recipes'.format(recipe_name))
return aiohttp.errors.HttpBadRequest('Recipe {0} not found in recipes'.format(recipe_name))
# fetch recipe
recipe = Configs.recipes[recipe_name]
# extrapolate the layers
if layers == 'all':
layers = list(recipe.layers.keys())
else:
layers = layers.split('+')
# compute bounds and extents
bounds = mercantile.bounds(x, y, zoom)
west, south = mercantile.xy(bounds.west, bounds.south)
east, north = mercantile.xy(bounds.east, bounds.north)
# process the layers
layer_data = []
for layer_name in layers:
if layer_name not in recipe.layers.keys():
logger.error('Layer {0} not found in recipe {1}'.format(layer_name, recipe_name))
return aiohttp.errors.HttpBadRequest('Layer {0} not found in layer config file'.format(layer_name))
else:
layer = recipe.layers[layer_name]
layer_data.append(await cls.query_layer(layer, zoom, west, south, east, north))
content_type, body = cls.post_process(layer_data)
response = Response(content_type=content_type, body=body, headers=generic_headers)
# TODO: develop and test some 'request' plugin hooks
response_hook_response = Configs.plugins.hook('request', response=response, request=request)
# if a response_hook_response is received, it overrides the regular response:
if response_hook_response:
return response_hook_response
else:
return response
def _write((tile, data)):
if not contains_data((tile, data)):
return
print("Writing", tile)
# Get the bounds of the tile.
ulx, uly = mercantile.xy(
*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(
*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
# TODO constantize
tmp_path = "/vsimem/tile"
# create GeoTIFF
meta = creation_options.copy()
meta["count"] = 1
meta["nodata"] = data.fill_value
meta["dtype"] = data.dtype
meta["width"] = CHUNK_SIZE
meta["height"] = CHUNK_SIZE
meta["transform"] = from_bounds(ulx, lry, lrx, uly, CHUNK_SIZE, CHUNK_SIZE)
with rasterio.drivers():
with rasterio.open(tmp_path, "w", **meta) as tmp:
tmp.write(data, 1)
# write out
output_uri = urlparse(out_dir)
contents = bytearray(virtual_file_to_buffer(tmp_path))
if output_uri.scheme == "s3":
# TODO use mapPartitions so that the client only needs to be
# instantiated once per partition
client = boto3.client("s3")
bucket = output_uri.netloc
# TODO strip out trailing slashes on the path if necessary
key = "%s/%d/%d/%d.tif" % (output_uri.path[1:], tile.z, tile.x, tile.y)
response = client.put_object(
ACL="public-read",
Body=bytes(contents),
Bucket=bucket,
# CacheControl="TODO",
ContentType="image/tiff",
Key=key
)
else:
output_path = os.path.join(out_dir, "%d/%d/%d.tif" % (tile.z, tile.x, tile.y))
mkdir_p(os.path.dirname(output_path))
f = open(output_path, "w")
f.write(contents)
f.close()
def test_xy():
ul = mercantile.ul(486, 332, 10)
xy = mercantile.xy(*ul)
expected = (-1017529.7205322663, 7044436.526761846)
for a, b in zip(expected, xy):
assert round(a - b, 7) == 0
xy = mercantile.xy(0.0, 0.0)
expected = (0.0, 0.0)
for a, b in zip(expected, xy):
assert round(a - b, 7) == 0
def make_transform(tilerange, zoom):
ulx, uly = mercantile.xy(
*mercantile.ul(tilerange["x"]["min"], tilerange["y"]["min"], zoom)
)
lrx, lry = mercantile.xy(
*mercantile.ul(tilerange["x"]["max"], tilerange["y"]["max"], zoom)
)
xcell = (lrx - ulx) / float(tilerange["x"]["max"] - tilerange["x"]["min"])
ycell = (uly - lry) / float(tilerange["y"]["max"] - tilerange["y"]["min"])
return Affine(xcell, 0, ulx, 0, -ycell, uly)
def test_xy():
ul = mercantile.ul(486, 332, 10)
xy = mercantile.xy(*ul)
expected = (-1017529.7205322663, 7044436.526761846)
for a, b in zip(expected, xy):
assert round(a-b, 7) == 0
xy = mercantile.xy(0.0, 0.0)
expected = (0.0, 0.0)
for a, b in zip(expected, xy):
assert round(a-b, 7) == 0
def test__sm2ll():
w, s, e, n = (-106.6495132446289, 25.845197677612305, -93.50721740722656,
36.49387741088867)
minX, minY = ctx.tile._sm2ll(w, s)
maxX, maxY = ctx.tile._sm2ll(e, n)
nw, ns = mt.xy(minX, minY)
ne, nn = mt.xy(maxX, maxY)
assert round(nw - w, TOL) == 0
assert round(ns - s, TOL) == 0
assert round(ne - e, TOL) == 0
assert round(nn - n, TOL) == 0
def __init__(self, center, bbox, zoom, basemap, opacity, size=None):
self._configure_event_loop()
point = center['geometry']['coordinates']
if size is None:
size = IMAGE_SIZE
self.num_tiles = [
math.ceil(size[x] / TILE_SIZE[x]) + 1 for x in (0, 1)
]
center_tile = mercantile.tile(point[0], point[1], zoom)
mercator = Proj(init='epsg:3857')
wgs84 = Proj(init='epsg:4326')
center_tile_bbox = BBox(mercantile.bounds(*center_tile),
projection=wgs84).project(mercator,
edge_points=0)
center_to_image = world_to_image(center_tile_bbox, TILE_SIZE)
center_to_world = image_to_world(center_tile_bbox, TILE_SIZE)
center_point_px = center_to_image(*mercantile.xy(*point))
self.ul_tile = mercantile.tile(*transform(
mercator, wgs84,
*center_to_world(center_point_px[0] -
math.ceil(IMAGE_SIZE[0] / 2), center_point_px[1] -
math.ceil(IMAGE_SIZE[1] / 2)), zoom))
lr_tile = mercantile.Tile(x=min(2**zoom,
self.ul_tile.x + self.num_tiles[0]),
y=min(2**zoom,
self.ul_tile.y + self.num_tiles[1]),
z=zoom)
ul = mercantile.xy(*mercantile.ul(*self.ul_tile))
lr = mercantile.xy(*mercantile.ul(*lr_tile))
self.image_bbox = BBox((ul[0], lr[1], lr[0], ul[1]))
self.image_size = (TILE_SIZE[0] * self.num_tiles[0],
TILE_SIZE[1] * self.num_tiles[1])
self.to_image = world_to_image(self.image_bbox, self.image_size)
self.to_world = image_to_world(self.image_bbox, self.image_size)
self.point_px = [
round(x) for x in self.to_image(*mercantile.xy(*point))
]
self.target_size = size
self.point = point
self.zoom = zoom
self.basemap = basemap
self._basemap_image = None
self.opacity = opacity
def vecttile_mask(tile, save_mask=True):
bounds = mercantile.bounds(tile.x, tile.y, tile.z)
ll = mercantile.xy(bounds.west, bounds.south)
ur = mercantile.xy(bounds.east, bounds.north)
extent = mapnik.Box2d(*(ll + ur))
m = mapnik.Map(256, 256)
m.srs = "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over"
s = mapnik.Style()
r = mapnik.Rule()
road_width = 75 # meters
pixel_size = mercantile.pixel_size(tile.z, bounds.north)
pixels = int(road_width/pixel_size) + 1
line_symbolizer = mapnik.LineSymbolizer(mapnik.Color('black'), pixels)
r.symbols.append(line_symbolizer)
s.rules.append(r)
m.append_style('RoadStyle', s)
layer = mapnik.Layer('Roads from PostGIS')
layer.srs = "+init=epsg:3857"
ROADS = """
(SELECT way
FROM planet_osm_line
WHERE roadbiking = TRUE) linestring
"""
layer.datasource = mapnik.PostGIS(host='localhost', user='******', password='******',
dbname='osm_uswest', table=ROADS, geometry_field='way')
layer.styles.append('RoadStyle')
m.layers.append(layer)
m.zoom_to_box(extent)
img = mapnik.Image(m.width, m.height)
mapnik.render(m, img)
imgpath = tile_path(os.path.join(tiledir, 'mask'), tile, 'png')
prep_dirs(imgpath)
if save_mask:
img.save(imgpath)
imgdata = np.frombuffer(img.tostring(), dtype=np.uint8).reshape((256, 256, 4))
# use alpha channel to get 0->1 scale for mask
alpha = imgdata[:,:,3] / 255.0
# invert for easy multiplication w/ strava alpha channel
# 1 = no road, keep strava heatmap values)
# 0 = road, make strava heatmap transparent
mask = 1 - alpha
return mask
def create_vectortile_sql(layer, bounds):
# Create extent
west, south = xy(bounds.west, bounds.south)
east, north = xy(bounds.east, bounds.north)
extent = "ST_MakeBox2D(ST_MakePoint({west}, {south}), ST_MakePoint({east}, {north}))".format(west=west, south=south, east=east, north=north)
# e.g. aus_census_2011_shapes.sa1
geom_table_name = "{schema_name}.{geometry_name}".format(geometry_name=layer["geometry"], schema_name=layer["schema"])
# e.g. aus_census_2011_shapes.sa1.geom_3857
geom_column_definition = "{}.geom_3857".format(geom_table_name)
# Replace the compiled geometry column definition with the zoom-level dependent version
# e.g. Replace "ST_AsEWKB(aus_census_2011_shapes.sa1.geom_3857)" with "ST_AsMVTGeom(ST_Simplify(aus_census_2011_shapes.sa1.geom_3857, TOLERANCE), EXTENT_OF_TILE)"
# Zoom 15 is our highest resolution (configured in OpenLayers), so we need to grab
# unsimplified geometries to allow us to re-use them as the user zooms in.
if z == 15:
data_query = layer["_postgis_query"].replace(
"ST_AsEWKB({})".format(geom_column_definition),
"ST_AsMVTGeom({geom_column_definition}, {extent})".format(geom_column_definition=geom_column_definition, extent=extent)
)
else:
# Bodge bodge
# Fudge the simplification tolerance so that collections of small and dense geometries (e.g. SA1s in capital cities)
# don't get dropped too soon
data_query = layer["_postgis_query"].replace(
"ST_AsEWKB({})".format(geom_column_definition),
"ST_AsMVTGeom(ST_Simplify({geom_column_definition}, {simplify_tolerance}), {extent})".format(geom_column_definition=geom_column_definition, simplify_tolerance=z_res(z + 2), extent=extent)
)
# Drop any geometries that are too small for the user to see in this tile (smaller than about a pixel)
area_filter = "{geom_table_name}.sqrt_area_geom_3857 >= {area_threshold} AND".format(geom_table_name=geom_table_name, area_threshold=z_res(z + 1.5))
# FIXME Build this whole query in SQLAlchemy instead
# We need to begin a WHERE clause if there's no filter on the layer
if " WHERE " not in layer["_postgis_query"]:
where_cause = "WHERE"
else:
where_cause = "AND"
return """
SELECT
ST_AsMVT(tile)
FROM
({data_query}
{where_cause}
{area_filter}
{geom_column_definition} && {extent}
) as tile""".format(data_query=data_query, where_cause=where_cause, area_filter=area_filter, geom_column_definition=geom_column_definition, extent=extent)
def process_tile(tile):
"""Process a single MBTiles tile
Parameters
----------
tile : mercantile.Tile
Returns:
tile : mercantile.Tile
The input tile.
bytes : bytearray
Image bytes corresponding to the tile.
"""
global base_kwds, resampling, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(
*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(
*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
src_nodata = kwds.pop('src_nodata', None)
dst_nodata = kwds.pop('dst_nodata', None)
with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
reproject(rasterio.band(src, src.indexes),
rasterio.band(tmp, tmp.indexes),
src_nodata=src_nodata,
dst_nodata=dst_nodata,
num_threads=1,
resampling=resampling)
data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
# Workaround for https://bugs.python.org/issue23349.
if sys.version_info[0] == 2 and sys.version_info[2] < 10:
# Check for backported bug fix before re-ordering
if kwds['driver'] == 'PNG' and data[0:8] == png_header:
# Properly constructed PNG, no need to re-order bytes
pass
elif kwds['driver'] == 'JPEG' and data[0:4] == jpeg_header:
# Properly constructed JPEG, no need to re-order bytes
pass
else:
data[:] = data[-1:] + data[:-1]
return tile, data
def test_xy_tile():
"""x, y for the 486-332-10 tile is correctly calculated"""
ul = mercantile.ul(486, 332, 10)
xy = mercantile.xy(*ul)
expected = (-1017529.7205322663, 7044436.526761846)
for a, b in zip(expected, xy):
assert round(a - b, 7) == 0
def project_geom(geom):
return {
'type':
geom['type'],
'coordinates': [[mercantile.xy(*coords) for coords in part]
for part in geom['coordinates']]
}
def test_xy_tile():
"""x, y for the 486-332-10 tile is correctly calculated"""
ul = mercantile.ul(486, 332, 10)
xy = mercantile.xy(*ul)
expected = (-1017529.7205322663, 7044436.526761846)
for a, b in zip(expected, xy):
assert round(a - b, 7) == 0
def draw_geometry(self, im, geometry, color, width):
canvas = ImageDraw.Draw(im)
canvas.line([
tuple(round(x) for x in self.to_image(*mercantile.xy(*p)))
for p in geometry
],
fill=color,
width=width)
def project_geom(geom):
return {
'type': geom['type'],
'coordinates': [
[mercantile.xy(*coords) for coords in part]
for part in geom['coordinates']
]
}
def process_tile(tile):
"""Process a single MBTiles tile."""
global base_kwds, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
# Reproject the src dataset into image tile.
for bidx in tmp.indexes:
reproject(rasterio.band(src, bidx), rasterio.band(tmp, bidx))
# Get contents of the virtual file and repair it.
contents = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
return tile, contents[-1:] + contents[:-1]
def serve(self):
bounds = mercantile.bounds(self.x, self.y, self.zoom)
self.west, self.south = mercantile.xy(bounds.west, bounds.south)
self.east, self.north = mercantile.xy(bounds.east, bounds.north)
self.layers = []
if self.namespace not in RECIPES:
msg = 'Recipe "{}" not found. Available recipes are: {}'
abort(400, msg.format(self.namespace, list(RECIPES.keys())))
self.recipe = RECIPES[self.namespace]
names = self.recipe.layers.keys() if self.ALL else self.names
for name in names:
if name not in self.recipe.layers:
abort(400, u'Layer "{}" not found in recipe {}'.format(
name, self.namespace))
self.process_layer(self.recipe.layers[name])
self.post_process()
return self.content, 200, {"Content-Type": self.CONTENT_TYPE}
def process_tile(tile):
"""Process a single MBTiles tile."""
global base_kwds, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds["transform"] = from_bounds(ulx, lry, lrx, uly, 256, 256)
with rasterio.open("/vsimem/tileimg", "w", **kwds) as tmp:
# Reproject the src dataset into image tile.
for bidx in tmp.indexes:
reproject(rasterio.band(src, bidx), rasterio.band(tmp, bidx))
# Get contents of the virtual file.
contents = bytearray(virtual_file_to_buffer("/vsimem/tileimg"))
return tile, contents
def process_chunk(tile, input, creation_options, resampling):
"""Process a single tile."""
from rasterio.warp import RESAMPLING
input = input.replace("s3://", "/vsicurl/http://s3.amazonaws.com/")
print("Chunking initial image for", tile)
# Get the bounds of the tile.
ulx, uly = mercantile.xy(
*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(
*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
tmp_path = "/vsimem/tile"
with rasterio.drivers():
with rasterio.open(input, "r") as src:
meta = src.meta.copy()
meta.update(creation_options)
meta["height"] = CHUNK_SIZE
meta["width"] = CHUNK_SIZE
meta["transform"] = from_bounds(ulx, lry, lrx, uly, CHUNK_SIZE, CHUNK_SIZE)
# write to a tmp file to allow GDAL to handle the transform
with rasterio.open(tmp_path, "w", **meta) as tmp:
# Reproject the src dataset into image tile.
for bidx in src.indexes:
reproject(
source=rasterio.band(src, bidx),
destination=rasterio.band(tmp, bidx),
resampling=getattr(RESAMPLING, resampling),
num_threads=multiprocessing.cpu_count(),
)
# check for chunks containing only NODATA
data = tmp.read(masked=True)
if data.mask.all():
return
# TODO hard-coded for the first band
return (tile, data[0])
def _tile_worker(tile):
"""
For each tile, and given an open rasterio src, plus a`global_args` dictionary
with attributes of `base_val`, `interval`, and a `writer_func`,
warp a continous single band raster to a 512 x 512 mercator tile,
then encode this tile into RGB.
Parameters
-----------
tile: list
[x, y, z] indices of tile
Returns
--------
tile, buffer
tuple with the input tile, and a bytearray with the data encoded into
the format created in the `writer_func`
"""
x, y, z = tile
bounds = [
c for i in (
mercantile.xy(*mercantile.ul(x, y + 1, z)),
mercantile.xy(*mercantile.ul(x + 1, y, z)),
) for c in i
]
toaffine = transform.from_bounds(*bounds + [512, 512])
out = np.empty((512, 512), dtype=src.meta["dtype"])
reproject(
rasterio.band(src, 1),
out,
dst_transform=toaffine,
dst_crs="epsg:3857",
resampling=Resampling.bilinear,
)
out = data_to_rgb(out, global_args["base_val"], global_args["interval"])
return tile, global_args["writer_func"](out, global_args["kwargs"].copy(),
toaffine)
def run(self):
try:
self.signals.logged.emit("Обработка {}".format(self.road.Name),
LogModel.INFO)
points = self.road.get_main_axe_coordinates(self.session)
west = min(p['lng'] for p in points)
east = max(p['lng'] for p in points)
south = min(p['lat'] for p in points)
north = max(p['lat'] for p in points)
img = geo2image.GeoImage(west,
south,
east,
north,
self.zoom,
pool_workers=4)
img.update()
with img.cairo_surface() as surface:
context = cairo.Context(surface)
with img.cairo_matrix_override(context):
context.set_source_rgb(1, 0.2, 0.2)
x, y = mercantile.xy(points[0]['lng'], points[0]['lat'])
context.arc(x, y, 11 / img.kx, 0, math.pi * 2)
context.fill()
context.set_source_rgb(1, 0.2, 0.2)
x, y = mercantile.xy(points[-1]['lng'], points[-1]['lat'])
context.arc(x, y, 11 / img.kx, 0, math.pi * 2)
context.fill()
context.set_source_rgba(1, 0.2, 0.2, 0.4)
context.set_line_width(10 / img.kx)
for p in points:
x, y = mercantile.xy(p['lng'], p['lat'])
context.line_to(x, y)
context.stroke()
surface.write_to_png(self.save_path)
self.signals.finished.emit(self)
except Exception as ex:
self.signals.logged.emit(
"{}:{}".format(self.road.Name, traceback.format_exc()),
LogModel.ERROR)
finally:
self.session.close()
def get_opts(self, count):
"""Return rasterio dataset creation options for the bounding tile."""
w, s, e, n = list(mercantile.bounds(*self.bounding_tile))
w, s = mercantile.xy(w, s)
e, n = mercantile.xy(e, n)
xcell = ((e - w) / self.tileshape)
ycell = ((n - s) / self.tileshape)
return {
'dtype': np.float32,
'driver': 'GTiff',
'height': self.tileshape,
'width': self.tileshape,
'count': count,
'compress': 'lzw',
'transform': affine.Affine(xcell, 0, w, 0, -ycell, n),
'crs': 'epsg:3857'}
def parse_coordinate(self, size_im, coordinate):
scale = 6324
h, w = size_im[:2]
y, x = coordinate
# 131.9022528224, 43.0294958224
x, y = mercantile.xy(x, y)
# рассчитываем координаты углов в веб-меркаторе
left_top = mercantile.xy(self.bounds['west'], self.bounds['north'])
right_bottom = mercantile.xy(self.bounds['east'], self.bounds['south'])
# расчитываем коэффициенты
kx = w / (right_bottom[0] - left_top[0])
ky = h / (right_bottom[1] - left_top[1])
x = (x - left_top[0]) * kx #* 1.198
y = (y - left_top[1]) * ky #* 0.915
return int(abs(x)), int(abs(y))
def test_make_tiles_tile_bounds(x, y):
'''
Test if children tiles from z10 are created correctly
'''
test_bounds = mercantile.bounds(x, y, 10)
test_bbox = list(mercantile.xy(test_bounds.west, test_bounds.south)) + list(mercantile.xy(test_bounds.east, test_bounds.north))
test_crs = 'epsg:3857'
test_minz = 10
test_maxz = 13
created_tiles_gen = _make_tiles(test_bbox, test_crs, test_minz, test_maxz)
assert isinstance(created_tiles_gen, types.GeneratorType)
created_tiles = list(created_tiles_gen)
assert len(created_tiles) == 85
def downsample((tile, data)):
if data is None:
return
print("Downsampling", tile)
# Get the bounds of the tile.
ulx, uly = mercantile.xy(
*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(
*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
# TODO constantize
tmp_path = "/vsimem/tile"
# create GeoTIFF
meta = {
"driver": "GTiff",
"crs": "EPSG:3857",
"nodata": data.fill_value,
"count": 1,
"dtype": data.dtype,
"width": CHUNK_SIZE,
"height": CHUNK_SIZE,
"transform": from_bounds(ulx, lry, lrx, uly, CHUNK_SIZE, CHUNK_SIZE),
}
with rasterio.drivers():
with rasterio.open(tmp_path, "w", **meta) as tmp:
# use GDAL to resample by writing an ndarray and immediately reading
# it out into a smaller array
tmp.write(data, 1)
resampled = tmp.read(
indexes=1,
masked=True,
out=ma.array(np.empty((CHUNK_SIZE / 2, CHUNK_SIZE / 2), data.dtype)),
)
if resampled.mask.all():
return
corner = CORNERS[(tile.x % 2, tile.y % 2)]
return (mercantile.parent(tile), (corner, resampled))
def map(self, coords):
# TODO: test this
m = np.empty(coords.shape)
x, y = coords[:, 0], coords[:, 1]
long, lat = offset_coord(self.longitude, self.latitude, x, y)
x, y = mercantile.xy(long, lat)
m[:, 0] = x
m[:, 1] = y
m[:, 2:] = coords[:, 2:]
return m
def project_geom(geom):
if geom["type"] == "Polygon":
return {
"type": geom["type"],
"coordinates": [
[mercantile.xy(*coords) for coords in part]
for part in geom["coordinates"]
],
}
elif geom["type"] == "LineString":
return {
"type": geom["type"],
"coordinates": [mercantile.xy(*coords) for coords in geom["coordinates"]],
}
elif geom["type"] == "Point":
return {
"type": geom["type"],
"coordinates": mercantile.xy(*geom["coordinates"]),
}
def tile_bbox(tile, mercator=False):
if isinstance(tile, mercantile.Tile):
if mercator:
return mercantile.xy_bounds(tile) # EPSG:3857
else:
return mercantile.bounds(tile) # EPSG:4326
else:
with open(rasterio_open(tile)) as r:
if mercator:
w, s, e, n = r.bounds
w, s = mercantile.xy(w, s)
e, n = mercantile.xy(e, n)
return w, s, e, n # EPSG:3857
else:
return r.bounds # EPSG:4326
assert False, "Unable to open tile"
def __init__(self, extent, map_dir, map_zoom=(MIN_ZOOM, MAX_ZOOM)):
self._map_dir = map_dir
self._min_zoom = map_zoom[0]
self._max_zoom = map_zoom[1]
# We need a manager to share the list of database names between processes
self._manager = multiprocessing.Manager()
self._database_names = self._manager.list()
# Get whole tiles that span the image's extent
self._tiles = list(mercantile.tiles(*extent, self._max_zoom))
tile_0 = self._tiles[0]
tile_N = self._tiles[-1]
self._tile_start_coords = (tile_0.x, tile_0.y)
self._tile_end_coords = (tile_N.x, tile_N.y)
# Tiled area in world coordinates (metres)
bounds_0 = mercantile.xy_bounds(tile_0)
bounds_N = mercantile.xy_bounds(tile_N)
tile_world = Rect(bounds_0.left, bounds_0.top, bounds_N.right,
bounds_N.bottom)
# Tiled area in tile pixel coordinates
tile_extent = Rect(0, 0, TILE_SIZE[0] * (tile_N.x - tile_0.x + 1),
TILE_SIZE[1] * (tile_N.y - tile_0.y + 1))
# Affine transform from world to tile pixel coordinates
sx = tile_extent.width / tile_world.width
sy = tile_extent.height / tile_world.height
world_to_tile = Affine((sx, -sy), (tile_world.x0, tile_world.y0),
(0, 0))
# Extent in world coordinates (metres)
sw = mercantile.xy(*extent[:2])
ne = mercantile.xy(*extent[2:])
# Converted to tile pixel coordinates
self._image_rect = Rect(world_to_tile.transform(sw[0], ne[1]),
world_to_tile.transform(ne[0], sw[1]))
self._processes = []
def merc2xy(x, y, z, sgeom):
""" Apply some affine transformations to input shapely geometry for coordinates
transformation from Mercatore to local tile pixels.
sgeom @shapely.geom : Input polygon
Returns a brand new shapely polygon in the new coordinates.
"""
MVT_EXTENT = 4096
X_min, Y_max = mercantile.xy(*mercantile.ul(x, y, z))
X_max, Y_min = mercantile.xy(*mercantile.ul(x + 1, y - 1, z))
geom_3857 = transform(transformer.transform, sgeom)
tx, ty = -X_min, -2 * Y_max + Y_min
geom_XY = translate(geom_3857, tx, ty)
x_scale_factor = MVT_EXTENT / (X_max - X_min)
y_scale_factor = -MVT_EXTENT / (Y_max - Y_min)
geom_xy = scale(geom_XY, x_scale_factor, y_scale_factor, origin=(0, 0, 0))
return geom_xy
def read_val_at_pixel(self, grid, lat, lon, zoom):
"""Interpolate the row/column of a webtile from a lat/lon/zoom and extract
the corresponding value from `grid`."""
tile = mercantile.tile(lon, lat, zoom)
size = len(grid)
box = mercantile.xy_bounds(tile)
x, y = mercantile.xy(lon, lat)
width = box.right - box.left
height = box.top - box.bottom
i = int(round(size * (box.top - y) / height))
j = int(round(size * (x - box.left) / width))
return grid[i][j]
def union(inputtiles, parsenames):
tiles = sutils.tile_parser(inputtiles, parsenames)
xmin, xmax, ymin, ymax = sutils.get_range(tiles)
zoom = sutils.get_zoom(tiles)
# make an array of shape (xrange + 3, yrange + 3)
burn = sutils.burnXYZs(tiles, xmin, xmax, ymin, ymax, 0)
nw = mercantile.xy(*mercantile.ul(xmin, ymin, zoom))
se = mercantile.xy(*mercantile.ul(xmax + 1, ymax + 1, zoom))
aff = Affine(
((se[0] - nw[0]) / float(xmax - xmin + 1)),
0.0,
nw[0],
0.0,
-((nw[1] - se[1]) / float(ymax - ymin + 1)),
nw[1],
)
unprojecter = sutils.Unprojecter()
unionedTiles = [
{
"geometry": unprojecter.unproject(feature),
"properties": {},
"type": "Feature",
}
for feature, shapes in features.shapes(
np.asarray(np.flipud(np.rot90(burn)).astype(np.uint8), order="C"),
transform=aff,
)
if shapes == 1
]
return unionedTiles
def tile_view(request, zoom, x, y):
"""
Returns an MVT tiles given zoom, x and y in TMS format
References:
https://www.mapbox.com/vector-tiles/
http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-mercator
"""
bounds = mercantile.bounds(int(x), int(y), int(zoom))
west, south = mercantile.xy(bounds.west, bounds.south)
east, north = mercantile.xy(bounds.east, bounds.north)
pixel = pixel_length(zoom)
buffer = 4 * pixel
bbox = Polygon.from_bbox(
(west - buffer, south - buffer, east + buffer, north + buffer))
departements = Departement.objects.filter(geom__intersects=bbox)
departements = departements.annotate(clipped=Intersection('geom', bbox))
tile = {
"name":
"departements",
"features": [{
"geometry":
departement.clipped.simplify(pixel, preserve_topology=True).wkt,
"properties": {
"numero": departement.code_dept,
"nom": departement.nom_dept,
},
} for departement in departements],
}
vector_tile = mapbox_vector_tile.encode(tile,
quantize_bounds=(west, south, east,
north))
return HttpResponse(vector_tile,
content_type="application/vnd.mapbox-vector-tile")
def get_basemap_image(self, basemap, bounds, size, zoom):
tiles = list(mercantile.tiles(*bounds, zoom, truncate=True))
bounds_merc = [*mercantile.xy(*bounds[:2]), *mercantile.xy(*bounds[2:])]
image = Image.new('RGBA', size)
to_image = world_to_image(BBox(bounds_merc, projection=WEB_MERCATOR), size)
async def fetch_tile(client, url, tile):
tile_bounds = mercantile.xy_bounds(tile)
if url.startswith('//'):
url = 'http:' + url
async with client.get(url.format(x=tile.x, y=tile.y, z=tile.z, s='server')) as r:
tile_im = Image.open(BytesIO(await r.read()))
image.paste(tile_im, [int(round(x)) for x in to_image(*tile_bounds[0:4:3])]) # [0:4:3] = [xmin, ymax] (upper-left)
async def fetch_tiles():
async with aiohttp.ClientSession() as client:
futures = [ensure_future(fetch_tile(client, basemap, tile)) for tile in tiles]
await asyncio.wait(futures, return_when=asyncio.ALL_COMPLETED)
asyncio.get_event_loop().run_until_complete(fetch_tiles())
return image
def _update_geoctx(self, change):
if self.map is None:
with self.hold_trait_notifications():
self.set_trait("xy_3857", ())
self.set_trait("geoctx", None)
return
lat, lon = self.location
tx, ty, tz = mercantile.tile(lon, lat, int(self.map.zoom))
ctx = GeoContext.from_xyz_tile(tx, ty, tz)
with self.hold_trait_notifications():
self.set_trait("xy_3857", mercantile.xy(lon, lat))
self.set_trait("geoctx", ctx)
def _tile_worker(tile):
"""
For each tile, and given an open rasterio src, plus a`global_args` dictionary
with attributes of `base_val`, `interval`, and a `writer_func`,
warp a continous single band raster to a 512 x 512 mercator tile,
then encode this tile into RGB.
Parameters
-----------
tile: list
[x, y, z] indices of tile
Returns
--------
tile, buffer
tuple with the input tile, and a bytearray with the data encoded into
the format created in the `writer_func`
"""
x, y, z = tile
bounds = [c for i in (mercantile.xy(*mercantile.ul(x, y + 1, z)),
mercantile.xy(*mercantile.ul(x + 1, y, z))) for c in i]
toaffine = transform.from_bounds(*bounds + [512, 512])
out = np.empty((512, 512), dtype=src.meta['dtype'])
reproject(
rasterio.band(src, 1), out,
dst_transform=toaffine,
dst_crs="epsg:3857",
resampling=RESAMPLING.bilinear)
out = data_to_rgb(out, global_args['base_val'], global_args['interval'])
return tile, global_args['writer_func'](out, global_args['kwargs'].copy(), toaffine)
def process_tile(tile):
"""Process a single MBTiles tile
Parameters
----------
tile : mercantile.Tile
Returns:
tile : mercantile.Tile
The input tile.
bytes : bytearray
Image bytes corresponding to the tile.
"""
global base_kwds, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds['transform'] = from_bounds(ulx, lry, lrx, uly, 256, 256)
src_nodata = kwds.pop('src_nodata', None)
dst_nodata = kwds.pop('dst_nodata', None)
with rasterio.open('/vsimem/tileimg', 'w', **kwds) as tmp:
reproject(rasterio.band(src, src.indexes),
rasterio.band(tmp, tmp.indexes),
src_nodata=src_nodata,
dst_nodata=dst_nodata,
num_threads=1)
data = bytearray(virtual_file_to_buffer('/vsimem/tileimg'))
# Workaround for https://bugs.python.org/issue23349.
if sys.version_info[0] == 2 and sys.version_info[2] < 10:
data[:] = data[-1:] + data[:-1]
return tile, data
def test_xy_south_pole():
"""Return -inf for y at South Pole"""
assert mercantile.xy(0.0, -90) == (0.0, float('-inf'))
def process_tile(tile):
"""Process a single MBTiles tile
Parameters
----------
tile : mercantile.Tile
Returns
-------
tile : mercantile.Tile
The input tile.
bytes : bytearray
Image bytes corresponding to the tile.
"""
global base_kwds, resampling, src
# Get the bounds of the tile.
ulx, uly = mercantile.xy(
*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(
*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
kwds = base_kwds.copy()
kwds['transform'] = transform_from_bounds(ulx, lry, lrx, uly,
kwds['width'], kwds['height'])
src_nodata = kwds.pop('src_nodata', None)
dst_nodata = kwds.pop('dst_nodata', None)
warnings.simplefilter('ignore')
with MemoryFile() as memfile:
with memfile.open(**kwds) as tmp:
# determine window of source raster corresponding to the tile
# image, with small buffer at edges
try:
west, south, east, north = transform_bounds(TILES_CRS, src.crs, ulx, lry, lrx, uly)
tile_window = window_from_bounds(west, south, east, north, transform=src.transform)
adjusted_tile_window = Window(
tile_window.col_off - 1, tile_window.row_off - 1,
tile_window.width + 2, tile_window.height + 2)
tile_window = adjusted_tile_window.round_offsets().round_shape()
# if no data in window, skip processing the tile
if not src.read_masks(1, window=tile_window).any():
return tile, None
except ValueError:
log.info("Tile %r will not be skipped, even if empty. This is harmless.", tile)
reproject(rasterio.band(src, tmp.indexes),
rasterio.band(tmp, tmp.indexes),
src_nodata=src_nodata,
dst_nodata=dst_nodata,
num_threads=1,
resampling=resampling)
return tile, memfile.read()
def test_xy_null_island():
"""x, y for (0, 0) is correctly calculated"""
xy = mercantile.xy(0.0, 0.0)
expected = (0.0, 0.0)
for a, b in zip(expected, xy):
assert round(a - b, 7) == 0
meta.update(
driver='GTiff',
dtype=rasterio.uint8,
compress="deflate",
predictor=1,
nodata=None,
tiled=True,
sparse_ok=True,
blockxsize=DST_BLOCK_SIZE,
blockysize=DST_BLOCK_SIZE,
height=DST_TILE_HEIGHT,
width=DST_TILE_WIDTH,
)
# Get the bounds of the tile.
ulx, uly = mercantile.xy(*mercantile.ul(tile.x, tile.y, tile.z))
lrx, lry = mercantile.xy(*mercantile.ul(tile.x + 1, tile.y + 1, tile.z))
meta["affine"] = src.window_transform(window)
ls = LightSource()
hs = ls.hillshade(data,
dx=dx,
dy=dy,
)
hs = (255.0 * hs).astype(np.uint8)
with rasterio.open("windowed.tif", "w", **meta) as dst:
# ignore the border pixels when writing
dst.write(hs[BUFFER:-BUFFER, BUFFER:-BUFFER], 1)
def test_xy_north_pole():
"""Return inf for y at North Pole"""
assert mercantile.xy(0.0, 90) == (0.0, float('inf'))
def test_xy_truncate():
"""Input is truncated"""
assert mercantile.xy(-181.0, 0.0, truncate=True) == mercantile.xy(-180.0, 0.0)
def shapes(
ctx, input, precision, indent, compact, projected,
seq, output_mode, collect, extents, buffer):
"""Reads one or more Web Mercator tile descriptions
from stdin and writes either a GeoJSON feature collection (the
default) or a JSON sequence of GeoJSON features/collections to
stdout.
tile descriptions may be either an [x, y, z] array or a JSON
object of the form
{"tile": [x, y, z], "properties": {"name": "foo", ...}}
In the latter case, the properties object will be used to update
the properties object of the output feature.
"""
verbosity = ctx.obj['verbosity']
logger = logging.getLogger('mercantile')
dump_kwds = {'sort_keys': True}
if indent:
dump_kwds['indent'] = indent
if compact:
dump_kwds['separators'] = (',', ':')
try:
src = click.open_file(input).__iter__()
except IOError:
src = [input]
try:
features = []
col_xs = []
col_ys = []
for i, line in enumerate(src):
line = line.strip()
obj = json.loads(line)
if isinstance(obj, dict):
x, y, z = obj['tile'][:3]
props = obj.get('properties')
fid = obj.get('id')
elif isinstance(obj, list):
x, y, z = obj[:3]
props = {}
fid = None
else:
raise ValueError("Invalid input: %r", obj)
west, south, east, north = mercantile.bounds(x, y, z)
if projected == 'mercator':
west, south = mercantile.xy(west, south, truncate=False)
east, north = mercantile.xy(east, north, truncate=False)
if buffer:
west -= buffer
south -= buffer
east += buffer
north += buffer
if precision and precision >= 0:
west, south, east, north = (
round(v, precision) for v in (west, south, east, north))
bbox = [
min(west, east), min(south, north),
max(west, east), max(south, north)]
col_xs.extend([west, east])
col_ys.extend([south, north])
geom = {
'type': 'Polygon',
'coordinates': [[
[west, south],
[west, north],
[east, north],
[east, south],
[west, south]]]}
xyz = str((x, y, z))
feature = {
'type': 'Feature',
'bbox': bbox,
'id': xyz,
'geometry': geom,
'properties': {'title': 'XYZ tile %s' % xyz}}
if props:
feature['properties'].update(props)
if fid:
feature['id'] = fid
if collect:
features.append(feature)
elif extents:
click.echo(" ".join(map(str, bbox)))
else:
if seq:
click.echo(u'\x1e')
if output_mode == 'bbox':
click.echo(json.dumps(bbox, **dump_kwds))
elif output_mode == 'feature':
click.echo(json.dumps(feature, **dump_kwds))
if collect and features:
bbox = [min(col_xs), min(col_ys), max(col_xs), max(col_ys)]
click.echo(json.dumps({
'type': 'FeatureCollection',
'bbox': bbox, 'features': features},
**dump_kwds))
sys.exit(0)
except Exception:
logger.exception("Failed. Exception caught")
sys.exit(1)
def test_lnglat_xy_roundtrip():
lnglat = (-105.0844, 40.5853)
roundtrip = mercantile.lnglat(*mercantile.xy(*lnglat))
for a, b in zip(roundtrip, lnglat):
assert round(a - b, 4) == 0