def coords2png(lat0, lng0, lat1, lng1, out_file):
# find desirable zoom level
zoom = 0
while True:
tiles_list = list(mercantile.tiles(lng0, lat0, lng1, lat1, [zoom]))
if len(tiles_list) > MAX_TILES:
zoom -= 1
break
zoom += 1
print('Zoom level %s' % zoom)
# get tiles list
tiles_list = list(mercantile.tiles(lng0, lat0, lng1, lat1, [zoom]))
xs = [tile.x for tile in tiles_list]
x0 = min(xs)
x1 = max(xs)
ys = [tile.y for tile in tiles_list]
y0 = min(ys)
y1 = max(ys)
x_tiles = x1 - x0 + 1
y_tiles = y1 - y0 + 1
x_width = x_tiles * TILE_SIDE
y_width = y_tiles * TILE_SIDE
# create image
img = Image.new("RGBA", (x_tiles * TILE_SIDE, y_tiles * TILE_SIDE), (0, 0, 0, 0))
for x in range(x0, x1 + 1):
for y in range(y0, y1 + 1):
print("processing tile %s, %s" % (x, y))
tile = tile_image(zoom, x, y)
tile_img = Image.open(io.BytesIO(tile))
img.paste(tile_img, ((x - x0) * TILE_SIDE, (y - y0) * TILE_SIDE))
# crop image
top_left = mercantile.bounds(x0, y0, zoom)
bottom_right = mercantile.bounds(x1, y1, zoom)
img_lat1 = top_left.north
img_lng0 = top_left.west
img_lat0 = bottom_right.south
img_lng1 = bottom_right.east
img_lat_delta = img_lat1 - img_lat0
img_lng_delta = img_lng1 - img_lng0
pixel_lat_ratio = float(x_width) / img_lat_delta
pixel_lng_ratio = float(y_width) / img_lng_delta
left = int(abs(lng0 - img_lng0) * pixel_lng_ratio)
right = x_width - int(abs(lng1 - img_lng1) * pixel_lng_ratio)
top = int(abs(lat1 - img_lat1) * pixel_lat_ratio)
bottom = y_width - int(abs(lat0 - img_lat0) * pixel_lat_ratio)
img = img.crop((left, top, right, bottom))
img.save(out_file, 'PNG')
def generate_chunk_tasks(image_source, tile_dim):
tasks = []
zoom = image_source.zoom
(min_col, max_col) = (image_source.tile_bounds[0], image_source.tile_bounds[2])
(min_row, max_row) = (image_source.tile_bounds[1], image_source.tile_bounds[3])
for tile_col in range(min_col, min(max_col + 1, 2**zoom)):
for tile_row in range(min_row, min(max_row + 1, 2**zoom)):
tile_bounds = mercantile.bounds(tile_col, tile_row, zoom)
(wm_left, wm_bottom, wm_right, wm_top) = warp.transform_bounds("EPSG:4326",
"EPSG:3857",
tile_bounds.west,
tile_bounds.south,
tile_bounds.east,
tile_bounds.north)
affine = transform.from_bounds(wm_left, wm_bottom, wm_right, wm_top, tile_dim, tile_dim)
target_meta = {
"transform": affine[:6],
"width": tile_dim,
"height": tile_dim
}
target = os.path.join(image_source.image_folder, "%d/%d/%d.tif" % (zoom, tile_col, tile_row))
task = ChunkTask(source_uri=image_source.source_uri,
target_meta=target_meta,
target=target)
tasks.append(task)
return tasks
def slice_geoms_to_tiles(grouped_countries):
tiles = {}
z = SPLIT_ZOOM
for x in range(2 ** z):
for y in range(2 ** z):
start = datetime.datetime.utcnow()
twest, tsouth, teast, tnorth = mercantile.bounds(x, y, z)
bound = shapely.geometry.box(twest, tsouth, teast, tnorth)
no_country_geom = shapely.geometry.box(twest, tsouth, teast, tnorth)
parts = []
for iso, geom, _ in grouped_countries:
if not bound.intersects(geom):
continue
geom_part = bound.intersection(geom)
no_country_geom = no_country_geom.difference(geom_part)
parts.append((iso, geom_part))
if not no_country_geom.buffer(0).is_empty:
parts.append(('??', no_country_geom))
time_spent = (datetime.datetime.utcnow() - start).total_seconds()
Stat().log('polygons in %s/%s/%s: %s - %s%s', z, x, y,
len(parts), '|'.join(iso for iso, _ in parts),
'' if time_spent < 1 else (' (%s sec.)' % time_spent))
tiles['%s/%s/%s' % (z, x, y)] = tuple((iso, geom, geom.bounds)
for iso, geom in parts)
return tiles
def triangulate(zoom, output, bounds, tile, tableid):
if bounds:
bounds = np.array(bounds).astype(np.float64)
elif tile:
epsilon = 1.0e-10
tile = np.array(tile).astype(np.uint16)
tBounds = mercantile.bounds(*tile)
bounds = np.array([
tBounds.west + epsilon,
tBounds.south + epsilon,
tBounds.east - epsilon,
tBounds.north - epsilon
])
else:
sys.exit('Error: A bounds or tile must be specified')
tileMin = mercantile.tile(bounds[0], bounds[3], zoom)
tileMax = mercantile.tile(bounds[2], bounds[1], zoom)
pGet = facetParent()
if tableid:
gJSON = createDBinit(tileMin, tileMax, zoom, pGet, tableid)
else:
gJSON = createFacets(tileMin, tileMax, zoom, pGet)
if output:
with open(output, 'w') as oFile:
for feat in gJSON:
oFile.write(json.dumps(feat) + '\n')
else:
for feat in gJSON:
click.echo(json.dumps(feat))
def createFacets(tileMin, tileMax, zoom, parentGet):
for r in range(tileMin.y, tileMax.y + 1):
for c in range(tileMin.x, tileMax.x + 1):
quad = tools.quadtree(c, r, zoom)
boolKey = (r+c) % 2 == 0
n = parentGet.getParents('n', c, r, zoom)
s = parentGet.getParents('s', c, r, zoom)
coords = getCorners(mercantile.bounds(c, r, zoom), boolKey)
nQT = ''.join(np.dstack((n, quad)).flatten()) + 'n'
sQT = ''.join(np.dstack((s, quad)).flatten()) + 's'
yield {
"type": "Feature",
"properties": {
"qt": nQT
},
"geometry": {
"type": "Polygon",
"coordinates": [coords[0].tolist()]
}
}
yield {
"type": "Feature",
"properties": {
"qt": sQT,
},
"geometry": {
"type": "Polygon",
"coordinates": [coords[1].tolist()]
}
}
def calculate_center(x, y, zoom):
bounds = mercantile.bounds(x, y, zoom)
height = bounds.north - bounds.south
width = bounds.east - bounds.west
center = (bounds.north + height / 2, bounds.west + width / 2)
return center
def get_image_tile(raster, x, y, z):
try:
bound = bounds(x, y, z)
with rasterio.open(raster) as src:
x_res, y_res = src.transform[0], src.transform[4]
p1 = Proj({'init': 'epsg:4326'})
p2 = Proj(**src.crs)
# project tile boundaries from lat/lng to source CRS
tile_ul_proj = transform(p1, p2, bound.west, bound.north)
tile_lr_proj = transform(p1, p2, bound.east, bound.south)
# get origin point from the TIF
tif_ul_proj = (src.bounds.left, src.bounds.top)
# use the above information to calculate the pixel indices of the window
top = int((tile_ul_proj[1] - tif_ul_proj[1]) / y_res)
left = int((tile_ul_proj[0] - tif_ul_proj[0]) / x_res)
bottom = int((tile_lr_proj[1] - tif_ul_proj[1]) / y_res)
right = int((tile_lr_proj[0] - tif_ul_proj[0]) / x_res)
window = ((top, bottom), (left, right))
# read the first three bands (assumed RGB) of the TIF into an array
data = np.empty(shape=(3, 256, 256)).astype(src.profile['dtype'])
for k in (1, 2, 3):
src.read(k, window=window, out=data[k - 1], boundless=True)
return Image.fromarray(np.moveaxis(data, 0, -1), mode='RGB')
except Exception as err:
raise TileNotFoundError(err)
def process_item(out, min_cache_zoom, cache, link,
part_zoom, tiled_countries, all_countries,
min_zoom, max_zoom, skip_empty_tiles):
stat = Stat()
date = get_date_from_link(link)
for line in lzma.LZMAFile(get_tile_usage_dump(link)):
path, count = line.decode().strip().split()
z, x, y = path.split('/')
x = int(x)
y = int(y)
z = int(z)
if min_zoom is not None and z < min_zoom:
continue
if max_zoom is not None and z > max_zoom:
continue
twest, tsouth, teast, tnorth = b = mercantile.bounds(x, y, z)
country = detect_country_with_cache(
path, b, x, y, z, part_zoom, tiled_countries, all_countries,
min_cache_zoom, cache, stat)
if skip_empty_tiles and country == '??':
continue
lat = tnorth + (tnorth - tsouth) / 2
lon = twest + (teast - twest) / 2
out.write(('%s,%s,%s,%s,%s,%s,%s,%s\n' % (
date, z, x, y, count, lat, lon, country)).encode())
stat.log_stats(date, cache)
async def serve(cls, request):
# TODO: develop and test some 'request' plugin hooks
endpoint = request.path.split('.')[-1]
path_args = request.match_info
request_hook_response = Configs.plugins.hook('request', endpoint=endpoint, request=request, **path_args)
# if a request_hook_response is received, return and skip the regular processing:
if request_hook_response:
return request_hook_response
# fetch query parameters from request info
x = int(request.match_info['x'])
y = int(request.match_info['y'])
zoom = int(request.match_info['z'])
layers = request.match_info['layers']
try:
recipe_name = request.match_info['recipe']
except KeyError:
recipe_name = 'default_recipe'
# check that recipe exists
if recipe_name not in Configs.recipes.keys():
logger.error('Recipe {0} not found in recipes'.format(recipe_name))
return aiohttp.errors.HttpBadRequest('Recipe {0} not found in recipes'.format(recipe_name))
# fetch recipe
recipe = Configs.recipes[recipe_name]
# extrapolate the layers
if layers == 'all':
layers = list(recipe.layers.keys())
else:
layers = layers.split('+')
# compute bounds and extents
bounds = mercantile.bounds(x, y, zoom)
west, south = mercantile.xy(bounds.west, bounds.south)
east, north = mercantile.xy(bounds.east, bounds.north)
# process the layers
layer_data = []
for layer_name in layers:
if layer_name not in recipe.layers.keys():
logger.error('Layer {0} not found in recipe {1}'.format(layer_name, recipe_name))
return aiohttp.errors.HttpBadRequest('Layer {0} not found in layer config file'.format(layer_name))
else:
layer = recipe.layers[layer_name]
layer_data.append(await cls.query_layer(layer, zoom, west, south, east, north))
content_type, body = cls.post_process(layer_data)
response = Response(content_type=content_type, body=body, headers=generic_headers)
# TODO: develop and test some 'request' plugin hooks
response_hook_response = Configs.plugins.hook('request', response=response, request=request)
# if a response_hook_response is received, it overrides the regular response:
if response_hook_response:
return response_hook_response
else:
return response
def test_bbox():
expected = (-9.140625, 53.12040528310657, -8.7890625, 53.33087298301705)
bbox = mercantile.bounds(486, 332, 10)
for a, b in zip(expected, bbox):
assert round(a-b, 7) == 0
assert bbox.west == bbox[0]
assert bbox.south == bbox[1]
assert bbox.east == bbox[2]
assert bbox.north == bbox[3]
def calculate_wkt(x, y, zoom):
bounds = mercantile.bounds(x, y, zoom)
text='POLYGON (('
text=text+str(bounds.west)+' '+str(bounds.north)+', '
text=text+str(bounds.east)+' '+str(bounds.north)+', '
text=text+str(bounds.east)+' '+str(bounds.south)+', '
text=text+str(bounds.west)+' '+str(bounds.south)
text=text+'))'
return text
def mvt(layer, x, y, z):
def create_vectortile_sql(layer, bounds):
# Create extent
west, south = xy(bounds.west, bounds.south)
east, north = xy(bounds.east, bounds.north)
extent = "ST_MakeBox2D(ST_MakePoint({west}, {south}), ST_MakePoint({east}, {north}))".format(west=west, south=south, east=east, north=north)
# e.g. aus_census_2011_shapes.sa1
geom_table_name = "{schema_name}.{geometry_name}".format(geometry_name=layer["geometry"], schema_name=layer["schema"])
# e.g. aus_census_2011_shapes.sa1.geom_3857
geom_column_definition = "{}.geom_3857".format(geom_table_name)
# Replace the compiled geometry column definition with the zoom-level dependent version
# e.g. Replace "ST_AsEWKB(aus_census_2011_shapes.sa1.geom_3857)" with "ST_AsMVTGeom(ST_Simplify(aus_census_2011_shapes.sa1.geom_3857, TOLERANCE), EXTENT_OF_TILE)"
# Zoom 15 is our highest resolution (configured in OpenLayers), so we need to grab
# unsimplified geometries to allow us to re-use them as the user zooms in.
if z == 15:
data_query = layer["_postgis_query"].replace(
"ST_AsEWKB({})".format(geom_column_definition),
"ST_AsMVTGeom({geom_column_definition}, {extent})".format(geom_column_definition=geom_column_definition, extent=extent)
)
else:
# Bodge bodge
# Fudge the simplification tolerance so that collections of small and dense geometries (e.g. SA1s in capital cities)
# don't get dropped too soon
data_query = layer["_postgis_query"].replace(
"ST_AsEWKB({})".format(geom_column_definition),
"ST_AsMVTGeom(ST_Simplify({geom_column_definition}, {simplify_tolerance}), {extent})".format(geom_column_definition=geom_column_definition, simplify_tolerance=z_res(z + 2), extent=extent)
)
# Drop any geometries that are too small for the user to see in this tile (smaller than about a pixel)
area_filter = "{geom_table_name}.sqrt_area_geom_3857 >= {area_threshold} AND".format(geom_table_name=geom_table_name, area_threshold=z_res(z + 1.5))
# FIXME Build this whole query in SQLAlchemy instead
# We need to begin a WHERE clause if there's no filter on the layer
if " WHERE " not in layer["_postgis_query"]:
where_cause = "WHERE"
else:
where_cause = "AND"
return """
SELECT
ST_AsMVT(tile)
FROM
({data_query}
{where_cause}
{area_filter}
{geom_column_definition} && {extent}
) as tile""".format(data_query=data_query, where_cause=where_cause, area_filter=area_filter, geom_column_definition=geom_column_definition, extent=extent)
# Wrap EALGIS query in a PostGIS query to produce a vector tile
mvt_query = create_vectortile_sql(layer, bounds=bounds(x, y, z))
with ealdb.access_data() as db:
tile = db.session.execute(mvt_query).fetchone()[0]
return BytesIO(tile).read()
def calculate_center(x, y, z):
k = (x, y, z)
if k not in cache_center:
bounds = mercantile.bounds(x, y, z)
height = bounds.north - bounds.south
width = bounds.east - bounds.west
center = (bounds.north + height / 2, bounds.west + width / 2)
cache_center[k] = center
return center
return cache_center[k]
def test_src_meta_making(expectedMeta):
bounds = merc.bounds(10, 10, 10)
src_meta = untiler.make_src_meta(bounds, 4096)
for k, e in zip(sorted(src_meta), sorted(expectedMeta)):
assert k == e
# assert src_meta[k] == expectedMeta[e]
print("# OK - %s " % (inspect.stack()[0][3]))
def tile_to_rect(zoom, x, y, v):
zoom = int(zoom)
x = int(x)
y = int(y)
box = mercantile.bounds(x, y, zoom)
return {
"type": "Feature",
"properties": {"t": "%s/%s/%s" % (zoom, x, y), "c": v, "k": "#%02X%02X%02X" % get_color(v)},
"geometry": shapely.geometry.mapping(shapely.geometry.box(*box)),
}
def pyramid_jobs(x, y, z, job_zoom):
"""
Generate pyramid jobs for a given job_zoom level
starting at with the parent tile defined by x, y, z.
"""
if z == job_zoom:
bounds = mercantile.bounds(x, y, z)
yield create_pyramid_job(
x=x, y=y,
min_zoom=z, max_zoom=14,
bounds=bounds
)
return
tiles = all_descendant_tiles(x, y, z, job_zoom)
pyramid_zoom_level_tiles = (t for t in tiles if t.z == job_zoom)
for tile in pyramid_zoom_level_tiles:
bounds = mercantile.bounds(tile.x, tile.y, tile.z)
yield create_pyramid_job(tile.x, tile.y, min_zoom=tile.z,
max_zoom=14, bounds=bounds)
def vecttile_mask(tile, save_mask=True):
bounds = mercantile.bounds(tile.x, tile.y, tile.z)
ll = mercantile.xy(bounds.west, bounds.south)
ur = mercantile.xy(bounds.east, bounds.north)
extent = mapnik.Box2d(*(ll + ur))
m = mapnik.Map(256, 256)
m.srs = "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over"
s = mapnik.Style()
r = mapnik.Rule()
road_width = 75 # meters
pixel_size = mercantile.pixel_size(tile.z, bounds.north)
pixels = int(road_width/pixel_size) + 1
line_symbolizer = mapnik.LineSymbolizer(mapnik.Color('black'), pixels)
r.symbols.append(line_symbolizer)
s.rules.append(r)
m.append_style('RoadStyle', s)
layer = mapnik.Layer('Roads from PostGIS')
layer.srs = "+init=epsg:3857"
ROADS = """
(SELECT way
FROM planet_osm_line
WHERE roadbiking = TRUE) linestring
"""
layer.datasource = mapnik.PostGIS(host='localhost', user='******', password='******',
dbname='osm_uswest', table=ROADS, geometry_field='way')
layer.styles.append('RoadStyle')
m.layers.append(layer)
m.zoom_to_box(extent)
img = mapnik.Image(m.width, m.height)
mapnik.render(m, img)
imgpath = tile_path(os.path.join(tiledir, 'mask'), tile, 'png')
prep_dirs(imgpath)
if save_mask:
img.save(imgpath)
imgdata = np.frombuffer(img.tostring(), dtype=np.uint8).reshape((256, 256, 4))
# use alpha channel to get 0->1 scale for mask
alpha = imgdata[:,:,3] / 255.0
# invert for easy multiplication w/ strava alpha channel
# 1 = no road, keep strava heatmap values)
# 0 = road, make strava heatmap transparent
mask = 1 - alpha
return mask
def rounding(z,x,y):
if z < 16:
print "Too big"
raise Exception
bb = mercantile.bounds(x,y,z)
bb = [str(f) for f in [bb.west,bb.south,bb.east,bb.north]]
url = "http://openstreetmap.org/api/0.6/map?bbox=" + ','.join(bb)
osm_api_response = requests.get(url)
changeset = task.rounding_changeset(BytesIO(osm_api_response.content))
filename = "rounding_{0}_{1}_{2}.osm".format(z,x,y)
return Response(etree.tostring(changeset,pretty_print=True),
mimetype="text/xml")
def changeset(z,x,y):
if z < 16:
print "Too big"
raise Exception
bb = mercantile.bounds(x,y,z)
bb = [str(f) for f in [bb.west,bb.south,bb.east,bb.north]]
url = "http://openstreetmap.org/api/0.6/map?bbox=" + ','.join(bb)
osm_api_response = requests.get(url)
changeset = task.changeset(BytesIO(osm_api_response.content),height_db)
filename = "sfbuildingheight_{0}_{1}_{2}.osm".format(z,x,y)
return Response(etree.tostring(changeset,pretty_print=True),
mimetype="text/xml",
headers={
'Content-Disposition':'attachment; filename={0}'.format(filename)
})
def serve(self):
bounds = mercantile.bounds(self.x, self.y, self.zoom)
self.west, self.south = mercantile.xy(bounds.west, bounds.south)
self.east, self.north = mercantile.xy(bounds.east, bounds.north)
self.layers = []
if self.namespace not in RECIPES:
msg = 'Recipe "{}" not found. Available recipes are: {}'
abort(400, msg.format(self.namespace, list(RECIPES.keys())))
self.recipe = RECIPES[self.namespace]
names = self.recipe.layers.keys() if self.ALL else self.names
for name in names:
if name not in self.recipe.layers:
abort(400, u'Layer "{}" not found in recipe {}'.format(
name, self.namespace))
self.process_layer(self.recipe.layers[name])
self.post_process()
return self.content, 200, {"Content-Type": self.CONTENT_TYPE}
def correct_altitude_mode(context, kmls: type_list_of_kmls):
"""
Check for KMLs with altitude relative to ground,
query Mapbox-terrain-rgb and change for relative
to sea level (absolute)
"""
for kml in tqdm(kmls, desc="KMLS"):
with open(kml, "r+") as f:
txt = f.read()
if re.search("(?<=altitudeMode>)relative(.+)?(?=\/altitudeMode>)", txt):
lat = round(float(find_with_re("latitude", txt)), 5)
lng = round(float(find_with_re("longitude", txt)), 5)
alt = round(float(find_with_re("altitude", txt)), 5)
z = 15
tile = mercantile.tile(lng, lat, z)
westmost, southmost, eastmost, northmost = mercantile.bounds(tile)
pixel_column = np.interp(lng, [westmost, eastmost], [0, 256])
pixel_row = np.interp(lat, [southmost, northmost], [256, 0])
tile_img = Image.open(
requests.get(
"https://api.mapbox.com/v4/mapbox.terrain-rgb/10/800/200.pngraw?access_token=pk.eyJ1IjoibWFydGltcGFzc29zIiwiYSI6ImNra3pmN2QxajBiYWUycW55N3E1dG1tcTEifQ.JFKSI85oP7M2gbeUTaUfQQ",
stream=True,
).raw
).load()
R, G, B, _ = tile_img[int(pixel_row), int(pixel_column)]
height = -10000 + ((R * 256 * 256 + G * 256 + B) * 0.1)
new_height = height + alt
txt = re.sub(
"(?<=<altitudeMode>).+(?=<\/altitudeMode>)", "absolute", txt
)
txt = re.sub("(?<=<altitude>).+(?=<\/altitude>)", f"{new_height}", txt)
txt = re.sub(
"(?<=<coordinates>).+(?=<\/coordinates>)",
f"{lng},{lat},{new_height}",
txt,
)
f.seek(0)
f.write(txt)
f.truncate()
else:
continue
return kmls
def _merge_tiles(tiles, arrays):
"""
Merge a set of tiles into a single array.
Parameters
---------
tiles : list of mercantile.Tile objects
The tiles to merge.
arrays : list of numpy arrays
The corresponding arrays (image pixels) of the tiles. This list
has the same length and order as the `tiles` argument.
Returns
-------
img : np.ndarray
Merged arrays.
extent : tuple
Bounding box [west, south, east, north] of the returned image
in long/lat.
"""
# create (n_tiles x 2) array with column for x and y coordinates
tile_xys = np.array([(t.x, t.y) for t in tiles])
# get indices starting at zero
indices = tile_xys - tile_xys.min(axis=0)
# the shape of individual tile images
h, w, d = arrays[0].shape
# number of rows and columns in the merged tile
n_x, n_y = (indices + 1).max(axis=0)
# empty merged tiles array to be filled in
img = np.zeros((h * n_y, w * n_x, d), dtype=np.uint8)
for ind, arr in zip(indices, arrays):
x, y = ind
img[y * h:(y + 1) * h, x * w:(x + 1) * w, :] = arr
bounds = np.array([mt.bounds(t) for t in tiles])
west, south, east, north = (min(bounds[:, 0]), min(bounds[:, 1]),
max(bounds[:, 2]), max(bounds[:, 3]))
return img, (west, south, east, north)
def get_opts(self, count):
"""Return rasterio dataset creation options for the bounding tile."""
w, s, e, n = list(mercantile.bounds(*self.bounding_tile))
w, s = mercantile.xy(w, s)
e, n = mercantile.xy(e, n)
xcell = ((e - w) / self.tileshape)
ycell = ((n - s) / self.tileshape)
return {
'dtype': np.float32,
'driver': 'GTiff',
'height': self.tileshape,
'width': self.tileshape,
'count': count,
'compress': 'lzw',
'transform': affine.Affine(xcell, 0, w, 0, -ycell, n),
'crs': 'epsg:3857'}
def test_make_tiles_tile_bounds(x, y):
'''
Test if children tiles from z10 are created correctly
'''
test_bounds = mercantile.bounds(x, y, 10)
test_bbox = list(mercantile.xy(test_bounds.west, test_bounds.south)) + list(mercantile.xy(test_bounds.east, test_bounds.north))
test_crs = 'epsg:3857'
test_minz = 10
test_maxz = 13
created_tiles_gen = _make_tiles(test_bbox, test_crs, test_minz, test_maxz)
assert isinstance(created_tiles_gen, types.GeneratorType)
created_tiles = list(created_tiles_gen)
assert len(created_tiles) == 85
def test_metadata():
"""Should return correct metadata."""
with rasterio.open(COG_CMAP) as src_dst:
meta = reader.metadata(src_dst)
assert meta["dtype"] == "int8"
assert meta["colorinterp"] == ["palette"]
assert not meta.get("scale")
assert not meta.get("ofsset")
assert meta.get("colormap")
with rasterio.open(COG_SCALE) as src_dst:
meta = reader.metadata(src_dst)
assert meta["dtype"] == "int16"
assert meta["colorinterp"] == ["gray"]
assert meta["scale"] == 0.0001
assert meta["offset"] == 1000.0
assert meta["band_descriptions"] == [(1, "Green")]
assert not meta.get("colormap")
assert meta["nodata_type"] == "Nodata"
meta = reader.metadata(src_dst, indexes=1)
assert meta["colorinterp"] == ["gray"]
bounds = mercantile.bounds(mercantile.Tile(x=218, y=99, z=8))
meta = reader.metadata(src_dst, bounds)
assert meta["colorinterp"] == ["gray"]
assert meta["bounds"] == bounds
with rasterio.open(S3_ALPHA_PATH) as src_dst:
with pytest.warns(AlphaBandWarning):
meta = reader.metadata(src_dst)
assert len(meta["band_descriptions"]) == 3
assert meta["colorinterp"] == ["red", "green", "blue"]
assert meta["nodata_type"] == "Alpha"
meta = reader.metadata(src_dst, indexes=(1, 2, 3, 4))
assert len(meta["band_descriptions"]) == 4
assert meta["colorinterp"] == ["red", "green", "blue", "alpha"]
assert meta["nodata_type"] == "Alpha"
with rasterio.open(S3_MASK_PATH) as src_dst:
meta = reader.metadata(src_dst)
assert meta["nodata_type"] == "Mask"
def intersects(self, file, tile):
ds = gdal.Open(file)
if ds is None:
return False
geo = ds.GetGeoTransform()
ulx = geo[0]
uly = geo[3]
lrx = ulx + ds.RasterXSize * geo[1]
lry = uly + ds.RasterYSize * geo[5]
del ds
f_bb = shapely.geometry.geo.box(min(ulx, lrx), min(uly, lry), max(ulx, lrx), max(uly, lry))
f_region = Polygon(f_bb)
x = mercantile.bounds(tile)
t_bb = shapely.geometry.geo.box(min(x.west, x.east), min(x.north, x.south),
max(x.west, x.east), max(x.north, x.south))
t_region = Polygon(t_bb)
return f_region.intersects(t_region)
def salehi():
with open(sys.argv[1], 'wb') as csvFile:
fileWriter = csv.writer(csvFile, delimiter=',')
numberOfRows = int(sys.argv[2])
sourceProj = Proj(init='epsg:4326')
targetProj = Proj(init='epsg:3857')
# x1, y1 = transform(sourceProj, targetProj, 43.0, 24.0)
# x2, y2 = transform(sourceProj, targetProj, 63.0, 40.0)
for i in mercantile.tiles(43.85, 25.25, 63.39, 39.87, [15], True):
bbox = mercantile.bounds(i)
assert isinstance(bbox, mercantile.LngLatBbox)
min_x, min_y = bbox.west, bbox.south
max_x, max_y = bbox.east, bbox.north
row = (min_x, min_y, max_x, max_y, 256, 256)
fileWriter.writerow(row)
numberOfRows = numberOfRows - 1
if numberOfRows <= 0:
break
def get_geojson(self):
"""Return geojson of the bounding tile."""
w, s, e, n = list(mercantile.bounds(*self.bounding_tile))
return {
"type": "Feature",
"properties": {},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[w, s],
[e, s],
[e, n],
[w, n],
[w, s]
]
]
}
}
def tile_bbox(tile, mercator=False):
if isinstance(tile, mercantile.Tile):
if mercator:
return mercantile.xy_bounds(tile) # EPSG:3857
else:
return mercantile.bounds(tile) # EPSG:4326
else:
with open(rasterio_open(tile)) as r:
if mercator:
w, s, e, n = r.bounds
w, s = mercantile.xy(w, s)
e, n = mercantile.xy(e, n)
return w, s, e, n # EPSG:3857
else:
return r.bounds # EPSG:4326
assert False, "Unable to open tile"
def to_geojson(input, collect):
"""Read MosaicJSON document and create GeoJSON features."""
features = []
with MosaicBackend(input) as mosaic:
for qk, assets in mosaic.mosaic_def.tiles.items():
tile = mercantile.quadkey_to_tile(qk)
west, south, east, north = mercantile.bounds(tile)
geom = {
"type":
"Polygon",
"coordinates": [[
[west, south],
[west, north],
[east, north],
[east, south],
[west, south],
]],
}
feature = {
"type": "Feature",
"id": str(tile),
"geometry": geom,
"properties": {
"nb_assets": len(assets),
"assets": assets
},
}
if collect:
features.append(feature)
else:
click.echo(json.dumps(feature))
if collect and features:
click.echo(
json.dumps({
"type": "FeatureCollection",
"features": features
}, ))
def generate_ground_points(zoom, num_pts, vert_sigma=10.0):
"""
This function creates a uniformly sampled grid of points across a single
Mercantile tile. Uses numpy.random.uniform to pick the 2D locations and
then assigns a height value based on vert_sigma. Returns 2 Nx3 ndarrays,
one of lon_lat_height, and the second is the NED locations in the tile.
Finally we return the Lon, Lat (deg), and HAE of the center point of
the tile used as the origin of the local level NED frame
"""
wpafb_lon = -84.049954
wpafb_lat = 39.8179055
tile = mercantile.tile(wpafb_lon, wpafb_lat, zoom)
bounds = mercantile.bounds(tile.x, tile.y, tile.z)
lons = np.random.uniform(bounds.west, bounds.east, num_pts)
lats = np.random.uniform(bounds.south, bounds.north, num_pts)
heights = np.random.standard_normal(num_pts) * vert_sigma
center = np.array([(bounds.west + bounds.east) / 2,
(bounds.north + bounds.south) / 2, 0.0])
pts_ned = navpy.lla2ned(lats, lons, heights, center[1], center[0], 0.0)
lon_lat_height = np.vstack((lons, lats, heights)).T
return lon_lat_height, pts_ned, center
def data_tile_meta(prefix, var, tile_pos):
z, x, y = tile_pos
tile_meta = mercantile.Tile(x=x, y=y, z=z)
min_lon, min_lat, max_lon, max_lat = mercantile.bounds(tile_meta)
data_tile = xr.open_zarr(store=get_store(prefix))[var]
data_tile = data_tile.sel(longitude=slice(min_lon, max_lon),
latitude=slice(max_lat, min_lat))
return {
"variable": var,
"z": z,
"x": x,
"y": y,
"lats": [min_lat, max_lat],
"lons": [min_lon, max_lon],
f"{var}Max": float(data_tile.max().compute()),
f"{var}Min": float(data_tile.min().compute()),
"units": data_tile.units,
"description": data_tile.long_name,
}
def quadkeys_to_bounds(quadkeys: List[str]):
"""Convert list of quadkeys to bounds
Args:
- quadkeys: List of quadkeys
"""
tile_bounds = [
mercantile.bounds(mercantile.quadkey_to_tile(qk)) for qk in quadkeys
]
minx = 180
miny = 90
maxx = -180
maxy = -90
for tb in tile_bounds:
minx = min(minx, tb[0])
miny = min(miny, tb[1])
maxx = max(maxx, tb[2])
maxy = max(maxy, tb[3])
return [minx, miny, maxx, maxy]
def createVRT(self, filespec=None, tile=None):
if filespec is None:
logging.error("Need to supply a file to process!")
return
logging.debug("Creating VRT for %s" % filespec)
self.drv = gdal.GetDriverByName("VRT")
base = os.path.splitext(filespec)
self.vrt = self.drv.Create(base[0] + ".vrt", self.tilesize, self.tilesize, bands=0)
self.metadata = self.drv.GetMetadata()
self.vrt.AddBand(gdal.GDT_Byte)
band = self.vrt.GetRasterBand(1)
#idx = "%s/%s/%s" % (tile.z, tile.x, tile.y)
simple = '<SourceFilename relativeToVRT=\"1\">%s</SourceFilename>' % os.path.basename(filespec)
band.SetMetadataItem("SimpleSource", simple)
bbox = mercantile.bounds(tile)
# x=0.0, y=0.0, z=0.0, pixel=0.0, line=0.0,
gcpList = [gdal.GCP(bbox.west,bbox.north,0,0,0),
gdal.GCP(bbox.east,bbox.north,0,256,0),
gdal.GCP(bbox.east,bbox.south,0,256,256),
gdal.GCP(bbox.west,bbox.south,0,0,256)]
self.vrt.SetGCPs(gcpList, str('')) # Add the GCPs to the VRT file
self.makeTileDir(tile)
tmpfile = base[0] + '-tmp.tif'
outfile = base[0] + '.tif'
imgfile = gdal.Open(filespec, gdal.GA_ReadOnly)
if os.path.exists(filespec):
# rgbExpand=rgba
# if "ERSI" not in filespec:
# opts = gdal.TranslateOptions(rgbExpand='RGBA', GCPs=gcpList, format='GTiff')
# else:
# opts = gdal.TranslateOptions(GCPs=gcpList, format='GTiff')
opts = gdal.TranslateOptions(GCPs=gcpList, format='GTiff')
ds1 = gdal.Translate(tmpfile, imgfile, options=opts)
#gdal.Warp(outfile, ds1, format='GTiff', xRes=30, yRes=30)
gdal.Warp(outfile, ds1, format='GTiff', dstSRS='EPSG:4326')
self.tifs.append(outfile)
os.remove(tmpfile)
def main(args):
config = load_config(args.config)
check_classes(config)
index = [
i for i in (list(range(len(config["classes"]))))
if config["classes"][i]["title"] == args.type
]
assert index, "Requested type {} not found among classes title in the config file.".format(
args.type)
print("RoboSat.pink - vectorize {} from {}".format(args.type, args.masks))
with open(args.out, "w", encoding="utf-8") as out:
first = True
out.write('{"type":"FeatureCollection","features":[')
for tile, path in tqdm(list(tiles_from_slippy_map(args.masks)),
ascii=True,
unit="mask"):
features = (np.array(Image.open(path).convert("P"),
dtype=np.uint8) == index).astype(np.uint8)
try:
C, W, H = features.shape
except:
W, H = features.shape
transform = rasterio.transform.from_bounds(
(*mercantile.bounds(tile.x, tile.y, tile.z)), W, H)
for shape, value in rasterio.features.shapes(features,
transform=transform):
prop = '"properties":{{"x":{},"y":{},"z":{}}}'.format(
int(tile.x), int(tile.y), int(tile.z))
geom = '"geometry":{{"type": "Polygon", "coordinates":{}}}'.format(
json.dumps(shape["coordinates"]))
out.write('{}{{"type":"Feature",{},{}}}'.format(
"," if not first else "", geom, prop))
first = False
out.write("]}")
def main(args):
config = load_config(args.config)
check_classes(config)
index = [
i for i in (list(range(len(config["classes"]))))
if config["classes"][i]["title"] == args.type
]
assert index, "Requested type {} not found among classes title in the config file.".format(
args.type)
print("RoboSat.pink - vectorize {} from {}".format(args.type, args.masks))
out = open(args.out, "w", encoding="utf-8")
assert out, "Unable to write in output file"
out.write('{"type":"FeatureCollection","features":[')
first = True
for tile, path in tqdm(list(tiles_from_dir(args.masks, xyz_path=True)),
ascii=True,
unit="mask"):
mask = (np.array(Image.open(path).convert("P"),
dtype=np.uint8) == index).astype(np.uint8)
try:
C, W, H = mask.shape
except:
W, H = mask.shape
transform = rasterio.transform.from_bounds(
(*mercantile.bounds(tile.x, tile.y, tile.z)), W, H)
for shape, value in rasterio.features.shapes(mask,
transform=transform,
mask=mask):
geom = '"geometry":{{"type": "Polygon", "coordinates":{}}}'.format(
json.dumps(shape["coordinates"]))
out.write('{}{{"type":"Feature",{}}}'.format(
"" if first else ",", geom))
first = False
out.write("]}")
def comptiles(filedir, comparedir, sampling, filetype):
# plotdir = '/Users/dnomadb/Documents/pcomp'
files = os.listdir(filedir)
cfiles = os.listdir(comparedir)
if plotdir:
import matplotlib.pyplot as plot
for f in files:
fileinfo = f.split("-")
if len(fileinfo[-1].split(".")) != 0 and fileinfo[-1].split(".")[-1] == filetype:
x, y, z = tiledelta.getXYZ(fileinfo)
bbox = mercantile.bounds(x, y, z)
with rio.drivers():
with rio.open(os.path.join(filedir, f), "r") as src:
greyimage_before = (
src.read(1).astype(np.uint16) + src.read(2).astype(np.uint16) + src.read(3).astype(np.uint16)
)
with rio.open(os.path.join(comparedir, f), "r") as src:
greyimage_after = (
src.read(1).astype(np.uint16) + src.read(2).astype(np.uint16) + src.read(3).astype(np.uint16)
)
pcplo = tiledelta.compareGreys(greyimage_after, greyimage_before, 10, 20)
pcplo = pcplo[::sampling, ::sampling]
if plotdir:
fig = plot.figure(figsize=(20, 10))
before = fig.add_subplot(131)
before.imshow(greyimage_after, cmap="Greys_r")
after = fig.add_subplot(132)
after.imshow(greyimage_before, cmap="Greys_r")
pc2 = fig.add_subplot(133)
pc2.imshow(pcplo, cmap="YlGnBu")
fig.savefig(os.path.join(plotdir, f))
else:
tiledelta.makeVectors(pcplo, tiledelta.makeAffine(pcplo.shape, bbox))
def get_tiles(change):
nonlocal url, tile_path
if dynamic:
new_tile_path = mkdtemp(prefix='xarray_leaflet_')
new_url = base_url + '/xarray_leaflet' + new_tile_path + '/{z}/{x}/{y}.png'
if l in m.layers:
m.remove_layer(l)
((south, west), (north, east)) = change['new']
tiles = list(mercantile.tiles(west, south, east, north, m.zoom))
if dynamic:
da_visible = da.sel(y=slice(north, south), x=slice(west, east))
else:
bbox = get_bbox_tiles(tiles)
da_visible = da.sel(y=slice(bbox.north, bbox.south), x=slice(bbox.west, bbox.east))
# check if we have visible data
if 0 not in da_visible.shape:
da_visible, transform1_args = get_transform(transform1(da_visible, *transform0_args))
if dynamic:
tile_path = new_tile_path
url = new_url
for tile in tiles:
path = f'{tile_path}/{int(tile.z)}/{tile.x}/{tile.y}.png'
# if static map, check if we already have the tile
# if dynamic map, new tiles are always created
if dynamic or not os.path.exists(path):
bbox = mercantile.bounds(tile)
da_tile = da_visible.sel(y=slice(bbox.north, bbox.south), x=slice(bbox.west, bbox.east))
# check if we have data for this tile
if 0 not in da_tile.shape:
da_tile = reindex(da_tile, bbox, dx, dy)
da_tile, transform2_args = get_transform(transform2(da_tile, *transform1_args))
np_tile = get_webmercator(da_tile.values, bbox.west, bbox.north, dx, dy)
np_tile, transform3_args = get_transform(transform3(np_tile, *transform2_args))
np_tile = colormap(np_tile)
write_image(np_tile, path, persist)
if dynamic:
l.path = url
m.add_layer(l)
l.redraw()
def tile_view(request, zoom, x, y):
"""
Returns an MVT tiles given zoom, x and y in TMS format
References:
https://www.mapbox.com/vector-tiles/
http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-mercator
"""
bounds = mercantile.bounds(int(x), int(y), int(zoom))
west, south = mercantile.xy(bounds.west, bounds.south)
east, north = mercantile.xy(bounds.east, bounds.north)
pixel = pixel_length(zoom)
buffer = 4 * pixel
bbox = Polygon.from_bbox(
(west - buffer, south - buffer, east + buffer, north + buffer))
departements = Departement.objects.filter(geom__intersects=bbox)
departements = departements.annotate(clipped=Intersection('geom', bbox))
tile = {
"name":
"departements",
"features": [{
"geometry":
departement.clipped.simplify(pixel, preserve_topology=True).wkt,
"properties": {
"numero": departement.code_dept,
"nom": departement.nom_dept,
},
} for departement in departements],
}
vector_tile = mapbox_vector_tile.encode(tile,
quantize_bounds=(west, south, east,
north))
return HttpResponse(vector_tile,
content_type="application/vnd.mapbox-vector-tile")
def build_source_index(tile, min_zoom, max_zoom):
source_cache = MemoryAdapter()
bbox = box(*mercantile.bounds(tile))
database_url = os.environ.get('DATABASE_URL')
with psycopg2.connect(database_url) as conn:
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(
"""
SELECT
filename, resolution, source, url,
min_zoom, max_zoom, priority, approximate_zoom,
wkb_geometry
FROM
footprints
WHERE
ST_Intersects(
wkb_geometry,
ST_GeomFromText(%s, 4326)
)
AND min_zoom <= %s
AND max_zoom >= %s
AND enabled = true
""", (bbox.to_wkt(), min_zoom, max_zoom))
logger.info("Found %s sources for tile %s, zoom %s-%s",
cur.rowcount, tile, min_zoom, max_zoom)
if not cur.rowcount:
raise ValueError("No sources found for this tile")
for row in cur:
row = dict(row)
shape = wkb.loads(row.pop('wkb_geometry').decode('hex'))
source_cache.add_source(shape, row)
return source_cache
def create_overview_mosaic(urls, quadkey_zoom, min_zoom, max_zoom):
"""Create mosaic representing overview
"""
# Input is file object
urls = [l.strip() for l in urls.readlines()]
quadkeys = [parse_url(url, quadkey_zoom) for url in urls]
# Find bounds of quadkeys
bboxes = [
mercantile.bounds(mercantile.quadkey_to_tile(qk)) for qk in quadkeys
]
minx = min(bboxes, key=lambda bbox: bbox[0])[0]
miny = min(bboxes, key=lambda bbox: bbox[1])[1]
maxx = max(bboxes, key=lambda bbox: bbox[2])[2]
maxy = max(bboxes, key=lambda bbox: bbox[3])[3]
bounds = [minx, miny, maxx, maxy]
# Find center
center = [(minx + maxx) / 2, (miny + maxy) / 2, min_zoom]
tiles = {}
for qk, url in zip(quadkeys, urls):
tiles[qk] = [url]
mosaic = {
"mosaicjson": "0.0.2",
"minzoom": min_zoom,
"maxzoom": max_zoom,
"quadkey_zoom": 6,
"bounds": bounds,
"center": center,
"tiles": tiles
}
# Validation
mosaic = MosaicJSON(**mosaic).dict(exclude_none=True)
print(json.dumps(mosaic, separators=(',', ':')))
def get_tile_tif(tile, imagery, dest_folder):
"""
Read a GeoTIFF with a window corresponding to a TMS tile
The TMS tile bounds are converted to the GeoTIFF source CRS. That bounding
box is converted to a pixel window which is read from the GeoTIFF. For
remote files which are internally tiled, this will take advantage of HTTP
GET Range Requests to avoid downloading the entire file. See more info at:
http://www.cogeo.org/in-depth.html
"""
bound = bounds(*[int(t) for t in tile.split('-')])
with rasterio.open(imagery) as src:
x_res, y_res = src.transform[0], src.transform[4]
proj_to = Proj(**src.crs)
# project tile boundaries from lat/lng to source CRS
tile_ul_proj = proj_to(bound.west, bound.north)
tile_lr_proj = proj_to(bound.east, bound.south)
# get origin point from the TIF
tif_ul_proj = (src.bounds.left, src.bounds.top)
# use the above information to calculate the pixel indices of the window
top = int((tile_ul_proj[1] - tif_ul_proj[1]) / y_res)
left = int((tile_ul_proj[0] - tif_ul_proj[0]) / x_res)
bottom = int((tile_lr_proj[1] - tif_ul_proj[1]) / y_res)
right = int((tile_lr_proj[0] - tif_ul_proj[0]) / x_res)
window = ((top, bottom), (left, right))
# read the first three bands (assumed RGB) of the TIF into an array
data = np.empty(shape=(3, 256, 256)).astype(src.profile['dtype'])
for k in (1, 2, 3):
src.read(k, window=window, out=data[k - 1], boundless=True)
# save
tile_img = op.join(dest_folder, 'tiles', '{}{}'.format(tile, '.jpg'))
img = Image.fromarray(np.moveaxis(data, 0, -1), mode='RGB')
img.save(tile_img)
async def predict_tile(self, session, tile):
quadkey = get_tile_quadkey(tile)
bounds = list(mercantile.bounds(tile))
bbox_str = ','.join(map(str, bounds))
tile_url = f'{self.tile_url}&bbox={bbox_str}'
try:
res = await session.get(tile_url)
if res.status != 200:
logging.warning(f'Unable to fetch tile {tile_url}')
raise Exception(f'Unable to fetch tile {tile_url}')
data = await res.json()
return {
'quadkey': quadkey,
'centroid': get_tile_center(tile),
'predictions': {
'ml_prediction': data['road_length_km']['total'],
'url': tile_url,
}
}
except Exception as e:
logging.error(str(e))
def get_render_configurations(self, request, **kwargs):
tile_bounds = list(mercantile.bounds(int(self.kwargs['x']), int(self.kwargs['y']), int(self.kwargs['z'])))
extent = BBox(tile_bounds, projection=Proj('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs')).project(
Proj(
'+proj=merc +lon_0=0 +k=1 +x_0=0 +y_0=0 +a=6378137 +b=6378137 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs'
)
)
base_config = ImageConfiguration(
extent=extent,
size=TILE_SIZE,
image_format='png',
background_color=TRANSPARENT_BACKGROUND_COLOR
)
return base_config, [
RenderConfiguration(
variable=x,
extent=extent,
size=TILE_SIZE,
image_format='png'
) for x in self.service.variable_set.all()
]
def create_cache(part_zoom, countries, splited_countries, min_cache_zoom):
stat = Stat()
cache = {}
if min_cache_zoom:
stat.log_stats('cache (%s)' % min_cache_zoom, cache)
return min_cache_zoom, cache
min_cache_zoom = MIN_ZOOM
for z in range(MAX_ZOOM + 1):
for x in range(2 ** z):
for y in range(2 ** z):
k = '%s/%s/%s' % (z, x, y)
b = mercantile.bounds(x, y, z)
country = detect_country_with_cache(
k, b, x, y, z, part_zoom, countries, splited_countries,
z, cache, stat)
if '|' not in country:
min_cache_zoom = z
if min_cache_zoom > 0:
break
stat.log_stats('cache (%s)' % min_cache_zoom, cache)
return min_cache_zoom, cache
async def test_cog_read(infile, create_cog_reader):
async with create_cog_reader(infile) as cog:
with rasterio.open(infile) as ds:
_, zoom = get_zooms(ds)
centroid = Polygon.from_bounds(
*transform_bounds(cog.epsg, "EPSG:4326", *cog.bounds)).centroid
tile = mercantile.tile(centroid.x, centroid.y, zoom)
tile_native_bounds = transform_bounds("EPSG:4326", cog.epsg,
*mercantile.bounds(tile))
arr = await cog.read(tile_native_bounds, (256, 256))
with cogeo_reader(infile) as ds:
rio_tile_arr, rio_tile_mask = ds.tile(tile.x,
tile.y,
tile.z,
tilesize=256,
resampling_method="bilinear")
if cog.is_masked:
tile_arr = np.ma.getdata(arr)
tile_mask = np.ma.getmask(arr)
# Make sure image data is the same
assert pytest.approx(tile_arr - rio_tile_arr,
1) == np.zeros(tile_arr.shape)
# Make sure mask data is the same
rio_mask_counts = np.unique(rio_tile_mask, return_counts=True)
tile_mask_counts = np.unique(tile_mask, return_counts=True)
assert len(rio_mask_counts[0]) == len(tile_mask_counts[0])
assert (rio_mask_counts[1][0] *
cog.profile["count"] == tile_mask_counts[1][0])
else:
assert pytest.approx(arr - rio_tile_arr, 1) == np.zeros(arr.shape)
def writeTile(self, path):
"""Write the image to disk"""
filespec = "%s/%s/%s/%s/%s.png" % (path, self.z, self.x, self.y, self.y)
if os.path.exists(filespec) is False:
tmp = ""
for i in os.path.dirname(filespec).split('/'):
tmp += i + '/'
if os.path.exists(tmp) is False:
os.mkdir(tmp)
file = open(filespec, "wb")
bytes = self.blob
logging.debug("Writing %r bytes to %r" % (len(bytes), filespec))
file.write(bytes)
file.close()
suffix = filetype.guess(filespec)
print('File extension: %s' % suffix.extension)
if suffix.extension == 'jpg':
dest = filespec.replace(".png", ".jpg")
logging.debug("Renaming %r to %r" % (filespec, dest))
os.rename(filespec, dest)
foo = mercantile.bounds(self.y, self.x, self.z + 12)
logging.info("Wrote %r" % filespec)
return True
def get_tile_wms(tile, imagery, folder, imagery_offset, kwargs):
"""
Read a WMS endpoint with query parameters corresponding to a TMS tile
Converts the tile boundaries to the spatial/coordinate reference system
(SRS or CRS) specified by the WMS query parameter.
"""
# retrieve the necessary parameters from the query string
query_dict = parse_qs(imagery.lower())
image_format = query_dict.get('format')[0].split('/')[1]
wms_version = query_dict.get('version')[0]
if wms_version == '1.3.0':
wms_srs = query_dict.get('crs')[0]
else:
wms_srs = query_dict.get('srs')[0]
# find our tile bounding box
bound = bounds(*[int(t) for t in tile.split('-')])
p1 = Proj({'init': 'epsg:4326'})
p2 = Proj({'init': wms_srs})
# project the tile bounding box from lat/lng to WMS SRS
tile_ll_proj = transform(p1, p2, bound.west, bound.south)
tile_ur_proj = transform(p1, p2, bound.east, bound.north)
if wms_version == '1.3.0':
bbox = tile_ll_proj[::-1] + tile_ur_proj[::-1]
else:
bbox = tile_ll_proj + tile_ur_proj
# request the image with the transformed bounding box and save
wms_url = imagery.replace('{bbox}', ','.join([str(b) for b in bbox]))
r = requests.get(wms_url, auth=kwargs.get('http_auth'))
tile_img = op.join(folder, '{}.{}'.format(tile, image_format))
with open(tile_img, 'wb') as w:
w.write(r.content)
return tile_img
def pixel_to_location(tile, dx, dy):
'''Converts a pixel in a tile to a coordinate.
Args:
tile: the mercantile tile to calculate the location in.
dx: the relative x offset in range [0, 1].
dy: the relative y offset in range [0, 1].
Returns:
The coordinate for the pixel in the tile.
'''
assert 0 <= dx <= 1, 'x offset is in [0, 1]'
assert 0 <= dy <= 1, 'y offset is in [0, 1]'
west, south, east, north = mercantile.bounds(tile)
def lerp(a, b, c):
return a + c * (b - a)
lon = lerp(west, east, dx)
lat = lerp(south, north, dy)
return lon, lat
def create_single_tile(self, tile, in_file, postfix='', interp='lanczos', border=1):
name = "{}_{}".format(tile.x, tile.y)
tile_folder = os.path.join(self.out_folder, name)
os.makedirs(tile_folder, exist_ok=True)
out_base_name = postfix + '.tif'
bb = mercantile.bounds(tile)
out_file = os.path.join(tile_folder, out_base_name)
if os.path.exists(out_file):
logger.info('Skipping tile for {}, file exists.'.format(out_file))
else:
if not self.intersects(in_file, tile):
logger.info('Skipping tile for {}, does not intersect.'.format(out_file))
return None
else:
x_res = abs(bb.west - bb.east) / (self.tile_x-2) # should be fixed
y_res = abs(bb.north - bb.south) / (self.tile_y-2) # changes with lat
# When we select the projection - we will grow by a pixel in all directions to handle partial pixel
r = ''
if interp is not None:
r = "-r {}".format(interp)
cmd = 'gdal_translate {} -tr {} {} -projwin {} {} {} {} -a_nodata {} -co COMPRESS=LZW {} {}'.format(
r, x_res, y_res,
bb.west - x_res * border,
bb.north + y_res * border,
bb.east + x_res * border,
bb.south - y_res * border,
self.no_data, in_file, out_file)
logging.info(cmd)
os.system(cmd)
return out_file
def main(outfile, xyz, **dump_kw):
x, y, z = map(int, xyz.split(','))
minlon, minlat, maxlon, maxlat = (
round(v, 6) for v in mercantile.bounds(x, y, z))
geom = {
'type': 'Polygon',
'coordinates': [[
[minlon, minlat],
[minlon, maxlat],
[maxlon, maxlat],
[maxlon, minlat],
[minlon, minlat] ]]}
feature = {
'type': 'Feature',
'id': xyz,
'geometry': geom,
'properties': {'title': 'XYZ tile %s' % xyz} }
collection = {'type': 'FeatureCollection', 'features': [feature]}
with open_output(args.outfile) as sink:
json.dump(collection, sink, **dump_kw)
return 0
def pixel_to_location(tile, dx, dy):
"""Converts a pixel in a tile to a coordinate.
Args:
tile: the mercantile tile to calculate the location in.
dx: the relative x offset in range [0, 1].
dy: the relative y offset in range [0, 1].
Returns:
The coordinate for the pixel in the tile.
"""
assert 0 <= dx <= 1, "x offset is in [0, 1]"
assert 0 <= dy <= 1, "y offset is in [0, 1]"
west, south, east, north = mercantile.bounds(tile)
def lerp(a, b, c):
return a + c * (b - a)
lon = lerp(west, east, dx)
lat = lerp(south, north, dy)
return lon, lat
def create_cache(part_zoom, countries, splited_countries, min_cache_zoom):
stat = Stat()
cache = {}
if min_cache_zoom:
stat.log_stats('cache (%s)' % min_cache_zoom, cache)
return min_cache_zoom, cache
min_cache_zoom = MIN_ZOOM
for z in range(MAX_ZOOM + 1):
for x in range(2**z):
for y in range(2**z):
k = '%s/%s/%s' % (z, x, y)
b = mercantile.bounds(x, y, z)
country = detect_country_with_cache(k, b, x, y, z, part_zoom,
countries,
splited_countries, z,
cache, stat)
if '|' not in country:
min_cache_zoom = z
if min_cache_zoom > 0:
break
stat.log_stats('cache (%s)' % min_cache_zoom, cache)
return min_cache_zoom, cache
async def get_image(url, available_projections, lon, lat, zoom, session,
messages):
"""Download image (tms tile for coordinate lon,lat on level zoom and calculate image hash
Parameters
----------
url : str
available_projections : collection
lon : float
lat : float
zoom : int
session : ClientSession
messages : list
Returns
-------
ImageHash or None
"""
tile = list(mercantile.tiles(lon, lat, lon, lat, zooms=zoom))[0]
bounds = list(mercantile.bounds(tile))
proj = None
if "EPSG:4326" in available_projections:
proj = "EPSG:4326"
elif "EPSG:3857" in available_projections:
proj = "EPSG:3857"
else:
for proj in sorted(available_projections):
try:
CRS.from_string(proj)
except:
continue
break
if proj is None:
messages.append("No projection left: {}".format(available_projections))
return None
crs_from = CRS.from_string("epsg:4326")
crs_to = CRS.from_string(proj)
if not proj == "EPSG:4326":
transformer = Transformer.from_crs(crs_from, crs_to, always_xy=True)
bounds = list(transformer.transform(bounds[0], bounds[1])) + list(
transformer.transform(bounds[2], bounds[3]))
# WMS < 1.3.0 assumes x,y coordinate ordering.
# WMS 1.3.0 expects coordinate ordering defined in CRS.
if crs_to.axis_info[0].direction == "north" and "=1.3.0" in url:
bbox = ",".join(map(str, [bounds[1], bounds[0], bounds[3], bounds[2]]))
else:
bbox = ",".join(map(str, bounds))
formatted_url = url.format(proj=proj, width=512, height=512, bbox=bbox)
messages.append("Image URL: {}".format(formatted_url))
for i in range(3):
try:
# Download image
async with session.request(method="GET",
url=formatted_url,
ssl=False) as response:
data = await response.read()
img = Image.open(io.BytesIO(data))
img_hash = imagehash.average_hash(img)
messages.append("ImageHash: {}".format(img_hash))
return img_hash
except Exception as e:
messages.append("Could not download image in try {}: {}".format(
i, str(e)))
await asyncio.sleep(5)
return None
def bbox_from_import_url(url):
match = re.search("sfbuildingheight_(\d+)_(\d+)_(\d+).osm",url)
z = int(match.group(1))
x = int(match.group(2))
y = int(match.group(3))
return mercantile.bounds(x,y,z)
raise ValueError("Could not connect to queue {}".format(queue_name))
return queue
def create_job(x, y, min_zoom, max_zoom, bounds):
body = {
"x": tile.x,
"y": tile.y,
"min_zoom": tile.z,
"max_zoom": 14,
"bounds": {"west": bounds.west, "south": bounds.south, "east": bounds.east, "north": bounds.north},
}
msg = Message()
msg.set_body(json.dumps(body))
return msg
if __name__ == "__main__":
queue = connect_job_queue()
zoom_level = int(os.getenv("TASK_ZOOM_LEVEL", "8"))
for tile in tiles_for_switzerland(zoom_level):
bounds = mercantile.bounds(tile.x, tile.y, tile.z)
job = create_job(tile.x, tile.y, min_zoom=0, max_zoom=tile.z, bounds=bounds)
queue.write(job)
print("{} {} {} {}".format(bounds.west, bounds.south, bounds.east, bounds.north))
def run(self, processes=4):
'''
Warp, encode, and tile
'''
# get the bounding box + crs of the file to tile
with rasterio.open(self.inpath) as src:
bbox = list(src.bounds)
src_crs = src.crs
# remove the output filepath if it exists
if os.path.exists(self.outpath):
os.unlink(self.outpath)
# create a connection to the mbtiles file
conn = sqlite3.connect(self.outpath)
cur = conn.cursor()
# create the tiles table
cur.execute(
"CREATE TABLE tiles "
"(zoom_level integer, tile_column integer, "
"tile_row integer, tile_data blob);")
# create empty metadata
cur.execute(
"CREATE TABLE metadata (name text, value text);")
conn.commit()
# populate metadata with required fields
cur.execute(
"INSERT INTO metadata "
"(name, value) "
"VALUES ('format', ?);",
(self.image_format, ))
cur.execute(
"INSERT INTO metadata "
"(name, value) "
"VALUES ('name', '');")
cur.execute(
"INSERT INTO metadata "
"(name, value) "
"VALUES ('description', '');")
cur.execute(
"INSERT INTO metadata "
"(name, value) "
"VALUES ('version', '1');")
cur.execute(
"INSERT INTO metadata "
"(name, value) "
"VALUES ('type', 'baselayer');")
conn.commit()
if processes == 1:
# use mock pool for profiling / debugging
self.pool = MockTub(_main_worker, (self.inpath, self.run_function, self.global_args))
else:
self.pool = Pool(processes, _main_worker, (self.inpath, self.run_function, self.global_args))
# generator of tiles to make
if self.bounding_tile is None:
tiles = _make_tiles(bbox, src_crs, self.min_z, self.max_z)
else:
constrained_bbox = list(mercantile.bounds(self.bounding_tile))
tiles = _make_tiles(constrained_bbox, 'epsg:4326', self.min_z, self.max_z)
for tile, contents in self.pool.imap_unordered(self.run_function,
tiles):
x, y, z = tile
# mbtiles use inverse y indexing
tiley = int(math.pow(2, z)) - y - 1
# insert tile object
cur.execute(
"INSERT INTO tiles "
"(zoom_level, tile_column, tile_row, tile_data) "
"VALUES (?, ?, ?, ?);",
(z, x, tiley, buffer(contents)))
conn.commit()
conn.close()
self.pool.close()
self.pool.join()
return None
def streaming_tile_worker(data):
size = 2 ** (data['zMax'] - globalArgs['compositezoom']) * globalArgs['tileResolution']
out_meta = make_src_meta(merc.bounds(data['x'], data['y'], data['z']), size, globalArgs['creation_opts'])
filename = globalArgs['sceneTemplate'] % (data['z'], data['x'], data['y'])
subtiler = tile_utils.TileUtils()
log = 'FILE: %s\n' % filename
try:
with rasterio.drivers():
with rasterio.open(filename, 'w', **out_meta) as dst:
if data['zMaxCov']:
superTiles = subtiler.get_super_tiles(data['zMaxTiles'], data['zMaxCov'])
fillbaseX, fillbaseY = subtiler.get_sub_base_zoom(data['x'], data['y'], data['z'], data['zMaxCov'])
## fill thresh == the number of sub tiles that would need to occur in a fill tile to not fill (eg completely covered)
fThresh = 4 ** (data['zMax'] - data['zMaxCov'])
fDiff = 2 ** (data['zMax'] - data['zMaxCov'])
toFaux, frFaux = affaux(fDiff)
if not globalArgs['no_fill']:
print('filling')
## Read and write the fill tiles first
for t in subtiler.get_fill_super_tiles(superTiles, data['maxCovTiles'], fThresh):
z, x, y = t
path = globalArgs['readTemplate'] % (z, x, y)
log += '%s %s %s\n' % (z, x, y)
with rasterio.open(path) as src:
imdata = src.read()
imdata = make_image_array(imdata, globalArgs['tileResolution'])
imdata = upsample(imdata, fDiff, frFaux, toFaux)
window = make_window(x, y, fillbaseX, fillbaseY, globalArgs['tileResolution'] * fDiff)
dst.write(imdata, window=window)
baseX, baseY = subtiler.get_sub_base_zoom(data['x'], data['y'], data['z'], data['zMax'])
for t in data['zMaxTiles']:
z, x, y = t
path = globalArgs['readTemplate'] % (z, x, y)
log += '%s %s %s\n' % (z, x, y)
with rasterio.open(path) as src:
imdata = src.read()
imdata = make_image_array(imdata, globalArgs['tileResolution'])
window = make_window(x, y, baseX, baseY, globalArgs['tileResolution'])
dst.write(imdata, window=window)
if globalArgs['logdir']:
with open(os.path.join(globalArgs['logdir'], '%s.log' % os.path.basename(filename)), 'w') as logger:
logwriter(logger, log)
return filename
except Exception as e:
click.echo("%s errored" % (path), err=True)
raise e
def fetch(self, bbox: typing.List, zoom: int = None, *args, **kwargs):
"""
The function fetching tiles from a Slippy Map provider, composing them into a single image, and cropping it
to match the given BBOX. Retrieval of each tile is repeated self.max_retries times, waiting self.retry_delay
seconds between consecutive requests.
:param bbox: bounding box of the background image, dataset compliant format: [west, east, south, north, CRS]
:param zoom: zoom with which to retrieve Slippy Map's tiles (by default, it's calculated based on width, height)
:return: None if the CRS is different from self.tiles_crs, or background Image
"""
if not self.url:
logger.error("Thumbnail background requires url to be configured.")
raise ThumbnailError("Tiled background improperly configured.")
if bbox[-1].lower() != self.crs.lower():
# background service is not available the requested CRS CRS
logger.debug(
f"Thumbnail background generation skipped. "
f"Clashing CRSs: requested {bbox[-1]}, supported {self.crs}")
return
bbox = [float(coord) for coord in bbox[0:4]]
# check if BBOX fits within the EPSG:3857 map, if not - return an empty background
if bbox[2] > self._epsg3857_max_y or bbox[3] < -self._epsg3857_max_y:
return Image.new("RGB",
(self.thumbnail_width, self.thumbnail_height),
(250, 250, 250))
bbox4326 = self.bbox3857to4326(*bbox)
# change bbox from dataset (left, right, bottom, top) to mercantile (left, bottom, right, top)
self._mercantile_bbox = [
bbox4326[0], bbox4326[2], bbox4326[1], bbox4326[3]
]
# calculate zoom level
if zoom is None:
zoom = self.calculate_zoom()
else:
zoom = int(zoom)
top_left_tile = mercantile.tile(bbox4326[0], bbox4326[3], zoom)
bottom_right_tile = mercantile.tile(bbox4326[1], bbox4326[2], zoom)
# rescaling factors - indicators of how west and east BBOX boundaries are offset in respect to the world's map;
# east and west boundaries may exceed the maximum coordinate of the world in EPSG:3857. In such case additinal
# number of tiles need to be fetched to compose the image and the boundary tiles' coordinates need to be
# rescaled to ensure the proper image cropping.
epsg3857_world_width = 2 * self._epsg3857_max_x
west_rescaling_factor = 0
if abs(bbox[0]) > self._epsg3857_max_x:
west_rescaling_factor = ceil(
(abs(bbox[0]) - self._epsg3857_max_x) /
epsg3857_world_width) * copysign(1, bbox[0])
east_rescaling_factor = 0
if abs(bbox[1]) > self._epsg3857_max_x:
east_rescaling_factor = ceil(
(abs(bbox[1]) - self._epsg3857_max_x) /
epsg3857_world_width) * copysign(1, bbox[1])
map_row_tiles = 2**zoom - 1 # number of tiles in the Map's row for a certain zoom level
map_worlds = int(east_rescaling_factor -
west_rescaling_factor) # number maps in an image
worlds_between = map_worlds - 1 # number of full maps in an image
if top_left_tile.x > bottom_right_tile.x or bbox[1] - bbox[
0] > epsg3857_world_width or map_worlds > 0:
# BBOX crosses Slippy Map's border
if worlds_between > 0:
tiles_rows = (list(range(top_left_tile.x, map_row_tiles + 1)) +
worlds_between * list(range(map_row_tiles + 1)) +
list(range(bottom_right_tile.x + 1)))
else:
tiles_rows = list(range(top_left_tile.x,
map_row_tiles + 1)) + list(
range(bottom_right_tile.x + 1))
else:
# BBOx is contained by the Slippy Map
if worlds_between > 0:
tiles_rows = list(
range(top_left_tile.x, bottom_right_tile.x +
1)) + worlds_between * list(range(map_row_tiles + 1))
else:
tiles_rows = list(
range(top_left_tile.x, bottom_right_tile.x + 1))
tiles_cols = list(range(top_left_tile.y, bottom_right_tile.y + 1))
# if latitude boundaries extend world's height - add background's height, and set constant Y offset for tiles
additional_height = 0
fixed_top_offset = 0
fixed_bottom_offset = 0
north_extension3857 = max(0, bbox[3] - self._epsg3857_max_y)
south_extension3857 = abs(min(0, bbox[2] + self._epsg3857_max_y))
extension3857 = north_extension3857 + south_extension3857
if extension3857:
# get single tile's height in ESPG:3857
tile_bounds = mercantile.bounds(tiles_rows[0], tiles_cols[0], zoom)
_, south = self.point4326to3857(getattr(tile_bounds, "west"),
getattr(tile_bounds, "south"))
_, north = self.point4326to3857(getattr(tile_bounds, "west"),
getattr(tile_bounds, "north"))
tile_hight3857 = north - south
additional_height = round(
self.tile_size * extension3857 /
tile_hight3857) # based on linear proportion
if north_extension3857:
fixed_top_offset = round(self.tile_size * north_extension3857 /
tile_hight3857)
if south_extension3857:
fixed_bottom_offset = round(
self.tile_size * south_extension3857 / tile_hight3857)
background = Image.new(
"RGB",
(len(tiles_rows) * self.tile_size,
len(tiles_cols) * self.tile_size + additional_height),
(250, 250, 250),
)
for offset_x, x in enumerate(tiles_rows):
for offset_y, y in enumerate(tiles_cols):
if self.tms:
y = (2**zoom) - y - 1
imgurl = self.url.format(x=x, y=y, z=zoom)
im = None
for retries in range(self.max_retries):
try:
resp, content = http_client.request(imgurl)
if resp.status_code > 400:
retries = self.max_retries - 1
raise Exception(f"{strip_tags(content)}")
im = BytesIO(content)
Image.open(
im).verify() # verify that it is, in fact an image
break
except Exception as e:
logger.error(
f"Thumbnail background fetching from {imgurl} failed {retries} time(s) with: {e}"
)
if retries + 1 == self.max_retries:
raise e
time.sleep(self.retry_delay)
continue
if im:
image = Image.open(
im
) # "re-open" the file (required after running verify method)
# add the fetched tile to the background image, placing it under proper coordinates
background.paste(
image, (offset_x * self.tile_size,
offset_y * self.tile_size + fixed_top_offset))
# get BBOX of the tiles
top_left_bounds = mercantile.bounds(top_left_tile)
bottom_right_bounds = mercantile.bounds(bottom_right_tile)
tiles_bbox3857 = self.bbox4326to3857(
getattr(top_left_bounds, "west"),
getattr(bottom_right_bounds, "east"),
getattr(bottom_right_bounds, "south"),
getattr(top_left_bounds, "north"),
)
# rescale tiles' boundaries - if space covered by the input BBOX extends the width of the world,
# (e.g. two "worlds" are present on the map), translation between tiles' BBOX and image's pixel requires
# additional rescaling, for tiles' BBOX coordinates to match input BBOX coordinates
west_coord = tiles_bbox3857[
0] + west_rescaling_factor * epsg3857_world_width
east_coord = tiles_bbox3857[
1] + east_rescaling_factor * epsg3857_world_width
# prepare translating function from received BBOX to pixel values of the background image
src_quad = (0, fixed_top_offset, background.size[0],
background.size[1] - fixed_bottom_offset)
to_src_px = utils.make_bbox_to_pixels_transf(
[west_coord, tiles_bbox3857[2], east_coord, tiles_bbox3857[3]],
src_quad)
# translate received BBOX to pixel values
minx, miny = to_src_px(bbox[0], bbox[2])
maxx, maxy = to_src_px(bbox[1], bbox[3])
# max and min function for Y axis were introduced to mitigate rounding errors
crop_box = (
ceil(minx),
max(ceil(maxy) + fixed_top_offset, 0),
floor(maxx),
min(floor(miny) + fixed_top_offset, background.size[1]),
)
if not all([
0 <= crop_x <= background.size[0]
for crop_x in [crop_box[0], crop_box[2]]
]):
raise ThumbnailError(
f"Tiled background cropping error. Boundaries outside of the image: {crop_box}"
)
# crop background image to the desired bbox and resize it
background = background.crop(box=crop_box)
background = background.resize(
(self.thumbnail_width, self.thumbnail_height))
if sum(background.convert("L").getextrema()) in (0, 2):
# either all black or all white
logger.error("Thumbnail background outside the allowed area.")
raise ThumbnailError(
"Thumbnail background outside the allowed area.")
return background
def _calc_tms_zoom(self, scale):
for z in range(15,20):
b = mercantile.bounds(0,0,z)
if scale > math.sqrt((b.north - b.south)*(b.east - b.west) / (256*256)):
return z
