def test_overwite_metadata(tmpdir):
filename = str(tmpdir.join("test.mbtiles"))
metadata = {"name": "test tiles", "version": "1.0.0"}
new_metadata = {"name": "new test tiles", "version": "100000.0.0"}
with MBtiles(filename, mode="w") as out:
out.meta = metadata
out.meta = new_metadata
with sqlite3.connect(filename) as db:
cursor = db.cursor()
cursor.execute("SELECT name, value from metadata")
out = {row[0]: row[1] for row in cursor.fetchall()}
assert out == new_metadata
# overwrite existing key, value
with MBtiles(filename, mode="r+") as out:
out.meta["name"] = "new new test tiles"
with sqlite3.connect(filename) as db:
cursor = db.cursor()
cursor.execute("SELECT value from metadata WHERE name='name' LIMIT 1")
row = cursor.fetchone()
assert row is not None
assert row[0] == "new new test tiles"
def difference(leftfilename, rightfilename, outfilename, batch_size=1000):
"""Create new tileset from tiles in left that are not in right.
Note: caller is responsible for updating metadata as needed. Metadata is copied from target.
Parameters
----------
leftfilename : str
first tileset filename
rightfilename : str
second tileset filename
outfilename : str
output tileset filename
batch_size : int, optional (default: BATCH_SIZE)
size of each batch to read from the source and write to the target
"""
with MBtiles(leftfilename) as left, MBtiles(
rightfilename) as right, MBtiles(outfilename, "w") as out:
out.meta = left.meta
for batch in left.list_tiles_batched(batch_size):
tiles_to_copy = [
tile for tile in batch if not right.has_tile(*tile)
]
if tiles_to_copy:
out.write_tiles(
Tile(*tile, data=left.read_tile(*tile))
for tile in tiles_to_copy)
def extend(source_filename, target_filename, batch_size=1000):
"""
Add tiles from source_filename to target_tileset that are not already in target.
Equivalent to the union of tiles from source and target, but
written to the target to cut down on additional IO of copying
both to a new file.
Note: caller is responsible for updating metadata as needed.
Parameters
----------
source_filename : str
name of source tiles mbtiles file for additional tiles
target_tileset : str
name of target tiles mbtiles file for adding tiles to
batch_size : int, optional (default: 1000)
size of each batch to read from the source and write to the target
"""
with MBtiles(target_filename,
"r+") as target, MBtiles(source_filename) as source:
for batch in source.list_tiles_batched(batch_size):
tiles_to_copy = [
tile for tile in batch if not target.has_tile(*tile)
]
if tiles_to_copy:
target.write_tiles(
Tile(*tile, data=source.read_tile(*tile))
for tile in tiles_to_copy)
def test_list_tiles_batched(tmpdir):
zoom = 2
filename = str(tmpdir.join("test.mbtiles"))
with MBtiles(filename, mode="w") as out:
tiles = []
num_per_edge = 2**zoom
for x in range(0, num_per_edge):
for y in range(0, num_per_edge):
tiles.append(Tile(zoom, x, y, b""))
out.write_tiles(tiles)
with MBtiles(filename, mode="r") as src:
batch = list(src.list_tiles_batched())
print("batch", batch)
assert len(batch) == 1
tiles = batch[0]
assert len(tiles) == (2**zoom)**2
assert isinstance(tiles[0], TileCoordinate)
# test batch size
batch = list(src.list_tiles_batched(2))
assert len(batch) == ((2**zoom)**2) / 2
tiles = batch[0]
assert len(tiles) == 2
def test_ranges(tmpdir):
zooms = [0, 1, 2]
filename = str(tmpdir.join("test.mbtiles"))
with MBtiles(filename, mode="w") as out:
tiles = []
for zoom in zooms:
num_per_edge = 2**zoom
for x in range(0, num_per_edge):
for y in range(0, num_per_edge):
tiles.append(Tile(zoom, x, y, b""))
out.write_tiles(tiles)
with MBtiles(filename, mode="r") as src:
assert src.zoom_range() == (0, 2)
assert src.row_range(0) == (0, 0)
assert src.col_range(0) == (0, 0)
assert src.row_range(1) == (0, 1)
assert src.col_range(1) == (0, 1)
assert src.row_range(2) == (0, 3)
assert src.col_range(2) == (0, 3)
# ranges beyond available zoom levels should be none
assert src.row_range(3) == (None, None)
assert src.col_range(3) == (None, None)
def test_read_missing_tile(tmpdir, blank_png_tile):
filename = str(tmpdir.join("test.mbtiles"))
# Create mbtiles file with a tile to read
with MBtiles(filename, mode="w") as out:
out.write_tile(0, 0, 0, blank_png_tile)
with MBtiles(filename, mode="r") as src:
assert src.read_tile(1, 0, 0) is None
def test_has_tile(tmpdir, blank_png_tile):
filename = str(tmpdir.join("test.mbtiles"))
# Create mbtiles file with a tile to read
with MBtiles(filename, mode="w") as out:
out.write_tile(0, 0, 0, blank_png_tile)
with MBtiles(filename, mode="r") as src:
assert src.has_tile(0, 0, 0) == True
assert src.has_tile(10, 10, 10) == False
def test_read_metadata(tmpdir):
filename = str(tmpdir.join("test.mbtiles"))
metadata = {"name": "test tiles", "version": "1.0.0"}
with MBtiles(filename, mode="w") as out:
out.meta = metadata
with MBtiles(filename, mode="r") as src:
src.meta == metadata
def test_overwrite_tile(tmpdir, blank_png_tile):
# Should not fail if we send in a duplicate tile
filename = str(tmpdir.join("test.mbtiles"))
with MBtiles(filename, mode="w") as out:
out.write_tile(0, 0, 0, blank_png_tile)
# overwrite tile previously written
out.write_tile(0, 0, 0, b"123")
with MBtiles(filename, mode="r") as src:
assert src.read_tile(0, 0, 0) == b"123"
def test_existing_file(tmpdir, blank_png_tile):
filename = str(tmpdir.join("test.mbtiles"))
# create a file first
with MBtiles(filename, mode="w") as out:
out.write_tile(0, 0, 0, blank_png_tile)
assert os.path.exists(filename)
# this should overwrite the previous
with MBtiles(filename, mode="w") as out:
out.write_tile(1, 0, 0, blank_png_tile)
with MBtiles(filename, mode="r") as src:
assert src.read_tile(0, 0, 0) is None
assert src.read_tile(1, 0, 0) == blank_png_tile
def test_difference(tmpdir):
left = str(tmpdir.join("left.mbtiles"))
right = str(tmpdir.join("right.mbtiles"))
outfilename = str(tmpdir.join("out.mbtiles"))
with MBtiles(left, mode="w") as out:
out.write_tiles([Tile(0, 0, 0, b""), Tile(1, 0, 0, b"")])
with MBtiles(right, mode="w") as out:
out.write_tiles([Tile(0, 0, 0, b""), Tile(2, 0, 0, b"")])
difference(left, right, outfilename)
with MBtiles(outfilename) as src:
tiles = set(src.list_tiles())
assert tiles == {(1, 0, 0)}
def test_extend(tmpdir):
source = str(tmpdir.join("source.mbtiles"))
target = str(tmpdir.join("target.mbtiles"))
with MBtiles(source, mode="w") as out:
out.write_tiles([Tile(0, 0, 0, b""), Tile(1, 0, 0, b"")])
with MBtiles(target, mode="w") as out:
out.write_tiles([Tile(0, 0, 0, b"123"), Tile(2, 0, 0, b"")])
extend(source, target)
with MBtiles(target) as src:
tiles = set(src.list_tiles())
assert tiles == {(0, 0, 0), (1, 0, 0), (2, 0, 0)}
# Make sure we didn't overwrite a tile
assert src.read_tile(0, 0, 0) == b"123"
def test_list_tiles(tmpdir):
zoom = 2
filename = str(tmpdir.join("test.mbtiles"))
with MBtiles(filename, mode="w") as out:
tiles = []
num_per_edge = 2**zoom
for x in range(0, num_per_edge):
for y in range(0, num_per_edge):
tiles.append(Tile(zoom, x, y, b""))
out.write_tiles(tiles)
with MBtiles(filename, mode="r") as src:
tiles = src.list_tiles()
assert len(tiles) == (2**zoom)**2
assert isinstance(tiles[0], TileCoordinate)
def test_ranges_nonstandard(tmpdir):
filename = str(tmpdir.join("test.mbtiles"))
with MBtiles(filename, mode="w") as out:
# outer bounds of tiles are rows [2, 5], columns [1, 7]
tiles = [
Tile(3, 1, 2, b""),
Tile(3, 3, 5, b""),
Tile(3, 6, 2, b""),
Tile(3, 7, 3, b""),
]
out.write_tiles(tiles)
with MBtiles(filename, mode="r") as src:
assert src.zoom_range() == (3, 3)
assert src.row_range(3) == (2, 5)
assert src.col_range(3) == (1, 7)
def test_union(tmpdir):
left = str(tmpdir.join("left.mbtiles"))
right = str(tmpdir.join("right.mbtiles"))
outfilename = str(tmpdir.join("out.mbtiles"))
with MBtiles(left, mode="w") as out:
out.write_tiles([Tile(0, 0, 0, b""), Tile(1, 0, 0, b"")])
with MBtiles(right, mode="w") as out:
out.write_tiles([Tile(0, 0, 0, b"123"), Tile(2, 0, 0, b"")])
union(left, right, outfilename)
with MBtiles(outfilename) as src:
tiles = set(src.list_tiles())
assert tiles == {(0, 0, 0), (1, 0, 0), (2, 0, 0)}
# Make sure we didn't overwrite a tile
assert src.read_tile(0, 0, 0) == b""
def tif_to_mbtiles(
infilename,
outfilename,
min_zoom,
max_zoom,
tile_size=256,
metadata=None,
tile_renderer=to_smallest_png,
):
"""Convert a tif to mbtiles, rendering each tile using tile_renderer.
By default, this renders tiles as data using the smallest PNG image type
for the data type of infilename.
Parameters
----------
infilename : path to input GeoTIFF file
outfilename : path to output mbtiles file
min_zoom : int
max_zoom : int
tile_size : int, optional (default: 256)
metadata : dict, optional
metadata dictionary to add to the mbtiles metadata
tile_renderer : function, optional (default: to_smallest_png)
function that takes as input the data array for the tile and returns a PNG
"""
with rasterio.Env() as env:
with rasterio.open(infilename) as src:
with MBtiles(outfilename, mode="w") as mbtiles:
meta = {
"tilejson": "2.0.0",
"version": "1.0.0",
"minzoom": min_zoom,
"maxzoom": max_zoom,
}
meta.update(get_mbtiles_meta(src, min_zoom))
if metadata is not None:
meta.update(metadata)
mbtiles.meta = meta
for tile, data, transform in read_tiles(src,
min_zoom=min_zoom,
max_zoom=max_zoom,
tile_size=tile_size):
# Only write out non-empty tiles
if not np.all(data == src.nodata):
# flip tile Y to match xyz scheme
tiley = int(math.pow(2, tile.z)) - tile.y - 1
mbtiles.write_tile(tile.z, tile.x, tiley,
tile_renderer(data))
def test_tile_bounds(tmpdir):
mbtiles_filename = str(tmpdir.join("test.mbtiles"))
with TPK("tests/data/states_filled.tpk") as tpk:
tpk.to_mbtiles(mbtiles_filename, zoom=[0], tile_bounds=True)
with MBtiles(mbtiles_filename) as mbtiles:
assert mbtiles.zoom_range() == (0, 0)
# bounds calculated from tile 0 should be world bounds in web mercator coordinates
assert mbtiles.meta[
"bounds"] == "-180.000000,-85.051129,180.000000,85.051129"
def test_write_tile(tmpdir, blank_png_tile):
filename = str(tmpdir.join("test.mbtiles"))
with MBtiles(filename, mode="w") as out:
out.write_tile(0, 0, 0, blank_png_tile)
with sqlite3.connect(filename) as db:
cursor = db.cursor()
cursor.execute(
"SELECT tile_data FROM tiles "
"where zoom_level=0 and tile_column=0 and tile_row=0 LIMIT 1")
row = cursor.fetchone()
assert row is not None
assert blank_png_tile == str(row[0]) if IS_PY2 else row[0]
def test_write_tiles(tmpdir, blank_png_tile):
filename = str(tmpdir.join("test.mbtiles"))
tiles = (Tile(1, 0, 0, blank_png_tile), Tile(1, 0, 1, blank_png_tile))
with MBtiles(filename, mode="w") as out:
out.write_tiles(tiles)
with sqlite3.connect(filename) as db:
cursor = db.cursor()
cursor.execute("SELECT tile_data FROM tiles")
rows = cursor.fetchall()
assert len(rows) == 2
for i, row in enumerate(rows):
assert tiles[i].data == str(row[0]) if IS_PY2 else row[0]
def test_write_metadata(tmpdir):
filename = str(tmpdir.join("test.mbtiles"))
metadata = {"name": "test tiles", "version": "1.0.0"}
with MBtiles(filename, mode="w") as out:
out.meta = metadata
with sqlite3.connect(filename) as db:
cursor = db.cursor()
cursor.execute("SELECT name, value from metadata")
out = {row[0]: row[1] for row in cursor.fetchall()}
assert out == metadata
# add a new key, value
with MBtiles(filename, mode="r+") as out:
out.meta["foo"] = "bar"
with sqlite3.connect(filename) as db:
cursor = db.cursor()
cursor.execute('SELECT value from metadata WHERE name="foo" LIMIT 1')
row = cursor.fetchone()
assert row is not None
assert row[0] == "bar"
def test_export_mbtiles_tile_bounds(runner, tmpdir):
tpk = "tests/data/states_filled.tpk"
mbtiles_filename = str(tmpdir.join("test.mbtiles"))
result = runner.invoke(cli, [
"export", "mbtiles", tpk, mbtiles_filename, "-z", "0", "--tile-bounds"
])
assert result.exit_code == 0
print(result.output)
assert os.path.exists(mbtiles_filename)
with MBtiles(mbtiles_filename) as mbtiles:
assert mbtiles.zoom_range() == (0, 0)
assert mbtiles.meta[
"bounds"] == "-180.000000,-85.051129,180.000000,85.051129"
def test_nonstandard_zoom_levels(tmpdir):
"""Not all TPK file are created with zoom levels that start at 0.
Make sure that we handle those where zoom level != level of detail ordinal value.
Also verify that it writes a proper mbtiles file.
"""
mbtiles_filename = str(tmpdir.join("test.mbtiles"))
with TPK("tests/data/nonstandard_zoom_levels.tpk") as tpk:
assert tpk.lods == [0, 1, 2]
assert tpk.zoom_levels == [2, 3, 4]
assert len(list(tpk.read_tiles())) == 22
tpk.to_mbtiles(mbtiles_filename)
with MBtiles(mbtiles_filename) as mbtiles:
assert mbtiles.zoom_range() == (2, 4)
assert mbtiles.row_range(2) == (2, 2)
assert mbtiles.col_range(2) == (0, 1)
assert mbtiles.row_range(3) == (4, 5)
assert mbtiles.col_range(3) == (1, 2)
assert mbtiles.row_range(4) == (8, 11)
assert mbtiles.col_range(4) == (2, 5)
def test_read_missing_file(tmpdir):
mbtiles_filename = str(tmpdir.join("test.mbtiles"))
with pytest.raises(IOError):
MBtiles(mbtiles_filename)
min_zoom = 0
max_zoom = 0
mode = "r+"
# Approx bounds of South America
bounds = [-95.273438, -57.326521, -32.695313, 13.239945]
start = time()
# clear out any progress files
if mode == "w":
for filename in glob.glob("progress-*.pickle"):
os.remove(filename)
with MBtiles("../data/elevation2.mbtiles", mode) as mbtiles: # FIXME: w => r+
mbtiles.meta = {
"name": "elevation",
"description": "Mapzen Terrarium Elevation Tiles",
"version": "1.0",
"attribution": "Mapzen",
"credits": "Mapzen",
"type": "overlay",
"format": "png",
"bounds": ",".join(str(x) for x in bounds or WORLD_BOUNDS),
"center": ",".join(str(x) for x in get_center(bounds or WORLD_BOUNDS)),
"minzoom": str(min_zoom),
"maxzoom": str(max_zoom),
}
download(
def to_mbtiles(self,
filename,
zoom=None,
tile_bounds=False,
drop_empty=False):
"""
Export tile package to mbtiles v1.1 file, optionally limited to zoom
levels. If filename exists, it will be overwritten. If filename
does not include the suffix '.mbtiles' it will be added.
Parameters
----------
filename: string
name of mbtiles file
zoom: int or list-like of ints, default: None (all tiles exported)
zoom levels to export to mbtiles
tile_bounds: bool
if True, will use the tile bounds of the highest zoom level exported to determine tileset bounds
drop_empty: bool, default False
if True, tiles that are empty will not be output
"""
if self.format.lower() == "mixed":
raise ValueError(
"Mixed format tiles are not supported for export to mbtiles")
if not filename.endswith(".mbtiles"):
filename = "{0}.mbtiles".format(filename)
with MBtiles(filename, "w") as mbtiles:
if zoom is None:
zoom = self.zoom_levels
elif isinstance(zoom, int):
zoom = [zoom]
zoom = list(zoom)
zoom.sort()
tiles = (tile for tile in self.read_tiles(zoom, flip_y=True)
if not (drop_empty and hashlib.sha1(
tile.data).hexdigest() in EMPTY_TILES))
mbtiles.write_tiles(tiles)
if tile_bounds:
# Calculate bounds based on maximum zoom to be exported
highest_zoom = zoom[-1]
min_row, max_row = mbtiles.row_range(highest_zoom)
min_col, max_col = mbtiles.col_range(highest_zoom)
# get upper left coordinate
xmin, ymax = mercantile.ul(min_col, min_row, highest_zoom)
# get bottom right coordinate
# since we are using ul(), we need to go 1 tile beyond the range to get the right side of the
# tiles we have
xmax, ymin = mercantile.ul(max_col + 1, max_row + 1,
highest_zoom)
bounds = (xmin, ymin, xmax, ymax)
else:
bounds = self.bounds
# Center zoom level is middle zoom level
center = "{0:4f},{1:4f},{2}".format(
bounds[0] + (bounds[2] - bounds[0]) / 2.0,
bounds[1] + (bounds[3] - bounds[1]) / 2.0,
(zoom[0] + zoom[-1]) // 2,
)
mbtiles.meta.update({
"name":
self.name,
"description":
self.summary, # not description, which is optional
"version":
self.version,
"attribution":
self.attribution,
"tags":
self.tags,
"credits":
self.credits,
"use_constraints":
self.use_constraints,
"type":
"overlay",
"format":
self.format.lower().replace("jpeg", "jpg")[:3],
"bounds":
",".join("{0:4f}".format(v) for v in bounds),
"center":
center,
"minzoom":
zoom[0],
"maxzoom":
zoom[-1],
"legend":
json.dumps(self.legend) if self.legend else "",
})
def test_invalid_mode(tmpdir):
mbtiles_filename = str(tmpdir.join("test.mbtiles"))
with pytest.raises(ValueError):
MBtiles(mbtiles_filename, mode="r+w")
def to_mbtiles(self, filename, zoom=None):
"""
Export tile package to mbtiles v1.1 file, optionally limited to zoom
levels. If filename exists, it will be overwritten. If filename
does not include the suffix '.mbtiles' it will be added.
Parameters
----------
filename: string
name of mbtiles file
zoom: int or list-like of ints, default: None (all tiles exported)
zoom levels to export to mbtiles
"""
if self.format.lower() == 'mixed':
raise ValueError(
'Mixed format tiles are not supported for export to mbtiles')
if not filename.endswith('.mbtiles'):
filename = '{0}.mbtiles'.format(filename)
with MBtiles(filename, 'w') as mbtiles:
if zoom is None:
zoom = self.zoom_levels
elif isinstance(zoom, int):
zoom = [zoom]
zoom = list(zoom)
zoom.sort()
mbtiles.write_tiles(self.read_tiles(zoom, flip_y=True))
bounds = self.bounds
center = '{0:4f},{1:4f},{2}'.format(
bounds[0] + (bounds[2] - bounds[0]) / 2.0,
bounds[1] + (bounds[3] - bounds[1]) / 2.0,
max(zoom[0], int((zoom[-1] - zoom[0]) / 4.0)) # Tune this
)
mbtiles.meta.update({
'name':
self.name,
'description':
self.summary, # not description, which is optional
'version':
self.version,
'attribution':
self.attribution,
'tags':
self.tags,
'credits':
self.credits,
'use_constraints':
self.use_constraints,
'type':
'overlay',
'format':
self.format.lower().replace('jpeg', 'jpg')[:3],
'bounds':
','.join('{0:4f}'.format(v) for v in self.bounds),
'center':
center,
'minzoom':
str(zoom[0]),
'maxzoom':
str(zoom[-1]),
'legend':
json.dumps(self.legend) if self.legend else ''
})