def __init__(self, tile_json, urls_key="tiles", **kwargs): # FIXME schema # FIXME version 1.0.0 support d = json.loads(tile_json) assert "tiles" in d assert isinstance(d["tiles"], list) assert len(d["tiles"]) > 0 for key in self.KEYS: kwargs.setdefault(key, d.get(key)) if "bounding_pyramid" not in kwargs: zmin, zmax = d.get("minzoom", 0), d.get("maxzoom", 22) if "bounds" in d: lonmin, latmin, lonmax, latmax = d["bounds"] bounding_pyramid = BoundingPyramid.from_wgs84( zmin, zmax, lonmin, lonmax, latmin, latmax) else: bounding_pyramid = BoundingPyramid.full(zmin, zmax) kwargs["bounding_pyramid"] = bounding_pyramid urls = d[urls_key] if "content_type" not in kwargs: exts = set( os.path.splitext(urlparse(url1).path)[1] for url1 in urls) content_types = set(mimetypes.types_map.get(ext) for ext in exts) assert len(content_types) == 1 kwargs["content_type"] = content_types.pop() templates = [ re.sub(r"\{([xyz])\}", lambda m: "%({0!s})d".format(m.group(1)), url2) for url2 in urls ] tilelayouts = map(TemplateTileLayout, templates) URLTileStore.__init__(self, tilelayouts, **kwargs)
def __init__(self, tile_json, urls_key='tiles', **kwargs): # FIXME schema # FIXME version 1.0.0 support d = json.loads(tile_json) assert 'tiles' in d assert isinstance(d['tiles'], list) assert len(d['tiles']) > 0 for key in self.KEYS: kwargs.setdefault(key, d.get(key, None)) if 'bounding_pyramid' not in kwargs: zmin, zmax = d.get('minzoom', 0), d.get('maxzoom', 22) if 'bounds' in d: lonmin, latmin, lonmax, latmax = d['bounds'] bounding_pyramid = BoundingPyramid.from_wgs84(zmin, zmax, lonmin, lonmax, latmin, latmax) else: bounding_pyramid = BoundingPyramid.full(zmin, zmax) kwargs['bounding_pyramid'] = bounding_pyramid urls = d[urls_key] if 'content_type' not in kwargs: exts = set(os.path.splitext(urlparse(url).path)[1] for url in urls) content_types = set(mimetypes.types_map.get(ext) for ext in exts) assert len(content_types) == 1 kwargs['content_type'] = content_types.pop() templates = [re.sub(r'\{([xyz])\}', lambda m: '%%(%s)d' % m.group(1), url) for url in urls] tile_layouts = map(TemplateTileLayout, templates) URLTileStore.__init__(self, tile_layouts, **kwargs)
def test_add(self): bp = BoundingPyramid() bp.add(TileCoord(1, 0, 0)) self.assertEqual(len(bp), 1) self.assertTrue(TileCoord(1, 0, 0) in bp) self.assertFalse(TileCoord(1, 0, 1) in bp) self.assertFalse(TileCoord(1, 1, 0) in bp) self.assertFalse(TileCoord(1, 1, 1) in bp) self.assertEqual(list(bp), [TileCoord(1, 0, 0)])
def test_full(self): bp = BoundingPyramid.full(1, 3) self.assertRaises(KeyError, bp.zget, 0) self.assertEqual(bp.zget(1), (Bounds(0, 2), Bounds(0, 2))) self.assertEqual(bp.zget(2), (Bounds(0, 4), Bounds(0, 4))) self.assertEqual(bp.zget(3), (Bounds(0, 8), Bounds(0, 8))) self.assertRaises(KeyError, bp.zget, 4)
def test_init_boundingpyramid(self): ts = TileStore( bounding_pyramid=BoundingPyramid.from_string('1/0/0:1/1')) self.assertTrue(Tile(TileCoord(1, 0, 0)) in ts) tiles = list(ts.list()) self.assertEqual(len(tiles), 1) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
def test_iterbottomup(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fill_up(0) self.assertEqual( list(bp.iterbottomup()), [TileCoord(2, 1, 3), TileCoord(1, 0, 1), TileCoord(0, 0, 0)] )
def test_metatilecoords(self): bp = BoundingPyramid.full(1, 2) metatilecoords = bp.metatilecoords(2) self.assertEqual(TileCoord(1, 0, 0, 2), next(metatilecoords)) self.assertEqual(TileCoord(2, 0, 0, 2), next(metatilecoords)) self.assertEqual(TileCoord(2, 0, 2, 2), next(metatilecoords)) self.assertEqual(TileCoord(2, 2, 0, 2), next(metatilecoords)) self.assertEqual(TileCoord(2, 2, 2, 2), next(metatilecoords)) self.assertRaises(StopIteration, next, metatilecoords)
def test_fill(self): bp = BoundingPyramid() bp.fill(xrange(0, 8), (572215.4395248143, 5684416.95917649, 1277662.36597472, 6145307.39552287)) self.assertEqual(bp.zget(0), (Bounds(0, 1), Bounds(0, 1))) self.assertEqual(bp.zget(1), (Bounds(1, 2), Bounds(0, 1))) self.assertEqual(bp.zget(2), (Bounds(2, 3), Bounds(1, 2))) self.assertEqual(bp.zget(3), (Bounds(4, 5), Bounds(2, 3))) self.assertEqual(bp.zget(4), (Bounds(8, 9), Bounds(5, 6))) self.assertEqual(bp.zget(5), (Bounds(16, 18), Bounds(11, 12))) self.assertEqual(bp.zget(6), (Bounds(32, 35), Bounds(22, 23))) self.assertEqual(bp.zget(7), (Bounds(65, 69), Bounds(44, 46)))
def test_itertopdown(self) -> None: bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fill_up(0) self.assertEqual( list(bp.itertopdown()), [TileCoord(0, 0, 0), TileCoord(1, 0, 1), TileCoord(2, 1, 3)])
def test_empty(self) -> None: ts = TileStore() self.assertEqual(ts.bounding_pyramid, None) self.assertEqual(ts.content_type, None) self.assertEqual(len(ts), 0) self.assertRaises(NotImplementedError, next, ts.delete((Tile(TileCoord(0, 0, 0)),))) self.assertRaises(NotImplementedError, ts.delete_one, None) self.assertEqual(ts.get_cheap_bounding_pyramid(), None) self.assertRaises(NotImplementedError, next, ts.get((Tile(TileCoord(0, 0, 0)),))) self.assertEqual(list(ts.get_all()), []) self.assertRaises(NotImplementedError, ts.get_one, None) self.assertEqual(list(ts.list()), []) self.assertRaises(NotImplementedError, next, ts.put((Tile(TileCoord(0, 0, 0)),))) self.assertRaises(NotImplementedError, ts.put_one, None) self.assertFalse(None in ts) self.assertEqual(ts.get_bounding_pyramid(), BoundingPyramid())
def __init__(self, z, bounds, file=None, **kwargs): TileStore.__init__(self, **kwargs) self.z = z self.xbounds, self.ybounds = bounds self.width = self.xbounds.stop - self.xbounds.start self.height = self.ybounds.stop - self.ybounds.start if "bounding_pyramid" not in kwargs: self.bounding_pyramid = BoundingPyramid( {self.z: (self.xbounds, self.ybounds)}) if file: self.image = PIL.Image.open(file) assert self.image.mode == "1" assert self.image.size == (self.width, self.height) else: self.image = PIL.Image.new("1", (self.width, self.height)) self.pixels = self.image.load()
def main(): # Create our input and output TileStores input_tilestore = TileStore.load('tiles.openstreetmap_org') output_tilestore = TileStore.load('local.mbtiles') # 1. Generate a list of tiles to download from a BoundingPyramid # 4/8/5 is the root tile, corresponding to Central Europe # +3/+1/+1 specifies up to zoom level 4 + 3 = 7 and an extent of one tile in the X and Y directions bounding_pyramid = BoundingPyramid.from_string('4/8/5:+3/+1/+1') bounding_pyramid_tilestore = BoundingPyramidTileStore(bounding_pyramid) tilestream = bounding_pyramid_tilestore.list() # 2. Filter out tiles that already downloaded tilestream = (tile for tile in tilestream if tile not in output_tilestore) # 3. Get the tile from openstreetmap.org tilestream = input_tilestore.get(tilestream) # 4. Save the tile to local.mbtiles tilestream = output_tilestore.put(tilestream) # 5. Log the fact that the tile was downloaded tilestream = imap(Logger(logger, logging.INFO, 'downloaded %(tilecoord)s'), tilestream) # Go! consume(tilestream, None)
def main(argv): # Create our input and output TileStores input_tilestore = TileStore.load('tiles.openstreetmap_org') output_tilestore = TileStore.load('local.mbtiles') # 1. Generate a list of tiles to download from a BoundingPyramid # 4/8/5 is the root tile, corresponding to Central Europe # +3/+1/+1 specifies up to zoom level 4 + 3 = 7 and an extent of one tile in the X and Y directions bounding_pyramid = BoundingPyramid.from_string('4/8/5:+3/+1/+1') bounding_pyramid_tilestore = BoundingPyramidTileStore(bounding_pyramid) tilestream = bounding_pyramid_tilestore.list() # 2. Filter out tiles that already downloaded tilestream = (tile for tile in tilestream if not tile in output_tilestore) # 3. Get the tile from openstreetmap.org tilestream = input_tilestore.get(tilestream) # 4. Save the tile to local.mbtiles tilestream = output_tilestore.put(tilestream) # 5. Log the fact that the tile was downloaded tilestream = imap(Logger(logger, logging.INFO, 'downloaded %(tilecoord)s'), tilestream) # Go! consume(tilestream, None)
def test_one(self): tilestore = MBTilesTileStore(sqlite3.connect(':memory:'), content_type='image/png') self.assertEqual(len(tilestore), 0) tilestream = [ Tile(TileCoord(1, 0, 0), data=b'data'), None, Tile(TileCoord(1, 0, 1), error=True) ] tilestream = tilestore.put(tilestream) tiles = list(tilestream) self.assertEqual(len(tilestore), 2) self.assertEqual(len(tiles), 2) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0)) self.assertEqual(tiles[0].data, b'data') self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1)) self.assertEqual(tiles[1].error, True) self.assertTrue(Tile(TileCoord(1, 0, 0)) in tilestore) self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore) tilestream = [Tile(TileCoord(1, 0, 0)), Tile(TileCoord(1, 0, 1))] tilestream = tilestore.get(tilestream) consume(tilestream, None) self.assertEqual(tilestore.get_cheap_bounding_pyramid(), BoundingPyramid({1: (Bounds(0, 1), Bounds(0, 2))})) self.assertEqual(len(tilestore), 2) tiles = list(tilestore.list()) self.assertEqual(len(tiles), 2) tiles = sorted(tilestore.get_all()) self.assertEqual(len(tiles), 2) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0)) self.assertEqual(bytes(tiles[0].data), b'data') self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1)) self.assertEqual(tiles[1].data, None) tilestream = [Tile(TileCoord(1, 0, 0))] tilestream = tilestore.delete(tilestream) consume(tilestream, None) self.assertEqual(len(tilestore), 1) tiles = list(tilestore.get_all()) self.assertEqual(len(tiles), 1) self.assertFalse(Tile(TileCoord(1, 0, 0)) in tilestore) self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
def __init__(self, z: int, bounds: Tuple[Bounds, Bounds], file: Optional[str] = None, **kwargs: Any): TileStore.__init__(self, **kwargs) self.z = z self.xbounds, self.ybounds = bounds assert self.xbounds.start is not None assert self.xbounds.stop is not None assert self.ybounds.start is not None assert self.ybounds.stop is not None self.width = self.xbounds.stop - self.xbounds.start self.height = self.ybounds.stop - self.ybounds.start if "bounding_pyramid" not in kwargs: self.bounding_pyramid = BoundingPyramid( {self.z: (self.xbounds, self.ybounds)}) if file: self.image = PIL.Image.open(file) assert self.image.mode == "1" assert self.image.size == (self.width, self.height) else: self.image = PIL.Image.new("1", (self.width, self.height)) self.pixels = self.image.load()
def __init__(self, bounding_pyramid=None, **kwargs): TileStore.__init__(self, **kwargs) self.bounding_pyramid = bounding_pyramid or BoundingPyramid()
def get_cheap_bounding_pyramid(self): bounds = {} for z, xstart, xstop, ystart, ystop in query( self.connection, self.BOUNDING_PYRAMID_SQL): bounds[z] = (Bounds(xstart, xstop), Bounds(ystart, ystop)) return BoundingPyramid(bounds)
def test_from_string_up(self): bp = BoundingPyramid.from_string('2/1/3:0/2/4') self.assertEqual(bp.zget(0), (Bounds(0, 1), Bounds(0, 1))) self.assertEqual(bp.zget(1), (Bounds(0, 1), Bounds(1, 2))) self.assertEqual(bp.zget(2), (Bounds(1, 2), Bounds(3, 4))) self.assertRaises(KeyError, bp.zget, 3)
def init_tilecoords(self, layer): resolutions = layer['grid_ref']['resolutions'] if self.options.time is not None and self.options.zoom is None: if 'min_resolution_seed' in layer: # pragma: no cover self.options.zoom = [ resolutions.index(layer['min_resolution_seed']) ] else: self.options.zoom = [len(resolutions) - 1] if self.options.zoom is not None: zoom_max = len(resolutions) - 1 for zoom in self.options.zoom: if zoom > zoom_max: logger.warning( "zoom %i is greater than the maximum zoom %i" " of grid %s of layer %s, ignored." % (zoom, zoom_max, layer['grid'], layer['name'])) self.options.zoom = [z for z in self.options.zoom if z <= zoom_max] if 'min_resolution_seed' in layer: if self.options.zoom is None: self.options.zoom = [] for z, resolution in enumerate(resolutions): if resolution >= layer['min_resolution_seed']: self.options.zoom.append(z) else: for zoom in self.options.zoom: resolution = resolutions[zoom] if resolution < layer['min_resolution_seed']: logger.warning( "zoom %i corresponds to resolution %s is smaller" " than the 'min_resolution_seed' %s of layer %s, ignored." % (zoom, resolution, layer['min_resolution_seed'], layer['name'])) self.options.zoom = [ z for z in self.options.zoom if resolutions[z] >= layer['min_resolution_seed'] ] if self.options.zoom is None: self.options.zoom = [z for z, r in enumerate(resolutions)] # fill the bounding pyramid tilegrid = layer['grid_ref']['obj'] bounding_pyramid = BoundingPyramid(tilegrid=tilegrid) for zoom in self.options.zoom: if zoom in self.geoms: extent = self.geoms[zoom].bounds if len(extent) == 0: logger.warning("bounds empty for zoom {}".format(zoom)) else: minx, miny, maxx, maxy = extent px_buffer = layer['px_buffer'] m_buffer = px_buffer * resolutions[zoom] minx -= m_buffer miny -= m_buffer maxx += m_buffer maxy += m_buffer bounding_pyramid.add( tilegrid.tilecoord( zoom, max(minx, tilegrid.max_extent[0]), max(miny, tilegrid.max_extent[1]), )) bounding_pyramid.add( tilegrid.tilecoord( zoom, min(maxx, tilegrid.max_extent[2]), min(maxy, tilegrid.max_extent[3]), )) if layer.get('meta', False): self.set_tilecoords( bounding_pyramid.metatilecoords(layer['meta_size']), layer) else: self.set_tilecoords(bounding_pyramid, layer)
def test_hash_metatile(self): bp = BoundingPyramid({4: (Bounds(0, 16), Bounds(0, 16))}) metatilecoords = list(bp.metatilecoords(2)) hashes = map(hash, metatilecoords) self.assertEqual(len(metatilecoords), len(set(hashes)))
def test_init_boundingpyramid(self): ts = TileStore(bounding_pyramid=BoundingPyramid.from_string('1/0/0:1/1')) self.assertTrue(Tile(TileCoord(1, 0, 0)) in ts) tiles = list(ts.list()) self.assertEqual(len(tiles), 1) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
def test_from_string_relative(self): bp = BoundingPyramid.from_string('2/1/3:+1/+1/+1') self.assertRaises(KeyError, bp.zget, 1) self.assertEqual(bp.zget(2), (Bounds(1, 2), Bounds(3, 4))) self.assertEqual(bp.zget(3), (Bounds(2, 4), Bounds(6, 8))) self.assertRaises(KeyError, bp.zget, 4)
def test_zs(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fill_up(0) self.assertEqual(sorted(bp.zs()), [0, 1, 2])
def __init__(self, bounding_pyramid: Optional[BoundingPyramid] = None, **kwargs: Any): TileStore.__init__(self, **kwargs) self.bounding_pyramid = bounding_pyramid or BoundingPyramid()
def test_eq(self): self.assertEqual(BoundingPyramid(), BoundingPyramid()) self.assertEqual(BoundingPyramid({5: (Bounds(2, 5), Bounds(6, 15))}), BoundingPyramid({5: (Bounds(2, 5), Bounds(6, 15))}))
def test_filldown(self): bp = BoundingPyramid() bp.add(TileCoord(1, 1, 0)) bp.filldown(3) self.assertEqual(bp.zget(2), (Bounds(2, 4), Bounds(0, 2))) self.assertEqual(bp.zget(3), (Bounds(4, 8), Bounds(0, 4)))
def init_tilecoords(self, layer): resolutions = layer['grid_ref']['resolutions'] if self.options.time is not None and self.options.zoom is None: if 'min_resolution_seed' in layer: # pragma: no cover self.options.zoom = [resolutions.index( layer['min_resolution_seed'] )] else: self.options.zoom = [len(resolutions) - 1] if self.options.zoom is not None: zoom_max = len(resolutions) - 1 for zoom in self.options.zoom: if zoom > zoom_max: logger.warning( "zoom %i is greater than the maximum zoom %i" " of grid %s of layer %s, ignored." % ( zoom, zoom_max, layer['grid'], layer['name'] ) ) self.options.zoom = [z for z in self.options.zoom if z <= zoom_max] if 'min_resolution_seed' in layer: if self.options.zoom is None: self.options.zoom = [] for z, resolution in enumerate(resolutions): if resolution >= layer['min_resolution_seed']: self.options.zoom.append(z) else: for zoom in self.options.zoom: resolution = resolutions[zoom] if resolution < layer['min_resolution_seed']: logger.warning( "zoom %i corresponds to resolution %s is smaller" " than the 'min_resolution_seed' %s of layer %s, ignored." % ( zoom, resolution, layer['min_resolution_seed'], layer['name'] ) ) self.options.zoom = [ z for z in self.options.zoom if resolutions[z] >= layer['min_resolution_seed'] ] if self.options.zoom is None: self.options.zoom = [z for z, r in enumerate(resolutions)] # fill the bounding pyramid tilegrid = layer['grid_ref']['obj'] bounding_pyramid = BoundingPyramid(tilegrid=tilegrid) for zoom in self.options.zoom: if zoom in self.geoms: extent = self.geoms[zoom].bounds if len(extent) == 0: logger.warning("bounds empty for zoom {}".format(zoom)) else: minx, miny, maxx, maxy = extent px_buffer = layer['px_buffer'] m_buffer = px_buffer * resolutions[zoom] minx -= m_buffer miny -= m_buffer maxx += m_buffer maxy += m_buffer bounding_pyramid.add(tilegrid.tilecoord( zoom, max(minx, tilegrid.max_extent[0]), max(miny, tilegrid.max_extent[1]), )) bounding_pyramid.add(tilegrid.tilecoord( zoom, min(maxx, tilegrid.max_extent[2]), min(maxy, tilegrid.max_extent[3]), )) if layer.get('meta', False): self.set_tilecoords(bounding_pyramid.metatilecoords(layer['meta_size']), layer) else: self.set_tilecoords(bounding_pyramid, layer)
def test_from_string_one_level(self): bp = BoundingPyramid.from_string('5/9/13:12/15') self.assertRaises(KeyError, bp.zget, 4) self.assertEqual(bp.zget(5), (Bounds(9, 12), Bounds(13, 15))) self.assertRaises(KeyError, bp.zget, 6)
def test_fill_down(self): bp = BoundingPyramid() bp.add(TileCoord(1, 1, 0)) bp.fill_down(3) self.assertEqual(bp.zget(2), (Bounds(2, 4), Bounds(0, 2))) self.assertEqual(bp.zget(3), (Bounds(4, 8), Bounds(0, 4)))
def test_iterbottomup(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fill_up(0) self.assertEqual(list(bp.iterbottomup()), [TileCoord(2, 1, 3), TileCoord(1, 0, 1), TileCoord(0, 0, 0)])
def test_fillup(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fillup(0) self.assertEqual(bp.zget(1), (Bounds(0, 1), Bounds(1, 2))) self.assertEqual(bp.zget(0), (Bounds(0, 1), Bounds(0, 1)))
def test_ziter(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fill_up(0) self.assertEqual(list(bp.ziter(1)), [TileCoord(1, 0, 1)])
def test_fill_up2(self): bp = BoundingPyramid({1: (Bounds(0, 2), Bounds(1, 2))}) bp.add(TileCoord(2, 1, 3)) bp.fill_up(0) self.assertEqual(bp.zget(1), (Bounds(0, 2), Bounds(1, 2))) self.assertEqual(bp.zget(0), (Bounds(0, 1), Bounds(0, 1)))
def test_itertopdown(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fillup(0) self.assertEqual(list(bp.itertopdown()), [TileCoord(0, 0, 0), TileCoord(1, 0, 1), TileCoord(2, 1, 3)])
def test_empty(self): bp = BoundingPyramid() self.assertEqual(len(bp), 0) self.assertFalse(TileCoord(0, 0, 0) in bp) self.assertRaises(StopIteration, next, iter(bp))
def test_ziter(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fillup(0) self.assertEqual(list(bp.ziter(1)), [TileCoord(1, 0, 1)])
def test_from_string_star(self): bp = BoundingPyramid.from_string('0/0/0:2/*/*') self.assertEqual(bp.zget(0), (Bounds(0, 1), Bounds(0, 1))) self.assertEqual(bp.zget(1), (Bounds(0, 2), Bounds(0, 2))) self.assertEqual(bp.zget(2), (Bounds(0, 4), Bounds(0, 4))) self.assertRaises(KeyError, bp.zget, 3)
def test_zs(self): bp = BoundingPyramid() bp.add(TileCoord(2, 1, 3)) bp.fillup(0) self.assertEqual(sorted(bp.zs()), [0, 1, 2])