def __init__(self, tile_json, urls_key="tiles", **kwargs): # FIXME schema # FIXME version 1.0.0 support d = json.loads(tile_json) assert "tiles" in d assert isinstance(d["tiles"], list) assert len(d["tiles"]) > 0 for key in self.KEYS: kwargs.setdefault(key, d.get(key)) if "bounding_pyramid" not in kwargs: zmin, zmax = d.get("minzoom", 0), d.get("maxzoom", 22) if "bounds" in d: lonmin, latmin, lonmax, latmax = d["bounds"] bounding_pyramid = BoundingPyramid.from_wgs84( zmin, zmax, lonmin, lonmax, latmin, latmax) else: bounding_pyramid = BoundingPyramid.full(zmin, zmax) kwargs["bounding_pyramid"] = bounding_pyramid urls = d[urls_key] if "content_type" not in kwargs: exts = set( os.path.splitext(urlparse(url1).path)[1] for url1 in urls) content_types = set(mimetypes.types_map.get(ext) for ext in exts) assert len(content_types) == 1 kwargs["content_type"] = content_types.pop() templates = [ re.sub(r"\{([xyz])\}", lambda m: "%({0!s})d".format(m.group(1)), url2) for url2 in urls ] tilelayouts = map(TemplateTileLayout, templates) URLTileStore.__init__(self, tilelayouts, **kwargs)
def __init__(self, tile_json, urls_key='tiles', **kwargs): # FIXME schema # FIXME version 1.0.0 support d = json.loads(tile_json) assert 'tiles' in d assert isinstance(d['tiles'], list) assert len(d['tiles']) > 0 for key in self.KEYS: kwargs.setdefault(key, d.get(key, None)) if 'bounding_pyramid' not in kwargs: zmin, zmax = d.get('minzoom', 0), d.get('maxzoom', 22) if 'bounds' in d: lonmin, latmin, lonmax, latmax = d['bounds'] bounding_pyramid = BoundingPyramid.from_wgs84(zmin, zmax, lonmin, lonmax, latmin, latmax) else: bounding_pyramid = BoundingPyramid.full(zmin, zmax) kwargs['bounding_pyramid'] = bounding_pyramid urls = d[urls_key] if 'content_type' not in kwargs: exts = set(os.path.splitext(urlparse(url).path)[1] for url in urls) content_types = set(mimetypes.types_map.get(ext) for ext in exts) assert len(content_types) == 1 kwargs['content_type'] = content_types.pop() templates = [re.sub(r'\{([xyz])\}', lambda m: '%%(%s)d' % m.group(1), url) for url in urls] tile_layouts = map(TemplateTileLayout, templates) URLTileStore.__init__(self, tile_layouts, **kwargs)
def __init__(self, url=None, layer=None, style=None, format=None, tile_matrix_set=None, tile_matrix=None, **kwargs): layout = WMTSTileLayout(url, layer, style, format, tile_matrix_set, tile_matrix) URLTileStore.__init__(self, (layout, ), **kwargs)
def __init__( self, url: str = "", layer: Optional[str] = None, style: Optional[str] = None, format: Optional[str] = None, tile_matrix_set: Optional[str] = None, tile_matrix: type = str, **kwargs: Any, ): layout = WMTSTileLayout(url, layer, style, format, tile_matrix_set, tile_matrix) URLTileStore.__init__(self, (layout,), **kwargs)
def _get_tilestore_for_layer(self, layer, gene): if layer['type'] == 'wms': params = layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join( layer['wmts_style'] for l in layer['layers'].split(',')) if layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(self.dimensions) # Get the metatile image from the WMS server return URLTileStore( tilelayouts=(WMSTileLayout( url=layer['url'], layers=layer['layers'], srs=layer['grid_ref']['srs'], format=layer['mime_type'], border=layer['meta_buffer'] if layer.get('meta') else 0, tilegrid=gene.get_grid(layer['grid'])['obj'], params=params, ), ), headers=layer['headers'], ) elif layer['type'] == 'mapnik': # pragma: no cover try: from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore except ImportError: if 'TRAVIS' not in os.environ: # pragma nocover logger.error("Mapnik is not available") return None grid = gene.get_grid(layer['grid']) if layer['output_format'] == 'grid': return MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=layer['mapfile'], image_buffer=layer['meta_buffer'] if layer.get('meta') else 0, data_buffer=layer['data_buffer'], output_format=layer['output_format'], resolution=layer['resolution'], layers_fields=layer['layers_fields'], drop_empty_utfgrid=layer['drop_empty_utfgrid'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=[self.count_tiles, self.count_tiles_dropped], proj4_literal=grid['proj4_literal'], ) else: return MapnikTileStore( tilegrid=grid['obj'], mapfile=layer['mapfile'], image_buffer=layer['meta_buffer'] if layer.get('meta') else 0, data_buffer=layer['data_buffer'], output_format=layer['output_format'], proj4_literal=grid['proj4_literal'], )
from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore tile_store = URLTileStore( (TemplateTileLayout( 'http://%s.tile.openstreetmap.org/%%(z)d/%%(x)d/%%(y)d.png' % server) for server in 'abc'), attribution='© OpenStreetMap contributors, CC-BY-SA', content_type='image/png')
def __call__(self, config_file: str, layer_name: str) -> Optional[TileStore]: config = self.gene._gene.get_config(config_file) layer = config.config["layers"][layer_name] if layer["type"] == "wms": params = layer.get("params", {}).copy() if "STYLES" not in params: params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(",")) if layer.get("generate_salt", False): params["SALT"] = str(random.randint(0, 999999)) # nosec # Get the metatile image from the WMS server return URLTileStore( tilelayouts=( WMSTileLayout( url=layer["url"], layers=layer["layers"], srs=config.config["grids"][layer["grid"]]["srs"], format=layer["mime_type"], border=layer["meta_buffer"] if layer["meta"] else 0, tilegrid=self.gene._gene.get_grid(config, layer["grid"]), params=params, ), ), headers=layer["headers"], ) elif layer["type"] == "mapnik": try: from tilecloud.store.mapnik_ import MapnikTileStore # pylint: disable=import-outside-toplevel from tilecloud_chain.mapnik_ import ( # pylint: disable=import-outside-toplevel MapnikDropActionTileStore, ) except ImportError: if os.environ.get("CI", "FALSE") == "FALSE": # pragma nocover logger.error("Mapnik is not available", exc_info=True) return None grid = config.config["grids"][layer["grid"]] if cast(str, layer.get("output_format", "png")) == "grid": assert self.gene._count_tiles assert self.gene._count_tiles_dropped return MapnikDropActionTileStore( tilegrid=self.gene._gene.get_grid(config, layer["grid"]), mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer.get("data_buffer", 128), output_format=layer.get("output_format", "png"), resolution=layer.get("resolution", 4), layers_fields=layer.get("layers_fields", {}), drop_empty_utfgrid=layer.get("drop_empty_utfgrid", False), store=self.gene._cache_tilestore, queue_store=self.gene._queue_tilestore, count=[self.gene._count_tiles, self.gene._count_tiles_dropped], proj4_literal=grid["proj4_literal"], ) else: return MapnikTileStore( tilegrid=self.gene._gene.get_grid(config, layer["grid"]), mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer.get("data_buffer", 128), output_format=cast(str, layer.get("output_format", "png")), proj4_literal=grid["proj4_literal"], ) return None
from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore tilestore = URLTileStore( (TemplateTileLayout( f"http://{server!s}.tile.stamen.com/terrain/%(z)d/%(x)d/%(y)d.jpg") for server in "abcd"), attribution= 'Map tiles by <a href="http://stamen.com">Stamen Design</a>, under <a href="http://creativecommons.org/licenses/by/3.0">CC BY 3.0</a>. Data by <a href="http://openstreetmap.org">OpenStreetMap</a>, under <a href="http://creativecommons.org/licenses/by-sa/3.0">CC BY SA</a>.', content_type="image/jpg", )
def load(cls, name): # pragma: no cover """ Construct a :class:`TileStore` from a name. :param name: Name :type name: string :rtype: :class:`TileStore` The following shortcuts are available: bounds://<bounding-pyramid> file://<template> http://<template> and https://<template> memcached://<server>:<port>/<template> s3://<bucket>/<template> sqs://<region>/<queue> <filename>.bsddb <filename>.mbtiles <filename>.zip <module> """ if name == 'null://': from tilecloud.store.null import NullTileStore return NullTileStore() if name.startswith('bounds://'): from tilecloud.store.boundingpyramid import BoundingPyramidTileStore return BoundingPyramidTileStore( BoundingPyramid.from_string(name[9:])) if name.startswith('file://'): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.filesystem import FilesystemTileStore return FilesystemTileStore(TemplateTileLayout(name[7:]), ) if name.startswith('http://') or name.startswith('https://'): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore return URLTileStore((TemplateTileLayout(name), )) if name.startswith('memcached://'): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.memcached import MemcachedTileStore from tilecloud.lib.memcached import MemcachedClient server, template = name[12:].split('/', 1) host, port = server.split(':', 1) client = MemcachedClient(host, int(port)) return MemcachedTileStore(client, TemplateTileLayout(template)) if name.startswith('s3://'): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.s3 import S3TileStore bucket, template = name[5:].split('/', 1) return S3TileStore(bucket, TemplateTileLayout(template)) if name.startswith('sqs://'): from tilecloud.store.sqs import SQSTileStore import boto.sqs from boto.sqs.jsonmessage import JSONMessage region_name, queue_name = name[6:].split('/', 1) connection = boto.sqs.connect_to_region(region_name) queue = connection.create_queue(queue_name) queue.set_message_class(JSONMessage) return SQSTileStore(queue) root, ext = os.path.splitext(name) if ext == '.bsddb': import bsddb from tilecloud.store.bsddb import BSDDBTileStore return BSDDBTileStore(bsddb.hashopen(name)) if ext == '.mbtiles': import sqlite3 from tilecloud.store.mbtiles import MBTilesTileStore return MBTilesTileStore(sqlite3.connect(name)) if ext == '.zip': import zipfile from tilecloud.store.zip import ZipTileStore return ZipTileStore(zipfile.ZipFile(name, 'a')) module = __import__(name) components = name.split('.') module = reduce(lambda module, attr: getattr(module, attr), components[1:], module) return getattr(module, 'tilestore')
from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore tilestore = URLTileStore( (TemplateTileLayout( 'http://%s.tile.stamen.com/toner/%%(z)d/%%(x)d/%%(y)d.png' % server) for server in 'abcd'), attribution= 'Map tiles by <a href="http://stamen.com">Stamen Design</a>, under <a href="http://creativecommons.org/licenses/by/3.0">CC BY 3.0</a>. Data by <a href="http://openstreetmap.org">OpenStreetMap</a>, under <a href="http://creativecommons.org/licenses/by-sa/3.0">CC BY SA</a>.', content_type='image/png')
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print \ "Tile bounds: [%i,%i,%i,%i]" % \ gene.layer['grid_ref']['obj'].extent(tilecoord) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit('the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.') params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % \ ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore tilestore = URLTileStore( ( TemplateTileLayout("http://{0!s}.tile.openstreetmap.org/%(z)d/%(x)d/%(y)d.png".format(server)) for server in "abc" ), attribution="© OpenStreetMap contributors, CC-BY-SA", content_type="image/png", )
from six.moves import xrange from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore tilestore = URLTileStore( (TemplateTileLayout( 'http://otile%d.mqcdn.com/tiles/1.0.0/osm/%%(z)d/%%(x)d/%%(y)d.png' % i) for i in xrange(1, 5)), attribution= 'Data, imagery and map information provided by MapQuest, <a href="http://www.openstreetmap.org/">Open Street Map</a> and contributors, <a href="http://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>.', content_type='image/png')
def _get_tilestore_for_layer(self, layer) -> Optional[TileStore]: if layer["type"] == "wms": params = layer["params"].copy() if "STYLES" not in params: params["STYLES"] = ",".join( layer["wmts_style"] for _ in layer["layers"].split(",")) if layer["generate_salt"]: params["SALT"] = str(random.randint(0, 999999)) # Get the metatile image from the WMS server return URLTileStore( tilelayouts=(WMSTileLayout( url=layer["url"], layers=layer["layers"], srs=layer["grid_ref"]["srs"], format=layer["mime_type"], border=layer["meta_buffer"] if layer.get("meta", False) else 0, tilegrid=self._gene.get_grid(layer)["obj"], params=params, ), ), headers=layer["headers"], ) elif layer["type"] == "mapnik": # pragma: no cover try: from tilecloud.store.mapnik_ import MapnikTileStore # pylint: disable=import-outside-toplevel from tilecloud_chain.mapnik_ import ( # pylint: disable=import-outside-toplevel MapnikDropActionTileStore, ) except ImportError: if os.environ.get("CI", "FALSE") == "FALSE": # pragma nocover logger.error("Mapnik is not available", exc_info=True) return None grid = self._gene.get_grid(layer) if layer["output_format"] == "grid": return MapnikDropActionTileStore( tilegrid=grid["obj"], mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer["data_buffer"], output_format=layer["output_format"], resolution=layer["resolution"], layers_fields=layer["layers_fields"], drop_empty_utfgrid=layer["drop_empty_utfgrid"], store=self._cache_tilestore, queue_store=self._queue_tilestore, count=[self._count_tiles, self._count_tiles_dropped], proj4_literal=grid["proj4_literal"], ) else: return MapnikTileStore( tilegrid=grid["obj"], mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer["data_buffer"], output_format=layer["output_format"], proj4_literal=grid["proj4_literal"], ) return None
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord)) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join( gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer[ 'output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()])), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format( count_metatiles.nb), "Nb metatiles dropped: {}".format( count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds)) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def __init__(self, url=None, layer=None, style=None, format=None, tile_matrix_set=None, tile_matrix=None, **kwargs): layout = WMTSTileLayout(url, layer, style, format, tile_matrix_set, tile_matrix) URLTileStore.__init__(self, (layout,), **kwargs)
def load(name): # pragma: no cover """ Construct a :class:`TileStore` from a name. :param name: Name :type name: string :rtype: :class:`TileStore` The following shortcuts are available: bounds://<bounding-pyramid> file://<template> http://<template> and https://<template> memcached://<server>:<port>/<template> s3://<bucket>/<template> sqs://<region>/<queue> <filename>.bsddb <filename>.mbtiles <filename>.zip <module> """ if name == "null://": from tilecloud.store.null import NullTileStore return NullTileStore() if name.startswith("bounds://"): from tilecloud.store.boundingpyramid import BoundingPyramidTileStore return BoundingPyramidTileStore( BoundingPyramid.from_string(name[9:])) if name.startswith("file://"): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.filesystem import FilesystemTileStore return FilesystemTileStore(TemplateTileLayout(name[7:]), ) if name.startswith("http://") or name.startswith("https://"): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore return URLTileStore((TemplateTileLayout(name), )) if name.startswith("memcached://"): from tilecloud.layout.template import TemplateTileLayout from tilecloud.lib.memcached import MemcachedClient from tilecloud.store.memcached import MemcachedTileStore server, template = name[12:].split("/", 1) host, port = server.split(":", 1) client = MemcachedClient(host, int(port)) return MemcachedTileStore(client, TemplateTileLayout(template)) if name.startswith("s3://"): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.s3 import S3TileStore bucket, template = name[5:].split("/", 1) return S3TileStore(bucket, TemplateTileLayout(template)) if name.startswith("sqs://"): import boto.sqs # pylint: disable=import-error from boto.sqs.jsonmessage import JSONMessage # pylint: disable=import-error from tilecloud.store.sqs import SQSTileStore region_name, queue_name = name[6:].split("/", 1) connection = boto.sqs.connect_to_region(region_name) queue = connection.create_queue(queue_name) queue.set_message_class(JSONMessage) return SQSTileStore(queue) if name.startswith("redis://"): from tilecloud.store.redis import RedisTileStore return RedisTileStore(name) _, ext = os.path.splitext(name) if ext == ".bsddb": import bsddb # pylint: disable=import-error from tilecloud.store.bsddb import BSDDBTileStore return BSDDBTileStore(bsddb.hashopen(name)) if ext == ".mbtiles": import sqlite3 from tilecloud.store.mbtiles import MBTilesTileStore return MBTilesTileStore(sqlite3.connect(name)) if ext == ".zip": import zipfile from tilecloud.store.zip import ZipTileStore return ZipTileStore(zipfile.ZipFile(name, "a")) module = __import__(name) components = name.split(".") module = reduce(getattr, components[1:], module) return getattr(module, "tilestore")
from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore tilestore = URLTileStore((TemplateTileLayout( 'http://datahive.minedata.cn/mergeddata/Villtown,Road,Railway,Ptline,Adminflag,Poi,Annotation,Worldannotation/%(z)d/%(x)d/%(y)d?token=449ba822788c46bea9f90dfba48e5269&solu=4024' .format(server)) for server in 'abc'), attribution='© Minedata contributors, willmap', content_type='image/pbf')