Esempio n. 1
0
 def __init__(self, tile_json, urls_key="tiles", **kwargs):
     # FIXME schema
     # FIXME version 1.0.0 support
     d = json.loads(tile_json)
     assert "tiles" in d
     assert isinstance(d["tiles"], list)
     assert len(d["tiles"]) > 0
     for key in self.KEYS:
         kwargs.setdefault(key, d.get(key))
     if "bounding_pyramid" not in kwargs:
         zmin, zmax = d.get("minzoom", 0), d.get("maxzoom", 22)
         if "bounds" in d:
             lonmin, latmin, lonmax, latmax = d["bounds"]
             bounding_pyramid = BoundingPyramid.from_wgs84(
                 zmin, zmax, lonmin, lonmax, latmin, latmax)
         else:
             bounding_pyramid = BoundingPyramid.full(zmin, zmax)
         kwargs["bounding_pyramid"] = bounding_pyramid
     urls = d[urls_key]
     if "content_type" not in kwargs:
         exts = set(
             os.path.splitext(urlparse(url1).path)[1] for url1 in urls)
         content_types = set(mimetypes.types_map.get(ext) for ext in exts)
         assert len(content_types) == 1
         kwargs["content_type"] = content_types.pop()
     templates = [
         re.sub(r"\{([xyz])\}", lambda m: "%({0!s})d".format(m.group(1)),
                url2) for url2 in urls
     ]
     tilelayouts = map(TemplateTileLayout, templates)
     URLTileStore.__init__(self, tilelayouts, **kwargs)
Esempio n. 2
0
 def __init__(self, tile_json, urls_key='tiles', **kwargs):
     # FIXME schema
     # FIXME version 1.0.0 support
     d = json.loads(tile_json)
     assert 'tiles' in d
     assert isinstance(d['tiles'], list)
     assert len(d['tiles']) > 0
     for key in self.KEYS:
         kwargs.setdefault(key, d.get(key, None))
     if 'bounding_pyramid' not in kwargs:
         zmin, zmax = d.get('minzoom', 0), d.get('maxzoom', 22)
         if 'bounds' in d:
             lonmin, latmin, lonmax, latmax = d['bounds']
             bounding_pyramid = BoundingPyramid.from_wgs84(zmin, zmax,
                                                           lonmin, lonmax,
                                                           latmin, latmax)
         else:
             bounding_pyramid = BoundingPyramid.full(zmin, zmax)
         kwargs['bounding_pyramid'] = bounding_pyramid
     urls = d[urls_key]
     if 'content_type' not in kwargs:
         exts = set(os.path.splitext(urlparse(url).path)[1] for url in urls)
         content_types = set(mimetypes.types_map.get(ext) for ext in exts)
         assert len(content_types) == 1
         kwargs['content_type'] = content_types.pop()
     templates = [re.sub(r'\{([xyz])\}', lambda m: '%%(%s)d' % m.group(1), url)
                  for url in urls]
     tile_layouts = map(TemplateTileLayout, templates)
     URLTileStore.__init__(self, tile_layouts, **kwargs)
Esempio n. 3
0
 def __init__(self,
              url=None,
              layer=None,
              style=None,
              format=None,
              tile_matrix_set=None,
              tile_matrix=None,
              **kwargs):
     layout = WMTSTileLayout(url, layer, style, format, tile_matrix_set,
                             tile_matrix)
     URLTileStore.__init__(self, (layout, ), **kwargs)
Esempio n. 4
0
 def __init__(
     self,
     url: str = "",
     layer: Optional[str] = None,
     style: Optional[str] = None,
     format: Optional[str] = None,
     tile_matrix_set: Optional[str] = None,
     tile_matrix: type = str,
     **kwargs: Any,
 ):
     layout = WMTSTileLayout(url, layer, style, format, tile_matrix_set, tile_matrix)
     URLTileStore.__init__(self, (layout,), **kwargs)
Esempio n. 5
0
    def _get_tilestore_for_layer(self, layer, gene):
        if layer['type'] == 'wms':
            params = layer['params'].copy()
            if 'STYLES' not in params:
                params['STYLES'] = ','.join(
                    layer['wmts_style'] for l in layer['layers'].split(','))
            if layer['generate_salt']:
                params['SALT'] = str(random.randint(0, 999999))
            params.update(self.dimensions)

            # Get the metatile image from the WMS server
            return URLTileStore(
                tilelayouts=(WMSTileLayout(
                    url=layer['url'],
                    layers=layer['layers'],
                    srs=layer['grid_ref']['srs'],
                    format=layer['mime_type'],
                    border=layer['meta_buffer'] if layer.get('meta') else 0,
                    tilegrid=gene.get_grid(layer['grid'])['obj'],
                    params=params,
                ), ),
                headers=layer['headers'],
            )
        elif layer['type'] == 'mapnik':  # pragma: no cover
            try:
                from tilecloud.store.mapnik_ import MapnikTileStore
                from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
            except ImportError:
                if 'TRAVIS' not in os.environ:  # pragma nocover
                    logger.error("Mapnik is not available")
                return None

            grid = gene.get_grid(layer['grid'])
            if layer['output_format'] == 'grid':
                return MapnikDropActionTileStore(
                    tilegrid=grid['obj'],
                    mapfile=layer['mapfile'],
                    image_buffer=layer['meta_buffer']
                    if layer.get('meta') else 0,
                    data_buffer=layer['data_buffer'],
                    output_format=layer['output_format'],
                    resolution=layer['resolution'],
                    layers_fields=layer['layers_fields'],
                    drop_empty_utfgrid=layer['drop_empty_utfgrid'],
                    store=self.cache_tilestore,
                    queue_store=self.sqs_tilestore,
                    count=[self.count_tiles, self.count_tiles_dropped],
                    proj4_literal=grid['proj4_literal'],
                )
            else:
                return MapnikTileStore(
                    tilegrid=grid['obj'],
                    mapfile=layer['mapfile'],
                    image_buffer=layer['meta_buffer']
                    if layer.get('meta') else 0,
                    data_buffer=layer['data_buffer'],
                    output_format=layer['output_format'],
                    proj4_literal=grid['proj4_literal'],
                )
Esempio n. 6
0
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore

tile_store = URLTileStore(
    (TemplateTileLayout(
        'http://%s.tile.openstreetmap.org/%%(z)d/%%(x)d/%%(y)d.png' % server)
     for server in 'abc'),
    attribution='© OpenStreetMap contributors, CC-BY-SA',
    content_type='image/png')
Esempio n. 7
0
    def __call__(self, config_file: str, layer_name: str) -> Optional[TileStore]:
        config = self.gene._gene.get_config(config_file)
        layer = config.config["layers"][layer_name]
        if layer["type"] == "wms":
            params = layer.get("params", {}).copy()
            if "STYLES" not in params:
                params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(","))
            if layer.get("generate_salt", False):
                params["SALT"] = str(random.randint(0, 999999))  # nosec

            # Get the metatile image from the WMS server
            return URLTileStore(
                tilelayouts=(
                    WMSTileLayout(
                        url=layer["url"],
                        layers=layer["layers"],
                        srs=config.config["grids"][layer["grid"]]["srs"],
                        format=layer["mime_type"],
                        border=layer["meta_buffer"] if layer["meta"] else 0,
                        tilegrid=self.gene._gene.get_grid(config, layer["grid"]),
                        params=params,
                    ),
                ),
                headers=layer["headers"],
            )
        elif layer["type"] == "mapnik":
            try:
                from tilecloud.store.mapnik_ import MapnikTileStore  # pylint: disable=import-outside-toplevel
                from tilecloud_chain.mapnik_ import (  # pylint: disable=import-outside-toplevel
                    MapnikDropActionTileStore,
                )
            except ImportError:
                if os.environ.get("CI", "FALSE") == "FALSE":  # pragma nocover
                    logger.error("Mapnik is not available", exc_info=True)
                return None

            grid = config.config["grids"][layer["grid"]]
            if cast(str, layer.get("output_format", "png")) == "grid":
                assert self.gene._count_tiles
                assert self.gene._count_tiles_dropped
                return MapnikDropActionTileStore(
                    tilegrid=self.gene._gene.get_grid(config, layer["grid"]),
                    mapfile=layer["mapfile"],
                    image_buffer=layer["meta_buffer"] if layer.get("meta") else 0,
                    data_buffer=layer.get("data_buffer", 128),
                    output_format=layer.get("output_format", "png"),
                    resolution=layer.get("resolution", 4),
                    layers_fields=layer.get("layers_fields", {}),
                    drop_empty_utfgrid=layer.get("drop_empty_utfgrid", False),
                    store=self.gene._cache_tilestore,
                    queue_store=self.gene._queue_tilestore,
                    count=[self.gene._count_tiles, self.gene._count_tiles_dropped],
                    proj4_literal=grid["proj4_literal"],
                )
            else:
                return MapnikTileStore(
                    tilegrid=self.gene._gene.get_grid(config, layer["grid"]),
                    mapfile=layer["mapfile"],
                    image_buffer=layer["meta_buffer"] if layer.get("meta") else 0,
                    data_buffer=layer.get("data_buffer", 128),
                    output_format=cast(str, layer.get("output_format", "png")),
                    proj4_literal=grid["proj4_literal"],
                )
        return None
Esempio n. 8
0
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore

tilestore = URLTileStore(
    (TemplateTileLayout(
        f"http://{server!s}.tile.stamen.com/terrain/%(z)d/%(x)d/%(y)d.jpg")
     for server in "abcd"),
    attribution=
    'Map tiles by <a href="http://stamen.com">Stamen Design</a>, under <a href="http://creativecommons.org/licenses/by/3.0">CC BY 3.0</a>. Data by <a href="http://openstreetmap.org">OpenStreetMap</a>, under <a href="http://creativecommons.org/licenses/by-sa/3.0">CC BY SA</a>.',
    content_type="image/jpg",
)
Esempio n. 9
0
    def load(cls, name):  # pragma: no cover
        """
        Construct a :class:`TileStore` from a name.

        :param name: Name
        :type name: string

        :rtype: :class:`TileStore`

        The following shortcuts are available:

        bounds://<bounding-pyramid>

        file://<template>

        http://<template> and https://<template>

        memcached://<server>:<port>/<template>

        s3://<bucket>/<template>

        sqs://<region>/<queue>

        <filename>.bsddb

        <filename>.mbtiles

        <filename>.zip

        <module>

        """
        if name == 'null://':
            from tilecloud.store.null import NullTileStore
            return NullTileStore()
        if name.startswith('bounds://'):
            from tilecloud.store.boundingpyramid import BoundingPyramidTileStore
            return BoundingPyramidTileStore(
                BoundingPyramid.from_string(name[9:]))
        if name.startswith('file://'):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.store.filesystem import FilesystemTileStore
            return FilesystemTileStore(TemplateTileLayout(name[7:]), )
        if name.startswith('http://') or name.startswith('https://'):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.store.url import URLTileStore
            return URLTileStore((TemplateTileLayout(name), ))
        if name.startswith('memcached://'):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.store.memcached import MemcachedTileStore
            from tilecloud.lib.memcached import MemcachedClient
            server, template = name[12:].split('/', 1)
            host, port = server.split(':', 1)
            client = MemcachedClient(host, int(port))
            return MemcachedTileStore(client, TemplateTileLayout(template))
        if name.startswith('s3://'):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.store.s3 import S3TileStore
            bucket, template = name[5:].split('/', 1)
            return S3TileStore(bucket, TemplateTileLayout(template))
        if name.startswith('sqs://'):
            from tilecloud.store.sqs import SQSTileStore
            import boto.sqs
            from boto.sqs.jsonmessage import JSONMessage
            region_name, queue_name = name[6:].split('/', 1)
            connection = boto.sqs.connect_to_region(region_name)
            queue = connection.create_queue(queue_name)
            queue.set_message_class(JSONMessage)
            return SQSTileStore(queue)
        root, ext = os.path.splitext(name)
        if ext == '.bsddb':
            import bsddb
            from tilecloud.store.bsddb import BSDDBTileStore
            return BSDDBTileStore(bsddb.hashopen(name))
        if ext == '.mbtiles':
            import sqlite3
            from tilecloud.store.mbtiles import MBTilesTileStore
            return MBTilesTileStore(sqlite3.connect(name))
        if ext == '.zip':
            import zipfile
            from tilecloud.store.zip import ZipTileStore
            return ZipTileStore(zipfile.ZipFile(name, 'a'))
        module = __import__(name)
        components = name.split('.')
        module = reduce(lambda module, attr: getattr(module, attr),
                        components[1:], module)
        return getattr(module, 'tilestore')
Esempio n. 10
0
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore

tilestore = URLTileStore(
    (TemplateTileLayout(
        'http://%s.tile.stamen.com/toner/%%(z)d/%%(x)d/%%(y)d.png' % server)
     for server in 'abcd'),
    attribution=
    'Map tiles by <a href="http://stamen.com">Stamen Design</a>, under <a href="http://creativecommons.org/licenses/by/3.0">CC BY 3.0</a>. Data by <a href="http://openstreetmap.org">OpenStreetMap</a>, under <a href="http://creativecommons.org/licenses/by-sa/3.0">CC BY SA</a>.',
    content_type='image/png')
Esempio n. 11
0
    def gene(self, options, gene, layer):
        count_metatiles = None
        count_metatiles_dropped = Count()
        count_tiles = None
        count_tiles_dropped = Count()

        if options.role == 'slave' or options.get_hash or options.get_bbox:
            gene.layer = gene.layers[layer]
        else:
            gene.set_layer(layer, options)

        if options.get_bbox:
            try:
                tilecoord = parse_tilecoord(options.get_bbox)
                print \
                    "Tile bounds: [%i,%i,%i,%i]" % \
                    gene.layer['grid_ref']['obj'].extent(tilecoord)
                exit()
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
                    % (options.get_bbox, e))

        if options.get_hash:
            options.role = 'hash'
            options.test = 1

        sqs_tilestore = None
        if options.role in ('master', 'slave'):
            # Create SQS queue
            sqs_tilestore = SQSTileStore(
                gene.get_sqs_queue())  # pragma: no cover

        cache_tilestore = None
        if options.role in ('local', 'slave'):
            cache_tilestore = gene.get_tilesstore(options.cache)

        meta = gene.layer['meta']
        if options.tiles:
            gene.set_store(TilesFileStore(options.tiles))

        elif options.role in ('local', 'master'):
            # Generate a stream of metatiles
            gene.init_tilecoords()
            gene.add_geom_filter()

        elif options.role == 'slave':
            # Get the metatiles from the SQS queue
            gene.set_store(sqs_tilestore)  # pragma: no cover

        elif options.role == 'hash':
            try:
                z, x, y = (int(v) for v in options.get_hash.split('/'))
                if meta:
                    gene.set_tilecoords(
                        [TileCoord(z, x, y, gene.layer['meta_size'])])
                else:
                    gene.set_tilecoords([TileCoord(z, x, y)])
            except ValueError as e:  # pragma: no cover
                exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
                     (options.get_hash, e))

        # At this stage, the tilestream contains metatiles that intersect geometry
        gene.add_logger()

        count_metatiles = gene.counter()

        if options.role == 'master':  # pragma: no cover
            # Put the metatiles into the SQS queue
            gene.put(sqs_tilestore)

        elif options.role in ('local', 'slave', 'hash'):
            if gene.layer['type'] == 'wms':
                params = gene.layer['params'].copy()
                if 'STYLES' not in params:
                    params['STYLES'] = ','.join(gene.layer['wmts_style']
                                                for l in gene.layer['layers'])
                if gene.layer['generate_salt']:
                    params['SALT'] = str(random.randint(0, sys.maxint))
                for dim in gene.layer['dimensions']:
                    params[dim['name']] = dim['value']
                for dim in gene.options.dimensions:
                    dim = dim.split('=')
                    if len(dim) != 2:  # pragma: no cover
                        exit('the DIMENTIONS option should be like this '
                             'DATE=2013 VERSION=13.')
                    params[dim[0]] = dim[1]

                # Get the metatile image from the WMS server
                gene.get(
                    URLTileStore(
                        tilelayouts=(WMSTileLayout(
                            url=gene.layer['url'],
                            layers=','.join(gene.layer['layers']),
                            srs=gene.layer['grid_ref']['srs'],
                            format=gene.layer['mime_type'],
                            border=gene.layer['meta_buffer'] if meta else 0,
                            tilegrid=gene.get_grid()['obj'],
                            params=params,
                        ), ),
                        headers=gene.layer['headers'],
                    ), "Get tile from WMS")
            elif gene.layer['type'] == 'mapnik':
                from tilecloud.store.mapnik_ import MapnikTileStore
                from tilecloud_chain.mapnik_ import MapnikDropActionTileStore

                grid = gene.get_grid()
                if gene.layer['output_format'] == 'grid':
                    gene.get(
                        MapnikDropActionTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            resolution=gene.layer['resolution'],
                            layers_fields=gene.layer['layers_fields'],
                            drop_empty_utfgrid=gene.
                            layer['drop_empty_utfgrid'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik grid tile")
                else:
                    gene.get(
                        MapnikTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik tile")

            def wrong_content_type_to_error(tile):
                if tile is not None and tile.content_type is not None \
                        and tile.content_type.find("image/") != 0:
                    if tile.content_type.find(
                            "application/vnd.ogc.se_xml") == 0:
                        tile.error = "WMS server error: %s" % (
                            self._re_rm_xml_tag.sub('', tile.data))
                    else:  # pragma: no cover
                        tile.error = "%s is not an image format, error: %s" % (
                            tile.content_type, tile.data)
                return tile

            gene.imap(wrong_content_type_to_error)

            # Handle errors
            gene.add_error_filters()

            if meta:
                if options.role == 'hash':
                    gene.imap(HashLogger('empty_metatile_detection'))
                elif not options.near:
                    # Discard tiles with certain content
                    if 'empty_metatile_detection' in gene.layer:
                        empty_tile = gene.layer['empty_metatile_detection']

                        gene.imap(
                            HashDropper(
                                empty_tile['size'],
                                empty_tile['hash'],
                                store=cache_tilestore,
                                queue_store=sqs_tilestore,
                                count=count_metatiles_dropped,
                            ))

                def add_elapsed_togenerate(metatile):
                    if metatile is not None:
                        metatile.elapsed_togenerate = metatile.tilecoord.n**2
                        return True
                    return False  # pragma: no cover

                gene.ifilter(add_elapsed_togenerate)

                # Split the metatile image into individual tiles
                gene.add_metatile_splitter()
                gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))

                # Handle errors
                gene.add_error_filters()

            self.count_tiles = gene.counter()

            if 'pre_hash_post_process' in gene.layer:
                gene.process(gene.layer['pre_hash_post_process'])

            if options.role == 'hash':
                gene.imap(HashLogger('empty_tile_detection'))
            elif not options.near:
                # Discard tiles with certain content
                if 'empty_tile_detection' in gene.layer:
                    empty_tile = gene.layer['empty_tile_detection']

                    gene.imap(
                        HashDropper(
                            empty_tile['size'],
                            empty_tile['hash'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                        ))

            gene.process()

        if options.role in ('local', 'slave'):
            gene.add_error_filters()
            gene.ifilter(DropEmpty(gene))
            count_tiles = gene.counter(size=True)

            if options.time:

                def log_size(tile):
                    sys.stdout.write('size: %i\n' % len(tile.data))
                    return tile

                gene.imap(log_size)

            gene.put(cache_tilestore, "Store the tile")
        else:
            count_tiles = gene.counter(size=True)

        gene.add_error_filters()
        if options.generated_tiles_file:  # pragma: no cover
            generated_tiles_file = open(options.generated_tiles_file, 'a')

            def do(tile):
                generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
                return tile

            gene.imap(do)

        if options.role == 'slave':  # pragma: no cover
            if meta:

                def decr_tile_in_metatile(tile):
                    tile.metatile.elapsed_togenerate -= 1
                    if tile.metatile.elapsed_togenerate == 0:
                        sqs_tilestore.delete_one(tile.metatile)
                    return True

                gene.ifilter(decr_tile_in_metatile)
            else:
                gene.delete(sqs_tilestore)

        if options.time is not None:

            class LogTime:
                n = 0
                t1 = None

                def __call__(self, tile):
                    self.n += 1
                    if self.n == options.time:
                        self.t1 = datetime.now()
                    elif self.n == 2 * options.time:
                        t2 = datetime.now()
                        d = (t2 - self.t1) / options.time
                        sys.stdout.write(
                            'time: %i\n' %
                            ((d.days * 24 * 3600 + d.seconds) * 1000000 +
                             d.microseconds))
                    return tile

            gene.imap(LogTime())

            gene.consume(options.time * 3)
        else:
            gene.consume()

            if not options.quiet and options.role in ('local', 'slave'):
                nb_tiles = count_tiles.nb + count_tiles_dropped.nb
                print """The tile generation of layer '%s' is finish
%sNb generated tiles: %i
Nb tiles dropped: %i
Nb tiles stored: %i
Nb error: %i
Total time: %s
Total size: %s
Time per tiles: %i ms
Size per tile: %i o
""" % \
                    (
                        gene.layer['name'],
                        """Nb generated metatiles: %i
Nb metatiles dropped: %i
""" %
                        (
                            count_metatiles.nb, count_metatiles_dropped.nb
                        ) if meta else '',
                        nb_tiles,
                        count_tiles_dropped.nb,
                        count_tiles.nb,
                        gene.error,
                        duration_format(gene.duration),
                        size_format(count_tiles.size),
                        (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
                        count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1
                    )

        if cache_tilestore is not None and hasattr(cache_tilestore,
                                                   'connection'):
            cache_tilestore.connection.close()

        if options.role != 'hash' and options.time is None and 'sns' in gene.config:  # pragma: no cover
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'], """The tile generation is finish
Layer: %(layer)s
Role: %(role)s
Host: %(host)s
Command: %(cmd)s

%(meta)sNb generated tiles: %(nb_tiles)i
Nb tiles dropped: %(nb_tiles_dropped)i
Total time: %(duration)s [s]
Time per tiles: %(tile_duration)i [ms]""" % {
                    'role':
                    options.role,
                    'layer':
                    gene.layer['name'],
                    'host':
                    socket.getfqdn(),
                    'cmd':
                    ' '.join([quote(arg) for arg in sys.argv]),
                    'meta':
                    """Nb generated metatiles: %(nb_metatiles)i
Nb metatiles dropped: %(nb_metatiles_dropped)i
""" % {
                        'nb_metatiles': count_metatiles.nb,
                        'nb_metatiles_dropped': count_metatiles_dropped.nb,
                    } if meta else '',
                    'nb_tiles':
                    nb_tiles if meta else count_metatiles.nb,
                    'nb_tiles_dropped':
                    count_tiles_dropped.nb
                    if meta else count_metatiles_dropped.nb,
                    'duration':
                    duration_format(gene.duration),
                    'tile_duration': (gene.duration / nb_tiles *
                                      1000).seconds if nb_tiles != 0 else 0,
                }, "Tile generation (%(layer)s - %(role)s)" % {
                    'role': options.role,
                    'layer': gene.layer['name']
                })
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore


tilestore = URLTileStore(
    (
        TemplateTileLayout("http://{0!s}.tile.openstreetmap.org/%(z)d/%(x)d/%(y)d.png".format(server))
        for server in "abc"
    ),
    attribution="&copy; OpenStreetMap contributors, CC-BY-SA",
    content_type="image/png",
)
Esempio n. 13
0
from six.moves import xrange

from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore

tilestore = URLTileStore(
    (TemplateTileLayout(
        'http://otile%d.mqcdn.com/tiles/1.0.0/osm/%%(z)d/%%(x)d/%%(y)d.png' %
        i) for i in xrange(1, 5)),
    attribution=
    'Data, imagery and map information provided by MapQuest, <a href="http://www.openstreetmap.org/">Open Street Map</a> and contributors, <a href="http://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>.',
    content_type='image/png')
Esempio n. 14
0
    def _get_tilestore_for_layer(self, layer) -> Optional[TileStore]:
        if layer["type"] == "wms":
            params = layer["params"].copy()
            if "STYLES" not in params:
                params["STYLES"] = ",".join(
                    layer["wmts_style"] for _ in layer["layers"].split(","))
            if layer["generate_salt"]:
                params["SALT"] = str(random.randint(0, 999999))

            # Get the metatile image from the WMS server
            return URLTileStore(
                tilelayouts=(WMSTileLayout(
                    url=layer["url"],
                    layers=layer["layers"],
                    srs=layer["grid_ref"]["srs"],
                    format=layer["mime_type"],
                    border=layer["meta_buffer"]
                    if layer.get("meta", False) else 0,
                    tilegrid=self._gene.get_grid(layer)["obj"],
                    params=params,
                ), ),
                headers=layer["headers"],
            )
        elif layer["type"] == "mapnik":  # pragma: no cover
            try:
                from tilecloud.store.mapnik_ import MapnikTileStore  # pylint: disable=import-outside-toplevel
                from tilecloud_chain.mapnik_ import (  # pylint: disable=import-outside-toplevel
                    MapnikDropActionTileStore, )
            except ImportError:
                if os.environ.get("CI", "FALSE") == "FALSE":  # pragma nocover
                    logger.error("Mapnik is not available", exc_info=True)
                return None

            grid = self._gene.get_grid(layer)
            if layer["output_format"] == "grid":
                return MapnikDropActionTileStore(
                    tilegrid=grid["obj"],
                    mapfile=layer["mapfile"],
                    image_buffer=layer["meta_buffer"]
                    if layer.get("meta") else 0,
                    data_buffer=layer["data_buffer"],
                    output_format=layer["output_format"],
                    resolution=layer["resolution"],
                    layers_fields=layer["layers_fields"],
                    drop_empty_utfgrid=layer["drop_empty_utfgrid"],
                    store=self._cache_tilestore,
                    queue_store=self._queue_tilestore,
                    count=[self._count_tiles, self._count_tiles_dropped],
                    proj4_literal=grid["proj4_literal"],
                )
            else:
                return MapnikTileStore(
                    tilegrid=grid["obj"],
                    mapfile=layer["mapfile"],
                    image_buffer=layer["meta_buffer"]
                    if layer.get("meta") else 0,
                    data_buffer=layer["data_buffer"],
                    output_format=layer["output_format"],
                    proj4_literal=grid["proj4_literal"],
                )
        return None
Esempio n. 15
0
    def _gene(self, options, gene, layer, dimensions={}):
        count_metatiles = None
        count_metatiles_dropped = Count()
        count_tiles = None
        count_tiles_dropped = Count()
        count_tiles_stored = None

        if options.get_bbox:
            try:
                tilecoord = parse_tilecoord(options.get_bbox)
                print("Tile bounds: [%i,%i,%i,%i]" %
                      gene.layer['grid_ref']['obj'].extent(tilecoord))
                exit()
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
                    % (options.get_bbox, e))

        if options.get_hash:
            options.role = 'hash'
            options.test = 1

        sqs_tilestore = None
        if options.role in ('master', 'slave'):
            # Create SQS queue
            sqs_tilestore = SQSTileStore(
                gene.get_sqs_queue())  # pragma: no cover

        cache_tilestore = None
        if options.role in ('local', 'slave'):
            cache_tilestore = gene.get_tilesstore(options.cache, dimensions)

        meta = gene.layer['meta']
        if options.tiles:
            gene.set_store(TilesFileStore(options.tiles))

        elif options.role in ('local', 'master'):
            # Generate a stream of metatiles
            gene.init_tilecoords()
            gene.add_geom_filter()

        if options.local_process_number is not None:  # pragma: no cover
            gene.add_local_process_filter()

        elif options.role == 'slave':
            # Get the metatiles from the SQS queue
            gene.set_store(sqs_tilestore)  # pragma: no cover

        elif options.role == 'hash':
            try:
                z, x, y = (int(v) for v in options.get_hash.split('/'))
                if meta:
                    gene.set_tilecoords(
                        [TileCoord(z, x, y, gene.layer['meta_size'])])
                else:
                    gene.set_tilecoords([TileCoord(z, x, y)])
            except ValueError as e:  # pragma: no cover
                exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
                     (options.get_hash, e))

        # At this stage, the tilestream contains metatiles that intersect geometry
        gene.add_logger()

        count_metatiles = gene.counter()

        if options.role == 'master':  # pragma: no cover
            # Put the metatiles into the SQS queue
            gene.put(sqs_tilestore)
            count_tiles = gene.counter()

        elif options.role in ('local', 'slave', 'hash'):
            if gene.layer['type'] == 'wms':
                params = gene.layer['params'].copy()
                if 'STYLES' not in params:
                    params['STYLES'] = ','.join(
                        gene.layer['wmts_style']
                        for l in gene.layer['layers'].split(','))
                if gene.layer['generate_salt']:
                    params['SALT'] = str(random.randint(0, 999999))
                params.update(dimensions)

                # Get the metatile image from the WMS server
                gene.get(
                    URLTileStore(
                        tilelayouts=(WMSTileLayout(
                            url=gene.layer['url'],
                            layers=gene.layer['layers'],
                            srs=gene.layer['grid_ref']['srs'],
                            format=gene.layer['mime_type'],
                            border=gene.layer['meta_buffer'] if meta else 0,
                            tilegrid=gene.get_grid()['obj'],
                            params=params,
                        ), ),
                        headers=gene.layer['headers'],
                    ), "Get tile from WMS")
            elif gene.layer['type'] == 'mapnik':  # pragma: no cover
                from tilecloud.store.mapnik_ import MapnikTileStore
                from tilecloud_chain.mapnik_ import MapnikDropActionTileStore

                grid = gene.get_grid()
                if gene.layer['output_format'] == 'grid':
                    count_tiles = gene.counter()
                    gene.get(
                        MapnikDropActionTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            resolution=gene.layer['resolution'],
                            layers_fields=gene.layer['layers_fields'],
                            drop_empty_utfgrid=gene.
                            layer['drop_empty_utfgrid'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik grid tile")
                else:
                    gene.get(
                        MapnikTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik tile")

            def wrong_content_type_to_error(tile):
                if tile is not None and tile.content_type is not None \
                        and tile.content_type.find("image/") != 0:
                    if tile.content_type.find(
                            "application/vnd.ogc.se_xml") == 0:
                        tile.error = "WMS server error: %s" % (
                            self._re_rm_xml_tag.sub(
                                '',
                                tile.data.decode('utf-8')
                                if PY3 else tile.data))
                    else:  # pragma: no cover
                        tile.error = "%s is not an image format, error: %s" % (
                            tile.content_type, tile.data)
                return tile

            gene.imap(wrong_content_type_to_error)

            # Handle errors
            gene.add_error_filters()

            if meta:
                if options.role == 'hash':
                    gene.imap(HashLogger('empty_metatile_detection'))
                elif not options.near:
                    # Discard tiles with certain content
                    if 'empty_metatile_detection' in gene.layer:
                        empty_tile = gene.layer['empty_metatile_detection']

                        gene.imap(
                            HashDropper(
                                empty_tile['size'],
                                empty_tile['hash'],
                                store=cache_tilestore,
                                queue_store=sqs_tilestore,
                                count=count_metatiles_dropped,
                            ))

                def add_elapsed_togenerate(metatile):
                    if metatile is not None:
                        metatile.elapsed_togenerate = metatile.tilecoord.n**2
                        return True
                    return False  # pragma: no cover

                gene.ifilter(add_elapsed_togenerate)

                # Split the metatile image into individual tiles
                gene.add_metatile_splitter()
                gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))

                # Handle errors
                gene.add_error_filters()

            if gene.layer['type'] != 'mapnik' or gene.layer[
                    'output_format'] != 'grid':
                count_tiles = gene.counter()

            if 'pre_hash_post_process' in gene.layer:  # pragma: no cover
                gene.process(gene.layer['pre_hash_post_process'])

            if options.role == 'hash':
                gene.imap(HashLogger('empty_tile_detection'))
            elif not options.near:
                # Discard tiles with certain content
                if 'empty_tile_detection' in gene.layer:
                    empty_tile = gene.layer['empty_tile_detection']

                    gene.imap(
                        HashDropper(
                            empty_tile['size'],
                            empty_tile['hash'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                        ))

            gene.process()
        else:  # pragma: no cover
            count_tiles = gene.counter()

        if options.role in ('local', 'slave'):
            gene.add_error_filters()
            gene.ifilter(DropEmpty(gene))
            count_tiles_stored = gene.counter(size=True)

            if options.time:

                def log_size(tile):
                    sys.stdout.write('size: %i\n' % len(tile.data))
                    return tile

                gene.imap(log_size)

            gene.put(cache_tilestore, "Store the tile")

        gene.add_error_filters()
        if options.generated_tiles_file:  # pragma: no cover
            generated_tiles_file = open(options.generated_tiles_file, 'a')

            def do(tile):
                generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
                return tile

            gene.imap(do)

        if options.role == 'slave':  # pragma: no cover
            if meta:

                def decr_tile_in_metatile(tile):
                    tile.metatile.elapsed_togenerate -= 1
                    if tile.metatile.elapsed_togenerate == 0:
                        sqs_tilestore.delete_one(tile.metatile)
                    return True

                gene.ifilter(decr_tile_in_metatile)
            else:
                gene.delete(sqs_tilestore)

        message = []
        if options.time is not None:

            class LogTime:
                n = 0
                t1 = None

                def __call__(self, tile):
                    self.n += 1
                    if self.n == options.time:
                        self.t1 = datetime.now()
                    elif self.n == 2 * options.time:
                        t2 = datetime.now()
                        d = (t2 - self.t1) / options.time
                        sys.stdout.write(
                            'time: %i\n' %
                            ((d.days * 24 * 3600 + d.seconds) * 1000000 +
                             d.microseconds))
                    return tile

            gene.imap(LogTime())

            gene.consume(options.time * 3)
        else:
            gene.consume()

            message = [
                "The tile generation of layer '{}{}' is finish".format(
                    gene.layer['name'], "" if len(dimensions) == 0
                    or gene.layer['type'] != 'wms' else " (%s)" %
                    ", ".join(["=".join(d) for d in dimensions.items()])),
            ]
            if options.role == "master":  # pragma: no cover
                message.append("Nb of generated jobs: {}".format(
                    count_tiles.nb))
            else:
                if meta:
                    message += [
                        "Nb generated metatiles: {}".format(
                            count_metatiles.nb),
                        "Nb metatiles dropped: {}".format(
                            count_metatiles_dropped.nb),
                    ]
                message += [
                    "Nb generated tiles: {}".format(count_tiles.nb),
                    "Nb tiles dropped: {}".format(count_tiles_dropped.nb),
                ]
                if options.role in ('local', 'slave'):
                    message += [
                        "Nb tiles stored: {}".format(count_tiles_stored.nb),
                        "Nb tiles in error: {}".format(gene.error),
                        "Total time: {}".format(duration_format(
                            gene.duration)),
                    ]
                    if count_tiles_stored.nb != 0:
                        message.append("Total size: {}".format(
                            size_format(count_tiles_stored.size)))
                    if count_tiles.nb != 0:
                        message.append("Time per tile: {:0.0f} ms".format(
                            (gene.duration / count_tiles.nb * 1000).seconds))
                    if count_tiles_stored.nb != 0:
                        message.append("Size per tile: {:0.0f} o".format(
                            count_tiles_stored.size / count_tiles_stored.nb))

            if not options.quiet and options.role in ('local', 'slave'):
                print("\n".join(message) + "\n")

        if cache_tilestore is not None and hasattr(cache_tilestore,
                                                   'connection'):
            cache_tilestore.connection.close()

        if options.role != 'hash' and options.time is None and 'sns' in gene.config:  # pragma: no cover
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            sns_message = [message[0]]
            sns_message += [
                "Layer: {}".format(gene.layer['name']),
                "Role: {}".format(options.role),
                "Host: {}".format(socket.getfqdn()),
                "Command: {}".format(' '.join([quote(arg)
                                               for arg in sys.argv])),
            ]
            sns_message += message[1:]
            connection.publish(
                gene.config['sns']['topic'], "\n".join(sns_message),
                "Tile generation (%(layer)s - %(role)s)" % {
                    'role': options.role,
                    'layer': gene.layer['name']
                })
Esempio n. 16
0
 def __init__(self, url=None, layer=None, style=None, format=None, tile_matrix_set=None, tile_matrix=None, **kwargs):
     layout = WMTSTileLayout(url, layer, style, format, tile_matrix_set, tile_matrix)
     URLTileStore.__init__(self, (layout,), **kwargs)
Esempio n. 17
0
    def load(name):  # pragma: no cover
        """
        Construct a :class:`TileStore` from a name.

        :param name: Name
        :type name: string

        :rtype: :class:`TileStore`

        The following shortcuts are available:

        bounds://<bounding-pyramid>

        file://<template>

        http://<template> and https://<template>

        memcached://<server>:<port>/<template>

        s3://<bucket>/<template>

        sqs://<region>/<queue>

        <filename>.bsddb

        <filename>.mbtiles

        <filename>.zip

        <module>

        """
        if name == "null://":
            from tilecloud.store.null import NullTileStore

            return NullTileStore()
        if name.startswith("bounds://"):
            from tilecloud.store.boundingpyramid import BoundingPyramidTileStore

            return BoundingPyramidTileStore(
                BoundingPyramid.from_string(name[9:]))
        if name.startswith("file://"):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.store.filesystem import FilesystemTileStore

            return FilesystemTileStore(TemplateTileLayout(name[7:]), )
        if name.startswith("http://") or name.startswith("https://"):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.store.url import URLTileStore

            return URLTileStore((TemplateTileLayout(name), ))
        if name.startswith("memcached://"):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.lib.memcached import MemcachedClient
            from tilecloud.store.memcached import MemcachedTileStore

            server, template = name[12:].split("/", 1)
            host, port = server.split(":", 1)
            client = MemcachedClient(host, int(port))
            return MemcachedTileStore(client, TemplateTileLayout(template))
        if name.startswith("s3://"):
            from tilecloud.layout.template import TemplateTileLayout
            from tilecloud.store.s3 import S3TileStore

            bucket, template = name[5:].split("/", 1)
            return S3TileStore(bucket, TemplateTileLayout(template))
        if name.startswith("sqs://"):
            import boto.sqs  # pylint: disable=import-error
            from boto.sqs.jsonmessage import JSONMessage  # pylint: disable=import-error

            from tilecloud.store.sqs import SQSTileStore

            region_name, queue_name = name[6:].split("/", 1)
            connection = boto.sqs.connect_to_region(region_name)
            queue = connection.create_queue(queue_name)
            queue.set_message_class(JSONMessage)
            return SQSTileStore(queue)
        if name.startswith("redis://"):
            from tilecloud.store.redis import RedisTileStore

            return RedisTileStore(name)
        _, ext = os.path.splitext(name)
        if ext == ".bsddb":
            import bsddb  # pylint: disable=import-error

            from tilecloud.store.bsddb import BSDDBTileStore

            return BSDDBTileStore(bsddb.hashopen(name))
        if ext == ".mbtiles":
            import sqlite3

            from tilecloud.store.mbtiles import MBTilesTileStore

            return MBTilesTileStore(sqlite3.connect(name))
        if ext == ".zip":
            import zipfile

            from tilecloud.store.zip import ZipTileStore

            return ZipTileStore(zipfile.ZipFile(name, "a"))
        module = __import__(name)
        components = name.split(".")
        module = reduce(getattr, components[1:], module)
        return getattr(module, "tilestore")
Esempio n. 18
0
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore

tilestore = URLTileStore((TemplateTileLayout(
    'http://datahive.minedata.cn/mergeddata/Villtown,Road,Railway,Ptline,Adminflag,Poi,Annotation,Worldannotation/%(z)d/%(x)d/%(y)d?token=449ba822788c46bea9f90dfba48e5269&solu=4024'
    .format(server)) for server in 'abc'),
                         attribution='&copy; Minedata contributors, willmap',
                         content_type='image/pbf')