Example #1
0
def main():
    parser = ArgumentParser(
        description='Used to calculate the generation cost',
        prog=sys.argv[0]
    )
    add_comon_options(parser, tile_pyramid=False)
    parser.add_argument(
        '--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo',
        choices=('area', 'count'),
        help="The algorithm use to calculate the cost default base on the 'area' "
        "of the generation geometry, can also be 'count', to be base on number of tiles to generate."
    )

    options = parser.parse_args()
    gene = TileGeneration(
        options.config, options,
        layer_name=options.layer, base_config={'cost': {}}
    )

    all_size = 0
    tile_size = 0
    all_tiles = 0
    if (options.layer):
        (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options)
        tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024)
    else:
        all_time = timedelta()
        all_price = 0
        for layer in gene.config['generation']['default_layers']:
            print("")
            print("===== %s =====" % layer)
            gene.set_layer(layer, options)
            (size, time, price, tiles) = _calculate_cost(gene, options)
            tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024)
            all_time += time
            all_price += price
            all_size += size
            all_tiles += tiles

        print("")
        print("===== GLOBAL =====")
        print("Total number of tiles: %i" % all_tiles)
        print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time)))
        print('Total generation cost: %0.2f [$]' % all_price)
    print("")
    print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024)))
    print('S3 get: %0.2f [$/month]' % (
        gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
        gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
    )
#    if 'cloudfront' in gene.config['cost']:
#        print('CloudFront: %0.2f [$/month]' % ()
#            gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
#            gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
    if 'ec2' in gene.config:
        print('ESB storage: %0.2f [$/month]' % (
            gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size'])
        )
    sys.exit(0)
Example #2
0
    def copy(self, options, gene, layer, source, dest, task_name):
        # desable metatiles
        gene.layers[layer]['meta'] = False
        count_tiles_dropped = Count()

        gene.set_layer(layer, options)
        source_tilestore = gene.get_tilesstore(source)
        dest_tilestore = gene.get_tilesstore(dest)
        gene.init_tilecoords()
        gene.add_geom_filter()
        gene.add_logger()
        gene.get(source_tilestore, "Get the tiles")
        gene.ifilter(DropEmpty(gene))
        # Discard tiles with certain content
        if 'empty_tile_detection' in gene.layer:
            empty_tile = gene.layer['empty_tile_detection']

            gene.imap(
                HashDropper(
                    empty_tile['size'],
                    empty_tile['hash'],
                    store=dest_tilestore,
                    count=count_tiles_dropped,
                ))

        if options.process:
            gene.process(options.process)

        gene.ifilter(DropEmpty(gene))
        self.count = gene.counter(size=True)
        gene.put(dest_tilestore, "Store the tiles")
        gene.add_error_filters()
        gene.consume()
        if not options.quiet:
            print """The tile %s of layer '%s' is finish
Nb %s tiles: %i
Nb errored tiles: %i
Nb dropped tiles: %i
Total time: %s
Total size: %s
Time per tiles: %i ms
Size per tile: %i o
""" % \
                (
                    task_name,
                    gene.layer['name'],
                    task_name,
                    self.count.nb,
                    count_tiles_dropped.nb,
                    gene.error,
                    duration_format(gene.duration),
                    size_format(self.count.size),
                    (gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0,
                    self.count.size / self.count.nb if self.count.nb != 0 else -1
                )
Example #3
0
    def _copy(self,
              options,
              gene,
              layer,
              source,
              dest,
              task_name,
              dimensions=None):
        # disable metatiles
        if dimensions is None:  # pragma: no cover
            dimensions = {}
        del gene.layers[layer]['meta']
        count_tiles_dropped = Count()

        gene.set_layer(layer, options)
        source_tilestore = gene.get_tilesstore(source, dimensions)
        dest_tilestore = gene.get_tilesstore(dest, dimensions)
        gene.init_tilecoords()
        gene.add_geom_filter()
        gene.add_logger()
        gene.get(source_tilestore, "Get the tiles")
        gene.ifilter(DropEmpty(gene))
        # Discard tiles with certain content
        if gene.layer and 'empty_tile_detection' in gene.layer:
            empty_tile = gene.layer['empty_tile_detection']

            gene.imap(
                HashDropper(empty_tile['size'],
                            empty_tile['hash'],
                            store=dest_tilestore,
                            count=count_tiles_dropped))

        if options.process:
            gene.process(options.process)

        gene.ifilter(DropEmpty(gene))
        self.count = gene.counter(size=True)
        gene.put(dest_tilestore, "Store the tiles")
        gene.add_error_filters()
        gene.consume()
        if not options.quiet:
            print("""The tile {} of layer '{}' is finish
Nb {} tiles: {}
Nb errored tiles: {}
Nb dropped tiles: {}
Total time: {}
Total size: {}
Time per tile: {} ms
Size per tile: {} o
""".format(task_name, gene.layer['name'], task_name, self.count.nb,
            count_tiles_dropped.nb, gene.error, duration_format(gene.duration),
            size_format(self.count.size),
            (gene.duration / self.count.nb *
            1000).seconds if self.count.nb != 0 else 0,
            self.count.size / self.count.nb if self.count.nb != 0 else -1))
Example #4
0
    def _copy(self, options, gene, layer, source, dest, task_name, dimensions={}):
        # disable metatiles
        del gene.layers[layer]['meta']
        count_tiles_dropped = Count()

        gene.set_layer(layer, options)
        source_tilestore = gene.get_tilesstore(source, dimensions)
        dest_tilestore = gene.get_tilesstore(dest, dimensions)
        gene.init_tilecoords()
        gene.add_geom_filter()
        gene.add_logger()
        gene.get(source_tilestore, "Get the tiles")
        gene.ifilter(DropEmpty(gene))
        # Discard tiles with certain content
        if 'empty_tile_detection' in gene.layer:
            empty_tile = gene.layer['empty_tile_detection']

            gene.imap(HashDropper(
                empty_tile['size'], empty_tile['hash'], store=dest_tilestore,
                count=count_tiles_dropped,
            ))

        if options.process:
            gene.process(options.process)

        gene.ifilter(DropEmpty(gene))
        self.count = gene.counter(size=True)
        gene.put(dest_tilestore, "Store the tiles")
        gene.add_error_filters()
        gene.consume()
        if not options.quiet:
            print(
                """The tile %s of layer '%s' is finish
Nb %s tiles: %i
Nb errored tiles: %i
Nb dropped tiles: %i
Total time: %s
Total size: %s
Time per tile: %i ms
Size per tile: %i o
""" % (
                    task_name,
                    gene.layer['name'],
                    task_name,
                    self.count.nb,
                    count_tiles_dropped.nb,
                    gene.error,
                    duration_format(gene.duration),
                    size_format(self.count.size),
                    (gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0,
                    self.count.size / self.count.nb if self.count.nb != 0 else -1
                )
            )
Example #5
0
    def _copy(self, options, gene, layer_name, source, dest, task_name):
        # disable metatiles
        layer = gene.layers[layer_name]
        del layer["meta"]
        count_tiles_dropped = Count()

        gene.init_layer(layer, options)
        source_tilestore = gene.get_tilesstore(source)
        dest_tilestore = gene.get_tilesstore(dest)
        gene.init_tilecoords(layer)
        gene.add_geom_filter()
        gene.add_logger()
        gene.get(source_tilestore, "Get the tiles")
        gene.imap(DropEmpty(gene))
        # Discard tiles with certain content
        if "empty_tile_detection" in layer:
            empty_tile = layer["empty_tile_detection"]

            gene.imap(
                HashDropper(empty_tile["size"],
                            empty_tile["hash"],
                            store=dest_tilestore,
                            count=count_tiles_dropped))

        if options.process:
            gene.process(options.process)

        gene.imap(DropEmpty(gene))
        self.count = gene.counter(size=True)
        gene.put(dest_tilestore, "Store the tiles")
        gene.consume()
        if not options.quiet:
            print("""The tile {} of layer '{}' is finish
Nb {} tiles: {}
Nb errored tiles: {}
Nb dropped tiles: {}
Total time: {}
Total size: {}
Time per tile: {} ms
Size per tile: {} o
""".format(
                task_name,
                layer["name"],
                task_name,
                self.count.nb,
                count_tiles_dropped.nb,
                gene.error,
                duration_format(gene.duration),
                size_format(self.count.size),
                (gene.duration / self.count.nb *
                 1000).seconds if self.count.nb != 0 else 0,
                self.count.size / self.count.nb if self.count.nb != 0 else -1,
            ))
Example #6
0
    def _gene(self, options, gene, layer, dimensions={}):
        count_metatiles = None
        count_metatiles_dropped = Count()
        count_tiles = None
        count_tiles_dropped = Count()
        count_tiles_stored = None

        if options.get_bbox:
            try:
                tilecoord = parse_tilecoord(options.get_bbox)
                print(
                    "Tile bounds: [%i,%i,%i,%i]" %
                    gene.layer['grid_ref']['obj'].extent(tilecoord)
                )
                exit()
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" %
                    (options.get_bbox, e)
                )

        if options.get_hash:
            options.role = 'hash'
            options.test = 1

        sqs_tilestore = None
        if options.role in ('master', 'slave'):
            # Create SQS queue
            sqs_tilestore = SQSTileStore(gene.get_sqs_queue())  # pragma: no cover

        cache_tilestore = None
        if options.role in ('local', 'slave'):
            cache_tilestore = gene.get_tilesstore(options.cache, dimensions)

        meta = gene.layer['meta']
        if options.tiles:
            gene.set_store(TilesFileStore(options.tiles))

        elif options.role in ('local', 'master'):
            # Generate a stream of metatiles
            gene.init_tilecoords()
            gene.add_geom_filter()

        if options.local_process_number is not None:  # pragma: no cover
            gene.add_local_process_filter()

        elif options.role == 'slave':
            # Get the metatiles from the SQS queue
            gene.set_store(sqs_tilestore)  # pragma: no cover

        elif options.role == 'hash':
            try:
                z, x, y = (int(v) for v in options.get_hash.split('/'))
                if meta:
                    gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])])
                else:
                    gene.set_tilecoords([TileCoord(z, x, y)])
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y'\n%r" %
                    (options.get_hash, e)
                )

        # At this stage, the tilestream contains metatiles that intersect geometry
        gene.add_logger()

        count_metatiles = gene.counter()

        if options.role == 'master':  # pragma: no cover
            # Put the metatiles into the SQS queue
            gene.put(sqs_tilestore)
            count_tiles = gene.counter()

        elif options.role in ('local', 'slave', 'hash'):
            if gene.layer['type'] == 'wms':
                params = gene.layer['params'].copy()
                if 'STYLES' not in params:
                    params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers'].split(','))
                if gene.layer['generate_salt']:
                    params['SALT'] = str(random.randint(0, 999999))
                params.update(dimensions)

                # Get the metatile image from the WMS server
                gene.get(URLTileStore(
                    tilelayouts=(WMSTileLayout(
                        url=gene.layer['url'],
                        layers=gene.layer['layers'],
                        srs=gene.layer['grid_ref']['srs'],
                        format=gene.layer['mime_type'],
                        border=gene.layer['meta_buffer'] if meta else 0,
                        tilegrid=gene.get_grid()['obj'],
                        params=params,
                    ),),
                    headers=gene.layer['headers'],
                ), "Get tile from WMS")
            elif gene.layer['type'] == 'mapnik':  # pragma: no cover
                from tilecloud.store.mapnik_ import MapnikTileStore
                from tilecloud_chain.mapnik_ import MapnikDropActionTileStore

                grid = gene.get_grid()
                if gene.layer['output_format'] == 'grid':
                    count_tiles = gene.counter()
                    gene.get(MapnikDropActionTileStore(
                        tilegrid=grid['obj'],
                        mapfile=gene.layer['mapfile'],
                        image_buffer=gene.layer['meta_buffer'] if meta else 0,
                        data_buffer=gene.layer['data_buffer'],
                        output_format=gene.layer['output_format'],
                        resolution=gene.layer['resolution'],
                        layers_fields=gene.layer['layers_fields'],
                        drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'],
                        store=cache_tilestore,
                        queue_store=sqs_tilestore,
                        count=count_tiles_dropped,
                        proj4_literal=grid['proj4_literal'],
                    ), "Create Mapnik grid tile")
                else:
                    gene.get(MapnikTileStore(
                        tilegrid=grid['obj'],
                        mapfile=gene.layer['mapfile'],
                        image_buffer=gene.layer['meta_buffer'] if meta else 0,
                        data_buffer=gene.layer['data_buffer'],
                        output_format=gene.layer['output_format'],
                        proj4_literal=grid['proj4_literal'],
                    ), "Create Mapnik tile")

            def wrong_content_type_to_error(tile):
                if tile is not None and tile.content_type is not None \
                        and tile.content_type.find("image/") != 0:
                    if tile.content_type.find("application/vnd.ogc.se_xml") == 0:
                        tile.error = "WMS server error: %s" % (
                            self._re_rm_xml_tag.sub(
                                '', tile.data.decode('utf-8') if PY3 else tile.data
                            )
                        )
                    else:  # pragma: no cover
                        tile.error = "%s is not an image format, error: %s" % (
                            tile.content_type,
                            tile.data
                        )
                return tile
            gene.imap(wrong_content_type_to_error)

            # Handle errors
            gene.add_error_filters()

            if meta:
                if options.role == 'hash':
                    gene.imap(HashLogger('empty_metatile_detection'))
                elif not options.near:
                    # Discard tiles with certain content
                    if 'empty_metatile_detection' in gene.layer:
                        empty_tile = gene.layer['empty_metatile_detection']

                        gene.imap(HashDropper(
                            empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_metatiles_dropped,
                        ))

                def add_elapsed_togenerate(metatile):
                    if metatile is not None:
                        metatile.elapsed_togenerate = metatile.tilecoord.n ** 2
                        return True
                    return False  # pragma: no cover
                gene.ifilter(add_elapsed_togenerate)

                # Split the metatile image into individual tiles
                gene.add_metatile_splitter()
                gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))

                # Handle errors
                gene.add_error_filters()

            if gene.layer['type'] != 'mapnik' or gene.layer['output_format'] != 'grid':
                count_tiles = gene.counter()

            if 'pre_hash_post_process' in gene.layer:  # pragma: no cover
                gene.process(gene.layer['pre_hash_post_process'])

            if options.role == 'hash':
                gene.imap(HashLogger('empty_tile_detection'))
            elif not options.near:
                # Discard tiles with certain content
                if 'empty_tile_detection' in gene.layer:
                    empty_tile = gene.layer['empty_tile_detection']

                    gene.imap(HashDropper(
                        empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
                        queue_store=sqs_tilestore,
                        count=count_tiles_dropped,
                    ))

            gene.process()
        else:  # pragma: no cover
            count_tiles = gene.counter()

        if options.role in ('local', 'slave'):
            gene.add_error_filters()
            gene.ifilter(DropEmpty(gene))
            count_tiles_stored = gene.counter(size=True)

            if options.time:
                def log_size(tile):
                    sys.stdout.write('size: %i\n' % len(tile.data))
                    return tile
                gene.imap(log_size)

            gene.put(cache_tilestore, "Store the tile")

        gene.add_error_filters()
        if options.generated_tiles_file:  # pragma: no cover
            generated_tiles_file = open(options.generated_tiles_file, 'a')

            def do(tile):
                generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
                return tile
            gene.imap(do)

        if options.role == 'slave':  # pragma: no cover
            if meta:
                def decr_tile_in_metatile(tile):
                    tile.metatile.elapsed_togenerate -= 1
                    if tile.metatile.elapsed_togenerate == 0:
                        sqs_tilestore.delete_one(tile.metatile)
                    return True
                gene.ifilter(decr_tile_in_metatile)
            else:
                gene.delete(sqs_tilestore)

        message = []
        if options.time is not None:
            class LogTime:
                n = 0
                t1 = None

                def __call__(self, tile):
                    self.n += 1
                    if self.n == options.time:
                        self.t1 = datetime.now()
                    elif self.n == 2 * options.time:
                        t2 = datetime.now()
                        d = (t2 - self.t1) / options.time
                        sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds))
                    return tile
            gene.imap(LogTime())

            gene.consume(options.time * 3)
        else:
            gene.consume()

            message = [
                "The tile generation of layer '{}{}' is finish".format(
                    gene.layer['name'],
                    "" if len(dimensions) == 0 or gene.layer['type'] != 'wms'
                    else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()])
                ),
            ]
            if options.role == "master":  # pragma: no cover
                message.append("Nb of generated jobs: {}".format(count_tiles.nb))
            else:
                if meta:
                    message += [
                        "Nb generated metatiles: {}".format(count_metatiles.nb),
                        "Nb metatiles dropped: {}".format(count_metatiles_dropped.nb),
                    ]
                message += [
                    "Nb generated tiles: {}".format(count_tiles.nb),
                    "Nb tiles dropped: {}".format(count_tiles_dropped.nb),
                ]
                if options.role in ('local', 'slave'):
                    message += [
                        "Nb tiles stored: {}".format(count_tiles_stored.nb),
                        "Nb tiles in error: {}".format(gene.error),
                        "Total time: {}".format(duration_format(gene.duration)),
                    ]
                    if count_tiles_stored.nb != 0:
                        message.append("Total size: {}".format(size_format(count_tiles_stored.size)))
                    if count_tiles.nb != 0:
                        message.append("Time per tile: {:0.0f} ms".format(
                            (gene.duration / count_tiles.nb * 1000).seconds)
                        )
                    if count_tiles_stored.nb != 0:
                        message.append("Size per tile: {:0.0f} o".format(
                            count_tiles_stored.size / count_tiles_stored.nb)
                        )

            if not options.quiet and options.role in ('local', 'slave'):
                print("\n".join(message) + "\n")

        if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'):
            cache_tilestore.connection.close()

        if options.role != 'hash' and options.time is None and 'sns' in gene.config:  # pragma: no cover
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            sns_message = [message[0]]
            sns_message += [
                "Layer: {}".format(gene.layer['name']),
                "Role: {}".format(options.role),
                "Host: {}".format(socket.getfqdn()),
                "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])),
            ]
            sns_message += message[1:]
            connection.publish(
                gene.config['sns']['topic'],
                "\n".join(sns_message),
                "Tile generation (%(layer)s - %(role)s)" % {
                    'role': options.role,
                    'layer': gene.layer['name']
                }
            )
Example #7
0
def main():
    parser = ArgumentParser(
        description="Used to calculate the generation cost", prog=sys.argv[0])
    add_comon_options(parser, tile_pyramid=False, dimensions=True)
    parser.add_argument(
        "--cost-algo",
        "--calculate-cost-algorithm",
        default="area",
        dest="cost_algo",
        choices=("area", "count"),
        help=
        "The algorithm use to calculate the cost default base on the 'area' "
        "of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config,
                          options,
                          layer_name=options.layer,
                          base_config={"cost": {}},
                          multi_thread=False)

    all_size = 0
    tile_size = 0
    all_tiles = 0
    if options.layer:
        layer = gene.layers[options.layer]
        (all_size, all_time, all_price,
         all_tiles) = _calculate_cost(gene, layer, options)
        tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
    else:
        all_time = timedelta()
        all_price = 0
        for layer_name in gene.config["generation"]["default_layers"]:
            print("")
            print("===== {} =====".format(layer_name))
            layer = gene.layers[layer_name]
            gene.init_layer(layer, options)
            (size, time, price, tiles) = _calculate_cost(gene, layer, options)
            tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
            all_time += time
            all_price += price
            all_size += size
            all_tiles += tiles

        print("")
        print("===== GLOBAL =====")
        print("Total number of tiles: {}".format(all_tiles))
        print("Total generation time: {} [d h:mm:ss]".format(
            (duration_format(all_time))))
        print("Total generation cost: {0:0.2f} [$]".format(all_price))
    print("")
    print("S3 Storage: {0:0.2f} [$/month]".format(
        all_size * gene.config["cost"]["s3"]["storage"] /
        (1024.0 * 1024 * 1024)))
    print("S3 get: {0:0.2f} [$/month]".format(
        (gene.config["cost"]["s3"]["get"] *
         gene.config["cost"]["request_per_layers"] / 10000.0 +
         gene.config["cost"]["s3"]["download"] *
         gene.config["cost"]["request_per_layers"] * tile_size)))
    #    if 'cloudfront' in gene.config['cost']:
    #        print('CloudFront: %0.2f [$/month]' % ()
    #            gene.config['cost']['cloudfront']['get'] *
    #            gene.config['cost']['request_per_layers'] / 10000.0 +
    #            gene.config['cost']['cloudfront']['download'] *
    #            gene.config['cost']['request_per_layers'] * tile_size)
    sys.exit(0)
Example #8
0
    def gene(self, options, gene, layer):
        count_metatiles = None
        count_metatiles_dropped = Count()
        count_tiles = None
        count_tiles_dropped = Count()

        if options.role == 'slave' or options.get_hash or options.get_bbox:
            gene.layer = gene.layers[layer]
        else:
            gene.set_layer(layer, options)

        if options.get_bbox:
            try:
                tilecoord = parse_tilecoord(options.get_bbox)
                print \
                    "Tile bounds: [%i,%i,%i,%i]" % \
                    gene.layer['grid_ref']['obj'].extent(tilecoord)
                exit()
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
                    % (options.get_bbox, e))

        if options.get_hash:
            options.role = 'hash'
            options.test = 1

        sqs_tilestore = None
        if options.role in ('master', 'slave'):
            # Create SQS queue
            sqs_tilestore = SQSTileStore(
                gene.get_sqs_queue())  # pragma: no cover

        cache_tilestore = None
        if options.role in ('local', 'slave'):
            cache_tilestore = gene.get_tilesstore(options.cache)

        meta = gene.layer['meta']
        if options.tiles:
            gene.set_store(TilesFileStore(options.tiles))

        elif options.role in ('local', 'master'):
            # Generate a stream of metatiles
            gene.init_tilecoords()
            gene.add_geom_filter()

        elif options.role == 'slave':
            # Get the metatiles from the SQS queue
            gene.set_store(sqs_tilestore)  # pragma: no cover

        elif options.role == 'hash':
            try:
                z, x, y = (int(v) for v in options.get_hash.split('/'))
                if meta:
                    gene.set_tilecoords(
                        [TileCoord(z, x, y, gene.layer['meta_size'])])
                else:
                    gene.set_tilecoords([TileCoord(z, x, y)])
            except ValueError as e:  # pragma: no cover
                exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
                     (options.get_hash, e))

        # At this stage, the tilestream contains metatiles that intersect geometry
        gene.add_logger()

        count_metatiles = gene.counter()

        if options.role == 'master':  # pragma: no cover
            # Put the metatiles into the SQS queue
            gene.put(sqs_tilestore)

        elif options.role in ('local', 'slave', 'hash'):
            if gene.layer['type'] == 'wms':
                params = gene.layer['params'].copy()
                if 'STYLES' not in params:
                    params['STYLES'] = ','.join(gene.layer['wmts_style']
                                                for l in gene.layer['layers'])
                if gene.layer['generate_salt']:
                    params['SALT'] = str(random.randint(0, sys.maxint))
                for dim in gene.layer['dimensions']:
                    params[dim['name']] = dim['value']
                for dim in gene.options.dimensions:
                    dim = dim.split('=')
                    if len(dim) != 2:  # pragma: no cover
                        exit('the DIMENTIONS option should be like this '
                             'DATE=2013 VERSION=13.')
                    params[dim[0]] = dim[1]

                # Get the metatile image from the WMS server
                gene.get(
                    URLTileStore(
                        tilelayouts=(WMSTileLayout(
                            url=gene.layer['url'],
                            layers=','.join(gene.layer['layers']),
                            srs=gene.layer['grid_ref']['srs'],
                            format=gene.layer['mime_type'],
                            border=gene.layer['meta_buffer'] if meta else 0,
                            tilegrid=gene.get_grid()['obj'],
                            params=params,
                        ), ),
                        headers=gene.layer['headers'],
                    ), "Get tile from WMS")
            elif gene.layer['type'] == 'mapnik':
                from tilecloud.store.mapnik_ import MapnikTileStore
                from tilecloud_chain.mapnik_ import MapnikDropActionTileStore

                grid = gene.get_grid()
                if gene.layer['output_format'] == 'grid':
                    gene.get(
                        MapnikDropActionTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            resolution=gene.layer['resolution'],
                            layers_fields=gene.layer['layers_fields'],
                            drop_empty_utfgrid=gene.
                            layer['drop_empty_utfgrid'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik grid tile")
                else:
                    gene.get(
                        MapnikTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik tile")

            def wrong_content_type_to_error(tile):
                if tile is not None and tile.content_type is not None \
                        and tile.content_type.find("image/") != 0:
                    if tile.content_type.find(
                            "application/vnd.ogc.se_xml") == 0:
                        tile.error = "WMS server error: %s" % (
                            self._re_rm_xml_tag.sub('', tile.data))
                    else:  # pragma: no cover
                        tile.error = "%s is not an image format, error: %s" % (
                            tile.content_type, tile.data)
                return tile

            gene.imap(wrong_content_type_to_error)

            # Handle errors
            gene.add_error_filters()

            if meta:
                if options.role == 'hash':
                    gene.imap(HashLogger('empty_metatile_detection'))
                elif not options.near:
                    # Discard tiles with certain content
                    if 'empty_metatile_detection' in gene.layer:
                        empty_tile = gene.layer['empty_metatile_detection']

                        gene.imap(
                            HashDropper(
                                empty_tile['size'],
                                empty_tile['hash'],
                                store=cache_tilestore,
                                queue_store=sqs_tilestore,
                                count=count_metatiles_dropped,
                            ))

                def add_elapsed_togenerate(metatile):
                    if metatile is not None:
                        metatile.elapsed_togenerate = metatile.tilecoord.n**2
                        return True
                    return False  # pragma: no cover

                gene.ifilter(add_elapsed_togenerate)

                # Split the metatile image into individual tiles
                gene.add_metatile_splitter()
                gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))

                # Handle errors
                gene.add_error_filters()

            self.count_tiles = gene.counter()

            if 'pre_hash_post_process' in gene.layer:
                gene.process(gene.layer['pre_hash_post_process'])

            if options.role == 'hash':
                gene.imap(HashLogger('empty_tile_detection'))
            elif not options.near:
                # Discard tiles with certain content
                if 'empty_tile_detection' in gene.layer:
                    empty_tile = gene.layer['empty_tile_detection']

                    gene.imap(
                        HashDropper(
                            empty_tile['size'],
                            empty_tile['hash'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                        ))

            gene.process()

        if options.role in ('local', 'slave'):
            gene.add_error_filters()
            gene.ifilter(DropEmpty(gene))
            count_tiles = gene.counter(size=True)

            if options.time:

                def log_size(tile):
                    sys.stdout.write('size: %i\n' % len(tile.data))
                    return tile

                gene.imap(log_size)

            gene.put(cache_tilestore, "Store the tile")
        else:
            count_tiles = gene.counter(size=True)

        gene.add_error_filters()
        if options.generated_tiles_file:  # pragma: no cover
            generated_tiles_file = open(options.generated_tiles_file, 'a')

            def do(tile):
                generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
                return tile

            gene.imap(do)

        if options.role == 'slave':  # pragma: no cover
            if meta:

                def decr_tile_in_metatile(tile):
                    tile.metatile.elapsed_togenerate -= 1
                    if tile.metatile.elapsed_togenerate == 0:
                        sqs_tilestore.delete_one(tile.metatile)
                    return True

                gene.ifilter(decr_tile_in_metatile)
            else:
                gene.delete(sqs_tilestore)

        if options.time is not None:

            class LogTime:
                n = 0
                t1 = None

                def __call__(self, tile):
                    self.n += 1
                    if self.n == options.time:
                        self.t1 = datetime.now()
                    elif self.n == 2 * options.time:
                        t2 = datetime.now()
                        d = (t2 - self.t1) / options.time
                        sys.stdout.write(
                            'time: %i\n' %
                            ((d.days * 24 * 3600 + d.seconds) * 1000000 +
                             d.microseconds))
                    return tile

            gene.imap(LogTime())

            gene.consume(options.time * 3)
        else:
            gene.consume()

            if not options.quiet and options.role in ('local', 'slave'):
                nb_tiles = count_tiles.nb + count_tiles_dropped.nb
                print """The tile generation of layer '%s' is finish
%sNb generated tiles: %i
Nb tiles dropped: %i
Nb tiles stored: %i
Nb error: %i
Total time: %s
Total size: %s
Time per tiles: %i ms
Size per tile: %i o
""" % \
                    (
                        gene.layer['name'],
                        """Nb generated metatiles: %i
Nb metatiles dropped: %i
""" %
                        (
                            count_metatiles.nb, count_metatiles_dropped.nb
                        ) if meta else '',
                        nb_tiles,
                        count_tiles_dropped.nb,
                        count_tiles.nb,
                        gene.error,
                        duration_format(gene.duration),
                        size_format(count_tiles.size),
                        (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
                        count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1
                    )

        if cache_tilestore is not None and hasattr(cache_tilestore,
                                                   'connection'):
            cache_tilestore.connection.close()

        if options.role != 'hash' and options.time is None and 'sns' in gene.config:  # pragma: no cover
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'], """The tile generation is finish
Layer: %(layer)s
Role: %(role)s
Host: %(host)s
Command: %(cmd)s

%(meta)sNb generated tiles: %(nb_tiles)i
Nb tiles dropped: %(nb_tiles_dropped)i
Total time: %(duration)s [s]
Time per tiles: %(tile_duration)i [ms]""" % {
                    'role':
                    options.role,
                    'layer':
                    gene.layer['name'],
                    'host':
                    socket.getfqdn(),
                    'cmd':
                    ' '.join([quote(arg) for arg in sys.argv]),
                    'meta':
                    """Nb generated metatiles: %(nb_metatiles)i
Nb metatiles dropped: %(nb_metatiles_dropped)i
""" % {
                        'nb_metatiles': count_metatiles.nb,
                        'nb_metatiles_dropped': count_metatiles_dropped.nb,
                    } if meta else '',
                    'nb_tiles':
                    nb_tiles if meta else count_metatiles.nb,
                    'nb_tiles_dropped':
                    count_tiles_dropped.nb
                    if meta else count_metatiles_dropped.nb,
                    'duration':
                    duration_format(gene.duration),
                    'tile_duration': (gene.duration / nb_tiles *
                                      1000).seconds if nb_tiles != 0 else 0,
                }, "Tile generation (%(layer)s - %(role)s)" % {
                    'role': options.role,
                    'layer': gene.layer['name']
                })
Example #9
0
def _calculate_cost(gene, options):
    validate_calculate_cost(gene)

    nb_metatiles = {}
    nb_tiles = {}

    meta = gene.layer['meta']
    if options.cost_algo == 'area':
        tile_size = gene.layer['grid_ref']['tile_size']
        for zoom, resolution in enumerate(gene.layer['grid_ref']['resolutions']):
            if 'min_resolution_seed' in gene.layer and resolution < gene.layer['min_resolution_seed']:
                continue

            print "Calculate zoom %i." % zoom

            px_buffer = gene.layer['px_buffer'] + \
                gene.layer['meta_buffer'] if meta else 0
            m_buffer = px_buffer * resolution
            if meta:
                size = tile_size * gene.layer['meta_size'] * resolution
                meta_buffer = size * 0.7 + m_buffer
                meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1)
                nb_metatiles[zoom] = int(round(meta_geom.area / size ** 2))
            size = tile_size * resolution
            tile_buffer = size * 0.7 + m_buffer
            geom = gene.geoms[zoom].buffer(tile_buffer, 1)
            nb_tiles[zoom] = int(round(geom.area / size ** 2))

    elif options.cost_algo == 'count':
        gene.init_tilecoords()
        gene.add_geom_filter()

        if meta:
            def count_metatile(tile):
                if tile:
                    if tile.tilecoord.z in nb_metatiles:
                        nb_metatiles[tile.tilecoord.z] += 1
                    else:
                        nb_metatiles[tile.tilecoord.z] = 1
                return tile
            gene.imap(count_metatile)

            class MetaTileSplitter(TileStore):
                def get(self, tiles):
                    for metatile in tiles:
                        for tilecoord in metatile.tilecoord:
                            yield Tile(tilecoord)
            gene.tilestream = MetaTileSplitter().get(gene.tilestream)

            # Only keep tiles that intersect geometry
            gene.add_geom_filter()

        def count_tile(tile):
            if tile:
                if tile.tilecoord.z in nb_tiles:
                    nb_tiles[tile.tilecoord.z] += 1
                else:
                    print "Calculate zoom %i." % tile.tilecoord.z
                    nb_tiles[tile.tilecoord.z] = 1
            return tile
        gene.imap(count_tile)

        consume(gene.tilestream, None)

    times = {}
    print
    for z in nb_metatiles:
        print "%i meta tiles in zoom %i." % (nb_metatiles[z], z)
        times[z] = gene.layer['cost']['metatile_generation_time'] * nb_metatiles[z]

    price = 0
    all_size = 0
    all_time = 0
    all_tiles = 0
    for z in nb_tiles:
        print
        print "%i tiles in zoom %i." % (nb_tiles[z], z)
        all_tiles += nb_tiles[z]
        if meta:
            time = times[z] + gene.layer['cost']['tile_generation_time'] * nb_tiles[z]
        else:
            time = gene.layer['cost']['tileonly_generation_time'] * nb_tiles[z]
        size = gene.layer['cost']['tile_size'] * nb_tiles[z]
        all_size += size

        all_time += time
        td = timedelta(milliseconds=time)
        print "Time to generate: %s [d h:mm:ss]" % (duration_format(td))
        c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0
        price += c
        print 'S3 PUT: %0.2f [$]' % c

        if 'ec2' in gene.config:
            c = time * gene.config['cost']['ec2']['usage'] / (1000.0 * 3600)
            price += c
            print 'EC2 usage: %0.2f [$]' % c

            c = gene.config['cost']['esb']['io'] * time / (1000.0 * 2600 * 24 * 30)
            price += c
            print 'ESB usage: %0.2f [$]' % c

        if 'sqs' in gene.layer:
            if meta:
                nb_sqs = nb_metatiles[z] * 3
            else:
                nb_sqs = nb_tiles[z] * 3
            c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0
            price += c
            print 'SQS usage: %0.2f [$]' % c

    print
    td = timedelta(milliseconds=all_time)
    print "Number of tiles: %i" % all_tiles
    print 'Generation time: %s [d h:mm:ss]' % (duration_format(td))
    print 'Generation cost: %0.2f [$]' % price

    return (all_size, td, price, all_tiles)
Example #10
0
    def generate_resume(self, layer):
        if self._options.time is None:
            if layer is not None:
                all_dimensions = self._gene.get_all_dimensions(layer)
                message = [
                    "The tile generation of layer '{}{}' is finish".format(
                        layer["name"],
                        "" if ((len(all_dimensions) == 1
                                and len(all_dimensions[0]) == 0)
                               or layer["type"] != "wms") else
                        " ({})".format(" - ".join([
                            ", ".join(
                                ["=".join(d) for d in dimensions.items()])
                            for dimensions in all_dimensions
                        ])),
                    ),
                ]
            else:
                message = ["The tile generation is finish"]
            if self._options.role == "master":  # pragma: no cover
                message.append("Nb of generated jobs: {}".format(
                    self._count_tiles.nb))
            elif layer.get(
                    "meta"
            ) if layer is not None else self._options.role == "slave":
                message += [
                    "Nb generated metatiles: {}".format(
                        self._count_metatiles.nb),
                    "Nb metatiles dropped: {}".format(
                        self._count_metatiles_dropped.nb),
                ]

            if self._options.role != "master":
                message += [
                    "Nb generated tiles: {}".format(self._count_tiles.nb),
                    "Nb tiles dropped: {}".format(
                        self._count_tiles_dropped.nb),
                ]
                if self._options.role in ("local", "slave"):
                    message += [
                        "Nb tiles stored: {}".format(
                            self._count_tiles_stored.nb),
                        "Nb tiles in error: {}".format(self._gene.error),
                        "Total time: {}".format(
                            duration_format(self._gene.duration)),
                    ]
                    if self._count_tiles_stored.nb != 0:
                        message.append("Total size: {}".format(
                            size_format(self._count_tiles_stored.size)))
                    if self._count_tiles.nb != 0:
                        message.append("Time per tile: {:0.0f} ms".format(
                            (self._gene.duration / self._count_tiles.nb *
                             1000).seconds))
                    if self._count_tiles_stored.nb != 0:
                        message.append("Size per tile: {:0.0f} o".format(
                            self._count_tiles_stored.size /
                            self._count_tiles_stored.nb))

            if not self._options.quiet and self._options.role in (
                    "local", "slave", "master") and message:
                print("\n".join(message) + "\n")

        if self._cache_tilestore is not None and hasattr(
                self._cache_tilestore, "connection"):
            self._cache_tilestore.connection.close()

        if (self._options.role != "hash" and self._options.time is None
                and "sns" in self._gene.config):  # pragma: no cover
            if "region" in self._gene.config["sns"]:
                sns_client = boto3.client(
                    "sns", region_name=self._gene.config["sns"]["region"])
            else:
                sns_client = boto3.client("sns")
            sns_message = [message[0]]
            sns_message += [
                "Layer: {}".format(
                    layer["name"] if layer is not None else "(All layers)"),
                "Role: {}".format(self._options.role),
                "Host: {}".format(socket.getfqdn()),
                "Command: {}".format(" ".join([quote(arg)
                                               for arg in sys.argv])),
            ]
            sns_message += message[1:]
            sns_client.publish(
                TopicArn=self._gene.config["sns"]["topic"],
                Message="\n".join(sns_message),
                Subject="Tile generation ({layer} - {role})".format(
                    role=self._options.role,
                    layer=layer["name"]
                    if layer is not None else "All layers"),
            )
Example #11
0
def _calculate_cost(gene, options):
    validate_calculate_cost(gene)

    nb_metatiles = {}
    nb_tiles = {}

    meta = gene.layer['meta']
    if options.cost_algo == 'area':
        tile_size = gene.layer['grid_ref']['tile_size']
        for zoom, resolution in enumerate(
                gene.layer['grid_ref']['resolutions']):
            if 'min_resolution_seed' in gene.layer and resolution < gene.layer[
                    'min_resolution_seed']:
                continue

            print "Calculate zoom %i." % zoom

            px_buffer = gene.layer['px_buffer'] + \
                gene.layer['meta_buffer'] if meta else 0
            m_buffer = px_buffer * resolution
            if meta:
                size = tile_size * gene.layer['meta_size'] * resolution
                meta_buffer = size * 0.7 + m_buffer
                meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1)
                nb_metatiles[zoom] = int(round(meta_geom.area / size**2))
            size = tile_size * resolution
            tile_buffer = size * 0.7 + m_buffer
            geom = gene.geoms[zoom].buffer(tile_buffer, 1)
            nb_tiles[zoom] = int(round(geom.area / size**2))

    elif options.cost_algo == 'count':
        gene.init_tilecoords()
        gene.add_geom_filter()

        if meta:

            def count_metatile(tile):
                if tile:
                    if tile.tilecoord.z in nb_metatiles:
                        nb_metatiles[tile.tilecoord.z] += 1
                    else:
                        nb_metatiles[tile.tilecoord.z] = 1
                return tile

            gene.imap(count_metatile)

            class MetaTileSplitter(TileStore):
                def get(self, tiles):
                    for metatile in tiles:
                        for tilecoord in metatile.tilecoord:
                            yield Tile(tilecoord)

            gene.tilestream = MetaTileSplitter().get(gene.tilestream)

            # Only keep tiles that intersect geometry
            gene.add_geom_filter()

        def count_tile(tile):
            if tile:
                if tile.tilecoord.z in nb_tiles:
                    nb_tiles[tile.tilecoord.z] += 1
                else:
                    print "Calculate zoom %i." % tile.tilecoord.z
                    nb_tiles[tile.tilecoord.z] = 1
            return tile

        gene.imap(count_tile)

        consume(gene.tilestream, None)

    times = {}
    print
    for z in nb_metatiles:
        print "%i meta tiles in zoom %i." % (nb_metatiles[z], z)
        times[z] = gene.layer['cost'][
            'metatile_generation_time'] * nb_metatiles[z]

    price = 0
    all_size = 0
    all_time = 0
    all_tiles = 0
    for z in nb_tiles:
        print
        print "%i tiles in zoom %i." % (nb_tiles[z], z)
        all_tiles += nb_tiles[z]
        if meta:
            time = times[
                z] + gene.layer['cost']['tile_generation_time'] * nb_tiles[z]
        else:
            time = gene.layer['cost']['tileonly_generation_time'] * nb_tiles[z]
        size = gene.layer['cost']['tile_size'] * nb_tiles[z]
        all_size += size

        all_time += time
        td = timedelta(milliseconds=time)
        print "Time to generate: %s [d h:mm:ss]" % (duration_format(td))
        c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0
        price += c
        print 'S3 PUT: %0.2f [$]' % c

        if 'ec2' in gene.config:
            c = time * gene.config['cost']['ec2']['usage'] / (1000.0 * 3600)
            price += c
            print 'EC2 usage: %0.2f [$]' % c

            c = gene.config['cost']['esb']['io'] * time / (1000.0 * 2600 * 24 *
                                                           30)
            price += c
            print 'ESB usage: %0.2f [$]' % c

        if 'sqs' in gene.layer:
            if meta:
                nb_sqs = nb_metatiles[z] * 3
            else:
                nb_sqs = nb_tiles[z] * 3
            c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0
            price += c
            print 'SQS usage: %0.2f [$]' % c

    print
    td = timedelta(milliseconds=all_time)
    print "Number of tiles: %i" % all_tiles
    print 'Generation time: %s [d h:mm:ss]' % (duration_format(td))
    print 'Generation cost: %0.2f [$]' % price

    return (all_size, td, price, all_tiles)
Example #12
0
def _calculate_cost(gene, layer, options):
    nb_metatiles = {}
    nb_tiles = {}

    meta = layer["meta"]
    if options.cost_algo == "area":
        tile_size = layer["grid_ref"]["tile_size"]
        for zoom, resolution in enumerate(layer["grid_ref"]["resolutions"]):
            if "min_resolution_seed" in layer and resolution < layer[
                    "min_resolution_seed"]:
                continue

            print("Calculate zoom {}.".format(zoom))

            px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0
            m_buffer = px_buffer * resolution
            if meta:
                size = tile_size * layer["meta_size"] * resolution
                meta_buffer = size * 0.7 + m_buffer
                meta_geom = gene.geoms[layer["name"]][zoom].buffer(
                    meta_buffer, 1)
                nb_metatiles[zoom] = int(round(meta_geom.area / size**2))
            size = tile_size * resolution
            tile_buffer = size * 0.7 + m_buffer
            geom = gene.geoms[layer["name"]][zoom].buffer(tile_buffer, 1)
            nb_tiles[zoom] = int(round(geom.area / size**2))

    elif options.cost_algo == "count":
        gene.init_tilecoords(layer)
        gene.add_geom_filter()

        if meta:

            def count_metatile(tile):
                if tile:
                    if tile.tilecoord.z in nb_metatiles:
                        nb_metatiles[tile.tilecoord.z] += 1
                    else:
                        nb_metatiles[tile.tilecoord.z] = 1
                return tile

            gene.imap(count_metatile)

            class MetaTileSplitter(TileStore):
                @staticmethod
                def get(tiles):
                    for metatile in tiles:
                        for tilecoord in metatile.tilecoord:
                            yield Tile(tilecoord)

            gene.add_metatile_splitter(MetaTileSplitter())

            # Only keep tiles that intersect geometry
            gene.add_geom_filter()

        def count_tile(tile):
            if tile:
                if tile.tilecoord.z in nb_tiles:
                    nb_tiles[tile.tilecoord.z] += 1
                else:
                    print("Calculate zoom {}.".format(tile.tilecoord.z))
                    nb_tiles[tile.tilecoord.z] = 1
            return tile

        gene.imap(count_tile)

        run = Run(gene, gene.functions_metatiles)
        for tile in gene.tilestream:
            tile.metadata["layer"] = layer["name"]
            run(tile)

    times = {}
    print("")
    for z in nb_metatiles:
        print("{} meta tiles in zoom {}.".format(nb_metatiles[z], z))
        times[z] = layer["cost"]["metatile_generation_time"] * nb_metatiles[z]

    price = 0
    all_size = 0
    all_time = 0
    all_tiles = 0
    for z in nb_tiles:
        print("")
        print("{} tiles in zoom {}.".format(nb_tiles[z], z))
        all_tiles += nb_tiles[z]
        if meta:
            time = times[
                z] + layer["cost"]["tile_generation_time"] * nb_tiles[z]
        else:
            time = layer["cost"]["tileonly_generation_time"] * nb_tiles[z]
        size = layer["cost"]["tile_size"] * nb_tiles[z]
        all_size += size

        all_time += time
        td = timedelta(milliseconds=time)
        print("Time to generate: {} [d h:mm:ss]".format((duration_format(td))))
        c = gene.config["cost"]["s3"]["put"] * nb_tiles[z] / 1000.0
        price += c
        print("S3 PUT: {0:0.2f} [$]".format(c))

        if "sqs" in gene.config:
            if meta:
                nb_sqs = nb_metatiles[z] * 3
            else:
                nb_sqs = nb_tiles[z] * 3
            c = nb_sqs * gene.config["cost"]["sqs"]["request"] / 1000000.0
            price += c
            print("SQS usage: {0:0.2f} [$]".format(c))

    print("")
    td = timedelta(milliseconds=all_time)
    print("Number of tiles: {}".format(all_tiles))
    print("Generation time: {} [d h:mm:ss]".format((duration_format(td))))
    print("Generation cost: {0:0.2f} [$]".format(price))

    return all_size, td, price, all_tiles
Example #13
0
    def _gene(self, options, gene, layer, dimensions={}):
        count_metatiles = None
        count_metatiles_dropped = Count()
        count_tiles = None
        count_tiles_dropped = Count()
        count_tiles_stored = None

        if options.get_bbox:
            try:
                tilecoord = parse_tilecoord(options.get_bbox)
                print("Tile bounds: [%i,%i,%i,%i]" %
                      gene.layer['grid_ref']['obj'].extent(tilecoord))
                exit()
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
                    % (options.get_bbox, e))

        if options.get_hash:
            options.role = 'hash'
            options.test = 1

        sqs_tilestore = None
        if options.role in ('master', 'slave'):
            # Create SQS queue
            sqs_tilestore = SQSTileStore(
                gene.get_sqs_queue())  # pragma: no cover

        cache_tilestore = None
        if options.role in ('local', 'slave'):
            cache_tilestore = gene.get_tilesstore(options.cache, dimensions)

        meta = gene.layer['meta']
        if options.tiles:
            gene.set_store(TilesFileStore(options.tiles))

        elif options.role in ('local', 'master'):
            # Generate a stream of metatiles
            gene.init_tilecoords()
            gene.add_geom_filter()

        if options.local_process_number is not None:  # pragma: no cover
            gene.add_local_process_filter()

        elif options.role == 'slave':
            # Get the metatiles from the SQS queue
            gene.set_store(sqs_tilestore)  # pragma: no cover

        elif options.role == 'hash':
            try:
                z, x, y = (int(v) for v in options.get_hash.split('/'))
                if meta:
                    gene.set_tilecoords(
                        [TileCoord(z, x, y, gene.layer['meta_size'])])
                else:
                    gene.set_tilecoords([TileCoord(z, x, y)])
            except ValueError as e:  # pragma: no cover
                exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
                     (options.get_hash, e))

        # At this stage, the tilestream contains metatiles that intersect geometry
        gene.add_logger()

        count_metatiles = gene.counter()

        if options.role == 'master':  # pragma: no cover
            # Put the metatiles into the SQS queue
            gene.put(sqs_tilestore)
            count_tiles = gene.counter()

        elif options.role in ('local', 'slave', 'hash'):
            if gene.layer['type'] == 'wms':
                params = gene.layer['params'].copy()
                if 'STYLES' not in params:
                    params['STYLES'] = ','.join(
                        gene.layer['wmts_style']
                        for l in gene.layer['layers'].split(','))
                if gene.layer['generate_salt']:
                    params['SALT'] = str(random.randint(0, 999999))
                params.update(dimensions)

                # Get the metatile image from the WMS server
                gene.get(
                    URLTileStore(
                        tilelayouts=(WMSTileLayout(
                            url=gene.layer['url'],
                            layers=gene.layer['layers'],
                            srs=gene.layer['grid_ref']['srs'],
                            format=gene.layer['mime_type'],
                            border=gene.layer['meta_buffer'] if meta else 0,
                            tilegrid=gene.get_grid()['obj'],
                            params=params,
                        ), ),
                        headers=gene.layer['headers'],
                    ), "Get tile from WMS")
            elif gene.layer['type'] == 'mapnik':  # pragma: no cover
                from tilecloud.store.mapnik_ import MapnikTileStore
                from tilecloud_chain.mapnik_ import MapnikDropActionTileStore

                grid = gene.get_grid()
                if gene.layer['output_format'] == 'grid':
                    count_tiles = gene.counter()
                    gene.get(
                        MapnikDropActionTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            resolution=gene.layer['resolution'],
                            layers_fields=gene.layer['layers_fields'],
                            drop_empty_utfgrid=gene.
                            layer['drop_empty_utfgrid'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik grid tile")
                else:
                    gene.get(
                        MapnikTileStore(
                            tilegrid=grid['obj'],
                            mapfile=gene.layer['mapfile'],
                            image_buffer=gene.layer['meta_buffer']
                            if meta else 0,
                            data_buffer=gene.layer['data_buffer'],
                            output_format=gene.layer['output_format'],
                            proj4_literal=grid['proj4_literal'],
                        ), "Create Mapnik tile")

            def wrong_content_type_to_error(tile):
                if tile is not None and tile.content_type is not None \
                        and tile.content_type.find("image/") != 0:
                    if tile.content_type.find(
                            "application/vnd.ogc.se_xml") == 0:
                        tile.error = "WMS server error: %s" % (
                            self._re_rm_xml_tag.sub(
                                '',
                                tile.data.decode('utf-8')
                                if PY3 else tile.data))
                    else:  # pragma: no cover
                        tile.error = "%s is not an image format, error: %s" % (
                            tile.content_type, tile.data)
                return tile

            gene.imap(wrong_content_type_to_error)

            # Handle errors
            gene.add_error_filters()

            if meta:
                if options.role == 'hash':
                    gene.imap(HashLogger('empty_metatile_detection'))
                elif not options.near:
                    # Discard tiles with certain content
                    if 'empty_metatile_detection' in gene.layer:
                        empty_tile = gene.layer['empty_metatile_detection']

                        gene.imap(
                            HashDropper(
                                empty_tile['size'],
                                empty_tile['hash'],
                                store=cache_tilestore,
                                queue_store=sqs_tilestore,
                                count=count_metatiles_dropped,
                            ))

                def add_elapsed_togenerate(metatile):
                    if metatile is not None:
                        metatile.elapsed_togenerate = metatile.tilecoord.n**2
                        return True
                    return False  # pragma: no cover

                gene.ifilter(add_elapsed_togenerate)

                # Split the metatile image into individual tiles
                gene.add_metatile_splitter()
                gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))

                # Handle errors
                gene.add_error_filters()

            if gene.layer['type'] != 'mapnik' or gene.layer[
                    'output_format'] != 'grid':
                count_tiles = gene.counter()

            if 'pre_hash_post_process' in gene.layer:  # pragma: no cover
                gene.process(gene.layer['pre_hash_post_process'])

            if options.role == 'hash':
                gene.imap(HashLogger('empty_tile_detection'))
            elif not options.near:
                # Discard tiles with certain content
                if 'empty_tile_detection' in gene.layer:
                    empty_tile = gene.layer['empty_tile_detection']

                    gene.imap(
                        HashDropper(
                            empty_tile['size'],
                            empty_tile['hash'],
                            store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_tiles_dropped,
                        ))

            gene.process()
        else:  # pragma: no cover
            count_tiles = gene.counter()

        if options.role in ('local', 'slave'):
            gene.add_error_filters()
            gene.ifilter(DropEmpty(gene))
            count_tiles_stored = gene.counter(size=True)

            if options.time:

                def log_size(tile):
                    sys.stdout.write('size: %i\n' % len(tile.data))
                    return tile

                gene.imap(log_size)

            gene.put(cache_tilestore, "Store the tile")

        gene.add_error_filters()
        if options.generated_tiles_file:  # pragma: no cover
            generated_tiles_file = open(options.generated_tiles_file, 'a')

            def do(tile):
                generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
                return tile

            gene.imap(do)

        if options.role == 'slave':  # pragma: no cover
            if meta:

                def decr_tile_in_metatile(tile):
                    tile.metatile.elapsed_togenerate -= 1
                    if tile.metatile.elapsed_togenerate == 0:
                        sqs_tilestore.delete_one(tile.metatile)
                    return True

                gene.ifilter(decr_tile_in_metatile)
            else:
                gene.delete(sqs_tilestore)

        message = []
        if options.time is not None:

            class LogTime:
                n = 0
                t1 = None

                def __call__(self, tile):
                    self.n += 1
                    if self.n == options.time:
                        self.t1 = datetime.now()
                    elif self.n == 2 * options.time:
                        t2 = datetime.now()
                        d = (t2 - self.t1) / options.time
                        sys.stdout.write(
                            'time: %i\n' %
                            ((d.days * 24 * 3600 + d.seconds) * 1000000 +
                             d.microseconds))
                    return tile

            gene.imap(LogTime())

            gene.consume(options.time * 3)
        else:
            gene.consume()

            message = [
                "The tile generation of layer '{}{}' is finish".format(
                    gene.layer['name'], "" if len(dimensions) == 0
                    or gene.layer['type'] != 'wms' else " (%s)" %
                    ", ".join(["=".join(d) for d in dimensions.items()])),
            ]
            if options.role == "master":  # pragma: no cover
                message.append("Nb of generated jobs: {}".format(
                    count_tiles.nb))
            else:
                if meta:
                    message += [
                        "Nb generated metatiles: {}".format(
                            count_metatiles.nb),
                        "Nb metatiles dropped: {}".format(
                            count_metatiles_dropped.nb),
                    ]
                message += [
                    "Nb generated tiles: {}".format(count_tiles.nb),
                    "Nb tiles dropped: {}".format(count_tiles_dropped.nb),
                ]
                if options.role in ('local', 'slave'):
                    message += [
                        "Nb tiles stored: {}".format(count_tiles_stored.nb),
                        "Nb tiles in error: {}".format(gene.error),
                        "Total time: {}".format(duration_format(
                            gene.duration)),
                    ]
                    if count_tiles_stored.nb != 0:
                        message.append("Total size: {}".format(
                            size_format(count_tiles_stored.size)))
                    if count_tiles.nb != 0:
                        message.append("Time per tile: {:0.0f} ms".format(
                            (gene.duration / count_tiles.nb * 1000).seconds))
                    if count_tiles_stored.nb != 0:
                        message.append("Size per tile: {:0.0f} o".format(
                            count_tiles_stored.size / count_tiles_stored.nb))

            if not options.quiet and options.role in ('local', 'slave'):
                print("\n".join(message) + "\n")

        if cache_tilestore is not None and hasattr(cache_tilestore,
                                                   'connection'):
            cache_tilestore.connection.close()

        if options.role != 'hash' and options.time is None and 'sns' in gene.config:  # pragma: no cover
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            sns_message = [message[0]]
            sns_message += [
                "Layer: {}".format(gene.layer['name']),
                "Role: {}".format(options.role),
                "Host: {}".format(socket.getfqdn()),
                "Command: {}".format(' '.join([quote(arg)
                                               for arg in sys.argv])),
            ]
            sns_message += message[1:]
            connection.publish(
                gene.config['sns']['topic'], "\n".join(sns_message),
                "Tile generation (%(layer)s - %(role)s)" % {
                    'role': options.role,
                    'layer': gene.layer['name']
                })
Example #14
0
    def gene(self, options, gene, layer):
        count_metatiles = None
        count_metatiles_dropped = Count()
        count_tiles = None
        count_tiles_dropped = Count()

        if options.role == 'slave' or options.get_hash or options.get_bbox:
            gene.layer = gene.layers[layer]
        else:
            gene.set_layer(layer, options)

        if options.get_bbox:
            try:
                tilecoord = parse_tilecoord(options.get_bbox)
                print(
                    "Tile bounds: [%i,%i,%i,%i]" %
                    gene.layer['grid_ref']['obj'].extent(tilecoord)
                )
                exit()
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" %
                    (options.get_bbox, e)
                )

        if options.get_hash:
            options.role = 'hash'
            options.test = 1

        sqs_tilestore = None
        if options.role in ('master', 'slave'):
            # Create SQS queue
            sqs_tilestore = SQSTileStore(gene.get_sqs_queue())  # pragma: no cover

        cache_tilestore = None
        if options.role in ('local', 'slave'):
            cache_tilestore = gene.get_tilesstore(options.cache)

        meta = gene.layer['meta']
        if options.tiles:
            gene.set_store(TilesFileStore(options.tiles))

        elif options.role in ('local', 'master'):
            # Generate a stream of metatiles
            gene.init_tilecoords()
            gene.add_geom_filter()

        if options.local_process_number is not None:  # pragma: no cover
            gene.add_local_process_filter()

        elif options.role == 'slave':
            # Get the metatiles from the SQS queue
            gene.set_store(sqs_tilestore)  # pragma: no cover

        elif options.role == 'hash':
            try:
                z, x, y = (int(v) for v in options.get_hash.split('/'))
                if meta:
                    gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])])
                else:
                    gene.set_tilecoords([TileCoord(z, x, y)])
            except ValueError as e:  # pragma: no cover
                exit(
                    "Tile '%s' is not in the format 'z/x/y'\n%r" %
                    (options.get_hash, e)
                )

        # At this stage, the tilestream contains metatiles that intersect geometry
        gene.add_logger()

        count_metatiles = gene.counter()

        if options.role == 'master':  # pragma: no cover
            # Put the metatiles into the SQS queue
            gene.put(sqs_tilestore)

        elif options.role in ('local', 'slave', 'hash'):
            if gene.layer['type'] == 'wms':
                params = gene.layer['params'].copy()
                if 'STYLES' not in params:
                    params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers'])
                if gene.layer['generate_salt']:
                    params['SALT'] = str(random.randint(0, sys.maxint))
                for dim in gene.layer['dimensions']:
                    params[dim['name']] = dim['value']
                for dim in gene.options.dimensions:
                    dim = dim.split('=')
                    if len(dim) != 2:  # pragma: no cover
                        exit(
                            'the DIMENTIONS option should be like this '
                            'DATE=2013 VERSION=13.'
                        )
                    params[dim[0]] = dim[1]

                # Get the metatile image from the WMS server
                gene.get(URLTileStore(
                    tilelayouts=(WMSTileLayout(
                        url=gene.layer['url'],
                        layers=','.join(gene.layer['layers']),
                        srs=gene.layer['grid_ref']['srs'],
                        format=gene.layer['mime_type'],
                        border=gene.layer['meta_buffer'] if meta else 0,
                        tilegrid=gene.get_grid()['obj'],
                        params=params,
                    ),),
                    headers=gene.layer['headers'],
                ), "Get tile from WMS")
            elif gene.layer['type'] == 'mapnik':
                from tilecloud.store.mapnik_ import MapnikTileStore
                from tilecloud_chain.mapnik_ import MapnikDropActionTileStore

                grid = gene.get_grid()
                if gene.layer['output_format'] == 'grid':
                    gene.get(MapnikDropActionTileStore(
                        tilegrid=grid['obj'],
                        mapfile=gene.layer['mapfile'],
                        image_buffer=gene.layer['meta_buffer'] if meta else 0,
                        data_buffer=gene.layer['data_buffer'],
                        output_format=gene.layer['output_format'],
                        resolution=gene.layer['resolution'],
                        layers_fields=gene.layer['layers_fields'],
                        drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'],
                        store=cache_tilestore,
                        queue_store=sqs_tilestore,
                        count=count_tiles_dropped,
                        proj4_literal=grid['proj4_literal'],
                    ), "Create Mapnik grid tile")
                else:
                    gene.get(MapnikTileStore(
                        tilegrid=grid['obj'],
                        mapfile=gene.layer['mapfile'],
                        image_buffer=gene.layer['meta_buffer'] if meta else 0,
                        data_buffer=gene.layer['data_buffer'],
                        output_format=gene.layer['output_format'],
                        proj4_literal=grid['proj4_literal'],
                    ), "Create Mapnik tile")

            def wrong_content_type_to_error(tile):
                if tile is not None and tile.content_type is not None \
                        and tile.content_type.find("image/") != 0:
                    if tile.content_type.find("application/vnd.ogc.se_xml") == 0:
                        tile.error = "WMS server error: %s" % (
                            self._re_rm_xml_tag.sub('', tile.data)
                        )
                    else:  # pragma: no cover
                        tile.error = "%s is not an image format, error: %s" % (
                            tile.content_type,
                            tile.data
                        )
                return tile
            gene.imap(wrong_content_type_to_error)

            # Handle errors
            gene.add_error_filters()

            if meta:
                if options.role == 'hash':
                    gene.imap(HashLogger('empty_metatile_detection'))
                elif not options.near:
                    # Discard tiles with certain content
                    if 'empty_metatile_detection' in gene.layer:
                        empty_tile = gene.layer['empty_metatile_detection']

                        gene.imap(HashDropper(
                            empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
                            queue_store=sqs_tilestore,
                            count=count_metatiles_dropped,
                        ))

                def add_elapsed_togenerate(metatile):
                    if metatile is not None:
                        metatile.elapsed_togenerate = metatile.tilecoord.n ** 2
                        return True
                    return False  # pragma: no cover
                gene.ifilter(add_elapsed_togenerate)

                # Split the metatile image into individual tiles
                gene.add_metatile_splitter()
                gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))

                # Handle errors
                gene.add_error_filters()

            self.count_tiles = gene.counter()

            if 'pre_hash_post_process' in gene.layer:
                gene.process(gene.layer['pre_hash_post_process'])

            if options.role == 'hash':
                gene.imap(HashLogger('empty_tile_detection'))
            elif not options.near:
                # Discard tiles with certain content
                if 'empty_tile_detection' in gene.layer:
                    empty_tile = gene.layer['empty_tile_detection']

                    gene.imap(HashDropper(
                        empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
                        queue_store=sqs_tilestore,
                        count=count_tiles_dropped,
                    ))

            gene.process()

        if options.role in ('local', 'slave'):
            gene.add_error_filters()
            gene.ifilter(DropEmpty(gene))
            count_tiles = gene.counter(size=True)

            if options.time:
                def log_size(tile):
                    sys.stdout.write('size: %i\n' % len(tile.data))
                    return tile
                gene.imap(log_size)

            gene.put(cache_tilestore, "Store the tile")
        else:
            count_tiles = gene.counter(size=True)

        gene.add_error_filters()
        if options.generated_tiles_file:  # pragma: no cover
            generated_tiles_file = open(options.generated_tiles_file, 'a')

            def do(tile):
                generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
                return tile
            gene.imap(do)

        if options.role == 'slave':  # pragma: no cover
            if meta:
                def decr_tile_in_metatile(tile):
                    tile.metatile.elapsed_togenerate -= 1
                    if tile.metatile.elapsed_togenerate == 0:
                        sqs_tilestore.delete_one(tile.metatile)
                    return True
                gene.ifilter(decr_tile_in_metatile)
            else:
                gene.delete(sqs_tilestore)

        if options.time is not None:
            class LogTime:
                n = 0
                t1 = None

                def __call__(self, tile):
                    self.n += 1
                    if self.n == options.time:
                        self.t1 = datetime.now()
                    elif self.n == 2 * options.time:
                        t2 = datetime.now()
                        d = (t2 - self.t1) / options.time
                        sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds))
                    return tile
            gene.imap(LogTime())

            gene.consume(options.time * 3)
        else:
            gene.consume()

            if not options.quiet and options.role in ('local', 'slave'):
                nb_tiles = count_tiles.nb + count_tiles_dropped.nb
                print(
                    """The tile generation of layer '%s' is finish
%sNb generated tiles: %i
Nb tiles dropped: %i
Nb tiles stored: %i
Nb error: %i
Total time: %s
Total size: %s
Time per tiles: %i ms
Size per tile: %i o
""" % (
                        gene.layer['name'],
                        """Nb generated metatiles: %i
Nb metatiles dropped: %i
""" %
                        (
                            count_metatiles.nb, count_metatiles_dropped.nb
                        ) if meta else '',
                        nb_tiles,
                        count_tiles_dropped.nb,
                        count_tiles.nb,
                        gene.error,
                        duration_format(gene.duration),
                        size_format(count_tiles.size),
                        (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
                        count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1
                    )
                )

        if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'):
            cache_tilestore.connection.close()

        if options.role != 'hash' and options.time is None and 'sns' in gene.config:  # pragma: no cover
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'],
                """The tile generation is finish
Layer: %(layer)s
Role: %(role)s
Host: %(host)s
Command: %(cmd)s

%(meta)sNb generated tiles: %(nb_tiles)i
Nb tiles dropped: %(nb_tiles_dropped)i
Total time: %(duration)s [s]
Time per tiles: %(tile_duration)i [ms]""" %
                {
                    'role': options.role,
                    'layer': gene.layer['name'],
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv]),
                    'meta': """Nb generated metatiles: %(nb_metatiles)i
Nb metatiles dropped: %(nb_metatiles_dropped)i
""" %
                    {
                        'nb_metatiles': count_metatiles.nb,
                        'nb_metatiles_dropped': count_metatiles_dropped.nb,
                    } if meta else '',
                    'nb_tiles': nb_tiles if meta else count_metatiles.nb,
                    'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb,
                    'duration': duration_format(gene.duration),
                    'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
                },
                "Tile generation (%(layer)s - %(role)s)" % {
                    'role': options.role,
                    'layer': gene.layer['name']
                }
            )
Example #15
0
def _calculate_cost(gene, layer, options):
    nb_metatiles = {}
    nb_tiles = {}

    meta = layer['meta']
    if options.cost_algo == 'area':
        tile_size = layer['grid_ref']['tile_size']
        for zoom, resolution in enumerate(layer['grid_ref']['resolutions']):
            if 'min_resolution_seed' in layer and resolution < layer['min_resolution_seed']:
                continue

            print("Calculate zoom {}.".format(zoom))

            px_buffer = layer['px_buffer'] + \
                layer['meta_buffer'] if meta else 0
            m_buffer = px_buffer * resolution
            if meta:
                size = tile_size * layer['meta_size'] * resolution
                meta_buffer = size * 0.7 + m_buffer
                meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1)
                nb_metatiles[zoom] = int(round(meta_geom.area / size ** 2))
            size = tile_size * resolution
            tile_buffer = size * 0.7 + m_buffer
            geom = gene.geoms[zoom].buffer(tile_buffer, 1)
            nb_tiles[zoom] = int(round(geom.area / size ** 2))

    elif options.cost_algo == 'count':
        gene.init_tilecoords(layer)
        gene.add_geom_filter(layer)

        if meta:
            def count_metatile(tile):
                if tile:
                    if tile.tilecoord.z in nb_metatiles:
                        nb_metatiles[tile.tilecoord.z] += 1
                    else:
                        nb_metatiles[tile.tilecoord.z] = 1
                return tile
            gene.imap(count_metatile)

            class MetaTileSplitter(TileStore):
                @staticmethod
                def get(tiles):
                    for metatile in tiles:
                        for tilecoord in metatile.tilecoord:
                            yield Tile(tilecoord)
            gene.tilestream = MetaTileSplitter().get(gene.tilestream)

            # Only keep tiles that intersect geometry
            gene.add_geom_filter(layer)

        def count_tile(tile):
            if tile:
                if tile.tilecoord.z in nb_tiles:
                    nb_tiles[tile.tilecoord.z] += 1
                else:
                    print("Calculate zoom {}.".format(tile.tilecoord.z))
                    nb_tiles[tile.tilecoord.z] = 1
            return tile
        gene.imap(count_tile)

        consume(gene.tilestream, None)

    times = {}
    print('')
    for z in nb_metatiles:
        print("{} meta tiles in zoom {}.".format(nb_metatiles[z], z))
        times[z] = layer['cost']['metatile_generation_time'] * nb_metatiles[z]

    price = 0
    all_size = 0
    all_time = 0
    all_tiles = 0
    for z in nb_tiles:
        print('')
        print("{} tiles in zoom {}.".format(nb_tiles[z], z))
        all_tiles += nb_tiles[z]
        if meta:
            time = times[z] + layer['cost']['tile_generation_time'] * nb_tiles[z]
        else:
            time = layer['cost']['tileonly_generation_time'] * nb_tiles[z]
        size = layer['cost']['tile_size'] * nb_tiles[z]
        all_size += size

        all_time += time
        td = timedelta(milliseconds=time)
        print("Time to generate: {} [d h:mm:ss]".format((duration_format(td))))
        c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0
        price += c
        print('S3 PUT: {0:0.2f} [$]'.format(c))

        if 'sqs' in gene.config:
            if meta:
                nb_sqs = nb_metatiles[z] * 3
            else:
                nb_sqs = nb_tiles[z] * 3
            c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0
            price += c
            print('SQS usage: {0:0.2f} [$]'.format(c))

    print("")
    td = timedelta(milliseconds=all_time)
    print("Number of tiles: {}".format(all_tiles))
    print('Generation time: {} [d h:mm:ss]'.format((duration_format(td))))
    print('Generation cost: {0:0.2f} [$]'.format(price))

    return all_size, td, price, all_tiles
Example #16
0
    def generate_resume(self, layer):
        if self._options.time is None:
            if layer is not None:
                all_dimensions = self._gene.get_all_dimensions(layer)
                message = [
                    "The tile generation of layer '{}{}' is finish".format(
                        layer['name'],
                        "" if (
                            (len(all_dimensions) == 1 and len(all_dimensions[0]) == 0)
                            or layer['type'] != 'wms'
                        )
                        else " ({})".format(
                            " - ".join([
                                ", ".join(
                                    ["=".join(d) for d in dimensions.items()]
                                )
                                for dimensions in all_dimensions
                            ])
                        )
                    ),
                ]
            else:
                message = [
                    "The tile generation is finish"
                ]
            if self._options.role == "master":  # pragma: no cover
                message.append("Nb of generated jobs: {}".format(self._count_tiles.nb))
            elif layer.get('meta') if layer is not None else self._options.role == "slave":
                message += [
                    "Nb generated metatiles: {}".format(self._count_metatiles.nb),
                    "Nb metatiles dropped: {}".format(self._count_metatiles_dropped.nb),
                ]

            if self._options.role != "master":
                message += [
                    "Nb generated tiles: {}".format(self._count_tiles.nb),
                    "Nb tiles dropped: {}".format(self._count_tiles_dropped.nb),
                ]
                if self._options.role in ('local', 'slave'):
                    message += [
                        "Nb tiles stored: {}".format(self._count_tiles_stored.nb),
                        "Nb tiles in error: {}".format(self._gene.error),
                        "Total time: {}".format(duration_format(self._gene.duration)),
                    ]
                    if self._count_tiles_stored.nb != 0:
                        message.append("Total size: {}".format(size_format(self._count_tiles_stored.size)))
                    if self._count_tiles.nb != 0:
                        message.append("Time per tile: {:0.0f} ms".format(
                            (self._gene.duration / self._count_tiles.nb * 1000).seconds)
                        )
                    if self._count_tiles_stored.nb != 0:
                        message.append("Size per tile: {:0.0f} o".format(
                            self._count_tiles_stored.size / self._count_tiles_stored.nb)
                        )

            if not self._options.quiet and self._options.role in ('local', 'slave', 'master') and message:
                print("\n".join(message) + "\n")

        if self._cache_tilestore is not None and hasattr(self._cache_tilestore, 'connection'):
            self._cache_tilestore.connection.close()

        if self._options.role != 'hash' and \
                self._options.time is None and \
                'sns' in self._gene.config:  # pragma: no cover
            if 'region' in self._gene.config['sns']:
                sns_client = boto3.client('sns', region_name=self._gene.config['sns']['region'])
            else:
                sns_client = boto3.client('sns')
            sns_message = [message[0]]
            sns_message += [
                "Layer: {}".format(layer['name'] if layer is not None else "(All layers)"),
                "Role: {}".format(self._options.role),
                "Host: {}".format(socket.getfqdn()),
                "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])),
            ]
            sns_message += message[1:]
            sns_client.publish(TopicArn=self._gene.config['sns']['topic'], Message="\n".join(sns_message),
                               Subject="Tile generation ({layer} - {role})".format(
                                   role=self._options.role,
                                   layer=layer['name'] if layer is not None else "All layers"))
Example #17
0
def _calculate_cost(gene, options):
    nb_metatiles = {}
    nb_tiles = {}

    meta = gene.layer['meta']
    if options.cost_algo == 'area':
        tile_size = gene.layer['grid_ref']['tile_size']
        for zoom, resolution in enumerate(gene.layer['grid_ref']['resolutions']):
            if 'min_resolution_seed' in gene.layer and resolution < gene.layer['min_resolution_seed']:
                continue

            print("Calculate zoom {}.".format(zoom))

            px_buffer = gene.layer['px_buffer'] + \
                gene.layer['meta_buffer'] if meta else 0
            m_buffer = px_buffer * resolution
            if meta:
                size = tile_size * gene.layer['meta_size'] * resolution
                meta_buffer = size * 0.7 + m_buffer
                meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1)
                nb_metatiles[zoom] = int(round(meta_geom.area / size ** 2))
            size = tile_size * resolution
            tile_buffer = size * 0.7 + m_buffer
            geom = gene.geoms[zoom].buffer(tile_buffer, 1)
            nb_tiles[zoom] = int(round(geom.area / size ** 2))

    elif options.cost_algo == 'count':
        gene.init_tilecoords()
        gene.add_geom_filter()

        if meta:
            def count_metatile(tile):
                if tile:
                    if tile.tilecoord.z in nb_metatiles:
                        nb_metatiles[tile.tilecoord.z] += 1
                    else:
                        nb_metatiles[tile.tilecoord.z] = 1
                return tile
            gene.imap(count_metatile)

            class MetaTileSplitter(TileStore):
                @staticmethod
                def get(tiles):
                    for metatile in tiles:
                        for tilecoord in metatile.tilecoord:
                            yield Tile(tilecoord)
            gene.tilestream = MetaTileSplitter().get(gene.tilestream)

            # Only keep tiles that intersect geometry
            gene.add_geom_filter()

        def count_tile(tile):
            if tile:
                if tile.tilecoord.z in nb_tiles:
                    nb_tiles[tile.tilecoord.z] += 1
                else:
                    print("Calculate zoom {}.".format(tile.tilecoord.z))
                    nb_tiles[tile.tilecoord.z] = 1
            return tile
        gene.imap(count_tile)

        consume(gene.tilestream, None)

    times = {}
    print('')
    for z in nb_metatiles:
        print("{} meta tiles in zoom {}.".format(nb_metatiles[z], z))
        times[z] = gene.layer['cost']['metatile_generation_time'] * nb_metatiles[z]

    price = 0
    all_size = 0
    all_time = 0
    all_tiles = 0
    for z in nb_tiles:
        print('')
        print("{} tiles in zoom {}.".format(nb_tiles[z], z))
        all_tiles += nb_tiles[z]
        if meta:
            time = times[z] + gene.layer['cost']['tile_generation_time'] * nb_tiles[z]
        else:
            time = gene.layer['cost']['tileonly_generation_time'] * nb_tiles[z]
        size = gene.layer['cost']['tile_size'] * nb_tiles[z]
        all_size += size

        all_time += time
        td = timedelta(milliseconds=time)
        print("Time to generate: {} [d h:mm:ss]".format((duration_format(td))))
        c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0
        price += c
        print('S3 PUT: {0:0.2f} [$]'.format(c))

        if 'sqs' in gene.config:
            if meta:
                nb_sqs = nb_metatiles[z] * 3
            else:
                nb_sqs = nb_tiles[z] * 3
            c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0
            price += c
            print('SQS usage: {0:0.2f} [$]'.format(c))

    print("")
    td = timedelta(milliseconds=all_time)
    print("Number of tiles: {}".format(all_tiles))
    print('Generation time: {} [d h:mm:ss]'.format((duration_format(td))))
    print('Generation cost: {0:0.2f} [$]'.format(price))

    return (all_size, td, price, all_tiles)
Example #18
0
    def _gene(self, options, gene, dimensions=None):
        if dimensions is None:  # pragma: no cover
            dimensions = {}
        self.dimensions = dimensions
        self.count_metatiles = None
        self.count_metatiles_dropped = Count()
        self.count_tiles = Count()
        self.count_tiles_dropped = Count()
        self.count_tiles_stored = None
        self.sqs_tilestore = None
        self.cache_tilestore = None

        if options.get_bbox:
            try:
                tilecoord = parse_tilecoord(options.get_bbox)
                print("Tile bounds: [{},{},{},{}]".format(*default_int(
                    gene.layer['grid_ref']['obj'].extent(tilecoord))))
                exit()
            except ValueError as e:  # pragma: no cover
                print(
                    "Tile '{}' is not in the format 'z/x/y' or z/x/y:+n/+n\n{}"
                    .format(options.get_bbox, repr(e)))
                exit(1)

        if options.get_hash:
            options.role = 'hash'
            options.test = 1

        if options.role in ('master', 'slave'):
            # Create SQS queue
            self.sqs_tilestore = SQSTileStore(
                gene.get_sqs_queue(),
                on_empty=await_message
                if options.daemon else maybe_stop)  # pragma: no cover

        if options.role in ('local', 'slave'):
            self.cache_tilestore = gene.get_tilesstore(options.cache,
                                                       dimensions)

        if options.tiles:
            gene.set_store(TilesFileStore(options.tiles, options.layer))

        elif options.role in ('local', 'master'):
            # Generate a stream of metatiles
            gene.init_tilecoords()
            gene.add_geom_filter()

        if options.role in ('local', 'master') and 'logging' in gene.config:
            gene.imap(
                DatabaseLoggerInit(gene.config['logging'], options is not None
                                   and options.daemon))

        if options.local_process_number is not None:  # pragma: no cover
            gene.add_local_process_filter()

        elif options.role == 'slave':
            # Get the metatiles from the SQS queue
            gene.set_store(self.sqs_tilestore)  # pragma: no cover

        elif options.role == 'hash':
            try:
                z, x, y = (int(v) for v in options.get_hash.split('/'))
                if gene.layer.get('meta'):
                    gene.set_tilecoords(
                        [TileCoord(z, x, y, gene.layer['meta_size'])])
                else:
                    gene.set_tilecoords([TileCoord(z, x, y)])
            except ValueError as e:  # pragma: no cover
                exit("Tile '{}' is not in the format 'z/x/y'\n{}".format(
                    options.get_hash, repr(e)))

        # At this stage, the tilestream contains metatiles that intersect geometry
        gene.add_logger()

        self.count_metatiles = gene.counter()

        if options.role == 'master':  # pragma: no cover
            # Put the metatiles into the SQS queue
            gene.put(self.sqs_tilestore)
            self.count_tiles = gene.counter()

        elif options.role in ('local', 'slave', 'hash'):
            gene.get(
                MultiTileStore({
                    name: self._get_tilestore_for_layer(layer, gene)
                    for name, layer in gene.layers.items()
                }), 'Get tile')

            def wrong_content_type_to_error(tile):
                if tile is not None and tile.content_type is not None \
                        and tile.content_type.find("image/") != 0:
                    if tile.content_type.find(
                            "application/vnd.ogc.se_xml") == 0:
                        tile.error = "WMS server error: {}".format(
                            (self._re_rm_xml_tag.sub('', tile.error)))
                    else:  # pragma: no cover
                        tile.error = "{} is not an image format, error: {}".format(
                            tile.content_type, tile.error)
                return tile

            gene.imap(wrong_content_type_to_error)
            gene.add_error_filters()

            if options.role == 'hash':
                if gene.layer.get('meta', False):
                    gene.imap(HashLogger('empty_metatile_detection'))
            elif not options.near:
                droppers = {}
                for lname, layer in gene.layers.items():
                    if 'empty_metatile_detection' in layer:
                        empty_tile = layer['empty_metatile_detection']
                        droppers[lname] = HashDropper(
                            empty_tile['size'],
                            empty_tile['hash'],
                            store=self.cache_tilestore,
                            queue_store=self.sqs_tilestore,
                            count=self.count_metatiles_dropped,
                        )
                if droppers:
                    gene.imap(MultiAction(droppers))

            def add_elapsed_togenerate(metatile):
                if metatile is not None:
                    metatile.elapsed_togenerate = metatile.tilecoord.n**2
                    return True
                return False  # pragma: no cover

            gene.ifilter(add_elapsed_togenerate)

            # Split the metatile image into individual tiles
            gene.add_metatile_splitter()
            gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))

            gene.imap(self.count_tiles)

            gene.process(key='pre_hash_post_process')

            if options.role == 'hash':
                gene.imap(HashLogger('empty_tile_detection'))
            elif not options.near:
                droppers = {}
                for lname, layer in gene.layers.items():
                    if 'empty_tile_detection' in layer:
                        empty_tile = layer['empty_tile_detection']
                        droppers[lname] = HashDropper(
                            empty_tile['size'],
                            empty_tile['hash'],
                            store=self.cache_tilestore,
                            queue_store=self.sqs_tilestore,
                            count=self.count_tiles_dropped,
                        )
                if droppers:
                    gene.imap(MultiAction(droppers))

            gene.process()
        else:  # pragma: no cover
            self.count_tiles = gene.counter()

        if options.role in ('local', 'slave'):
            self.count_tiles_stored = gene.counter(size=True)

            if options.time:

                def log_size(tile):
                    sys.stdout.write('size: {}\n'.format(len(tile.data)))
                    return tile

                gene.imap(log_size)

            gene.put(self.cache_tilestore, "Store the tile")

        if options.generated_tiles_file:  # pragma: no cover
            generated_tiles_file = open(options.generated_tiles_file, 'a')

            def do(tile):
                generated_tiles_file.write('{}\n'.format(tile.tilecoord))
                return tile

            gene.imap(do)

        if options.role == 'slave':  # pragma: no cover

            def delete_from_store(tile):
                if hasattr(tile, 'metatile'):
                    tile.metatile.elapsed_togenerate -= 1
                    if tile.metatile.elapsed_togenerate == 0:
                        self.sqs_tilestore.delete_one(tile.metatile)
                else:
                    self.sqs_tilestore.delete_one(tile)
                return True

            gene.ifilter(delete_from_store)

        if options.role in ('local', 'slave') and 'logging' in gene.config:
            gene.imap(
                DatabaseLogger(gene.config['logging'], options is not None
                               and options.daemon))
        gene.add_error_filters()

        message = []
        if options.time is not None:

            class LogTime:
                n = 0
                t1 = None

                def __call__(self, tile):
                    self.n += 1
                    if self.n == options.time:
                        self.t1 = datetime.now()
                    elif self.n == 2 * options.time:
                        t2 = datetime.now()
                        d = (t2 - self.t1) / options.time
                        sys.stdout.write('time: {}\n'.format(
                            ((d.days * 24 * 3600 + d.seconds) * 1000000 +
                             d.microseconds)))
                    return tile

            gene.imap(LogTime())

            gene.consume(options.time * 3)
        else:
            gene.consume()

            if gene.layer is not None:
                message = [
                    "The tile generation of layer '{}{}' is finish".format(
                        gene.layer['name'], ""
                        if len(dimensions) == 0 or gene.layer['type'] != 'wms'
                        else " ({})".format(", ".join(
                            ["=".join(d) for d in dimensions.items()]))),
                ]
                if options.role == "master":  # pragma: no cover
                    message.append("Nb of generated jobs: {}".format(
                        self.count_tiles.nb))
                else:
                    if gene.layer.get('meta'):
                        message += [
                            "Nb generated metatiles: {}".format(
                                self.count_metatiles.nb),
                            "Nb metatiles dropped: {}".format(
                                self.count_metatiles_dropped.nb),
                        ]
            else:
                message = ["The tile generation is finish"]

            if options.role != "master":
                message += [
                    "Nb generated tiles: {}".format(self.count_tiles.nb),
                    "Nb tiles dropped: {}".format(self.count_tiles_dropped.nb),
                ]
                if options.role in ('local', 'slave'):
                    message += [
                        "Nb tiles stored: {}".format(
                            self.count_tiles_stored.nb),
                        "Nb tiles in error: {}".format(gene.error),
                        "Total time: {}".format(duration_format(
                            gene.duration)),
                    ]
                    if self.count_tiles_stored.nb != 0:
                        message.append("Total size: {}".format(
                            size_format(self.count_tiles_stored.size)))
                    if self.count_tiles.nb != 0:
                        message.append("Time per tile: {:0.0f} ms".format(
                            (gene.duration / self.count_tiles.nb *
                             1000).seconds))
                    if self.count_tiles_stored.nb != 0:
                        message.append("Size per tile: {:0.0f} o".format(
                            self.count_tiles_stored.size /
                            self.count_tiles_stored.nb))

            if not options.quiet and options.role in ('local', 'slave'):
                print("\n".join(message) + "\n")

        if self.cache_tilestore is not None and hasattr(
                self.cache_tilestore, 'connection'):
            self.cache_tilestore.connection.close()

        if options.role != 'hash' and options.time is None and 'sns' in gene.config:  # pragma: no cover
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            sns_message = [message[0]]
            sns_message += [
                "Layer: {}".format(gene.layer['name'] if gene.
                                   layer is not None else "(All layers)"),
                "Role: {}".format(options.role),
                "Host: {}".format(socket.getfqdn()),
                "Command: {}".format(' '.join([quote(arg)
                                               for arg in sys.argv])),
            ]
            sns_message += message[1:]
            connection.publish(
                gene.config['sns']['topic'], "\n".join(sns_message),
                "Tile generation ({layer!s} - {role!s})".format(
                    **{
                        'role':
                        options.role,
                        'layer':
                        gene.layer['name'] if gene.
                        layer is not None else "All layers"
                    }))