def __init__( self, store: Optional[TileStore] = None, queue_store: Optional[TileStore] = None, count: Optional[List[Callable[[Optional[Tile]], Any]]] = None, **kwargs: Any, ) -> None: """Initialize.""" self.store = store self.queue_store = queue_store self.count = count or [] MapnikTileStore.__init__(self, **kwargs)
def _get_tilestore_for_layer(self, layer, gene): if layer['type'] == 'wms': params = layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join( layer['wmts_style'] for l in layer['layers'].split(',')) if layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(self.dimensions) # Get the metatile image from the WMS server return URLTileStore( tilelayouts=(WMSTileLayout( url=layer['url'], layers=layer['layers'], srs=layer['grid_ref']['srs'], format=layer['mime_type'], border=layer['meta_buffer'] if layer.get('meta') else 0, tilegrid=gene.get_grid(layer['grid'])['obj'], params=params, ), ), headers=layer['headers'], ) elif layer['type'] == 'mapnik': # pragma: no cover try: from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore except ImportError: if 'TRAVIS' not in os.environ: # pragma nocover logger.error("Mapnik is not available") return None grid = gene.get_grid(layer['grid']) if layer['output_format'] == 'grid': return MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=layer['mapfile'], image_buffer=layer['meta_buffer'] if layer.get('meta') else 0, data_buffer=layer['data_buffer'], output_format=layer['output_format'], resolution=layer['resolution'], layers_fields=layer['layers_fields'], drop_empty_utfgrid=layer['drop_empty_utfgrid'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=[self.count_tiles, self.count_tiles_dropped], proj4_literal=grid['proj4_literal'], ) else: return MapnikTileStore( tilegrid=grid['obj'], mapfile=layer['mapfile'], image_buffer=layer['meta_buffer'] if layer.get('meta') else 0, data_buffer=layer['data_buffer'], output_format=layer['output_format'], proj4_literal=grid['proj4_literal'], )
def get_one(self, tile): result = MapnikTileStore.get_one(self, tile) if result is None: if self.store is not None: if tile.tilecoord.n != 1: # pragma: no cover for tilecoord in tile.tilecoord: self.store.delete_one(Tile(tilecoord)) else: self.store.delete_one(tile) logger.info("The tile {} {} is dropped".format(tile.tilecoord, tile.formated_metadata)) if hasattr(tile, 'metatile'): # pragma: no cover tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0 and self.queue_store is not None: self.queue_store.delete_one(tile.metatile) elif self.queue_store is not None: # pragma: no cover self.queue_store.delete_one(tile) for count in self.count: count() return result
def get_one(self, tile): result = MapnikTileStore.get_one(self, tile) if result is None: if self.store is not None: if tile.tilecoord.n != 1: # pragma: no cover for tilecoord in tile.tilecoord: self.store.delete_one(Tile(tilecoord)) else: self.store.delete_one(tile) logger.info("The tile %s is dropped" % str(tile.tilecoord)) if hasattr(tile, 'metatile'): # pragma: no cover tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0 and self.queue_store is not None: self.queue_store.delete_one(tile.metatile) elif self.queue_store is not None: # pragma: no cover self.queue_store.delete_one(tile) if self.count: self.count() return result
def get_one(self, tile: Tile) -> Optional[Tile]: """See in superclass.""" result = MapnikTileStore.get_one(self, tile) if result is None: if self.store is not None: if tile.tilecoord.n != 1: for tilecoord in tile.tilecoord: self.store.delete_one(Tile(tilecoord)) else: self.store.delete_one(tile) logger.info("The tile %s %s is dropped", tile.tilecoord, tile.formated_metadata) if hasattr(tile, "metatile"): metatile: Tile = tile.metatile # type: ignore metatile.elapsed_togenerate -= 1 # type: ignore if metatile.elapsed_togenerate == 0 and self.queue_store is not None: # type: ignore self.queue_store.delete_one(metatile) elif self.queue_store is not None: self.queue_store.delete_one(tile) for count in self.count: count(None) return result
def __init__(self, store=None, queue_store=None, count=None, **kwargs): self.store = store self.queue_store = queue_store self.count = count or [] MapnikTileStore.__init__(self, **kwargs)
def __call__(self, config_file: str, layer_name: str) -> Optional[TileStore]: config = self.gene._gene.get_config(config_file) layer = config.config["layers"][layer_name] if layer["type"] == "wms": params = layer.get("params", {}).copy() if "STYLES" not in params: params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(",")) if layer.get("generate_salt", False): params["SALT"] = str(random.randint(0, 999999)) # nosec # Get the metatile image from the WMS server return URLTileStore( tilelayouts=( WMSTileLayout( url=layer["url"], layers=layer["layers"], srs=config.config["grids"][layer["grid"]]["srs"], format=layer["mime_type"], border=layer["meta_buffer"] if layer["meta"] else 0, tilegrid=self.gene._gene.get_grid(config, layer["grid"]), params=params, ), ), headers=layer["headers"], ) elif layer["type"] == "mapnik": try: from tilecloud.store.mapnik_ import MapnikTileStore # pylint: disable=import-outside-toplevel from tilecloud_chain.mapnik_ import ( # pylint: disable=import-outside-toplevel MapnikDropActionTileStore, ) except ImportError: if os.environ.get("CI", "FALSE") == "FALSE": # pragma nocover logger.error("Mapnik is not available", exc_info=True) return None grid = config.config["grids"][layer["grid"]] if cast(str, layer.get("output_format", "png")) == "grid": assert self.gene._count_tiles assert self.gene._count_tiles_dropped return MapnikDropActionTileStore( tilegrid=self.gene._gene.get_grid(config, layer["grid"]), mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer.get("data_buffer", 128), output_format=layer.get("output_format", "png"), resolution=layer.get("resolution", 4), layers_fields=layer.get("layers_fields", {}), drop_empty_utfgrid=layer.get("drop_empty_utfgrid", False), store=self.gene._cache_tilestore, queue_store=self.gene._queue_tilestore, count=[self.gene._count_tiles, self.gene._count_tiles_dropped], proj4_literal=grid["proj4_literal"], ) else: return MapnikTileStore( tilegrid=self.gene._gene.get_grid(config, layer["grid"]), mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer.get("data_buffer", 128), output_format=cast(str, layer.get("output_format", "png")), proj4_literal=grid["proj4_literal"], ) return None
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print \ "Tile bounds: [%i,%i,%i,%i]" % \ gene.layer['grid_ref']['obj'].extent(tilecoord) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit('the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.') params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % \ ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def _get_tilestore_for_layer(self, layer) -> Optional[TileStore]: if layer["type"] == "wms": params = layer["params"].copy() if "STYLES" not in params: params["STYLES"] = ",".join( layer["wmts_style"] for _ in layer["layers"].split(",")) if layer["generate_salt"]: params["SALT"] = str(random.randint(0, 999999)) # Get the metatile image from the WMS server return URLTileStore( tilelayouts=(WMSTileLayout( url=layer["url"], layers=layer["layers"], srs=layer["grid_ref"]["srs"], format=layer["mime_type"], border=layer["meta_buffer"] if layer.get("meta", False) else 0, tilegrid=self._gene.get_grid(layer)["obj"], params=params, ), ), headers=layer["headers"], ) elif layer["type"] == "mapnik": # pragma: no cover try: from tilecloud.store.mapnik_ import MapnikTileStore # pylint: disable=import-outside-toplevel from tilecloud_chain.mapnik_ import ( # pylint: disable=import-outside-toplevel MapnikDropActionTileStore, ) except ImportError: if os.environ.get("CI", "FALSE") == "FALSE": # pragma nocover logger.error("Mapnik is not available", exc_info=True) return None grid = self._gene.get_grid(layer) if layer["output_format"] == "grid": return MapnikDropActionTileStore( tilegrid=grid["obj"], mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer["data_buffer"], output_format=layer["output_format"], resolution=layer["resolution"], layers_fields=layer["layers_fields"], drop_empty_utfgrid=layer["drop_empty_utfgrid"], store=self._cache_tilestore, queue_store=self._queue_tilestore, count=[self._count_tiles, self._count_tiles_dropped], proj4_literal=grid["proj4_literal"], ) else: return MapnikTileStore( tilegrid=grid["obj"], mapfile=layer["mapfile"], image_buffer=layer["meta_buffer"] if layer.get("meta") else 0, data_buffer=layer["data_buffer"], output_format=layer["output_format"], proj4_literal=grid["proj4_literal"], ) return None
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord)) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join( gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer[ 'output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()])), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format( count_metatiles.nb), "Nb metatiles dropped: {}".format( count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds)) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def __init__(self, store=None, queue_store=None, count=None, **kwargs): self.store = store self.queue_store = queue_store self.count = count MapnikTileStore.__init__(self, **kwargs)