def _generate_queue(self, layer): assert layer is not None if self._options.get_bbox: try: tilecoord = parse_tilecoord(self._options.get_bbox) print("Tile bounds: [{},{},{},{}]".format( *default_int(layer['grid_ref']['obj'].extent(tilecoord)) )) exit() except ValueError: # pragma: no cover logger.error("Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n", self._options.get_bbox, exc_info=True) exit(1) if self._options.get_hash: self._options.role = 'hash' self._options.test = 1 if self._options.tiles: self._gene.set_store(TilesFileStore( self._options.tiles, layer['name'], self._gene.get_all_dimensions(layer) )) elif self._options.role in ('local', 'master'): # Generate a stream of metatiles self._gene.init_tilecoords(layer) self._gene.add_geom_filter(layer) if self._options.role in ('local', 'master') and 'logging' in self._gene.config: self._gene.imap(DatabaseLoggerInit( self._gene.config['logging'], self._options is not None and self._options.daemon, )) if self._options.local_process_number is not None: # pragma: no cover self._gene.add_local_process_filter() elif self._options.role == 'hash': try: z, x, y = (int(v) for v in self._options.get_hash.split('/')) if layer.get('meta'): self._gene.set_tilecoords([TileCoord(z, x, y, layer['meta_size'])], layer) else: self._gene.set_tilecoords([TileCoord(z, x, y)], layer) except ValueError as e: # pragma: no cover exit( "Tile '{}' is not in the format 'z/x/y'\n{}".format( self._options.get_hash, repr(e)) ) # At this stage, the tilestream contains metatiles that intersect geometry self._gene.add_logger() self._count_metatiles = self._gene.counter() if self._options.role == 'master': # pragma: no cover # Put the metatiles into the SQS or Redis queue self._gene.put(self._queue_tilestore) self._count_tiles = self._gene.counter()
def _generate_queue(self, layer_name: Optional[str]) -> None: if self._options.tiles: self._gene.set_store(TilesFileStore(self._options.tiles)) return assert layer_name is not None assert self._gene.config_file is not None config = self._gene.get_config(self._gene.config_file) layer = config.config["layers"][layer_name] if self._options.get_bbox: try: tilecoord = parse_tilecoord(self._options.get_bbox) bounds = default_int(self._gene.get_grid(config, layer["grid"]).extent(tilecoord)) print(f"Tile bounds: [{','.join([str(b) for b in bounds])}]") sys.exit() except ValueError: logger.exception( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n", self._options.get_bbox, ) sys.exit(1) if self._options.role in ("local", "master"): # Generate a stream of metatiles self._gene.init_tilecoords(config, layer_name) elif self._options.role == "hash": layer = config.config["layers"][layer_name] try: z, x, y = (int(v) for v in self._options.get_hash.split("/")) if layer.get("meta"): self._gene.set_tilecoords(config, [TileCoord(z, x, y, layer["meta_size"])], layer_name) else: self._gene.set_tilecoords(config, [TileCoord(z, x, y)], layer_name) except ValueError: logger.exception("Tile '%s' is not in the format 'z/x/y'", self._options.get_hash) sys.exit(1)
def _generate_queue(self, layer): assert layer is not None if self._options.get_bbox: try: tilecoord = parse_tilecoord(self._options.get_bbox) print("Tile bounds: [{},{},{},{}]".format( *default_int(layer["grid_ref"]["obj"].extent(tilecoord)))) sys.exit() except ValueError: # pragma: no cover logger.error( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n", self._options.get_bbox, exc_info=True, ) sys.exit(1) if self._options.tiles: self._gene.set_store( TilesFileStore(self._options.tiles, layer["name"], self._gene.get_all_dimensions(layer))) elif self._options.role in ("local", "master"): # Generate a stream of metatiles self._gene.init_tilecoords(layer) elif self._options.role == "hash": try: z, x, y = (int(v) for v in self._options.get_hash.split("/")) if layer.get("meta"): self._gene.set_tilecoords( [TileCoord(z, x, y, layer["meta_size"])], layer) else: self._gene.set_tilecoords([TileCoord(z, x, y)], layer) except ValueError as e: # pragma: no cover sys.exit("Tile '{}' is not in the format 'z/x/y'\n{}".format( self._options.get_hash, repr(e)))
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print( "Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord) ) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e) ) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e) ) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get(URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ),), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get(MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get(MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find("application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data ) ) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data ) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n ** 2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer['output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()]) ), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format(count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format(count_metatiles.nb), "Nb metatiles dropped: {}".format(count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format(gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format(size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds) ) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb) ) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] } )
def main(): parser = ArgumentParser( description= "Used to import the osm2pgsql expire-tiles file to Postgres", prog=sys.argv[0]) parser.add_argument( "--buffer", type=float, default=0.0, help="Extent buffer to the tiles [m], default is 0", ) parser.add_argument( "--simplify", type=float, default=0.0, help="Simplify the result geometry [m], default is 0", ) parser.add_argument( "--create", default=False, action="store_true", help="create the table if not exists", ) parser.add_argument( "--delete", default=False, action="store_true", help="empty the table", ) parser.add_argument( "file", metavar="FILE", help="The osm2pgsql expire-tiles file", ) parser.add_argument( "connection", metavar="CONNECTION", help=("The PostgreSQL connection string e.g. " '"user=www-data password=www-data dbname=sig host=localhost"'), ) parser.add_argument( "table", metavar="TABLE", help="The PostgreSQL table to fill", ) parser.add_argument( "--schema", default="public", help= "The PostgreSQL schema to use (should already exists), default is public", ) parser.add_argument( "column", metavar="COLUMN", default="geom", nargs="?", help='The PostgreSQL column, default is "geom"', ) parser.add_argument( "--srid", type=int, default=3857, nargs="?", help="The stored geometry SRID, no conversion by default (3857)", ) options = parser.parse_args() connection = psycopg2.connect(options.connection) cursor = connection.cursor() if options.create: cursor.execute( "SELECT count(*) FROM pg_tables WHERE schemaname='{}' AND tablename='{}'" .format(options.schema, options.table)) if cursor.fetchone()[0] == 0: cursor.execute( 'CREATE TABLE IF NOT EXISTS "{}"."{}" (id serial)'.format( options.schema, options.table)) cursor.execute( "SELECT AddGeometryColumn('{}', '{}', '{}', {}, 'MULTIPOLYGON', 2)" .format(options.schema, options.table, options.column, options.srid)) if options.delete: cursor.execute('DELETE FROM "{}"'.format((options.table))) geoms = [] grid = QuadTileGrid(max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), ) with open(options.file, "r") as f: for coord in f: extent = grid.extent(parse_tilecoord(coord), options.buffer) geoms.append( Polygon(( (extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1]), ))) if len(geoms) == 0: print("No coords found") connection.commit() cursor.close() connection.close() sys.exit(0) geom = cascaded_union(geoms) if geom.geom_type == "Polygon": geom = MultiPolygon((geom, )) if options.simplify > 0: geom.simplify(options.simplify) sql_geom = "ST_GeomFromText('{}', 3857)".format(geom.wkt) if options.srid <= 0: sql_geom = "ST_GeomFromText('{}')".format(geom.wkt) # pragma: no cover elif options.srid != 3857: sql_geom = "ST_Transform({}, {})".format(sql_geom, options.srid) cursor.execute('INSERT INTO "{}" ("{}") VALUES ({})'.format( options.table, options.column, sql_geom)) connection.commit() cursor.close() connection.close() print("Import successful")
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print \ "Tile bounds: [%i,%i,%i,%i]" % \ gene.layer['grid_ref']['obj'].extent(tilecoord) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit('the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.') params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % \ ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def main(): parser = ArgumentParser( description= 'Used to import the osm2pgsql expire-tiles file to Postgres', prog='./buildout/bin/import_expire_tiles', ) parser.add_argument( '--buffer', type=float, default=0.0, help='Extent buffer to the tiles [m], default is 0', ) parser.add_argument( '--simplify', type=float, default=0.0, help='Simplify the result geometry [m], default is 0', ) parser.add_argument( '--create', default=False, action="store_true", help='create the table if not exists', ) parser.add_argument( '--delete', default=False, action="store_true", help='empty the table', ) parser.add_argument( 'file', metavar='FILE', help='The osm2pgsql expire-tiles file', ) parser.add_argument( 'connection', metavar='CONNECTION', help= 'The PostgreSQL connection string e.g. "user=www-data password=www-data dbname=sig host=localhost"', ) parser.add_argument( 'table', metavar='TABLE', help='The PostgreSQL table to fill', ) parser.add_argument( '--schema', default='public', help= 'The PostgreSQL schema to use (should already exists), default is public', ) parser.add_argument( 'column', metavar='COLUMN', default='geom', nargs='?', help='The PostgreSQL column, default is "geom"', ) parser.add_argument( '--srid', type=int, default=3857, nargs='?', help='The stored geometry SRID, no conversion by default (3857)', ) options = parser.parse_args() connection = psycopg2.connect(options.connection) cursor = connection.cursor() if options.create: cursor.execute( "SELECT count(*) FROM pg_tables WHERE schemaname='%s' AND tablename='%s'" % ( options.schema, options.table, )) if cursor.fetchone()[0] == 0: cursor.execute('CREATE TABLE IF NOT EXISTS "%s"."%s" (id serial)' % ( options.schema, options.table, )) cursor.execute( "SELECT AddGeometryColumn('%s', '%s', '%s', %s, 'MULTIPOLYGON', 2)" % ( options.schema, options.table, options.column, options.srid, )) if options.delete: cursor.execute('DELETE FROM "%s"' % (options.table)) geoms = [] f = file(options.file, 'r') grid = QuadTileGrid(max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), ) for coord in f: extent = grid.extent(parse_tilecoord(coord), options.buffer) geoms.append( Polygon(((extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1])))) f.close() if len(geoms) == 0: print "No coords found" connection.commit() cursor.close() connection.close() exit(0) geom = cascaded_union(geoms) if geom.geom_type == 'Polygon': geom = MultiPolygon((geom, )) if options.simplify > 0: geom.simplify(options.simplify) sql_geom = "ST_GeomFromText('%s', 3857)" % geom.wkt if options.srid <= 0: sql_geom = "ST_GeomFromText('%s')" % geom.wkt # pragma: no cover elif options.srid != 3857: sql_geom = 'ST_Transform(%s, %i)' % (sql_geom, options.srid) cursor.execute('INSERT INTO "%s" ("%s") VALUES (%s)' % (options.table, options.column, sql_geom)) connection.commit() cursor.close() connection.close() print 'Import successful'
def main() -> None: """Import the osm2pgsql expire-tiles file to Postgres.""" try: parser = ArgumentParser( description= "Used to import the osm2pgsql expire-tiles file to Postgres", prog=sys.argv[0]) parser.add_argument( "--buffer", type=float, default=0.0, help="Extent buffer to the tiles [m], default is 0", ) parser.add_argument( "--simplify", type=float, default=0.0, help="Simplify the result geometry [m], default is 0", ) parser.add_argument( "--create", default=False, action="store_true", help="create the table if not exists", ) parser.add_argument( "--delete", default=False, action="store_true", help="empty the table", ) parser.add_argument( "file", metavar="FILE", help="The osm2pgsql expire-tiles file", ) parser.add_argument( "connection", metavar="CONNECTION", help=( "The PostgreSQL connection string e.g. " '"user=www-data password=www-data dbname=sig host=localhost"'), ) parser.add_argument( "table", metavar="TABLE", help="The PostgreSQL table to fill", ) parser.add_argument( "--schema", default="public", help= "The PostgreSQL schema to use (should already exists), default is public", ) parser.add_argument( "column", metavar="COLUMN", default="geom", nargs="?", help='The PostgreSQL column, default is "geom"', ) parser.add_argument( "--srid", type=int, default=3857, nargs="?", help="The stored geometry SRID, no conversion by default (3857)", ) options = parser.parse_args() connection = psycopg2.connect(options.connection) cursor = connection.cursor() if options.create: cursor.execute( "SELECT count(*) FROM pg_tables WHERE schemaname=%(schema)s AND tablename=%(table)s", { "schema": options.schema, "table": options.table }, ) if cursor.fetchone()[0] == 0: cursor.execute( f'CREATE TABLE IF NOT EXISTS "{options.schema}"."{options.table}" (id serial)' ) cursor.execute( "SELECT AddGeometryColumn(%(schema)s, %(table)s, %(column)s, %(srid)s, 'MULTIPOLYGON', 2)", { "schema": options.schema, "table": options.table, "column": options.column, "srid": options.srid, }, ) if options.delete: cursor.execute( psycopg2.sql.SQL("DELETE FROM {}").format( psycopg2.sql.Identifier(options.table))) geoms = [] grid = QuadTileGrid(max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), ) with open(options.file, encoding="utf-8") as f: for coord in f: extent = grid.extent(parse_tilecoord(coord), options.buffer) geoms.append( Polygon(( (extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1]), ))) if len(geoms) == 0: print("No coords found") connection.commit() cursor.close() connection.close() sys.exit(0) geom = unary_union(geoms) if geom.geom_type == "Polygon": geom = MultiPolygon((geom, )) if options.simplify > 0: geom.simplify(options.simplify) sql_geom = f"ST_GeomFromText('{geom.wkt}', 3857)" if options.srid <= 0: sql_geom = f"ST_GeomFromText('{geom.wkt}')" elif options.srid != 3857: sql_geom = f"ST_Transform({sql_geom}, {options.srid})" cursor.execute( f'INSERT INTO "{options.table}" ("{options.column}") VALUES ({sql_geom})' ) connection.commit() cursor.close() connection.close() print("Import successful") except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") sys.exit(1)
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord)) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join( gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer[ 'output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()])), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format( count_metatiles.nb), "Nb metatiles dropped: {}".format( count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds)) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def main(): parser = ArgumentParser( description='Used to import the osm2pgsql expire-tiles file to Postgres', prog='./buildout/bin/import_expire_tiles', ) parser.add_argument( '--buffer', type=float, default=0.0, help='Extent buffer to the tiles [m], default is 0', ) parser.add_argument( '--simplify', type=float, default=0.0, help='Simplify the result geometry [m], default is 0', ) parser.add_argument( '--create', default=False, action="store_true", help='create the table if not exists', ) parser.add_argument( '--delete', default=False, action="store_true", help='empty the table', ) parser.add_argument( 'file', metavar='FILE', help='The osm2pgsql expire-tiles file', ) parser.add_argument( 'connection', metavar='CONNECTION', help='The PostgreSQL connection string e.g. "user=www-data password=www-data dbname=sig host=localhost"', ) parser.add_argument( 'table', metavar='TABLE', help='The PostgreSQL table to fill', ) parser.add_argument( '--schema', default='public', help='The PostgreSQL schema to use (should already exists), default is public', ) parser.add_argument( 'column', metavar='COLUMN', default='geom', nargs='?', help='The PostgreSQL column, default is "geom"', ) parser.add_argument( '--srid', type=int, default=3857, nargs='?', help='The stored geometry SRID, no conversion by default (3857)', ) options = parser.parse_args() connection = psycopg2.connect(options.connection) cursor = connection.cursor() if options.create: cursor.execute( "SELECT count(*) FROM pg_tables WHERE schemaname='%s' AND tablename='%s'" % ( options.schema, options.table, ) ) if cursor.fetchone()[0] == 0: cursor.execute('CREATE TABLE IF NOT EXISTS "%s"."%s" (id serial)' % ( options.schema, options.table, )) cursor.execute("SELECT AddGeometryColumn('%s', '%s', '%s', %s, 'MULTIPOLYGON', 2)" % ( options.schema, options.table, options.column, options.srid, )) if options.delete: cursor.execute('DELETE FROM "%s"' % (options.table)) geoms = [] f = file(options.file, 'r') grid = QuadTileGrid( max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), ) for coord in f: extent = grid.extent(parse_tilecoord(coord), options.buffer) geoms.append(Polygon(( (extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1]) ))) f.close() if len(geoms) == 0: print "No coords found" connection.commit() cursor.close() connection.close() exit(0) geom = cascaded_union(geoms) if geom.geom_type == 'Polygon': geom = MultiPolygon((geom,)) if options.simplify > 0: geom.simplify(options.simplify) sql_geom = "ST_GeomFromText('%s', 3857)" % geom.wkt if options.srid <= 0: sql_geom = "ST_GeomFromText('%s')" % geom.wkt # pragma: no cover elif options.srid != 3857: sql_geom = 'ST_Transform(%s, %i)' % (sql_geom, options.srid) cursor.execute('INSERT INTO "%s" ("%s") VALUES (%s)' % ( options.table, options.column, sql_geom )) connection.commit() cursor.close() connection.close() print 'Import successful'
def _gene(self, options, gene, dimensions=None): if dimensions is None: # pragma: no cover dimensions = {} self.dimensions = dimensions self.count_metatiles = None self.count_metatiles_dropped = Count() self.count_tiles = Count() self.count_tiles_dropped = Count() self.count_tiles_stored = None self.sqs_tilestore = None self.cache_tilestore = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [{},{},{},{}]".format(*default_int( gene.layer['grid_ref']['obj'].extent(tilecoord)))) exit() except ValueError as e: # pragma: no cover print( "Tile '{}' is not in the format 'z/x/y' or z/x/y:+n/+n\n{}" .format(options.get_bbox, repr(e))) exit(1) if options.get_hash: options.role = 'hash' options.test = 1 if options.role in ('master', 'slave'): # Create SQS queue self.sqs_tilestore = SQSTileStore( gene.get_sqs_queue(), on_empty=await_message if options.daemon else maybe_stop) # pragma: no cover if options.role in ('local', 'slave'): self.cache_tilestore = gene.get_tilesstore(options.cache, dimensions) if options.tiles: gene.set_store(TilesFileStore(options.tiles, options.layer)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.role in ('local', 'master') and 'logging' in gene.config: gene.imap( DatabaseLoggerInit(gene.config['logging'], options is not None and options.daemon)) if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(self.sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if gene.layer.get('meta'): gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '{}' is not in the format 'z/x/y'\n{}".format( options.get_hash, repr(e))) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() self.count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(self.sqs_tilestore) self.count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): gene.get( MultiTileStore({ name: self._get_tilestore_for_layer(layer, gene) for name, layer in gene.layers.items() }), 'Get tile') def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: {}".format( (self._re_rm_xml_tag.sub('', tile.error))) else: # pragma: no cover tile.error = "{} is not an image format, error: {}".format( tile.content_type, tile.error) return tile gene.imap(wrong_content_type_to_error) gene.add_error_filters() if options.role == 'hash': if gene.layer.get('meta', False): gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: droppers = {} for lname, layer in gene.layers.items(): if 'empty_metatile_detection' in layer: empty_tile = layer['empty_metatile_detection'] droppers[lname] = HashDropper( empty_tile['size'], empty_tile['hash'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=self.count_metatiles_dropped, ) if droppers: gene.imap(MultiAction(droppers)) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) gene.imap(self.count_tiles) gene.process(key='pre_hash_post_process') if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: droppers = {} for lname, layer in gene.layers.items(): if 'empty_tile_detection' in layer: empty_tile = layer['empty_tile_detection'] droppers[lname] = HashDropper( empty_tile['size'], empty_tile['hash'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=self.count_tiles_dropped, ) if droppers: gene.imap(MultiAction(droppers)) gene.process() else: # pragma: no cover self.count_tiles = gene.counter() if options.role in ('local', 'slave'): self.count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: {}\n'.format(len(tile.data))) return tile gene.imap(log_size) gene.put(self.cache_tilestore, "Store the tile") if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('{}\n'.format(tile.tilecoord)) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover def delete_from_store(tile): if hasattr(tile, 'metatile'): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: self.sqs_tilestore.delete_one(tile.metatile) else: self.sqs_tilestore.delete_one(tile) return True gene.ifilter(delete_from_store) if options.role in ('local', 'slave') and 'logging' in gene.config: gene.imap( DatabaseLogger(gene.config['logging'], options is not None and options.daemon)) gene.add_error_filters() message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: {}\n'.format( ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds))) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if gene.layer is not None: message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " ({})".format(", ".join( ["=".join(d) for d in dimensions.items()]))), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( self.count_tiles.nb)) else: if gene.layer.get('meta'): message += [ "Nb generated metatiles: {}".format( self.count_metatiles.nb), "Nb metatiles dropped: {}".format( self.count_metatiles_dropped.nb), ] else: message = ["The tile generation is finish"] if options.role != "master": message += [ "Nb generated tiles: {}".format(self.count_tiles.nb), "Nb tiles dropped: {}".format(self.count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format( self.count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if self.count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(self.count_tiles_stored.size))) if self.count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / self.count_tiles.nb * 1000).seconds)) if self.count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( self.count_tiles_stored.size / self.count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if self.cache_tilestore is not None and hasattr( self.cache_tilestore, 'connection'): self.cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name'] if gene. layer is not None else "(All layers)"), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation ({layer!s} - {role!s})".format( **{ 'role': options.role, 'layer': gene.layer['name'] if gene. layer is not None else "All layers" }))
def _generate_queue(self, layer): assert layer is not None if self._options.get_bbox: try: tilecoord = parse_tilecoord(self._options.get_bbox) print("Tile bounds: [{},{},{},{}]".format( *default_int(layer['grid_ref']['obj'].extent(tilecoord)))) exit() except ValueError as e: # pragma: no cover logger.error( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n", self._options.get_bbox, exc_info=True) exit(1) if self._options.get_hash: self._options.role = 'hash' self._options.test = 1 if self._options.tiles: self._gene.set_store( TilesFileStore(self._options.tiles, layer['name'], self._gene.get_all_dimensions(layer))) elif self._options.role in ('local', 'master'): # Generate a stream of metatiles self._gene.init_tilecoords(layer) self._gene.add_geom_filter(layer) if self._options.role in ('local', 'master') and 'logging' in self._gene.config: self._gene.imap( DatabaseLoggerInit( self._gene.config['logging'], self._options is not None and self._options.daemon, )) if self._options.local_process_number is not None: # pragma: no cover self._gene.add_local_process_filter() elif self._options.role == 'hash': try: z, x, y = (int(v) for v in self._options.get_hash.split('/')) if layer.get('meta'): self._gene.set_tilecoords( [TileCoord(z, x, y, layer['meta_size'])], layer) else: self._gene.set_tilecoords([TileCoord(z, x, y)], layer) except ValueError as e: # pragma: no cover exit("Tile '{}' is not in the format 'z/x/y'\n{}".format( self._options.get_hash, repr(e))) # At this stage, the tilestream contains metatiles that intersect geometry self._gene.add_logger() self._count_metatiles = self._gene.counter() if self._options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue self._gene.put(self._sqs_tilestore) self._count_tiles = self._gene.counter()
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print( "Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord) ) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e) ) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e) ) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit( 'the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.' ) params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get(URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ),), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get(MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get(MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find("application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data) ) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data ) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n ** 2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print( """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] } )