def main(): parser = ArgumentParser( description='Used to calculate the generation cost', prog=sys.argv[0] ) add_comon_options(parser, tile_pyramid=False) parser.add_argument( '--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo', choices=('area', 'count'), help="The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate." ) options = parser.parse_args() gene = TileGeneration( options.config, options, layer_name=options.layer, base_config={'cost': {}} ) all_size = 0 tile_size = 0 all_tiles = 0 if (options.layer): (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options) tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer in gene.config['generation']['default_layers']: print("") print("===== %s =====" % layer) gene.set_layer(layer, options) (size, time, price, tiles) = _calculate_cost(gene, options) tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print("") print("===== GLOBAL =====") print("Total number of tiles: %i" % all_tiles) print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time))) print('Total generation cost: %0.2f [$]' % all_price) print("") print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024))) print('S3 get: %0.2f [$/month]' % ( gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 + gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size) ) # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size) if 'ec2' in gene.config: print('ESB storage: %0.2f [$/month]' % ( gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size']) ) sys.exit(0)
def copy(self, options, gene, layer, source, dest, task_name): # desable metatiles gene.layers[layer]['meta'] = False count_tiles_dropped = Count() gene.set_layer(layer, options) source_tilestore = gene.get_tilesstore(source) dest_tilestore = gene.get_tilesstore(dest) gene.init_tilecoords() gene.add_geom_filter() gene.add_logger() gene.get(source_tilestore, "Get the tiles") gene.ifilter(DropEmpty(gene)) # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=dest_tilestore, count=count_tiles_dropped, )) if options.process: gene.process(options.process) gene.ifilter(DropEmpty(gene)) self.count = gene.counter(size=True) gene.put(dest_tilestore, "Store the tiles") gene.add_error_filters() gene.consume() if not options.quiet: print """The tile %s of layer '%s' is finish Nb %s tiles: %i Nb errored tiles: %i Nb dropped tiles: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % \ ( task_name, gene.layer['name'], task_name, self.count.nb, count_tiles_dropped.nb, gene.error, duration_format(gene.duration), size_format(self.count.size), (gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0, self.count.size / self.count.nb if self.count.nb != 0 else -1 )
def _copy(self, options, gene, layer, source, dest, task_name, dimensions=None): # disable metatiles if dimensions is None: # pragma: no cover dimensions = {} del gene.layers[layer]['meta'] count_tiles_dropped = Count() gene.set_layer(layer, options) source_tilestore = gene.get_tilesstore(source, dimensions) dest_tilestore = gene.get_tilesstore(dest, dimensions) gene.init_tilecoords() gene.add_geom_filter() gene.add_logger() gene.get(source_tilestore, "Get the tiles") gene.ifilter(DropEmpty(gene)) # Discard tiles with certain content if gene.layer and 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper(empty_tile['size'], empty_tile['hash'], store=dest_tilestore, count=count_tiles_dropped)) if options.process: gene.process(options.process) gene.ifilter(DropEmpty(gene)) self.count = gene.counter(size=True) gene.put(dest_tilestore, "Store the tiles") gene.add_error_filters() gene.consume() if not options.quiet: print("""The tile {} of layer '{}' is finish Nb {} tiles: {} Nb errored tiles: {} Nb dropped tiles: {} Total time: {} Total size: {} Time per tile: {} ms Size per tile: {} o """.format(task_name, gene.layer['name'], task_name, self.count.nb, count_tiles_dropped.nb, gene.error, duration_format(gene.duration), size_format(self.count.size), (gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0, self.count.size / self.count.nb if self.count.nb != 0 else -1))
def _copy(self, options, gene, layer, source, dest, task_name, dimensions={}): # disable metatiles del gene.layers[layer]['meta'] count_tiles_dropped = Count() gene.set_layer(layer, options) source_tilestore = gene.get_tilesstore(source, dimensions) dest_tilestore = gene.get_tilesstore(dest, dimensions) gene.init_tilecoords() gene.add_geom_filter() gene.add_logger() gene.get(source_tilestore, "Get the tiles") gene.ifilter(DropEmpty(gene)) # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=dest_tilestore, count=count_tiles_dropped, )) if options.process: gene.process(options.process) gene.ifilter(DropEmpty(gene)) self.count = gene.counter(size=True) gene.put(dest_tilestore, "Store the tiles") gene.add_error_filters() gene.consume() if not options.quiet: print( """The tile %s of layer '%s' is finish Nb %s tiles: %i Nb errored tiles: %i Nb dropped tiles: %i Total time: %s Total size: %s Time per tile: %i ms Size per tile: %i o """ % ( task_name, gene.layer['name'], task_name, self.count.nb, count_tiles_dropped.nb, gene.error, duration_format(gene.duration), size_format(self.count.size), (gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0, self.count.size / self.count.nb if self.count.nb != 0 else -1 ) )
def _copy(self, options, gene, layer_name, source, dest, task_name): # disable metatiles layer = gene.layers[layer_name] del layer["meta"] count_tiles_dropped = Count() gene.init_layer(layer, options) source_tilestore = gene.get_tilesstore(source) dest_tilestore = gene.get_tilesstore(dest) gene.init_tilecoords(layer) gene.add_geom_filter() gene.add_logger() gene.get(source_tilestore, "Get the tiles") gene.imap(DropEmpty(gene)) # Discard tiles with certain content if "empty_tile_detection" in layer: empty_tile = layer["empty_tile_detection"] gene.imap( HashDropper(empty_tile["size"], empty_tile["hash"], store=dest_tilestore, count=count_tiles_dropped)) if options.process: gene.process(options.process) gene.imap(DropEmpty(gene)) self.count = gene.counter(size=True) gene.put(dest_tilestore, "Store the tiles") gene.consume() if not options.quiet: print("""The tile {} of layer '{}' is finish Nb {} tiles: {} Nb errored tiles: {} Nb dropped tiles: {} Total time: {} Total size: {} Time per tile: {} ms Size per tile: {} o """.format( task_name, layer["name"], task_name, self.count.nb, count_tiles_dropped.nb, gene.error, duration_format(gene.duration), size_format(self.count.size), (gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0, self.count.size / self.count.nb if self.count.nb != 0 else -1, ))
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print( "Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord) ) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e) ) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e) ) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get(URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ),), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get(MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get(MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find("application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data ) ) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data ) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n ** 2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer['output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()]) ), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format(count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format(count_metatiles.nb), "Nb metatiles dropped: {}".format(count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format(gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format(size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds) ) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb) ) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] } )
def main(): parser = ArgumentParser( description="Used to calculate the generation cost", prog=sys.argv[0]) add_comon_options(parser, tile_pyramid=False, dimensions=True) parser.add_argument( "--cost-algo", "--calculate-cost-algorithm", default="area", dest="cost_algo", choices=("area", "count"), help= "The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate.", ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer, base_config={"cost": {}}, multi_thread=False) all_size = 0 tile_size = 0 all_tiles = 0 if options.layer: layer = gene.layers[options.layer] (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, layer, options) tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer_name in gene.config["generation"]["default_layers"]: print("") print("===== {} =====".format(layer_name)) layer = gene.layers[layer_name] gene.init_layer(layer, options) (size, time, price, tiles) = _calculate_cost(gene, layer, options) tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print("") print("===== GLOBAL =====") print("Total number of tiles: {}".format(all_tiles)) print("Total generation time: {} [d h:mm:ss]".format( (duration_format(all_time)))) print("Total generation cost: {0:0.2f} [$]".format(all_price)) print("") print("S3 Storage: {0:0.2f} [$/month]".format( all_size * gene.config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024))) print("S3 get: {0:0.2f} [$/month]".format( (gene.config["cost"]["s3"]["get"] * gene.config["cost"]["request_per_layers"] / 10000.0 + gene.config["cost"]["s3"]["download"] * gene.config["cost"]["request_per_layers"] * tile_size))) # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * # gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * # gene.config['cost']['request_per_layers'] * tile_size) sys.exit(0)
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print \ "Tile bounds: [%i,%i,%i,%i]" % \ gene.layer['grid_ref']['obj'].extent(tilecoord) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit('the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.') params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % \ ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def _calculate_cost(gene, options): validate_calculate_cost(gene) nb_metatiles = {} nb_tiles = {} meta = gene.layer['meta'] if options.cost_algo == 'area': tile_size = gene.layer['grid_ref']['tile_size'] for zoom, resolution in enumerate(gene.layer['grid_ref']['resolutions']): if 'min_resolution_seed' in gene.layer and resolution < gene.layer['min_resolution_seed']: continue print "Calculate zoom %i." % zoom px_buffer = gene.layer['px_buffer'] + \ gene.layer['meta_buffer'] if meta else 0 m_buffer = px_buffer * resolution if meta: size = tile_size * gene.layer['meta_size'] * resolution meta_buffer = size * 0.7 + m_buffer meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1) nb_metatiles[zoom] = int(round(meta_geom.area / size ** 2)) size = tile_size * resolution tile_buffer = size * 0.7 + m_buffer geom = gene.geoms[zoom].buffer(tile_buffer, 1) nb_tiles[zoom] = int(round(geom.area / size ** 2)) elif options.cost_algo == 'count': gene.init_tilecoords() gene.add_geom_filter() if meta: def count_metatile(tile): if tile: if tile.tilecoord.z in nb_metatiles: nb_metatiles[tile.tilecoord.z] += 1 else: nb_metatiles[tile.tilecoord.z] = 1 return tile gene.imap(count_metatile) class MetaTileSplitter(TileStore): def get(self, tiles): for metatile in tiles: for tilecoord in metatile.tilecoord: yield Tile(tilecoord) gene.tilestream = MetaTileSplitter().get(gene.tilestream) # Only keep tiles that intersect geometry gene.add_geom_filter() def count_tile(tile): if tile: if tile.tilecoord.z in nb_tiles: nb_tiles[tile.tilecoord.z] += 1 else: print "Calculate zoom %i." % tile.tilecoord.z nb_tiles[tile.tilecoord.z] = 1 return tile gene.imap(count_tile) consume(gene.tilestream, None) times = {} print for z in nb_metatiles: print "%i meta tiles in zoom %i." % (nb_metatiles[z], z) times[z] = gene.layer['cost']['metatile_generation_time'] * nb_metatiles[z] price = 0 all_size = 0 all_time = 0 all_tiles = 0 for z in nb_tiles: print print "%i tiles in zoom %i." % (nb_tiles[z], z) all_tiles += nb_tiles[z] if meta: time = times[z] + gene.layer['cost']['tile_generation_time'] * nb_tiles[z] else: time = gene.layer['cost']['tileonly_generation_time'] * nb_tiles[z] size = gene.layer['cost']['tile_size'] * nb_tiles[z] all_size += size all_time += time td = timedelta(milliseconds=time) print "Time to generate: %s [d h:mm:ss]" % (duration_format(td)) c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0 price += c print 'S3 PUT: %0.2f [$]' % c if 'ec2' in gene.config: c = time * gene.config['cost']['ec2']['usage'] / (1000.0 * 3600) price += c print 'EC2 usage: %0.2f [$]' % c c = gene.config['cost']['esb']['io'] * time / (1000.0 * 2600 * 24 * 30) price += c print 'ESB usage: %0.2f [$]' % c if 'sqs' in gene.layer: if meta: nb_sqs = nb_metatiles[z] * 3 else: nb_sqs = nb_tiles[z] * 3 c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0 price += c print 'SQS usage: %0.2f [$]' % c print td = timedelta(milliseconds=all_time) print "Number of tiles: %i" % all_tiles print 'Generation time: %s [d h:mm:ss]' % (duration_format(td)) print 'Generation cost: %0.2f [$]' % price return (all_size, td, price, all_tiles)
def generate_resume(self, layer): if self._options.time is None: if layer is not None: all_dimensions = self._gene.get_all_dimensions(layer) message = [ "The tile generation of layer '{}{}' is finish".format( layer["name"], "" if ((len(all_dimensions) == 1 and len(all_dimensions[0]) == 0) or layer["type"] != "wms") else " ({})".format(" - ".join([ ", ".join( ["=".join(d) for d in dimensions.items()]) for dimensions in all_dimensions ])), ), ] else: message = ["The tile generation is finish"] if self._options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( self._count_tiles.nb)) elif layer.get( "meta" ) if layer is not None else self._options.role == "slave": message += [ "Nb generated metatiles: {}".format( self._count_metatiles.nb), "Nb metatiles dropped: {}".format( self._count_metatiles_dropped.nb), ] if self._options.role != "master": message += [ "Nb generated tiles: {}".format(self._count_tiles.nb), "Nb tiles dropped: {}".format( self._count_tiles_dropped.nb), ] if self._options.role in ("local", "slave"): message += [ "Nb tiles stored: {}".format( self._count_tiles_stored.nb), "Nb tiles in error: {}".format(self._gene.error), "Total time: {}".format( duration_format(self._gene.duration)), ] if self._count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(self._count_tiles_stored.size))) if self._count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (self._gene.duration / self._count_tiles.nb * 1000).seconds)) if self._count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( self._count_tiles_stored.size / self._count_tiles_stored.nb)) if not self._options.quiet and self._options.role in ( "local", "slave", "master") and message: print("\n".join(message) + "\n") if self._cache_tilestore is not None and hasattr( self._cache_tilestore, "connection"): self._cache_tilestore.connection.close() if (self._options.role != "hash" and self._options.time is None and "sns" in self._gene.config): # pragma: no cover if "region" in self._gene.config["sns"]: sns_client = boto3.client( "sns", region_name=self._gene.config["sns"]["region"]) else: sns_client = boto3.client("sns") sns_message = [message[0]] sns_message += [ "Layer: {}".format( layer["name"] if layer is not None else "(All layers)"), "Role: {}".format(self._options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(" ".join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] sns_client.publish( TopicArn=self._gene.config["sns"]["topic"], Message="\n".join(sns_message), Subject="Tile generation ({layer} - {role})".format( role=self._options.role, layer=layer["name"] if layer is not None else "All layers"), )
def _calculate_cost(gene, options): validate_calculate_cost(gene) nb_metatiles = {} nb_tiles = {} meta = gene.layer['meta'] if options.cost_algo == 'area': tile_size = gene.layer['grid_ref']['tile_size'] for zoom, resolution in enumerate( gene.layer['grid_ref']['resolutions']): if 'min_resolution_seed' in gene.layer and resolution < gene.layer[ 'min_resolution_seed']: continue print "Calculate zoom %i." % zoom px_buffer = gene.layer['px_buffer'] + \ gene.layer['meta_buffer'] if meta else 0 m_buffer = px_buffer * resolution if meta: size = tile_size * gene.layer['meta_size'] * resolution meta_buffer = size * 0.7 + m_buffer meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1) nb_metatiles[zoom] = int(round(meta_geom.area / size**2)) size = tile_size * resolution tile_buffer = size * 0.7 + m_buffer geom = gene.geoms[zoom].buffer(tile_buffer, 1) nb_tiles[zoom] = int(round(geom.area / size**2)) elif options.cost_algo == 'count': gene.init_tilecoords() gene.add_geom_filter() if meta: def count_metatile(tile): if tile: if tile.tilecoord.z in nb_metatiles: nb_metatiles[tile.tilecoord.z] += 1 else: nb_metatiles[tile.tilecoord.z] = 1 return tile gene.imap(count_metatile) class MetaTileSplitter(TileStore): def get(self, tiles): for metatile in tiles: for tilecoord in metatile.tilecoord: yield Tile(tilecoord) gene.tilestream = MetaTileSplitter().get(gene.tilestream) # Only keep tiles that intersect geometry gene.add_geom_filter() def count_tile(tile): if tile: if tile.tilecoord.z in nb_tiles: nb_tiles[tile.tilecoord.z] += 1 else: print "Calculate zoom %i." % tile.tilecoord.z nb_tiles[tile.tilecoord.z] = 1 return tile gene.imap(count_tile) consume(gene.tilestream, None) times = {} print for z in nb_metatiles: print "%i meta tiles in zoom %i." % (nb_metatiles[z], z) times[z] = gene.layer['cost'][ 'metatile_generation_time'] * nb_metatiles[z] price = 0 all_size = 0 all_time = 0 all_tiles = 0 for z in nb_tiles: print print "%i tiles in zoom %i." % (nb_tiles[z], z) all_tiles += nb_tiles[z] if meta: time = times[ z] + gene.layer['cost']['tile_generation_time'] * nb_tiles[z] else: time = gene.layer['cost']['tileonly_generation_time'] * nb_tiles[z] size = gene.layer['cost']['tile_size'] * nb_tiles[z] all_size += size all_time += time td = timedelta(milliseconds=time) print "Time to generate: %s [d h:mm:ss]" % (duration_format(td)) c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0 price += c print 'S3 PUT: %0.2f [$]' % c if 'ec2' in gene.config: c = time * gene.config['cost']['ec2']['usage'] / (1000.0 * 3600) price += c print 'EC2 usage: %0.2f [$]' % c c = gene.config['cost']['esb']['io'] * time / (1000.0 * 2600 * 24 * 30) price += c print 'ESB usage: %0.2f [$]' % c if 'sqs' in gene.layer: if meta: nb_sqs = nb_metatiles[z] * 3 else: nb_sqs = nb_tiles[z] * 3 c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0 price += c print 'SQS usage: %0.2f [$]' % c print td = timedelta(milliseconds=all_time) print "Number of tiles: %i" % all_tiles print 'Generation time: %s [d h:mm:ss]' % (duration_format(td)) print 'Generation cost: %0.2f [$]' % price return (all_size, td, price, all_tiles)
def _calculate_cost(gene, layer, options): nb_metatiles = {} nb_tiles = {} meta = layer["meta"] if options.cost_algo == "area": tile_size = layer["grid_ref"]["tile_size"] for zoom, resolution in enumerate(layer["grid_ref"]["resolutions"]): if "min_resolution_seed" in layer and resolution < layer[ "min_resolution_seed"]: continue print("Calculate zoom {}.".format(zoom)) px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0 m_buffer = px_buffer * resolution if meta: size = tile_size * layer["meta_size"] * resolution meta_buffer = size * 0.7 + m_buffer meta_geom = gene.geoms[layer["name"]][zoom].buffer( meta_buffer, 1) nb_metatiles[zoom] = int(round(meta_geom.area / size**2)) size = tile_size * resolution tile_buffer = size * 0.7 + m_buffer geom = gene.geoms[layer["name"]][zoom].buffer(tile_buffer, 1) nb_tiles[zoom] = int(round(geom.area / size**2)) elif options.cost_algo == "count": gene.init_tilecoords(layer) gene.add_geom_filter() if meta: def count_metatile(tile): if tile: if tile.tilecoord.z in nb_metatiles: nb_metatiles[tile.tilecoord.z] += 1 else: nb_metatiles[tile.tilecoord.z] = 1 return tile gene.imap(count_metatile) class MetaTileSplitter(TileStore): @staticmethod def get(tiles): for metatile in tiles: for tilecoord in metatile.tilecoord: yield Tile(tilecoord) gene.add_metatile_splitter(MetaTileSplitter()) # Only keep tiles that intersect geometry gene.add_geom_filter() def count_tile(tile): if tile: if tile.tilecoord.z in nb_tiles: nb_tiles[tile.tilecoord.z] += 1 else: print("Calculate zoom {}.".format(tile.tilecoord.z)) nb_tiles[tile.tilecoord.z] = 1 return tile gene.imap(count_tile) run = Run(gene, gene.functions_metatiles) for tile in gene.tilestream: tile.metadata["layer"] = layer["name"] run(tile) times = {} print("") for z in nb_metatiles: print("{} meta tiles in zoom {}.".format(nb_metatiles[z], z)) times[z] = layer["cost"]["metatile_generation_time"] * nb_metatiles[z] price = 0 all_size = 0 all_time = 0 all_tiles = 0 for z in nb_tiles: print("") print("{} tiles in zoom {}.".format(nb_tiles[z], z)) all_tiles += nb_tiles[z] if meta: time = times[ z] + layer["cost"]["tile_generation_time"] * nb_tiles[z] else: time = layer["cost"]["tileonly_generation_time"] * nb_tiles[z] size = layer["cost"]["tile_size"] * nb_tiles[z] all_size += size all_time += time td = timedelta(milliseconds=time) print("Time to generate: {} [d h:mm:ss]".format((duration_format(td)))) c = gene.config["cost"]["s3"]["put"] * nb_tiles[z] / 1000.0 price += c print("S3 PUT: {0:0.2f} [$]".format(c)) if "sqs" in gene.config: if meta: nb_sqs = nb_metatiles[z] * 3 else: nb_sqs = nb_tiles[z] * 3 c = nb_sqs * gene.config["cost"]["sqs"]["request"] / 1000000.0 price += c print("SQS usage: {0:0.2f} [$]".format(c)) print("") td = timedelta(milliseconds=all_time) print("Number of tiles: {}".format(all_tiles)) print("Generation time: {} [d h:mm:ss]".format((duration_format(td)))) print("Generation cost: {0:0.2f} [$]".format(price)) return all_size, td, price, all_tiles
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord)) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join( gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer[ 'output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()])), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format( count_metatiles.nb), "Nb metatiles dropped: {}".format( count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds)) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print( "Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord) ) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e) ) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e) ) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit( 'the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.' ) params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get(URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ),), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get(MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get(MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find("application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data) ) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data ) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n ** 2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print( """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] } )
def _calculate_cost(gene, layer, options): nb_metatiles = {} nb_tiles = {} meta = layer['meta'] if options.cost_algo == 'area': tile_size = layer['grid_ref']['tile_size'] for zoom, resolution in enumerate(layer['grid_ref']['resolutions']): if 'min_resolution_seed' in layer and resolution < layer['min_resolution_seed']: continue print("Calculate zoom {}.".format(zoom)) px_buffer = layer['px_buffer'] + \ layer['meta_buffer'] if meta else 0 m_buffer = px_buffer * resolution if meta: size = tile_size * layer['meta_size'] * resolution meta_buffer = size * 0.7 + m_buffer meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1) nb_metatiles[zoom] = int(round(meta_geom.area / size ** 2)) size = tile_size * resolution tile_buffer = size * 0.7 + m_buffer geom = gene.geoms[zoom].buffer(tile_buffer, 1) nb_tiles[zoom] = int(round(geom.area / size ** 2)) elif options.cost_algo == 'count': gene.init_tilecoords(layer) gene.add_geom_filter(layer) if meta: def count_metatile(tile): if tile: if tile.tilecoord.z in nb_metatiles: nb_metatiles[tile.tilecoord.z] += 1 else: nb_metatiles[tile.tilecoord.z] = 1 return tile gene.imap(count_metatile) class MetaTileSplitter(TileStore): @staticmethod def get(tiles): for metatile in tiles: for tilecoord in metatile.tilecoord: yield Tile(tilecoord) gene.tilestream = MetaTileSplitter().get(gene.tilestream) # Only keep tiles that intersect geometry gene.add_geom_filter(layer) def count_tile(tile): if tile: if tile.tilecoord.z in nb_tiles: nb_tiles[tile.tilecoord.z] += 1 else: print("Calculate zoom {}.".format(tile.tilecoord.z)) nb_tiles[tile.tilecoord.z] = 1 return tile gene.imap(count_tile) consume(gene.tilestream, None) times = {} print('') for z in nb_metatiles: print("{} meta tiles in zoom {}.".format(nb_metatiles[z], z)) times[z] = layer['cost']['metatile_generation_time'] * nb_metatiles[z] price = 0 all_size = 0 all_time = 0 all_tiles = 0 for z in nb_tiles: print('') print("{} tiles in zoom {}.".format(nb_tiles[z], z)) all_tiles += nb_tiles[z] if meta: time = times[z] + layer['cost']['tile_generation_time'] * nb_tiles[z] else: time = layer['cost']['tileonly_generation_time'] * nb_tiles[z] size = layer['cost']['tile_size'] * nb_tiles[z] all_size += size all_time += time td = timedelta(milliseconds=time) print("Time to generate: {} [d h:mm:ss]".format((duration_format(td)))) c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0 price += c print('S3 PUT: {0:0.2f} [$]'.format(c)) if 'sqs' in gene.config: if meta: nb_sqs = nb_metatiles[z] * 3 else: nb_sqs = nb_tiles[z] * 3 c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0 price += c print('SQS usage: {0:0.2f} [$]'.format(c)) print("") td = timedelta(milliseconds=all_time) print("Number of tiles: {}".format(all_tiles)) print('Generation time: {} [d h:mm:ss]'.format((duration_format(td)))) print('Generation cost: {0:0.2f} [$]'.format(price)) return all_size, td, price, all_tiles
def generate_resume(self, layer): if self._options.time is None: if layer is not None: all_dimensions = self._gene.get_all_dimensions(layer) message = [ "The tile generation of layer '{}{}' is finish".format( layer['name'], "" if ( (len(all_dimensions) == 1 and len(all_dimensions[0]) == 0) or layer['type'] != 'wms' ) else " ({})".format( " - ".join([ ", ".join( ["=".join(d) for d in dimensions.items()] ) for dimensions in all_dimensions ]) ) ), ] else: message = [ "The tile generation is finish" ] if self._options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format(self._count_tiles.nb)) elif layer.get('meta') if layer is not None else self._options.role == "slave": message += [ "Nb generated metatiles: {}".format(self._count_metatiles.nb), "Nb metatiles dropped: {}".format(self._count_metatiles_dropped.nb), ] if self._options.role != "master": message += [ "Nb generated tiles: {}".format(self._count_tiles.nb), "Nb tiles dropped: {}".format(self._count_tiles_dropped.nb), ] if self._options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(self._count_tiles_stored.nb), "Nb tiles in error: {}".format(self._gene.error), "Total time: {}".format(duration_format(self._gene.duration)), ] if self._count_tiles_stored.nb != 0: message.append("Total size: {}".format(size_format(self._count_tiles_stored.size))) if self._count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (self._gene.duration / self._count_tiles.nb * 1000).seconds) ) if self._count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( self._count_tiles_stored.size / self._count_tiles_stored.nb) ) if not self._options.quiet and self._options.role in ('local', 'slave', 'master') and message: print("\n".join(message) + "\n") if self._cache_tilestore is not None and hasattr(self._cache_tilestore, 'connection'): self._cache_tilestore.connection.close() if self._options.role != 'hash' and \ self._options.time is None and \ 'sns' in self._gene.config: # pragma: no cover if 'region' in self._gene.config['sns']: sns_client = boto3.client('sns', region_name=self._gene.config['sns']['region']) else: sns_client = boto3.client('sns') sns_message = [message[0]] sns_message += [ "Layer: {}".format(layer['name'] if layer is not None else "(All layers)"), "Role: {}".format(self._options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] sns_client.publish(TopicArn=self._gene.config['sns']['topic'], Message="\n".join(sns_message), Subject="Tile generation ({layer} - {role})".format( role=self._options.role, layer=layer['name'] if layer is not None else "All layers"))
def _calculate_cost(gene, options): nb_metatiles = {} nb_tiles = {} meta = gene.layer['meta'] if options.cost_algo == 'area': tile_size = gene.layer['grid_ref']['tile_size'] for zoom, resolution in enumerate(gene.layer['grid_ref']['resolutions']): if 'min_resolution_seed' in gene.layer and resolution < gene.layer['min_resolution_seed']: continue print("Calculate zoom {}.".format(zoom)) px_buffer = gene.layer['px_buffer'] + \ gene.layer['meta_buffer'] if meta else 0 m_buffer = px_buffer * resolution if meta: size = tile_size * gene.layer['meta_size'] * resolution meta_buffer = size * 0.7 + m_buffer meta_geom = gene.geoms[zoom].buffer(meta_buffer, 1) nb_metatiles[zoom] = int(round(meta_geom.area / size ** 2)) size = tile_size * resolution tile_buffer = size * 0.7 + m_buffer geom = gene.geoms[zoom].buffer(tile_buffer, 1) nb_tiles[zoom] = int(round(geom.area / size ** 2)) elif options.cost_algo == 'count': gene.init_tilecoords() gene.add_geom_filter() if meta: def count_metatile(tile): if tile: if tile.tilecoord.z in nb_metatiles: nb_metatiles[tile.tilecoord.z] += 1 else: nb_metatiles[tile.tilecoord.z] = 1 return tile gene.imap(count_metatile) class MetaTileSplitter(TileStore): @staticmethod def get(tiles): for metatile in tiles: for tilecoord in metatile.tilecoord: yield Tile(tilecoord) gene.tilestream = MetaTileSplitter().get(gene.tilestream) # Only keep tiles that intersect geometry gene.add_geom_filter() def count_tile(tile): if tile: if tile.tilecoord.z in nb_tiles: nb_tiles[tile.tilecoord.z] += 1 else: print("Calculate zoom {}.".format(tile.tilecoord.z)) nb_tiles[tile.tilecoord.z] = 1 return tile gene.imap(count_tile) consume(gene.tilestream, None) times = {} print('') for z in nb_metatiles: print("{} meta tiles in zoom {}.".format(nb_metatiles[z], z)) times[z] = gene.layer['cost']['metatile_generation_time'] * nb_metatiles[z] price = 0 all_size = 0 all_time = 0 all_tiles = 0 for z in nb_tiles: print('') print("{} tiles in zoom {}.".format(nb_tiles[z], z)) all_tiles += nb_tiles[z] if meta: time = times[z] + gene.layer['cost']['tile_generation_time'] * nb_tiles[z] else: time = gene.layer['cost']['tileonly_generation_time'] * nb_tiles[z] size = gene.layer['cost']['tile_size'] * nb_tiles[z] all_size += size all_time += time td = timedelta(milliseconds=time) print("Time to generate: {} [d h:mm:ss]".format((duration_format(td)))) c = gene.config['cost']['s3']['put'] * nb_tiles[z] / 1000.0 price += c print('S3 PUT: {0:0.2f} [$]'.format(c)) if 'sqs' in gene.config: if meta: nb_sqs = nb_metatiles[z] * 3 else: nb_sqs = nb_tiles[z] * 3 c = nb_sqs * gene.config['cost']['sqs']['request'] / 1000000.0 price += c print('SQS usage: {0:0.2f} [$]'.format(c)) print("") td = timedelta(milliseconds=all_time) print("Number of tiles: {}".format(all_tiles)) print('Generation time: {} [d h:mm:ss]'.format((duration_format(td)))) print('Generation cost: {0:0.2f} [$]'.format(price)) return (all_size, td, price, all_tiles)
def _gene(self, options, gene, dimensions=None): if dimensions is None: # pragma: no cover dimensions = {} self.dimensions = dimensions self.count_metatiles = None self.count_metatiles_dropped = Count() self.count_tiles = Count() self.count_tiles_dropped = Count() self.count_tiles_stored = None self.sqs_tilestore = None self.cache_tilestore = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [{},{},{},{}]".format(*default_int( gene.layer['grid_ref']['obj'].extent(tilecoord)))) exit() except ValueError as e: # pragma: no cover print( "Tile '{}' is not in the format 'z/x/y' or z/x/y:+n/+n\n{}" .format(options.get_bbox, repr(e))) exit(1) if options.get_hash: options.role = 'hash' options.test = 1 if options.role in ('master', 'slave'): # Create SQS queue self.sqs_tilestore = SQSTileStore( gene.get_sqs_queue(), on_empty=await_message if options.daemon else maybe_stop) # pragma: no cover if options.role in ('local', 'slave'): self.cache_tilestore = gene.get_tilesstore(options.cache, dimensions) if options.tiles: gene.set_store(TilesFileStore(options.tiles, options.layer)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.role in ('local', 'master') and 'logging' in gene.config: gene.imap( DatabaseLoggerInit(gene.config['logging'], options is not None and options.daemon)) if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(self.sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if gene.layer.get('meta'): gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '{}' is not in the format 'z/x/y'\n{}".format( options.get_hash, repr(e))) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() self.count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(self.sqs_tilestore) self.count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): gene.get( MultiTileStore({ name: self._get_tilestore_for_layer(layer, gene) for name, layer in gene.layers.items() }), 'Get tile') def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: {}".format( (self._re_rm_xml_tag.sub('', tile.error))) else: # pragma: no cover tile.error = "{} is not an image format, error: {}".format( tile.content_type, tile.error) return tile gene.imap(wrong_content_type_to_error) gene.add_error_filters() if options.role == 'hash': if gene.layer.get('meta', False): gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: droppers = {} for lname, layer in gene.layers.items(): if 'empty_metatile_detection' in layer: empty_tile = layer['empty_metatile_detection'] droppers[lname] = HashDropper( empty_tile['size'], empty_tile['hash'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=self.count_metatiles_dropped, ) if droppers: gene.imap(MultiAction(droppers)) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) gene.imap(self.count_tiles) gene.process(key='pre_hash_post_process') if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: droppers = {} for lname, layer in gene.layers.items(): if 'empty_tile_detection' in layer: empty_tile = layer['empty_tile_detection'] droppers[lname] = HashDropper( empty_tile['size'], empty_tile['hash'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=self.count_tiles_dropped, ) if droppers: gene.imap(MultiAction(droppers)) gene.process() else: # pragma: no cover self.count_tiles = gene.counter() if options.role in ('local', 'slave'): self.count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: {}\n'.format(len(tile.data))) return tile gene.imap(log_size) gene.put(self.cache_tilestore, "Store the tile") if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('{}\n'.format(tile.tilecoord)) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover def delete_from_store(tile): if hasattr(tile, 'metatile'): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: self.sqs_tilestore.delete_one(tile.metatile) else: self.sqs_tilestore.delete_one(tile) return True gene.ifilter(delete_from_store) if options.role in ('local', 'slave') and 'logging' in gene.config: gene.imap( DatabaseLogger(gene.config['logging'], options is not None and options.daemon)) gene.add_error_filters() message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: {}\n'.format( ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds))) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if gene.layer is not None: message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " ({})".format(", ".join( ["=".join(d) for d in dimensions.items()]))), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( self.count_tiles.nb)) else: if gene.layer.get('meta'): message += [ "Nb generated metatiles: {}".format( self.count_metatiles.nb), "Nb metatiles dropped: {}".format( self.count_metatiles_dropped.nb), ] else: message = ["The tile generation is finish"] if options.role != "master": message += [ "Nb generated tiles: {}".format(self.count_tiles.nb), "Nb tiles dropped: {}".format(self.count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format( self.count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if self.count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(self.count_tiles_stored.size))) if self.count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / self.count_tiles.nb * 1000).seconds)) if self.count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( self.count_tiles_stored.size / self.count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if self.cache_tilestore is not None and hasattr( self.cache_tilestore, 'connection'): self.cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name'] if gene. layer is not None else "(All layers)"), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation ({layer!s} - {role!s})".format( **{ 'role': options.role, 'layer': gene.layer['name'] if gene. layer is not None else "All layers" }))