def process() -> None: """Copy the tiles from a cache to an other.""" try: parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_common_options(parser, near=False, time=False, dimensions=True) parser.add_argument("process", metavar="PROCESS", help="The process name to do") options = parser.parse_args() gene = TileGeneration(options.config, options, multi_thread=False) copy = Copy() if options.layer: copy.copy(options, gene, options.layer, options.cache, options.cache, "process") else: assert gene.config_file config = gene.get_config(gene.config_file) layers_name = (config.config["generation"]["default_layers"] if "default_layers" in config.config.get( "generation", {}) else config.config["layers"].keys()) for layer in layers_name: copy.copy(options, gene, layer, options.cache, options.cache, "process") except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") sys.exit(1)
def _fill_legend( gene: TileGeneration, cache: tilecloud_chain.configuration.Cache, server: Optional[tilecloud_chain.configuration.Server], base_urls: List[str], ) -> None: assert gene.config_file config = gene.get_config(gene.config_file) for layer_name, layer in config.config["layers"].items(): previous_legend: Optional[tilecloud_chain.Legend] = None previous_resolution = None if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends: gene.layer_legends[layer_name] = [] legends = gene.layer_legends[layer_name] for zoom, resolution in enumerate( config.config["grids"][layer["grid"]]["resolutions"]): path = "/".join([ "1.0.0", layer_name, layer["wmts_style"], f"legend{zoom}.{layer['legend_extension']}", ]) img = _get(path, cache) if img is not None: new_legend: tilecloud_chain.Legend = { "mime_type": layer["legend_mime"], "href": os.path.join( base_urls[0], server.get("static_path", "static") + "/" if server else "", path), } legends.append(new_legend) if previous_legend is not None: assert previous_resolution is not None middle_res = exp( (log(previous_resolution) + log(resolution)) / 2) previous_legend[ # pylint: disable=unsupported-assignment-operation "min_resolution"] = middle_res new_legend["max_resolution"] = middle_res try: pil_img = Image.open(BytesIO(img)) new_legend["width"] = pil_img.size[0] new_legend["height"] = pil_img.size[1] except Exception: # pragma: nocover logger.warning( "Unable to read legend image '%s', with '%s'", path, repr(img), exc_info=True, ) previous_legend = new_legend previous_resolution = resolution
def _copy( self, options: Namespace, gene: TileGeneration, layer_name: str, source: str, dest: str, task_name: str, ) -> None: # disable metatiles assert gene.config_file config = gene.get_config(gene.config_file) layer = config.config["layers"][layer_name] cast(tilecloud_chain.configuration.LayerWms, layer)["meta"] = False count_tiles_dropped = Count() gene.create_log_tiles_error(layer_name) source_tilestore = gene.get_tilesstore(source) dest_tilestore = gene.get_tilesstore(dest) gene.init_tilecoords(config, layer_name) gene.add_geom_filter() gene.add_logger() gene.get(source_tilestore, "Get the tiles") gene.imap(DropEmpty(gene)) # Discard tiles with certain content if "empty_tile_detection" in layer: empty_tile = layer["empty_tile_detection"] gene.imap( HashDropper(empty_tile["size"], empty_tile["hash"], store=dest_tilestore, count=count_tiles_dropped)) if options.process: gene.process(options.process) gene.imap(DropEmpty(gene)) self.count = gene.counter_size() gene.put(dest_tilestore, "Store the tiles") gene.consume() if not options.quiet: print(f"""The tile {task_name} of layer '{layer_name}' is finish Nb {task_name} tiles: {self.count.nb} Nb errored tiles: {gene.error} Nb dropped tiles: {count_tiles_dropped.nb} Total time: {duration_format(gene.duration)} Total size: {size_format(self.count.size)} Time per tile: {(gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0} ms Size per tile: {self.count.size / self.count.nb if self.count.nb != 0 else -1} o """)
def main() -> None: """Copy the tiles from a cache to an other.""" try: parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_common_options(parser, near=False, time=False, dimensions=True, cache=False) parser.add_argument("--process", dest="process", metavar="NAME", help="The process name to do") parser.add_argument("source", metavar="SOURCE", help="The source cache") parser.add_argument("dest", metavar="DEST", help="The destination cache") options = parser.parse_args() gene = TileGeneration(options.config, options) assert gene.config_file config = gene.get_config(gene.config_file) if options.layer: copy = Copy() copy.copy(options, gene, options.layer, options.source, options.dest, "copy") else: layers = (config.config["generation"]["default_layers"] if "default_layers" in config.config["generation"] else config.config["layers"].keys()) for layer in layers: copy = Copy() copy.copy(options, gene, layer, options.source, options.dest, "copy") except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise sys.exit(1)
def get_wmts_capabilities(gene: TileGeneration, cache_name: str, exit_: bool = False) -> Optional[str]: """Get the WMTS capabilities for a configuration file.""" assert gene.config_file config = gene.get_config(gene.config_file) cache = config.config["caches"][cache_name] if _validate_generate_wmts_capabilities(cache, cache_name, exit_): server = gene.get_main_config().config.get("server") base_urls = _get_base_urls(cache) _fill_legend(gene, cache, server, base_urls) data = pkgutil.get_data("tilecloud_chain", "wmts_get_capabilities.jinja") assert data return cast( str, jinja2_template( data.decode("utf-8"), layers=config.config["layers"], layer_legends=gene.layer_legends, grids=config.config["grids"], getcapabilities=urljoin( # type: ignore base_urls[0], (server.get("wmts_path", "wmts") + "/1.0.0/WMTSCapabilities.xml" if server is not None else cache.get("wmtscapabilities_file", "1.0.0/WMTSCapabilities.xml")), ), base_urls=base_urls, base_url_postfix=(server.get("wmts_path", "wmts") + "/") if server is not None else "", get_tile_matrix_identifier=get_tile_matrix_identifier, server=server is not None, has_metadata="metadata" in config.config, metadata=config.config.get("metadata"), has_provider="provider" in config.config, provider=config.config.get("provider"), enumerate=enumerate, ceil=math.ceil, int=int, sorted=sorted, ), ) return None
def main() -> None: """Run the tiles generation.""" try: stats.init_backends({}) parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0]) add_common_options(parser, dimensions=True) parser.add_argument( "--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y" ) parser.add_argument( "--get-bbox", metavar="TILE", help="get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles", ) parser.add_argument( "--role", default="local", choices=("local", "master", "slave"), help="local/master/slave, master to file the queue and slave to generate the tiles", ) parser.add_argument( "--local-process-number", default=None, help="The number of process that we run in parallel" ) parser.add_argument( "--detach", default=False, action="store_true", help="run detached from the terminal" ) parser.add_argument( "--daemon", default=False, action="store_true", help="run continuously as a daemon" ) parser.add_argument( "--tiles", metavar="FILE", help="Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles", ) options = parser.parse_args() if options.detach: detach() gene = TileGeneration( config_file=options.config, options=options, multi_thread=options.get_hash is None ) if ( options.get_hash is None and options.get_bbox is None and options.config is not None and "authorised_user" in gene.get_main_config().config.get("generation", {}) and gene.get_main_config().config["generation"]["authorised_user"] != getuser() ): logger.error( "not authorised, authorised user is: %s.", gene.get_main_config().config["generation"]["authorised_user"], ) sys.exit(1) if options.config: config = gene.get_config(options.config) if options.cache is None and options.config: options.cache = config.config["generation"]["default_cache"] if options.tiles is not None and options.role not in ["local", "master"]: logger.error("The --tiles option work only with role local or master") sys.exit(1) try: generate = Generate(options, gene) if options.role == "slave": generate.gene() elif options.layer: generate.gene(options.layer) elif options.get_bbox: logger.error("With --get-bbox option you need to specify a layer") sys.exit(1) elif options.get_hash: logger.error("With --get-hash option you need to specify a layer") sys.exit(1) else: if options.config: for layer in config.config["generation"].get( "default_layers", config.config["layers"].keys() ): generate.gene(layer) except tilecloud.filter.error.TooManyErrors: logger.exception("Too many errors") sys.exit(1) finally: gene.close() except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise sys.exit(1)
def _calculate_cost(gene: TileGeneration, layer_name: str, options: Namespace) -> Tuple[float, timedelta, float, int]: nb_metatiles = {} nb_tiles = {} config = gene.get_config(options.config) layer = config.config["layers"][layer_name] meta = layer["meta"] if options.cost_algo == "area": tile_size = config.config["grids"][layer["grid"]]["tile_size"] for zoom, resolution in enumerate( config.config["grids"][layer["grid"]]["resolutions"]): if "min_resolution_seed" in layer and resolution < layer[ "min_resolution_seed"]: continue print(f"Calculate zoom {zoom}.") px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0 m_buffer = px_buffer * resolution if meta: size = tile_size * layer["meta_size"] * resolution meta_buffer = size * 0.7 + m_buffer meta_geom = gene.get_geoms(config, layer_name)[zoom].buffer( meta_buffer, 1) nb_metatiles[zoom] = int(round(meta_geom.area / size**2)) size = tile_size * resolution tile_buffer = size * 0.7 + m_buffer geom = gene.get_geoms(config, layer_name)[zoom].buffer(tile_buffer, 1) nb_tiles[zoom] = int(round(geom.area / size**2)) elif options.cost_algo == "count": gene.init_tilecoords(config, layer_name) gene.add_geom_filter() if meta: def count_metatile(tile: Tile) -> Tile: if tile: if tile.tilecoord.z in nb_metatiles: nb_metatiles[tile.tilecoord.z] += 1 else: nb_metatiles[tile.tilecoord.z] = 1 return tile gene.imap(count_metatile) class MetaTileSplitter(TileStore): """Convert the metatile flow to tile flow.""" @staticmethod def get(tiles: Iterable[Tile]) -> Iterator[Tile]: for metatile in tiles: for tilecoord in metatile.tilecoord: yield Tile(tilecoord) gene.add_metatile_splitter(MetaTileSplitter()) # Only keep tiles that intersect geometry gene.add_geom_filter() def count_tile(tile: Tile) -> Tile: if tile: if tile.tilecoord.z in nb_tiles: nb_tiles[tile.tilecoord.z] += 1 else: print(f"Calculate zoom {tile.tilecoord.z}.") nb_tiles[tile.tilecoord.z] = 1 return tile gene.imap(count_tile) run = Run(gene, gene.functions_metatiles) assert gene.tilestream for tile in gene.tilestream: tile.metadata["layer"] = layer_name run(tile) times = {} print() for z, nb_metatile in nb_metatiles.items(): print(f"{nb_metatile} meta tiles in zoom {z}.") times[z] = layer["cost"]["metatile_generation_time"] * nb_metatile price: float = 0 all_size: float = 0 all_time: float = 0 all_tiles = 0 for z, nb_tile in nb_tiles.items(): print() print(f"{nb_tile} tiles in zoom {z}.") all_tiles += nb_tile if meta: time = times[z] + layer["cost"]["tile_generation_time"] * nb_tile else: time = layer["cost"]["tileonly_generation_time"] * nb_tile size = layer["cost"]["tile_size"] * nb_tile all_size += size all_time += time td = timedelta(milliseconds=time) print(f"Time to generate: {duration_format(td)} [d h:mm:ss]") c = gene.get_main_config( ).config["cost"]["s3"]["put"] * nb_tile / 1000.0 price += c print(f"S3 PUT: {c:0.2f} [$]") if "sqs" in gene.get_main_config().config: if meta: nb_sqs = nb_metatiles[z] * 3 else: nb_sqs = nb_tile * 3 c = nb_sqs * gene.get_main_config( ).config["cost"]["sqs"]["request"] / 1000000.0 price += c print(f"SQS usage: {c:0.2f} [$]") print() td = timedelta(milliseconds=all_time) print(f"Number of tiles: {all_tiles}") print(f"Generation time: {duration_format(td)} [d h:mm:ss]") print(f"Generation cost: {price:0.2f} [$]") return all_size, td, price, all_tiles
def main() -> None: """Calculate the cost, main function.""" try: parser = ArgumentParser( description="Used to calculate the generation cost", prog=sys.argv[0]) add_common_options(parser, tile_pyramid=False, dimensions=True) parser.add_argument( "--cost-algo", "--calculate-cost-algorithm", default="area", dest="cost_algo", choices=("area", "count"), help= "The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate.", ) options = parser.parse_args() gene = TileGeneration( options.config, options=options, layer_name=options.layer, base_config={"cost": {}}, multi_thread=False, ) config = gene.get_config(options.config) all_size: float = 0 tile_size: float = 0 all_tiles = 0 if options.layer: layer = config.config["layers"][options.layer] (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options.layer, options) tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer_name in gene.get_config( options.config).config["generation"]["default_layers"]: print() print(f"===== {layer_name} =====") layer = config.config["layers"][layer_name] gene.create_log_tiles_error(layer_name) (size, time, price, tiles) = _calculate_cost(gene, layer_name, options) tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print() print("===== GLOBAL =====") print(f"Total number of tiles: {all_tiles}") print( f"Total generation time: {duration_format(all_time)} [d h:mm:ss]" ) print(f"Total generation cost: {all_price:0.2f} [$]") print() s3_cost = all_size * gene.get_main_config( ).config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024) print(f"S3 Storage: {s3_cost:0.2f} [$/month]") s3_get_cost = ( gene.get_main_config().config["cost"]["s3"]["get"] * config.config["cost"]["request_per_layers"] / 10000.0 + gene.get_main_config().config["cost"]["s3"]["download"] * config.config["cost"]["request_per_layers"] * tile_size) print(f"S3 get: {s3_get_cost:0.2f} [$/month]") # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * # gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * # gene.config['cost']['request_per_layers'] * tile_size) except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") sys.exit(1)
def _generate_legend_images(gene: TileGeneration) -> None: assert gene.config_file config = gene.get_config(gene.config_file) cache = config.config["caches"][gene.options.cache] for layer_name, layer in config.config["layers"].items(): if "legend_mime" in layer and "legend_extension" in layer: if layer["type"] == "wms": session = requests.session() session.headers.update(layer["headers"]) previous_hash = None for zoom, resolution in enumerate( config.config["grids"][layer["grid"]]["resolutions"]): legends = [] for wmslayer in layer["layers"].split(","): response = session.get(layer["url"] + "?" + urlencode({ "SERVICE": "WMS", "VERSION": layer.get("version", "1.0.0"), "REQUEST": "GetLegendGraphic", "LAYER": wmslayer, "FORMAT": layer["legend_mime"], "TRANSPARENT": "TRUE" if layer["legend_mime"] == "image/png" else "FALSE", "STYLE": layer["wmts_style"], "SCALE": resolution / 0.00028, })) try: legends.append( Image.open(BytesIO(response.content))) except Exception: # pragma: nocover logger.warning( "Unable to read legend image for layer '%s'-'%s', resolution '%s': %s", layer_name, wmslayer, resolution, response.content, exc_info=True, ) width = max(i.size[0] for i in legends) height = sum(i.size[1] for i in legends) image = Image.new("RGBA", (width, height)) y = 0 for i in legends: image.paste(i, (0, y)) y += i.size[1] string_io = BytesIO() image.save(string_io, FORMAT_BY_CONTENT_TYPE[layer["legend_mime"]]) result = string_io.getvalue() new_hash = sha1(result).hexdigest() # nosec if new_hash != previous_hash: previous_hash = new_hash _send( result, f"1.0.0/{layer_name}/{layer['wmts_style']}/" f"legend{zoom}.{layer['legend_extension']}", layer["legend_mime"], cache, )
def main() -> None: """Generate the contextual file like the legends.""" try: stats.init_backends({}) parser = ArgumentParser( description= "Used to generate the contextual file like the capabilities, the legends, " "the OpenLayers example", prog=sys.argv[0], ) add_common_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument("--status", default=False, action="store_true", help="Display the SQS queue status and exit") parser.add_argument( "--legends", "--generate-legend-images", default=False, action="store_true", dest="legends", help="Generate the legend images", ) parser.add_argument( "--dump-config", default=False, action="store_true", help="Dump the used config with default values and exit", ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) assert gene.config_file config = gene.get_config(gene.config_file) if options.status: status(gene) sys.exit(0) if options.cache is None: options.cache = config.config["generation"]["default_cache"] if options.dump_config: _validate_generate_wmts_capabilities( config.config["caches"][options.cache], options.cache, True) yaml = ruamel.yaml.YAML() # type: ignore out = StringIO() yaml.dump(config.config, out) print(out.getvalue()) sys.exit(0) if options.legends: _generate_legend_images(gene) except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise sys.exit(1)