def get_status(gene: TileGeneration) -> List[str]:
"""Get the tile generation status."""
config = gene.get_main_config()
store = get_queue_store(config, False)
prefix = "redis" if "redis" in config.config else "sqs"
conf = config.config[
"redis"] if "redis" in config.config else config.config["sqs"]
stats_prefix = [prefix, conf["queue"]]
with stats.timer_context(stats_prefix + ["get_stats"]):
status_ = store.get_status()
return [name + ": " + str(value) for name, value in status_.items()]
def get_wmts_capabilities(gene: TileGeneration,
cache_name: str,
exit_: bool = False) -> Optional[str]:
"""Get the WMTS capabilities for a configuration file."""
assert gene.config_file
config = gene.get_config(gene.config_file)
cache = config.config["caches"][cache_name]
if _validate_generate_wmts_capabilities(cache, cache_name, exit_):
server = gene.get_main_config().config.get("server")
base_urls = _get_base_urls(cache)
_fill_legend(gene, cache, server, base_urls)
data = pkgutil.get_data("tilecloud_chain",
"wmts_get_capabilities.jinja")
assert data
return cast(
str,
jinja2_template(
data.decode("utf-8"),
layers=config.config["layers"],
layer_legends=gene.layer_legends,
grids=config.config["grids"],
getcapabilities=urljoin( # type: ignore
base_urls[0],
(server.get("wmts_path", "wmts") +
"/1.0.0/WMTSCapabilities.xml" if server is not None else
cache.get("wmtscapabilities_file",
"1.0.0/WMTSCapabilities.xml")),
),
base_urls=base_urls,
base_url_postfix=(server.get("wmts_path", "wmts") +
"/") if server is not None else "",
get_tile_matrix_identifier=get_tile_matrix_identifier,
server=server is not None,
has_metadata="metadata" in config.config,
metadata=config.config.get("metadata"),
has_provider="provider" in config.config,
provider=config.config.get("provider"),
enumerate=enumerate,
ceil=math.ceil,
int=int,
sorted=sorted,
),
)
return None
def main() -> None:
"""Run the tiles generation."""
try:
stats.init_backends({})
parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0])
add_common_options(parser, dimensions=True)
parser.add_argument(
"--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y"
)
parser.add_argument(
"--get-bbox",
metavar="TILE",
help="get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
)
parser.add_argument(
"--role",
default="local",
choices=("local", "master", "slave"),
help="local/master/slave, master to file the queue and slave to generate the tiles",
)
parser.add_argument(
"--local-process-number", default=None, help="The number of process that we run in parallel"
)
parser.add_argument(
"--detach", default=False, action="store_true", help="run detached from the terminal"
)
parser.add_argument(
"--daemon", default=False, action="store_true", help="run continuously as a daemon"
)
parser.add_argument(
"--tiles",
metavar="FILE",
help="Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
)
options = parser.parse_args()
if options.detach:
detach()
gene = TileGeneration(
config_file=options.config, options=options, multi_thread=options.get_hash is None
)
if (
options.get_hash is None
and options.get_bbox is None
and options.config is not None
and "authorised_user" in gene.get_main_config().config.get("generation", {})
and gene.get_main_config().config["generation"]["authorised_user"] != getuser()
):
logger.error(
"not authorised, authorised user is: %s.",
gene.get_main_config().config["generation"]["authorised_user"],
)
sys.exit(1)
if options.config:
config = gene.get_config(options.config)
if options.cache is None and options.config:
options.cache = config.config["generation"]["default_cache"]
if options.tiles is not None and options.role not in ["local", "master"]:
logger.error("The --tiles option work only with role local or master")
sys.exit(1)
try:
generate = Generate(options, gene)
if options.role == "slave":
generate.gene()
elif options.layer:
generate.gene(options.layer)
elif options.get_bbox:
logger.error("With --get-bbox option you need to specify a layer")
sys.exit(1)
elif options.get_hash:
logger.error("With --get-hash option you need to specify a layer")
sys.exit(1)
else:
if options.config:
for layer in config.config["generation"].get(
"default_layers", config.config["layers"].keys()
):
generate.gene(layer)
except tilecloud.filter.error.TooManyErrors:
logger.exception("Too many errors")
sys.exit(1)
finally:
gene.close()
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def _calculate_cost(gene: TileGeneration, layer_name: str,
options: Namespace) -> Tuple[float, timedelta, float, int]:
nb_metatiles = {}
nb_tiles = {}
config = gene.get_config(options.config)
layer = config.config["layers"][layer_name]
meta = layer["meta"]
if options.cost_algo == "area":
tile_size = config.config["grids"][layer["grid"]]["tile_size"]
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
if "min_resolution_seed" in layer and resolution < layer[
"min_resolution_seed"]:
continue
print(f"Calculate zoom {zoom}.")
px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0
m_buffer = px_buffer * resolution
if meta:
size = tile_size * layer["meta_size"] * resolution
meta_buffer = size * 0.7 + m_buffer
meta_geom = gene.get_geoms(config, layer_name)[zoom].buffer(
meta_buffer, 1)
nb_metatiles[zoom] = int(round(meta_geom.area / size**2))
size = tile_size * resolution
tile_buffer = size * 0.7 + m_buffer
geom = gene.get_geoms(config,
layer_name)[zoom].buffer(tile_buffer, 1)
nb_tiles[zoom] = int(round(geom.area / size**2))
elif options.cost_algo == "count":
gene.init_tilecoords(config, layer_name)
gene.add_geom_filter()
if meta:
def count_metatile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_metatiles:
nb_metatiles[tile.tilecoord.z] += 1
else:
nb_metatiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_metatile)
class MetaTileSplitter(TileStore):
"""Convert the metatile flow to tile flow."""
@staticmethod
def get(tiles: Iterable[Tile]) -> Iterator[Tile]:
for metatile in tiles:
for tilecoord in metatile.tilecoord:
yield Tile(tilecoord)
gene.add_metatile_splitter(MetaTileSplitter())
# Only keep tiles that intersect geometry
gene.add_geom_filter()
def count_tile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_tiles:
nb_tiles[tile.tilecoord.z] += 1
else:
print(f"Calculate zoom {tile.tilecoord.z}.")
nb_tiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_tile)
run = Run(gene, gene.functions_metatiles)
assert gene.tilestream
for tile in gene.tilestream:
tile.metadata["layer"] = layer_name
run(tile)
times = {}
print()
for z, nb_metatile in nb_metatiles.items():
print(f"{nb_metatile} meta tiles in zoom {z}.")
times[z] = layer["cost"]["metatile_generation_time"] * nb_metatile
price: float = 0
all_size: float = 0
all_time: float = 0
all_tiles = 0
for z, nb_tile in nb_tiles.items():
print()
print(f"{nb_tile} tiles in zoom {z}.")
all_tiles += nb_tile
if meta:
time = times[z] + layer["cost"]["tile_generation_time"] * nb_tile
else:
time = layer["cost"]["tileonly_generation_time"] * nb_tile
size = layer["cost"]["tile_size"] * nb_tile
all_size += size
all_time += time
td = timedelta(milliseconds=time)
print(f"Time to generate: {duration_format(td)} [d h:mm:ss]")
c = gene.get_main_config(
).config["cost"]["s3"]["put"] * nb_tile / 1000.0
price += c
print(f"S3 PUT: {c:0.2f} [$]")
if "sqs" in gene.get_main_config().config:
if meta:
nb_sqs = nb_metatiles[z] * 3
else:
nb_sqs = nb_tile * 3
c = nb_sqs * gene.get_main_config(
).config["cost"]["sqs"]["request"] / 1000000.0
price += c
print(f"SQS usage: {c:0.2f} [$]")
print()
td = timedelta(milliseconds=all_time)
print(f"Number of tiles: {all_tiles}")
print(f"Generation time: {duration_format(td)} [d h:mm:ss]")
print(f"Generation cost: {price:0.2f} [$]")
return all_size, td, price, all_tiles
def main() -> None:
"""Calculate the cost, main function."""
try:
parser = ArgumentParser(
description="Used to calculate the generation cost",
prog=sys.argv[0])
add_common_options(parser, tile_pyramid=False, dimensions=True)
parser.add_argument(
"--cost-algo",
"--calculate-cost-algorithm",
default="area",
dest="cost_algo",
choices=("area", "count"),
help=
"The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
)
options = parser.parse_args()
gene = TileGeneration(
options.config,
options=options,
layer_name=options.layer,
base_config={"cost": {}},
multi_thread=False,
)
config = gene.get_config(options.config)
all_size: float = 0
tile_size: float = 0
all_tiles = 0
if options.layer:
layer = config.config["layers"][options.layer]
(all_size, all_time, all_price,
all_tiles) = _calculate_cost(gene, options.layer, options)
tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer_name in gene.get_config(
options.config).config["generation"]["default_layers"]:
print()
print(f"===== {layer_name} =====")
layer = config.config["layers"][layer_name]
gene.create_log_tiles_error(layer_name)
(size, time, price,
tiles) = _calculate_cost(gene, layer_name, options)
tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print()
print("===== GLOBAL =====")
print(f"Total number of tiles: {all_tiles}")
print(
f"Total generation time: {duration_format(all_time)} [d h:mm:ss]"
)
print(f"Total generation cost: {all_price:0.2f} [$]")
print()
s3_cost = all_size * gene.get_main_config(
).config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024)
print(f"S3 Storage: {s3_cost:0.2f} [$/month]")
s3_get_cost = (
gene.get_main_config().config["cost"]["s3"]["get"] *
config.config["cost"]["request_per_layers"] / 10000.0 +
gene.get_main_config().config["cost"]["s3"]["download"] *
config.config["cost"]["request_per_layers"] * tile_size)
print(f"S3 get: {s3_get_cost:0.2f} [$/month]")
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] *
# gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] *
# gene.config['cost']['request_per_layers'] * tile_size)
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
sys.exit(1)
