def _copy(
self,
options: Namespace,
gene: TileGeneration,
layer_name: str,
source: str,
dest: str,
task_name: str,
) -> None:
# disable metatiles
assert gene.config_file
config = gene.get_config(gene.config_file)
layer = config.config["layers"][layer_name]
cast(tilecloud_chain.configuration.LayerWms, layer)["meta"] = False
count_tiles_dropped = Count()
gene.create_log_tiles_error(layer_name)
source_tilestore = gene.get_tilesstore(source)
dest_tilestore = gene.get_tilesstore(dest)
gene.init_tilecoords(config, layer_name)
gene.add_geom_filter()
gene.add_logger()
gene.get(source_tilestore, "Get the tiles")
gene.imap(DropEmpty(gene))
# Discard tiles with certain content
if "empty_tile_detection" in layer:
empty_tile = layer["empty_tile_detection"]
gene.imap(
HashDropper(empty_tile["size"],
empty_tile["hash"],
store=dest_tilestore,
count=count_tiles_dropped))
if options.process:
gene.process(options.process)
gene.imap(DropEmpty(gene))
self.count = gene.counter_size()
gene.put(dest_tilestore, "Store the tiles")
gene.consume()
if not options.quiet:
print(f"""The tile {task_name} of layer '{layer_name}' is finish
Nb {task_name} tiles: {self.count.nb}
Nb errored tiles: {gene.error}
Nb dropped tiles: {count_tiles_dropped.nb}
Total time: {duration_format(gene.duration)}
Total size: {size_format(self.count.size)}
Time per tile: {(gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0} ms
Size per tile: {self.count.size / self.count.nb if self.count.nb != 0 else -1} o
""")
def _calculate_cost(gene: TileGeneration, layer_name: str,
options: Namespace) -> Tuple[float, timedelta, float, int]:
nb_metatiles = {}
nb_tiles = {}
config = gene.get_config(options.config)
layer = config.config["layers"][layer_name]
meta = layer["meta"]
if options.cost_algo == "area":
tile_size = config.config["grids"][layer["grid"]]["tile_size"]
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
if "min_resolution_seed" in layer and resolution < layer[
"min_resolution_seed"]:
continue
print(f"Calculate zoom {zoom}.")
px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0
m_buffer = px_buffer * resolution
if meta:
size = tile_size * layer["meta_size"] * resolution
meta_buffer = size * 0.7 + m_buffer
meta_geom = gene.get_geoms(config, layer_name)[zoom].buffer(
meta_buffer, 1)
nb_metatiles[zoom] = int(round(meta_geom.area / size**2))
size = tile_size * resolution
tile_buffer = size * 0.7 + m_buffer
geom = gene.get_geoms(config,
layer_name)[zoom].buffer(tile_buffer, 1)
nb_tiles[zoom] = int(round(geom.area / size**2))
elif options.cost_algo == "count":
gene.init_tilecoords(config, layer_name)
gene.add_geom_filter()
if meta:
def count_metatile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_metatiles:
nb_metatiles[tile.tilecoord.z] += 1
else:
nb_metatiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_metatile)
class MetaTileSplitter(TileStore):
"""Convert the metatile flow to tile flow."""
@staticmethod
def get(tiles: Iterable[Tile]) -> Iterator[Tile]:
for metatile in tiles:
for tilecoord in metatile.tilecoord:
yield Tile(tilecoord)
gene.add_metatile_splitter(MetaTileSplitter())
# Only keep tiles that intersect geometry
gene.add_geom_filter()
def count_tile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_tiles:
nb_tiles[tile.tilecoord.z] += 1
else:
print(f"Calculate zoom {tile.tilecoord.z}.")
nb_tiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_tile)
run = Run(gene, gene.functions_metatiles)
assert gene.tilestream
for tile in gene.tilestream:
tile.metadata["layer"] = layer_name
run(tile)
times = {}
print()
for z, nb_metatile in nb_metatiles.items():
print(f"{nb_metatile} meta tiles in zoom {z}.")
times[z] = layer["cost"]["metatile_generation_time"] * nb_metatile
price: float = 0
all_size: float = 0
all_time: float = 0
all_tiles = 0
for z, nb_tile in nb_tiles.items():
print()
print(f"{nb_tile} tiles in zoom {z}.")
all_tiles += nb_tile
if meta:
time = times[z] + layer["cost"]["tile_generation_time"] * nb_tile
else:
time = layer["cost"]["tileonly_generation_time"] * nb_tile
size = layer["cost"]["tile_size"] * nb_tile
all_size += size
all_time += time
td = timedelta(milliseconds=time)
print(f"Time to generate: {duration_format(td)} [d h:mm:ss]")
c = gene.get_main_config(
).config["cost"]["s3"]["put"] * nb_tile / 1000.0
price += c
print(f"S3 PUT: {c:0.2f} [$]")
if "sqs" in gene.get_main_config().config:
if meta:
nb_sqs = nb_metatiles[z] * 3
else:
nb_sqs = nb_tile * 3
c = nb_sqs * gene.get_main_config(
).config["cost"]["sqs"]["request"] / 1000000.0
price += c
print(f"SQS usage: {c:0.2f} [$]")
print()
td = timedelta(milliseconds=all_time)
print(f"Number of tiles: {all_tiles}")
print(f"Generation time: {duration_format(td)} [d h:mm:ss]")
print(f"Generation cost: {price:0.2f} [$]")
return all_size, td, price, all_tiles