def process() -> None:
"""Copy the tiles from a cache to an other."""
try:
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_common_options(parser, near=False, time=False, dimensions=True)
parser.add_argument("process",
metavar="PROCESS",
help="The process name to do")
options = parser.parse_args()
gene = TileGeneration(options.config, options, multi_thread=False)
copy = Copy()
if options.layer:
copy.copy(options, gene, options.layer, options.cache,
options.cache, "process")
else:
assert gene.config_file
config = gene.get_config(gene.config_file)
layers_name = (config.config["generation"]["default_layers"]
if "default_layers" in config.config.get(
"generation", {}) else
config.config["layers"].keys())
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
"process")
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
sys.exit(1)
def _fill_legend(
gene: TileGeneration,
cache: tilecloud_chain.configuration.Cache,
server: Optional[tilecloud_chain.configuration.Server],
base_urls: List[str],
) -> None:
assert gene.config_file
config = gene.get_config(gene.config_file)
for layer_name, layer in config.config["layers"].items():
previous_legend: Optional[tilecloud_chain.Legend] = None
previous_resolution = None
if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends:
gene.layer_legends[layer_name] = []
legends = gene.layer_legends[layer_name]
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
path = "/".join([
"1.0.0",
layer_name,
layer["wmts_style"],
f"legend{zoom}.{layer['legend_extension']}",
])
img = _get(path, cache)
if img is not None:
new_legend: tilecloud_chain.Legend = {
"mime_type":
layer["legend_mime"],
"href":
os.path.join(
base_urls[0],
server.get("static_path", "static") +
"/" if server else "", path),
}
legends.append(new_legend)
if previous_legend is not None:
assert previous_resolution is not None
middle_res = exp(
(log(previous_resolution) + log(resolution)) / 2)
previous_legend[ # pylint: disable=unsupported-assignment-operation
"min_resolution"] = middle_res
new_legend["max_resolution"] = middle_res
try:
pil_img = Image.open(BytesIO(img))
new_legend["width"] = pil_img.size[0]
new_legend["height"] = pil_img.size[1]
except Exception: # pragma: nocover
logger.warning(
"Unable to read legend image '%s', with '%s'",
path,
repr(img),
exc_info=True,
)
previous_legend = new_legend
previous_resolution = resolution
def _copy(
self,
options: Namespace,
gene: TileGeneration,
layer_name: str,
source: str,
dest: str,
task_name: str,
) -> None:
# disable metatiles
assert gene.config_file
config = gene.get_config(gene.config_file)
layer = config.config["layers"][layer_name]
cast(tilecloud_chain.configuration.LayerWms, layer)["meta"] = False
count_tiles_dropped = Count()
gene.create_log_tiles_error(layer_name)
source_tilestore = gene.get_tilesstore(source)
dest_tilestore = gene.get_tilesstore(dest)
gene.init_tilecoords(config, layer_name)
gene.add_geom_filter()
gene.add_logger()
gene.get(source_tilestore, "Get the tiles")
gene.imap(DropEmpty(gene))
# Discard tiles with certain content
if "empty_tile_detection" in layer:
empty_tile = layer["empty_tile_detection"]
gene.imap(
HashDropper(empty_tile["size"],
empty_tile["hash"],
store=dest_tilestore,
count=count_tiles_dropped))
if options.process:
gene.process(options.process)
gene.imap(DropEmpty(gene))
self.count = gene.counter_size()
gene.put(dest_tilestore, "Store the tiles")
gene.consume()
if not options.quiet:
print(f"""The tile {task_name} of layer '{layer_name}' is finish
Nb {task_name} tiles: {self.count.nb}
Nb errored tiles: {gene.error}
Nb dropped tiles: {count_tiles_dropped.nb}
Total time: {duration_format(gene.duration)}
Total size: {size_format(self.count.size)}
Time per tile: {(gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0} ms
Size per tile: {self.count.size / self.count.nb if self.count.nb != 0 else -1} o
""")
def main() -> None:
"""Copy the tiles from a cache to an other."""
try:
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_common_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument("--process",
dest="process",
metavar="NAME",
help="The process name to do")
parser.add_argument("source",
metavar="SOURCE",
help="The source cache")
parser.add_argument("dest",
metavar="DEST",
help="The destination cache")
options = parser.parse_args()
gene = TileGeneration(options.config, options)
assert gene.config_file
config = gene.get_config(gene.config_file)
if options.layer:
copy = Copy()
copy.copy(options, gene, options.layer, options.source,
options.dest, "copy")
else:
layers = (config.config["generation"]["default_layers"]
if "default_layers" in config.config["generation"] else
config.config["layers"].keys())
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
"copy")
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def get_wmts_capabilities(gene: TileGeneration,
cache_name: str,
exit_: bool = False) -> Optional[str]:
"""Get the WMTS capabilities for a configuration file."""
assert gene.config_file
config = gene.get_config(gene.config_file)
cache = config.config["caches"][cache_name]
if _validate_generate_wmts_capabilities(cache, cache_name, exit_):
server = gene.get_main_config().config.get("server")
base_urls = _get_base_urls(cache)
_fill_legend(gene, cache, server, base_urls)
data = pkgutil.get_data("tilecloud_chain",
"wmts_get_capabilities.jinja")
assert data
return cast(
str,
jinja2_template(
data.decode("utf-8"),
layers=config.config["layers"],
layer_legends=gene.layer_legends,
grids=config.config["grids"],
getcapabilities=urljoin( # type: ignore
base_urls[0],
(server.get("wmts_path", "wmts") +
"/1.0.0/WMTSCapabilities.xml" if server is not None else
cache.get("wmtscapabilities_file",
"1.0.0/WMTSCapabilities.xml")),
),
base_urls=base_urls,
base_url_postfix=(server.get("wmts_path", "wmts") +
"/") if server is not None else "",
get_tile_matrix_identifier=get_tile_matrix_identifier,
server=server is not None,
has_metadata="metadata" in config.config,
metadata=config.config.get("metadata"),
has_provider="provider" in config.config,
provider=config.config.get("provider"),
enumerate=enumerate,
ceil=math.ceil,
int=int,
sorted=sorted,
),
)
return None
def main() -> None:
"""Run the tiles generation."""
try:
stats.init_backends({})
parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0])
add_common_options(parser, dimensions=True)
parser.add_argument(
"--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y"
)
parser.add_argument(
"--get-bbox",
metavar="TILE",
help="get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
)
parser.add_argument(
"--role",
default="local",
choices=("local", "master", "slave"),
help="local/master/slave, master to file the queue and slave to generate the tiles",
)
parser.add_argument(
"--local-process-number", default=None, help="The number of process that we run in parallel"
)
parser.add_argument(
"--detach", default=False, action="store_true", help="run detached from the terminal"
)
parser.add_argument(
"--daemon", default=False, action="store_true", help="run continuously as a daemon"
)
parser.add_argument(
"--tiles",
metavar="FILE",
help="Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
)
options = parser.parse_args()
if options.detach:
detach()
gene = TileGeneration(
config_file=options.config, options=options, multi_thread=options.get_hash is None
)
if (
options.get_hash is None
and options.get_bbox is None
and options.config is not None
and "authorised_user" in gene.get_main_config().config.get("generation", {})
and gene.get_main_config().config["generation"]["authorised_user"] != getuser()
):
logger.error(
"not authorised, authorised user is: %s.",
gene.get_main_config().config["generation"]["authorised_user"],
)
sys.exit(1)
if options.config:
config = gene.get_config(options.config)
if options.cache is None and options.config:
options.cache = config.config["generation"]["default_cache"]
if options.tiles is not None and options.role not in ["local", "master"]:
logger.error("The --tiles option work only with role local or master")
sys.exit(1)
try:
generate = Generate(options, gene)
if options.role == "slave":
generate.gene()
elif options.layer:
generate.gene(options.layer)
elif options.get_bbox:
logger.error("With --get-bbox option you need to specify a layer")
sys.exit(1)
elif options.get_hash:
logger.error("With --get-hash option you need to specify a layer")
sys.exit(1)
else:
if options.config:
for layer in config.config["generation"].get(
"default_layers", config.config["layers"].keys()
):
generate.gene(layer)
except tilecloud.filter.error.TooManyErrors:
logger.exception("Too many errors")
sys.exit(1)
finally:
gene.close()
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def _calculate_cost(gene: TileGeneration, layer_name: str,
options: Namespace) -> Tuple[float, timedelta, float, int]:
nb_metatiles = {}
nb_tiles = {}
config = gene.get_config(options.config)
layer = config.config["layers"][layer_name]
meta = layer["meta"]
if options.cost_algo == "area":
tile_size = config.config["grids"][layer["grid"]]["tile_size"]
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
if "min_resolution_seed" in layer and resolution < layer[
"min_resolution_seed"]:
continue
print(f"Calculate zoom {zoom}.")
px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0
m_buffer = px_buffer * resolution
if meta:
size = tile_size * layer["meta_size"] * resolution
meta_buffer = size * 0.7 + m_buffer
meta_geom = gene.get_geoms(config, layer_name)[zoom].buffer(
meta_buffer, 1)
nb_metatiles[zoom] = int(round(meta_geom.area / size**2))
size = tile_size * resolution
tile_buffer = size * 0.7 + m_buffer
geom = gene.get_geoms(config,
layer_name)[zoom].buffer(tile_buffer, 1)
nb_tiles[zoom] = int(round(geom.area / size**2))
elif options.cost_algo == "count":
gene.init_tilecoords(config, layer_name)
gene.add_geom_filter()
if meta:
def count_metatile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_metatiles:
nb_metatiles[tile.tilecoord.z] += 1
else:
nb_metatiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_metatile)
class MetaTileSplitter(TileStore):
"""Convert the metatile flow to tile flow."""
@staticmethod
def get(tiles: Iterable[Tile]) -> Iterator[Tile]:
for metatile in tiles:
for tilecoord in metatile.tilecoord:
yield Tile(tilecoord)
gene.add_metatile_splitter(MetaTileSplitter())
# Only keep tiles that intersect geometry
gene.add_geom_filter()
def count_tile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_tiles:
nb_tiles[tile.tilecoord.z] += 1
else:
print(f"Calculate zoom {tile.tilecoord.z}.")
nb_tiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_tile)
run = Run(gene, gene.functions_metatiles)
assert gene.tilestream
for tile in gene.tilestream:
tile.metadata["layer"] = layer_name
run(tile)
times = {}
print()
for z, nb_metatile in nb_metatiles.items():
print(f"{nb_metatile} meta tiles in zoom {z}.")
times[z] = layer["cost"]["metatile_generation_time"] * nb_metatile
price: float = 0
all_size: float = 0
all_time: float = 0
all_tiles = 0
for z, nb_tile in nb_tiles.items():
print()
print(f"{nb_tile} tiles in zoom {z}.")
all_tiles += nb_tile
if meta:
time = times[z] + layer["cost"]["tile_generation_time"] * nb_tile
else:
time = layer["cost"]["tileonly_generation_time"] * nb_tile
size = layer["cost"]["tile_size"] * nb_tile
all_size += size
all_time += time
td = timedelta(milliseconds=time)
print(f"Time to generate: {duration_format(td)} [d h:mm:ss]")
c = gene.get_main_config(
).config["cost"]["s3"]["put"] * nb_tile / 1000.0
price += c
print(f"S3 PUT: {c:0.2f} [$]")
if "sqs" in gene.get_main_config().config:
if meta:
nb_sqs = nb_metatiles[z] * 3
else:
nb_sqs = nb_tile * 3
c = nb_sqs * gene.get_main_config(
).config["cost"]["sqs"]["request"] / 1000000.0
price += c
print(f"SQS usage: {c:0.2f} [$]")
print()
td = timedelta(milliseconds=all_time)
print(f"Number of tiles: {all_tiles}")
print(f"Generation time: {duration_format(td)} [d h:mm:ss]")
print(f"Generation cost: {price:0.2f} [$]")
return all_size, td, price, all_tiles
def main() -> None:
"""Calculate the cost, main function."""
try:
parser = ArgumentParser(
description="Used to calculate the generation cost",
prog=sys.argv[0])
add_common_options(parser, tile_pyramid=False, dimensions=True)
parser.add_argument(
"--cost-algo",
"--calculate-cost-algorithm",
default="area",
dest="cost_algo",
choices=("area", "count"),
help=
"The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
)
options = parser.parse_args()
gene = TileGeneration(
options.config,
options=options,
layer_name=options.layer,
base_config={"cost": {}},
multi_thread=False,
)
config = gene.get_config(options.config)
all_size: float = 0
tile_size: float = 0
all_tiles = 0
if options.layer:
layer = config.config["layers"][options.layer]
(all_size, all_time, all_price,
all_tiles) = _calculate_cost(gene, options.layer, options)
tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer_name in gene.get_config(
options.config).config["generation"]["default_layers"]:
print()
print(f"===== {layer_name} =====")
layer = config.config["layers"][layer_name]
gene.create_log_tiles_error(layer_name)
(size, time, price,
tiles) = _calculate_cost(gene, layer_name, options)
tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print()
print("===== GLOBAL =====")
print(f"Total number of tiles: {all_tiles}")
print(
f"Total generation time: {duration_format(all_time)} [d h:mm:ss]"
)
print(f"Total generation cost: {all_price:0.2f} [$]")
print()
s3_cost = all_size * gene.get_main_config(
).config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024)
print(f"S3 Storage: {s3_cost:0.2f} [$/month]")
s3_get_cost = (
gene.get_main_config().config["cost"]["s3"]["get"] *
config.config["cost"]["request_per_layers"] / 10000.0 +
gene.get_main_config().config["cost"]["s3"]["download"] *
config.config["cost"]["request_per_layers"] * tile_size)
print(f"S3 get: {s3_get_cost:0.2f} [$/month]")
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] *
# gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] *
# gene.config['cost']['request_per_layers'] * tile_size)
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
sys.exit(1)
def _generate_legend_images(gene: TileGeneration) -> None:
assert gene.config_file
config = gene.get_config(gene.config_file)
cache = config.config["caches"][gene.options.cache]
for layer_name, layer in config.config["layers"].items():
if "legend_mime" in layer and "legend_extension" in layer:
if layer["type"] == "wms":
session = requests.session()
session.headers.update(layer["headers"])
previous_hash = None
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
legends = []
for wmslayer in layer["layers"].split(","):
response = session.get(layer["url"] + "?" + urlencode({
"SERVICE":
"WMS",
"VERSION":
layer.get("version", "1.0.0"),
"REQUEST":
"GetLegendGraphic",
"LAYER":
wmslayer,
"FORMAT":
layer["legend_mime"],
"TRANSPARENT":
"TRUE" if layer["legend_mime"] ==
"image/png" else "FALSE",
"STYLE":
layer["wmts_style"],
"SCALE":
resolution / 0.00028,
}))
try:
legends.append(
Image.open(BytesIO(response.content)))
except Exception: # pragma: nocover
logger.warning(
"Unable to read legend image for layer '%s'-'%s', resolution '%s': %s",
layer_name,
wmslayer,
resolution,
response.content,
exc_info=True,
)
width = max(i.size[0] for i in legends)
height = sum(i.size[1] for i in legends)
image = Image.new("RGBA", (width, height))
y = 0
for i in legends:
image.paste(i, (0, y))
y += i.size[1]
string_io = BytesIO()
image.save(string_io,
FORMAT_BY_CONTENT_TYPE[layer["legend_mime"]])
result = string_io.getvalue()
new_hash = sha1(result).hexdigest() # nosec
if new_hash != previous_hash:
previous_hash = new_hash
_send(
result,
f"1.0.0/{layer_name}/{layer['wmts_style']}/"
f"legend{zoom}.{layer['legend_extension']}",
layer["legend_mime"],
cache,
)
def main() -> None:
"""Generate the contextual file like the legends."""
try:
stats.init_backends({})
parser = ArgumentParser(
description=
"Used to generate the contextual file like the capabilities, the legends, "
"the OpenLayers example",
prog=sys.argv[0],
)
add_common_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument("--status",
default=False,
action="store_true",
help="Display the SQS queue status and exit")
parser.add_argument(
"--legends",
"--generate-legend-images",
default=False,
action="store_true",
dest="legends",
help="Generate the legend images",
)
parser.add_argument(
"--dump-config",
default=False,
action="store_true",
help="Dump the used config with default values and exit",
)
options = parser.parse_args()
gene = TileGeneration(options.config,
options,
layer_name=options.layer)
assert gene.config_file
config = gene.get_config(gene.config_file)
if options.status:
status(gene)
sys.exit(0)
if options.cache is None:
options.cache = config.config["generation"]["default_cache"]
if options.dump_config:
_validate_generate_wmts_capabilities(
config.config["caches"][options.cache], options.cache, True)
yaml = ruamel.yaml.YAML() # type: ignore
out = StringIO()
yaml.dump(config.config, out)
print(out.getvalue())
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)