def _build_retrieval_requests(
tiles: Dict[int, Dict[int, List[int]]],
url_template: str,
image_formats: List[str],
file_extension: str,
) -> List[RetrievalRequest]:
requests = list()
url_format = _determine_format(url_template)
for z, xs in tiles.items():
for x, ys in xs.items():
for y in ys:
path = _build_tile_path(z, x, y, url_template, file_extension)
if skip_file_creation(path):
continue
requests.append(
RetrievalRequest(
url=_build_tile_url(url_format, url_template, z, x, y),
path=path,
expected_types=image_formats,
)
)
return requests
url=
f"https://pub.data.gov.bc.ca/datasets/177864/tif/bcalb/{cell_parent}/{cell_name}.zip",
path=get_cache_path((CACHE_DIR_NAME, f"{cell_name}.zip")),
expected_types=["application/zip"],
cell_name=cell_name,
tif_name=f"{cell_name}.tif",
tif_path=get_cache_path((CACHE_DIR_NAME, f"{cell_name}.tif")),
prj_path=get_cache_path(
(CACHE_DIR_NAME, f"{cell_name}_prj.tif")),
run_path=get_run_data_path(
run_id, (CACHE_DIR_NAME, f"{cell_name}.tif")),
))
to_generate = list(
filter(
lambda generation_request: not skip_file_creation(
generation_request.prj_path),
bbox_cells,
))
retrieve(to_generate, HTTP_RETRIEVAL_CONCURRENCY)
for generation_request in to_generate:
try:
with zipfile.ZipFile(generation_request.path, "r") as zip_ref:
zip_ref.extract(generation_request.tif_name,
get_cache_path((CACHE_DIR_NAME, )))
Warp(
generation_request.prj_path,
generation_request.tif_path,
cutlineDSName=get_data_path(("grids.gpkg", )),
cutlineLayer="BC-20000",
cutlineWhere=f"MAP_TILE = '{generation_request.cell_name}'",
def _filter_grid_for_missing(
grid: List[PartialCoverageTile],
) -> List[PartialCoverageTile]:
return list(filter(lambda tile: not skip_file_creation(tile.tif_path), grid))