def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("GPKG")
grid_datasource = driver.Open(get_data_path(("grids.gpkg", )))
grid_layer = grid_datasource.GetLayerByName("BC-20000")
grid_layer.SetSpatialFilterRect(bbox.min_x, bbox.min_y, bbox.max_x,
bbox.max_y)
bbox_cells = list()
while grid_cell := grid_layer.GetNextFeature():
cell_name = grid_cell.GetFieldAsString("MAP_TILE")
cell_parent = re.search(r"^\d{2,3}[a-z]", cell_name, re.IGNORECASE)[0]
bbox_cells.append(
GenerationRequest(
url=
f"https://pub.data.gov.bc.ca/datasets/177864/tif/bcalb/{cell_parent}/{cell_name}.zip",
path=get_cache_path((CACHE_DIR_NAME, f"{cell_name}.zip")),
expected_types=["application/zip"],
cell_name=cell_name,
tif_name=f"{cell_name}.tif",
tif_path=get_cache_path((CACHE_DIR_NAME, f"{cell_name}.tif")),
prj_path=get_cache_path(
(CACHE_DIR_NAME, f"{cell_name}_prj.tif")),
run_path=get_run_data_path(
run_id, (CACHE_DIR_NAME, f"{cell_name}.tif")),
))
def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("GPKG")
grid_datasource = driver.Open(get_data_path(("grids.gpkg", )))
grid_layer = grid_datasource.GetLayerByName("Canada-50000")
grid_layer.SetSpatialFilterRect(bbox.min_x, bbox.min_y, bbox.max_x,
bbox.max_y)
bbox_cells = list()
while grid_cell := grid_layer.GetNextFeature():
cell_name = grid_cell.GetFieldAsString("NTS_SNRC")
cell_parent = re.sub(
"^0", "",
re.search(r"^\d{2,3}[a-z]", cell_name, re.IGNORECASE)[0])
for cardinal in ("e", "w"):
cell_part_name = f"{cell_name.lower()}_{cardinal}"
zip_file_name = f"{cell_part_name}.dem.zip"
bbox_cells.append(
GenerationRequest(
url=
f"https://pub.data.gov.bc.ca/datasets/175624/{cell_parent.lower()}/{zip_file_name}",
path=get_cache_path((CACHE_DIR_NAME, zip_file_name)),
expected_type="application/zip",
dem_path=get_cache_path(
(CACHE_DIR_NAME, f"{cell_part_name}.dem")),
prj_path=get_cache_path(
(CACHE_DIR_NAME, f"{cell_part_name}_prj.tif")),
hs_path=get_cache_path(
(CACHE_DIR_NAME, f"{cell_part_name}_hs.tif")),
run_path=get_run_data_path(
run_id, (CACHE_DIR_NAME, f"{cell_part_name}.tif")),
))
def provision(
bbox: BBOX,
base_url: str,
wms_properties: WmsProperties,
wms_crs_code: str,
layers: Tuple[str],
styles: Tuple[str],
scales: Tuple[int],
image_format: str,
cache_dir_name: str,
run_id: str,
) -> Dict[int, List[str]]:
cache_directory = get_cache_path((cache_dir_name,))
run_directory = get_run_data_path(run_id, (cache_dir_name,))
os.makedirs(run_directory)
grid = _build_grid_for_bbox(bbox, wms_crs_code, scales, wms_properties)
grid_for_retrieval = _update_grid_for_retrieval(
base_url,
grid,
layers,
styles,
wms_crs_code,
image_format,
cache_directory,
run_directory,
)
grid_for_missing = _filter_grid_for_missing(grid_for_retrieval)
requests = _convert_grid_to_requests(grid_for_missing, image_format)
retrieve(requests)
if image_format != TARGET_FILE_FORMAT:
_convert_to_tif(grid_for_missing, wms_crs_code)
return _create_run_output(bbox, grid_for_retrieval, run_id)
def provision(
bbox: BBOX,
run_id: str,
src_layer_name: str,
dst_layer_name: str,
) -> List[str]:
kmz_driver = ogr.GetDriverByName("LIBKML")
kmz_datasets = [
kmz_driver.Open(filename) for filename in glob.iglob(
get_data_path(("avcan-ates-areas-2020-06-23", "**", "*.kmz")),
recursive=True,
)
]
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory, exist_ok=True)
path = os.path.join(run_directory, f"{dst_layer_name}.shp")
ogr_to_shp(
bbox,
[ds.GetLayerByName(src_layer_name) for ds in kmz_datasets],
path,
dst_layer_name,
OUTPUT_CRS_CODE,
)
kmz_datasets = None
return [path]
def execute(bbox: BBOX, run_id: str, args: Dict[str, object] = dict()) -> None:
xyz_result = xyz_provisioner(
bbox,
args["xyz_url"],
ZOOM_MIN,
ZOOM_MAX,
["image/png", "image/jpeg"],
OUTPUT_FORMAT,
)
tmp_dir = get_run_data_path(run_id, (NAME, ))
tmp_tile_paths = list()
logging.info(
f"Copying {len(xyz_result.tile_paths)} {NAME} tiles to tmp dir for edge clipping"
)
for tile_path in xyz_result.tile_paths:
tmp_tile_path = tile_path.replace(xyz_result.tile_dir, tmp_dir)
tmp_tile_paths.append(tmp_tile_path)
os.makedirs(os.path.dirname(tmp_tile_path), exist_ok=True)
copyfile(tile_path, tmp_tile_path)
transparent_clip_to_bbox(
[
os.path.join(tmp_dir, tile_path)
for tile_path in get_edge_tiles(tmp_dir)
],
bbox,
)
add_or_update(tmp_dir, get_result_path((NAME, )))
def provision(bbox: BBOX, run_id: str) -> List[str]:
logging.info(
"Retrieving BC Freshwater Atlas - this could take a while the first time"
)
zip_path = fetch("FWA_BC.zip", "ftp.geobc.gov.bc.ca",
"/sections/outgoing/bmgs/FWA_Public")
fgdb_dir = os.path.dirname(zip_path)
fgdb = os.path.join(fgdb_dir, "FWA_BC.gdb")
if not os.path.exists(fgdb):
with zipfile.ZipFile(zip_path, "r") as zip_ref:
zip_ref.extractall(get_cache_path((fgdb, )))
logging.info("Retrieved BC Freshwater Atlas")
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("OpenFileGDB")
datasource = driver.Open(fgdb)
path = os.path.join(run_directory, "bc_wetlands.shp")
ogr_to_shp(
bbox,
[datasource.GetLayerByName("FWA_WETLANDS_POLY")],
path,
"bc_wetlands",
OUTPUT_CRS_CODE,
)
datasource = None
return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("GPKG")
datasource = driver.Open(get_local_features_path())
result = ogr_to_shp(
bbox,
[datasource.GetLayerByName("trails")],
os.path.join(run_directory, "trails.shp"),
"trails",
OUTPUT_CRS_CODE,
)
datasource = None
return result
def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("GPKG")
datasource = driver.Open(LOCAL_FEATURES_PATH)
path = os.path.join(run_directory, "trails.shp")
ogr_to_shp(
bbox,
[datasource.GetLayerByName("trails")],
path,
"trails",
OUTPUT_CRS_CODE,
)
datasource = None
return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("ESRI Shapefile")
datasource = driver.Open(
get_data_path(("FTEN_ROAD_SECTION_LINES_SVW", "FTEN_RS_LN_line.shp")))
result = ogr_to_shp(
bbox,
[datasource.GetLayerByIndex(0)],
os.path.join(run_directory, "bc_resource_roads.shp"),
"bc_resource_roads",
OUTPUT_CRS_CODE,
)
datasource = None
return result
def provision(arg: ProvisionArg) -> ProvisionResult:
bbox, profile_name, xyz_url = arg.bbox, arg.profile_name, arg.xyz_url
bbox_exists = has_prior_run(get_result_path((profile_name, )), bbox)
if bbox_exists and arg.skippable:
logging.info(
f"Skipping {profile_name} {bbox.min_x},{bbox.min_y} {bbox.max_x},{bbox.max_y} as it already exists"
)
return ProvisionResult.SKIPPED
else:
logging.info(
f"Provisioning {profile_name} {bbox.min_x},{bbox.min_y} {bbox.max_x},{bbox.max_y}"
)
run_id = str(uuid.uuid4())
profiles = {
profile.NAME: {
"execute": profile.execute,
"zoom_min": profile.ZOOM_MIN,
"zoom_max": profile.ZOOM_MAX,
"format": profile.OUTPUT_FORMAT,
}
for profile in [xyz, topo, xyzsummer, xyzwinter, xyzhunting]
}
profiles[profile_name]["execute"](bbox, run_id, {"xyz_url": xyz_url})
if int(os.environ.get("BVSAR_HEAD_VALIDATE", 0)) == 1:
logging.info("Validating result")
check_exists(
xyz_check_builder(
bbox,
"{0}/{1}/{{z}}/{{x}}/{{y}}.png".format(
os.environ.get("HTTP_URL", "http://rpi/tile/file"),
profile_name,
),
profiles[profile_name]["zoom_min"],
profiles[profile_name]["zoom_max"],
"image/{0}".format(profiles[profile_name]["format"]),
))
record_run(get_result_path((profile_name, )), bbox)
if remove_intermediaries():
run_dir = get_run_data_path(run_id, None)
result_temp_dir = get_result_path((run_id, ))
if os.path.exists(run_dir):
rmtree(run_dir)
if os.path.exists(result_temp_dir):
rmtree(result_temp_dir)
logging.info("Finished")
return ProvisionResult.SUCCESS
def provision(bbox: BBOX, profile_name: str, xyz_url: str) -> None:
bbox_repeat_if_exists = int(os.environ.get("BBOX_REPEAT_IF_EXISTS",
0)) == 1
bbox_exists = has_prior_run(get_result_path((profile_name, )), bbox)
if bbox_exists and not bbox_repeat_if_exists:
logging.info(
f"Skipping {profile_name} {bbox.min_x},{bbox.min_y} {bbox.max_x},{bbox.max_y} as it already exists"
)
return
run_id = str(uuid.uuid4())
profiles = {
profile.NAME: {
"execute": profile.execute,
"zoom_min": profile.ZOOM_MIN,
"zoom_max": profile.ZOOM_MAX,
"format": profile.OUTPUT_FORMAT,
}
for profile in [xyz, topo, xyzsummer, xyzwinter]
}
profiles[profile_name]["execute"](bbox, run_id, {"xyz_url": xyz_url})
logging.info("Validating result")
check_exists(
xyz_check_builder(
bbox,
"{0}/{1}/{{z}}/{{x}}/{{y}}.png".format(
os.environ.get("HTTP_URL",
"http://localhost:9000/tiles/files"),
profile_name,
),
profiles[profile_name]["zoom_min"],
profiles[profile_name]["zoom_max"],
"image/{0}".format(profiles[profile_name]["format"]),
))
record_run(get_result_path((profile_name, )), bbox)
if remove_intermediaries():
run_dir = get_run_data_path(run_id, None)
result_temp_dir = get_result_path((run_id, ))
if os.path.exists(run_dir):
rmtree(run_dir)
if os.path.exists(result_temp_dir):
rmtree(result_temp_dir)
logging.info("Finished")
def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("OpenFileGDB")
datasource = driver.Open(
get_data_path(("FTEN_ROAD_SEGMENT_LINES_SVW.gdb", )))
path = os.path.join(run_directory, "bc_resource_roads.shp")
ogr_to_shp(
bbox,
[
datasource.GetLayerByName(
"WHSE_FOREST_TENURE_FTEN_ROAD_SEGMENT_LINES_SVW")
],
path,
"bc_resource_roads",
OUTPUT_CRS_CODE,
)
datasource = None
return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]:
logging.info(
"Retrieving BC Freshwater Atlas - this could take a while the first time"
)
fgdb = retrieve_directory("ftp.geobc.gov.bc.ca",
"/sections/outgoing/bmgs/FWA_Public/FWA_BC.gdb")
logging.info("Retrieved BC Freshwater Atlas")
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("OpenFileGDB")
datasource = driver.Open(fgdb)
result = ogr_to_shp(
bbox,
[datasource.GetLayerByName("FWA_WETLANDS_POLY")],
os.path.join(run_directory, "bc_wetlands.shp"),
"bc_wetlands",
OUTPUT_CRS_CODE,
)
datasource = None
return result
def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("OpenFileGDB")
datasource = driver.Open(get_data_path(("FTEN_RECREATION_POLY_SVW.gdb", )))
rec_sites_layer = datasource.GetLayerByName(
"WHSE_FOREST_TENURE_FTEN_RECREATION_POLY_SVW")
rec_sites_layer.SetAttributeFilter(
"LIFE_CYCLE_STATUS_CODE IN ('ACTIVE','PENDING')")
path = os.path.join(run_directory, "bc_rec_sites.shp")
ogr_to_shp(
bbox,
[rec_sites_layer],
path,
"bc_rec_sites",
OUTPUT_CRS_CODE,
)
rec_sites_layer = None
datasource = None
return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]:
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
driver = ogr.GetDriverByName("OpenFileGDB")
datasource = driver.Open(
get_data_path(("pmbc_parcel_fabric_poly_svw.gdb", )))
parcels_layer = datasource.GetLayerByName("pmbc_parcel_fabric_poly_svw")
parcels_layer.SetAttributeFilter(
"OWNER_TYPE IN ('First Nation','Mixed Ownership','Municipal','Private','Unknown')"
)
path = os.path.join(run_directory, "bc_parcels.shp")
ogr_to_shp(
bbox,
[parcels_layer],
path,
"bc_parcels",
OUTPUT_CRS_CODE,
)
parcels_layer = None
datasource = None
return [path]
def _create_run_output(
bbox: BBOX, grid: List[PartialCoverageTile], run_id: str
) -> Dict[int, List[str]]:
file_list = dict()
run_directory = get_run_data_path(run_id, None)
for tile in grid:
if tile.scale not in file_list:
file_list[tile.scale] = list()
try:
Warp(
tile.final_path,
tile.tif_path,
cutlineDSName=get_datasource_from_bbox(bbox, run_directory),
cutlineLayer=BBOX_LAYER_NAME,
cropToCutline=True,
dstNodata=-1,
)
file_list[tile.scale].append(tile.final_path)
except Exception as ex:
swallow_unimportant_warp_error(ex)
return file_list
def provision(bbox: BBOX, run_id: str) -> List[str]:
logging.info(
"Retrieving BC Freshwater Atlas - this could take a while the first time"
)
zip_path = fetch("FWA_BC.zip", "ftp.geobc.gov.bc.ca",
"/sections/outgoing/bmgs/FWA_Public")
fgdb_dir = os.path.dirname(zip_path)
fgdb = os.path.join(fgdb_dir, "FWA_BC.gdb")
if not os.path.exists(fgdb):
with zipfile.ZipFile(zip_path, "r") as zip_ref:
zip_ref.extractall(get_cache_path((fgdb_dir, )))
logging.info("Retrieved BC Freshwater Atlas")
run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, ))
os.makedirs(run_directory)
src_driver = ogr.GetDriverByName("OpenFileGDB")
src_datasource = src_driver.Open(fgdb)
mem_driver = ogr.GetDriverByName("Memory")
mem_datasource = mem_driver.CreateDataSource("")
logging.info("Clipping waterways for bbox")
ogr_to_provided(
bbox,
[src_datasource.GetLayerByName("FWA_ROUTES_SP")],
mem_datasource,
"waterways",
OUTPUT_CRS_CODE,
)
logging.info("Clipping lakes for bbox")
ogr_to_provided(
bbox,
[src_datasource.GetLayerByName("FWA_LAKES_POLY")],
mem_datasource,
"lakes",
OUTPUT_CRS_CODE,
)
logging.info("Clipping rivers for bbox")
ogr_to_provided(
bbox,
[src_datasource.GetLayerByName("FWA_RIVERS_POLY")],
mem_datasource,
"rivers",
OUTPUT_CRS_CODE,
)
logging.info("Clipping wetlands for bbox")
ogr_to_provided(
bbox,
[src_datasource.GetLayerByName("FWA_WETLANDS_POLY")],
mem_datasource,
"wetlands",
OUTPUT_CRS_CODE,
)
waterways_layer = mem_datasource.GetLayerByName("waterways")
lakes_layer = mem_datasource.GetLayerByName("lakes")
rivers_layer = mem_datasource.GetLayerByName("rivers")
wetlands_layer = mem_datasource.GetLayerByName("wetlands")
dst_srs = ogr.osr.SpatialReference()
dst_srs.ImportFromEPSG(int(OUTPUT_CRS_CODE.split(":")[-1]))
no_lakes_layer = mem_datasource.CreateLayer(
"no_lakes", dst_srs,
waterways_layer.GetLayerDefn().GetGeomType())
no_rivers_layer = mem_datasource.CreateLayer(
"no_rivers", dst_srs,
waterways_layer.GetLayerDefn().GetGeomType())
no_wetlands_layer = mem_datasource.CreateLayer(
"no_wetlands", dst_srs,
waterways_layer.GetLayerDefn().GetGeomType())
logging.info("Erasing intersections - lakes")
waterways_layer.Erase(lakes_layer, no_lakes_layer)
logging.info("Erasing intersections - rivers")
no_lakes_layer.Erase(rivers_layer, no_rivers_layer)
logging.info("Erasing intersections - wetlands")
no_rivers_layer.Erase(wetlands_layer, no_wetlands_layer)
logging.info("Writing waterways")
dst_path = os.path.join(run_directory, "bc_waterways.shp")
dst_driver = ogr.GetDriverByName("ESRI Shapefile")
dst_datasource = dst_driver.CreateDataSource(dst_path)
dst_datasource.CopyLayer(no_wetlands_layer, "bc_waterways")
src_datasource = None
mem_datasource = None
dst_datasource = None
return [dst_path]
dstSRS=OUTPUT_CRS_CODE,
resampleAlg="lanczos",
)
if remove_intermediaries():
os.remove(generation_request.path)
os.remove(generation_request.tif_path)
except Exception as ex:
logging.error(ex)
for generation_request in bbox_cells:
try:
Warp(
generation_request.run_path,
generation_request.prj_path,
cutlineDSName=get_datasource_from_bbox(
bbox, get_run_data_path(run_id, None)),
cutlineLayer=BBOX_LAYER_NAME,
cropToCutline=False,
cutlineBlend=1,
dstNodata=-1,
)
except Exception as ex:
swallow_unimportant_warp_error(ex)
return list(
filter(
lambda run_path: os.path.exists(run_path),
map(lambda generation_request: generation_request.run_path,
bbox_cells),
))
