def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("GPKG") grid_datasource = driver.Open(get_data_path(("grids.gpkg", ))) grid_layer = grid_datasource.GetLayerByName("BC-20000") grid_layer.SetSpatialFilterRect(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y) bbox_cells = list() while grid_cell := grid_layer.GetNextFeature(): cell_name = grid_cell.GetFieldAsString("MAP_TILE") cell_parent = re.search(r"^\d{2,3}[a-z]", cell_name, re.IGNORECASE)[0] bbox_cells.append( GenerationRequest( url= f"https://pub.data.gov.bc.ca/datasets/177864/tif/bcalb/{cell_parent}/{cell_name}.zip", path=get_cache_path((CACHE_DIR_NAME, f"{cell_name}.zip")), expected_types=["application/zip"], cell_name=cell_name, tif_name=f"{cell_name}.tif", tif_path=get_cache_path((CACHE_DIR_NAME, f"{cell_name}.tif")), prj_path=get_cache_path( (CACHE_DIR_NAME, f"{cell_name}_prj.tif")), run_path=get_run_data_path( run_id, (CACHE_DIR_NAME, f"{cell_name}.tif")), ))
def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("GPKG") grid_datasource = driver.Open(get_data_path(("grids.gpkg", ))) grid_layer = grid_datasource.GetLayerByName("Canada-50000") grid_layer.SetSpatialFilterRect(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y) bbox_cells = list() while grid_cell := grid_layer.GetNextFeature(): cell_name = grid_cell.GetFieldAsString("NTS_SNRC") cell_parent = re.sub( "^0", "", re.search(r"^\d{2,3}[a-z]", cell_name, re.IGNORECASE)[0]) for cardinal in ("e", "w"): cell_part_name = f"{cell_name.lower()}_{cardinal}" zip_file_name = f"{cell_part_name}.dem.zip" bbox_cells.append( GenerationRequest( url= f"https://pub.data.gov.bc.ca/datasets/175624/{cell_parent.lower()}/{zip_file_name}", path=get_cache_path((CACHE_DIR_NAME, zip_file_name)), expected_type="application/zip", dem_path=get_cache_path( (CACHE_DIR_NAME, f"{cell_part_name}.dem")), prj_path=get_cache_path( (CACHE_DIR_NAME, f"{cell_part_name}_prj.tif")), hs_path=get_cache_path( (CACHE_DIR_NAME, f"{cell_part_name}_hs.tif")), run_path=get_run_data_path( run_id, (CACHE_DIR_NAME, f"{cell_part_name}.tif")), ))
def provision( bbox: BBOX, base_url: str, wms_properties: WmsProperties, wms_crs_code: str, layers: Tuple[str], styles: Tuple[str], scales: Tuple[int], image_format: str, cache_dir_name: str, run_id: str, ) -> Dict[int, List[str]]: cache_directory = get_cache_path((cache_dir_name,)) run_directory = get_run_data_path(run_id, (cache_dir_name,)) os.makedirs(run_directory) grid = _build_grid_for_bbox(bbox, wms_crs_code, scales, wms_properties) grid_for_retrieval = _update_grid_for_retrieval( base_url, grid, layers, styles, wms_crs_code, image_format, cache_directory, run_directory, ) grid_for_missing = _filter_grid_for_missing(grid_for_retrieval) requests = _convert_grid_to_requests(grid_for_missing, image_format) retrieve(requests) if image_format != TARGET_FILE_FORMAT: _convert_to_tif(grid_for_missing, wms_crs_code) return _create_run_output(bbox, grid_for_retrieval, run_id)
def provision( bbox: BBOX, run_id: str, src_layer_name: str, dst_layer_name: str, ) -> List[str]: kmz_driver = ogr.GetDriverByName("LIBKML") kmz_datasets = [ kmz_driver.Open(filename) for filename in glob.iglob( get_data_path(("avcan-ates-areas-2020-06-23", "**", "*.kmz")), recursive=True, ) ] run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory, exist_ok=True) path = os.path.join(run_directory, f"{dst_layer_name}.shp") ogr_to_shp( bbox, [ds.GetLayerByName(src_layer_name) for ds in kmz_datasets], path, dst_layer_name, OUTPUT_CRS_CODE, ) kmz_datasets = None return [path]
def execute(bbox: BBOX, run_id: str, args: Dict[str, object] = dict()) -> None: xyz_result = xyz_provisioner( bbox, args["xyz_url"], ZOOM_MIN, ZOOM_MAX, ["image/png", "image/jpeg"], OUTPUT_FORMAT, ) tmp_dir = get_run_data_path(run_id, (NAME, )) tmp_tile_paths = list() logging.info( f"Copying {len(xyz_result.tile_paths)} {NAME} tiles to tmp dir for edge clipping" ) for tile_path in xyz_result.tile_paths: tmp_tile_path = tile_path.replace(xyz_result.tile_dir, tmp_dir) tmp_tile_paths.append(tmp_tile_path) os.makedirs(os.path.dirname(tmp_tile_path), exist_ok=True) copyfile(tile_path, tmp_tile_path) transparent_clip_to_bbox( [ os.path.join(tmp_dir, tile_path) for tile_path in get_edge_tiles(tmp_dir) ], bbox, ) add_or_update(tmp_dir, get_result_path((NAME, )))
def provision(bbox: BBOX, run_id: str) -> List[str]: logging.info( "Retrieving BC Freshwater Atlas - this could take a while the first time" ) zip_path = fetch("FWA_BC.zip", "ftp.geobc.gov.bc.ca", "/sections/outgoing/bmgs/FWA_Public") fgdb_dir = os.path.dirname(zip_path) fgdb = os.path.join(fgdb_dir, "FWA_BC.gdb") if not os.path.exists(fgdb): with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(get_cache_path((fgdb, ))) logging.info("Retrieved BC Freshwater Atlas") run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("OpenFileGDB") datasource = driver.Open(fgdb) path = os.path.join(run_directory, "bc_wetlands.shp") ogr_to_shp( bbox, [datasource.GetLayerByName("FWA_WETLANDS_POLY")], path, "bc_wetlands", OUTPUT_CRS_CODE, ) datasource = None return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("GPKG") datasource = driver.Open(get_local_features_path()) result = ogr_to_shp( bbox, [datasource.GetLayerByName("trails")], os.path.join(run_directory, "trails.shp"), "trails", OUTPUT_CRS_CODE, ) datasource = None return result
def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("GPKG") datasource = driver.Open(LOCAL_FEATURES_PATH) path = os.path.join(run_directory, "trails.shp") ogr_to_shp( bbox, [datasource.GetLayerByName("trails")], path, "trails", OUTPUT_CRS_CODE, ) datasource = None return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("ESRI Shapefile") datasource = driver.Open( get_data_path(("FTEN_ROAD_SECTION_LINES_SVW", "FTEN_RS_LN_line.shp"))) result = ogr_to_shp( bbox, [datasource.GetLayerByIndex(0)], os.path.join(run_directory, "bc_resource_roads.shp"), "bc_resource_roads", OUTPUT_CRS_CODE, ) datasource = None return result
def provision(arg: ProvisionArg) -> ProvisionResult: bbox, profile_name, xyz_url = arg.bbox, arg.profile_name, arg.xyz_url bbox_exists = has_prior_run(get_result_path((profile_name, )), bbox) if bbox_exists and arg.skippable: logging.info( f"Skipping {profile_name} {bbox.min_x},{bbox.min_y} {bbox.max_x},{bbox.max_y} as it already exists" ) return ProvisionResult.SKIPPED else: logging.info( f"Provisioning {profile_name} {bbox.min_x},{bbox.min_y} {bbox.max_x},{bbox.max_y}" ) run_id = str(uuid.uuid4()) profiles = { profile.NAME: { "execute": profile.execute, "zoom_min": profile.ZOOM_MIN, "zoom_max": profile.ZOOM_MAX, "format": profile.OUTPUT_FORMAT, } for profile in [xyz, topo, xyzsummer, xyzwinter, xyzhunting] } profiles[profile_name]["execute"](bbox, run_id, {"xyz_url": xyz_url}) if int(os.environ.get("BVSAR_HEAD_VALIDATE", 0)) == 1: logging.info("Validating result") check_exists( xyz_check_builder( bbox, "{0}/{1}/{{z}}/{{x}}/{{y}}.png".format( os.environ.get("HTTP_URL", "http://rpi/tile/file"), profile_name, ), profiles[profile_name]["zoom_min"], profiles[profile_name]["zoom_max"], "image/{0}".format(profiles[profile_name]["format"]), )) record_run(get_result_path((profile_name, )), bbox) if remove_intermediaries(): run_dir = get_run_data_path(run_id, None) result_temp_dir = get_result_path((run_id, )) if os.path.exists(run_dir): rmtree(run_dir) if os.path.exists(result_temp_dir): rmtree(result_temp_dir) logging.info("Finished") return ProvisionResult.SUCCESS
def provision(bbox: BBOX, profile_name: str, xyz_url: str) -> None: bbox_repeat_if_exists = int(os.environ.get("BBOX_REPEAT_IF_EXISTS", 0)) == 1 bbox_exists = has_prior_run(get_result_path((profile_name, )), bbox) if bbox_exists and not bbox_repeat_if_exists: logging.info( f"Skipping {profile_name} {bbox.min_x},{bbox.min_y} {bbox.max_x},{bbox.max_y} as it already exists" ) return run_id = str(uuid.uuid4()) profiles = { profile.NAME: { "execute": profile.execute, "zoom_min": profile.ZOOM_MIN, "zoom_max": profile.ZOOM_MAX, "format": profile.OUTPUT_FORMAT, } for profile in [xyz, topo, xyzsummer, xyzwinter] } profiles[profile_name]["execute"](bbox, run_id, {"xyz_url": xyz_url}) logging.info("Validating result") check_exists( xyz_check_builder( bbox, "{0}/{1}/{{z}}/{{x}}/{{y}}.png".format( os.environ.get("HTTP_URL", "http://localhost:9000/tiles/files"), profile_name, ), profiles[profile_name]["zoom_min"], profiles[profile_name]["zoom_max"], "image/{0}".format(profiles[profile_name]["format"]), )) record_run(get_result_path((profile_name, )), bbox) if remove_intermediaries(): run_dir = get_run_data_path(run_id, None) result_temp_dir = get_result_path((run_id, )) if os.path.exists(run_dir): rmtree(run_dir) if os.path.exists(result_temp_dir): rmtree(result_temp_dir) logging.info("Finished")
def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("OpenFileGDB") datasource = driver.Open( get_data_path(("FTEN_ROAD_SEGMENT_LINES_SVW.gdb", ))) path = os.path.join(run_directory, "bc_resource_roads.shp") ogr_to_shp( bbox, [ datasource.GetLayerByName( "WHSE_FOREST_TENURE_FTEN_ROAD_SEGMENT_LINES_SVW") ], path, "bc_resource_roads", OUTPUT_CRS_CODE, ) datasource = None return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]: logging.info( "Retrieving BC Freshwater Atlas - this could take a while the first time" ) fgdb = retrieve_directory("ftp.geobc.gov.bc.ca", "/sections/outgoing/bmgs/FWA_Public/FWA_BC.gdb") logging.info("Retrieved BC Freshwater Atlas") run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("OpenFileGDB") datasource = driver.Open(fgdb) result = ogr_to_shp( bbox, [datasource.GetLayerByName("FWA_WETLANDS_POLY")], os.path.join(run_directory, "bc_wetlands.shp"), "bc_wetlands", OUTPUT_CRS_CODE, ) datasource = None return result
def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("OpenFileGDB") datasource = driver.Open(get_data_path(("FTEN_RECREATION_POLY_SVW.gdb", ))) rec_sites_layer = datasource.GetLayerByName( "WHSE_FOREST_TENURE_FTEN_RECREATION_POLY_SVW") rec_sites_layer.SetAttributeFilter( "LIFE_CYCLE_STATUS_CODE IN ('ACTIVE','PENDING')") path = os.path.join(run_directory, "bc_rec_sites.shp") ogr_to_shp( bbox, [rec_sites_layer], path, "bc_rec_sites", OUTPUT_CRS_CODE, ) rec_sites_layer = None datasource = None return [path]
def provision(bbox: BBOX, run_id: str) -> List[str]: run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) driver = ogr.GetDriverByName("OpenFileGDB") datasource = driver.Open( get_data_path(("pmbc_parcel_fabric_poly_svw.gdb", ))) parcels_layer = datasource.GetLayerByName("pmbc_parcel_fabric_poly_svw") parcels_layer.SetAttributeFilter( "OWNER_TYPE IN ('First Nation','Mixed Ownership','Municipal','Private','Unknown')" ) path = os.path.join(run_directory, "bc_parcels.shp") ogr_to_shp( bbox, [parcels_layer], path, "bc_parcels", OUTPUT_CRS_CODE, ) parcels_layer = None datasource = None return [path]
def _create_run_output( bbox: BBOX, grid: List[PartialCoverageTile], run_id: str ) -> Dict[int, List[str]]: file_list = dict() run_directory = get_run_data_path(run_id, None) for tile in grid: if tile.scale not in file_list: file_list[tile.scale] = list() try: Warp( tile.final_path, tile.tif_path, cutlineDSName=get_datasource_from_bbox(bbox, run_directory), cutlineLayer=BBOX_LAYER_NAME, cropToCutline=True, dstNodata=-1, ) file_list[tile.scale].append(tile.final_path) except Exception as ex: swallow_unimportant_warp_error(ex) return file_list
def provision(bbox: BBOX, run_id: str) -> List[str]: logging.info( "Retrieving BC Freshwater Atlas - this could take a while the first time" ) zip_path = fetch("FWA_BC.zip", "ftp.geobc.gov.bc.ca", "/sections/outgoing/bmgs/FWA_Public") fgdb_dir = os.path.dirname(zip_path) fgdb = os.path.join(fgdb_dir, "FWA_BC.gdb") if not os.path.exists(fgdb): with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(get_cache_path((fgdb_dir, ))) logging.info("Retrieved BC Freshwater Atlas") run_directory = get_run_data_path(run_id, (CACHE_DIR_NAME, )) os.makedirs(run_directory) src_driver = ogr.GetDriverByName("OpenFileGDB") src_datasource = src_driver.Open(fgdb) mem_driver = ogr.GetDriverByName("Memory") mem_datasource = mem_driver.CreateDataSource("") logging.info("Clipping waterways for bbox") ogr_to_provided( bbox, [src_datasource.GetLayerByName("FWA_ROUTES_SP")], mem_datasource, "waterways", OUTPUT_CRS_CODE, ) logging.info("Clipping lakes for bbox") ogr_to_provided( bbox, [src_datasource.GetLayerByName("FWA_LAKES_POLY")], mem_datasource, "lakes", OUTPUT_CRS_CODE, ) logging.info("Clipping rivers for bbox") ogr_to_provided( bbox, [src_datasource.GetLayerByName("FWA_RIVERS_POLY")], mem_datasource, "rivers", OUTPUT_CRS_CODE, ) logging.info("Clipping wetlands for bbox") ogr_to_provided( bbox, [src_datasource.GetLayerByName("FWA_WETLANDS_POLY")], mem_datasource, "wetlands", OUTPUT_CRS_CODE, ) waterways_layer = mem_datasource.GetLayerByName("waterways") lakes_layer = mem_datasource.GetLayerByName("lakes") rivers_layer = mem_datasource.GetLayerByName("rivers") wetlands_layer = mem_datasource.GetLayerByName("wetlands") dst_srs = ogr.osr.SpatialReference() dst_srs.ImportFromEPSG(int(OUTPUT_CRS_CODE.split(":")[-1])) no_lakes_layer = mem_datasource.CreateLayer( "no_lakes", dst_srs, waterways_layer.GetLayerDefn().GetGeomType()) no_rivers_layer = mem_datasource.CreateLayer( "no_rivers", dst_srs, waterways_layer.GetLayerDefn().GetGeomType()) no_wetlands_layer = mem_datasource.CreateLayer( "no_wetlands", dst_srs, waterways_layer.GetLayerDefn().GetGeomType()) logging.info("Erasing intersections - lakes") waterways_layer.Erase(lakes_layer, no_lakes_layer) logging.info("Erasing intersections - rivers") no_lakes_layer.Erase(rivers_layer, no_rivers_layer) logging.info("Erasing intersections - wetlands") no_rivers_layer.Erase(wetlands_layer, no_wetlands_layer) logging.info("Writing waterways") dst_path = os.path.join(run_directory, "bc_waterways.shp") dst_driver = ogr.GetDriverByName("ESRI Shapefile") dst_datasource = dst_driver.CreateDataSource(dst_path) dst_datasource.CopyLayer(no_wetlands_layer, "bc_waterways") src_datasource = None mem_datasource = None dst_datasource = None return [dst_path]
dstSRS=OUTPUT_CRS_CODE, resampleAlg="lanczos", ) if remove_intermediaries(): os.remove(generation_request.path) os.remove(generation_request.tif_path) except Exception as ex: logging.error(ex) for generation_request in bbox_cells: try: Warp( generation_request.run_path, generation_request.prj_path, cutlineDSName=get_datasource_from_bbox( bbox, get_run_data_path(run_id, None)), cutlineLayer=BBOX_LAYER_NAME, cropToCutline=False, cutlineBlend=1, dstNodata=-1, ) except Exception as ex: swallow_unimportant_warp_error(ex) return list( filter( lambda run_path: os.path.exists(run_path), map(lambda generation_request: generation_request.run_path, bbox_cells), ))