def transparent_clip_to_bbox(
tile_paths: List[str], bbox: BBOX, quantize: bool = True
) -> None:
logging.info("Clipping edge tiles to bbox")
min_x, max_x, min_y, max_y = bbox.transform_as_geom("EPSG:3857").GetEnvelope()
_transparent_clip_to_bbox_executor(
min_x, min_y, max_x, max_y, tile_paths, quantize
).parallel(get_process_pool_count())
def _create_run_output(
bbox: BBOX, grid: List[PartialCoverageTile], wms_crs_code: str
) -> Dict[int, List[str]]:
file_list = defaultdict(list)
wms_crs = CRS(wms_crs_code)
for tile in grid:
try:
adjusted_envelope = gdal_window_intersection(
bbox.as_gdal_win(wms_crs_code),
(tile.x_min, tile.y_max, tile.x_max, tile.y_min),
)
(
int_win_x_min,
int_win_y_max,
int_win_x_max,
int_win_y_min,
) = adjusted_envelope
if int_win_x_min > tile.x_min:
pixels_changed = _map_units_to_pixels(
int_win_x_min - tile.x_min, tile.scale, wms_crs
)
int_win_x_min -= _pixels_to_map_units(
pixels_changed % 1, tile.scale, wms_crs
)
if int_win_y_max < tile.y_max:
pixels_changed = _map_units_to_pixels(
tile.y_max - int_win_y_max, tile.scale, wms_crs
)
int_win_y_max += _pixels_to_map_units(
pixels_changed % 1, tile.scale, wms_crs
)
if int_win_x_max < tile.x_max:
pixels_changed = _map_units_to_pixels(
tile.x_max - int_win_x_max, tile.scale, wms_crs
)
int_win_x_max += _pixels_to_map_units(
pixels_changed % 1, tile.scale, wms_crs
)
if int_win_y_min > tile.x_min:
pixels_changed = _map_units_to_pixels(
int_win_y_min - tile.y_min, tile.scale, wms_crs
)
int_win_y_min -= _pixels_to_map_units(
pixels_changed % 1, tile.scale, wms_crs
)
Translate(
tile.final_path,
tile.tif_path,
projWin=[int_win_x_min, int_win_y_max, int_win_x_max, int_win_y_min,],
)
file_list[tile.scale].append(tile.final_path)
except Exception as ex:
swallow_unimportant_warp_error(ex)
return file_list
def _getExtentFromShp(path: str, crs_code: str) -> BBOX:
driver = gdal.ogr.GetDriverByName("ESRI Shapefile")
shp_datasource = driver.Open(path)
shp_layer = shp_datasource.GetLayerByIndex(0)
shp_extent = shp_layer.GetExtent()
shp_crs = CRS(crs_code)
bbox_crs = CRS("EPSG:4326")
transformer = Transformer.from_crs(shp_crs, bbox_crs, always_xy=True)
llx, lly = transformer.transform(shp_extent[0], shp_extent[2])
urx, ury = transformer.transform(shp_extent[1], shp_extent[3])
return BBOX(min_x=llx, min_y=lly, max_x=urx, max_y=ury)
def _getExtentFromRaster(path: str, crs_code: str) -> BBOX:
image = gdal.Open(path)
ulx, xres, _, uly, _, yres = image.GetGeoTransform()
lrx = ulx + (image.RasterXSize * xres)
lry = uly + (image.RasterYSize * yres)
destCrs = CRS("EPSG:4326")
srcCrs = CRS(crs_code)
transformer = Transformer.from_crs(srcCrs, destCrs, always_xy=True)
lowerRight = transformer.transform(lrx, lry)
upperLeft = transformer.transform(ulx, uly)
return BBOX(min_x=upperLeft[0],
min_y=lowerRight[1],
max_x=lowerRight[0],
max_y=upperLeft[1])
def transparent_clip_to_bbox(tile_paths: List[str], bbox: BBOX) -> None:
logging.info("Clipping edge tiles to bbox")
min_x, max_x, min_y, max_y = bbox.transform_as_geom(
"EPSG:3857").GetEnvelope()
def inspect_and_clip(tile_path: str) -> None:
match_groups = re.match(r".+/(\d+)/(\d+)/(\d+)\.png$", tile_path)
z, x, y = (
int(match_groups.group(1)),
int(match_groups.group(2)),
int(match_groups.group(3)),
)
tile_min_x = EXTENT_LIMIT * -1 + METRES_PER_TILE[z] * x
tile_max_x = tile_min_x + METRES_PER_TILE[z]
tile_min_y = EXTENT_LIMIT - METRES_PER_TILE[z] * (y + 1)
tile_max_y = tile_min_y + METRES_PER_TILE[z]
if not (tile_min_x >= max_x or tile_max_x <= min_x
or tile_min_y >= max_y or tile_max_y <= min_y):
left_pixels = (math.floor(
(min_x - tile_min_x) /
METRES_PER_PIXEL[z]) if min_x > tile_min_x else 0)
bottom_pixels = (math.floor(
(min_y - tile_min_y) /
METRES_PER_PIXEL[z]) if min_y > tile_min_y else 0)
right_pixels = (math.floor(
(tile_max_x - max_x) /
METRES_PER_PIXEL[z]) if tile_max_x > max_x else 0)
top_pixels = (math.floor(
(tile_max_y - max_y) /
METRES_PER_PIXEL[z]) if tile_max_y > max_y else 0)
logging.debug(
f"{tile_path} needs clipping by {left_pixels},{bottom_pixels} {right_pixels},{top_pixels}"
)
tile_src = Image.open(tile_path)
tile_has_palette = tile_src.getpalette is not None
tile = tile_src.convert("RGBA") if tile_has_palette else tile_src
for i in range(TILE_SIZE):
for j in range(TILE_SIZE):
coord = (i, j)
if (i < left_pixels or i >= (TILE_SIZE - right_pixels)
or j < top_pixels or j >=
(TILE_SIZE - bottom_pixels)):
new_values = (0, 0, 0, 0)
tile.putpixel(coord, new_values)
tile.quantize(method=2).save(tile_path)
ThreadPool(int(os.environ.get("TRANSPARENT_CLIP_CONCURRENCY",
4))).map(inspect_and_clip, tile_paths)
def get_prior_runs(result_dir: str) -> List[BBOX]:
path = _get_gpkg_path(result_dir)
if os.path.exists(path):
datasource = GPKG_DRIVER.Open(path, 0)
layer = datasource.GetLayerByName(LAYER_NAME)
runs = list()
while area_feature := layer.GetNextFeature():
run_envelope = area_feature.GetGeometryRef().GetEnvelope()
runs.append(
BBOX(
min_x=run_envelope[0],
min_y=run_envelope[2],
max_x=run_envelope[1],
max_y=run_envelope[3],
))
return runs
def ogr_to_shp(
bbox: BBOX,
src_layers: List[ogr.Layer],
dst_path: str,
dst_layer_name: str,
dst_crs_code: str,
) -> List[str]:
gen_driver = ogr.GetDriverByName("ESRI Shapefile")
gen_datasource = gen_driver.CreateDataSource(dst_path)
gen_srs = ogr.osr.SpatialReference()
gen_srs.ImportFromEPSG(int(dst_crs_code.split(":")[-1]))
for i, src_layer in enumerate(src_layers):
if src_layer.GetGeomType() == ogr.wkbNone:
logging.debug(
f"Layer {src_layer.GetName()} does not contain geometries, skipping"
)
continue
src_layer_srs = src_layer.GetSpatialRef()
clip_geometry = bbox.transform_as_geom(
f"{src_layer_srs.GetAuthorityName(None)}:{src_layer_srs.GetAuthorityCode(None)}"
)
if i == 0:
gen_layer = gen_datasource.CreateLayer(
dst_layer_name, gen_srs,
src_layer.GetLayerDefn().GetGeomType())
for j in range(src_layer.GetLayerDefn().GetFieldCount()):
field_defn = src_layer.GetLayerDefn().GetFieldDefn(j)
gen_layer.CreateField(field_defn)
src_layer.SetSpatialFilter(clip_geometry)
logging.debug(
f"Clipped src_layer to {src_layer.GetFeatureCount()} features")
while filtered_feature := src_layer.GetNextFeature():
contained_feature = filtered_feature.Clone()
contained_geometry = contained_feature.GetGeometryRef(
).Intersection(clip_geometry)
if contained_geometry:
contained_geometry.AssignSpatialReference(
contained_feature.GetGeometryRef().GetSpatialReference()
) # geometry loses its spatial ref during Intersection
contained_geometry.TransformTo(gen_srs)
contained_feature.SetGeometryDirectly(contained_geometry)
gen_layer.CreateFeature(contained_feature)
def get_datasource_from_bbox(bbox: BBOX, output_dir: str) -> None:
driver = ogr.GetDriverByName("GPKG")
gpkg_path = os.path.join(output_dir, BBOX_GPKG_NAME)
datasource = driver.Open(gpkg_path)
if not datasource:
datasource = driver.CreateDataSource(gpkg_path)
layer = datasource.GetLayerByName(BBOX_LAYER_NAME)
srs = osr.SpatialReference()
srs.SetFromUserInput(bbox.crs_code)
if not layer:
layer = datasource.CreateLayer(BBOX_LAYER_NAME, srs, ogr.wkbPolygon)
if layer.GetFeatureCount() == 0:
geometry = ogr.CreateGeometryFromWkt(bbox.get_wkt())
feature_defn = layer.GetLayerDefn()
feature = ogr.Feature(feature_defn)
feature.SetGeometry(geometry)
layer.CreateFeature(feature)
feature = None
layer, datasource = None, None
return gpkg_path
def record_run(result_dir: str, bbox: BBOX) -> None:
gpkg_path = _get_gpkg_path(result_dir)
gpkg_datasource = GPKG_DRIVER.Open(gpkg_path, 1)
if not gpkg_datasource:
gpkg_datasource = GPKG_DRIVER.CreateDataSource(gpkg_path)
cumulative_layer = gpkg_datasource.GetLayerByName(LAYER_NAME)
if not cumulative_layer:
srs = osr.SpatialReference()
srs.SetFromUserInput("CRS:84")
cumulative_layer = gpkg_datasource.CreateLayer(LAYER_NAME, srs,
ogr.wkbPolygon)
geometry = ogr.CreateGeometryFromWkt(bbox.get_wkt())
feature_defn = cumulative_layer.GetLayerDefn()
feature = ogr.Feature(feature_defn)
feature.SetGeometryDirectly(geometry)
cumulative_layer.CreateFeature(feature)
kml_path = os.path.join(result_dir, "coverage.kml")
if os.path.exists(kml_path):
os.remove(kml_path)
kml_driver = ogr.GetDriverByName("KML")
kml_datasource = kml_driver.CreateDataSource(kml_path)
kml_datasource.CopyLayer(cumulative_layer, "areas")
geojson_path = os.path.join(result_dir, "coverage.geojson")
if os.path.exists(geojson_path):
os.remove(geojson_path)
geojson_driver = ogr.GetDriverByName("GeoJSON")
geojson_datasource = geojson_driver.CreateDataSource(geojson_path)
geojson_datasource.CopyLayer(cumulative_layer, "areas")
cumulative_layer, gpkg_datasource, kml_datasource, geojson_datasource = (
None,
None,
None,
None,
)
def has_prior_run(result_dir: str, bbox: BBOX) -> bool:
wkts = [prior_run.get_wkt() for prior_run in get_prior_runs(result_dir)]
return bbox.get_wkt() in wkts
while min_y < max_y:
increment_y = min(BBOX_DIVISION, max_y - min_y)
this_max_x = min_x + increment_x
this_max_y = min_y + increment_y
logging.info(
f"area division {min_x},{this_max_x} {min_y},{this_max_y}. x diff: {this_max_x - min_x}, y diff: {this_max_y - min_y}"
)
area_division_args.append((
min_x,
this_max_x,
min_y,
this_max_y,
profile_name,
xyz_url,
))
min_y += increment_y
min_x += increment_x
logging.info(
f"Require {len(area_division_args)} export(s) at {BBOX_DIVISION}x{BBOX_DIVISION}"
)
for idx, args in enumerate(area_division_args):
logging.info(
f"Export {idx + 1} of {len(area_division_args)}: {args[0]},{args[2]} {args[1]},{args[3]}"
)
provision(
BBOX(min_x=args[0], max_x=args[1], min_y=args[2], max_y=args[3]),
args[4],
args[5],
)
result_temp_dir = get_result_path((run_id, ))
if os.path.exists(run_dir):
rmtree(run_dir)
if os.path.exists(result_temp_dir):
rmtree(result_temp_dir)
logging.info("Finished")
return ProvisionResult.SUCCESS
if __name__ == "__main__":
# has been directly invoked, likely debugging
configure_logging()
parser = argparse.ArgumentParser()
parser.add_argument("min_x", type=float)
parser.add_argument("min_y", type=float)
parser.add_argument("max_x", type=float)
parser.add_argument("max_y", type=float)
parser.add_argument("profile", type=str)
parser.add_argument("xyz_url", type=str)
args = vars(parser.parse_args())
provision(
BBOX(
**{
key: value
for key, value in args.items()
if key in ("min_x", "min_y", "max_x", "max_y")
}),
args["profile"],
args["xyz_url"],
)
while cell_y_min < grid_max_y:
cell_y_max = round_for_increment(cell_y_min + walk_increment,
walk_increment)
next_cell_geom = ogr.CreateGeometryFromWkt(
f"POLYGON (({cell_x_min} {cell_y_min}, {cell_x_max} {cell_y_min}, {cell_x_max} {cell_y_max}, {cell_x_min} {cell_y_max}, {cell_x_min} {cell_y_min}))"
)
next_cell_geom.AssignSpatialReference(grid_srs)
intersection_geom = area_feature.GetGeometryRef().Intersection(
next_cell_geom)
if intersection_geom is not None and not intersection_geom.IsEmpty(
):
provision_args.extend([
ProvisionArg(
bbox=BBOX(
min_x=cell_x_min,
min_y=cell_y_min,
max_x=cell_x_max,
max_y=cell_y_max,
),
profile_name=profile_name,
xyz_url=get_xyz_url_for_feature(area_feature),
skippable=int(
os.environ.get("GRIDDED_REPEAT_IF_EXISTS", 0))
!= 1,
) for profile_name in get_profile_names_for_feature(
area_feature)
])
cell_y_min = cell_y_max
cell_x_min = cell_x_max
area_layer.SetAttributeFilter(f"strategy = '{RunStrategy.ENVELOPE.value}'")
while area_feature := area_layer.GetNextFeature():