def test_tiles_in_index(self, bag3d_db, features_idx_sch):
if features_idx_sch["index"]["table"] == "bag_index_identical":
to_process = ["25gn1_10", "25gn1_11", "25gn1_6", "not_in_index"]
expectation = ["25gn1_10", "25gn1_11", "25gn1_6"]
tiles = tileconfig.DbTiles(
bag3d_db,
tile_index_schema=db.Schema(features_idx_sch),
features_schema=None,
)
result = tiles.tiles_in_index(to_process)
assert set(result) == set(expectation)
elif features_idx_sch["index"]["table"] == "bag_index":
to_process = ["u1", "u2", "u5", "not_in_index"]
expectation = ["u1", "u2", "u5"]
tiles = tileconfig.DbTiles(
bag3d_db,
tile_index_schema=db.Schema(features_idx_sch),
features_schema=None,
)
result = tiles.tiles_in_index(to_process)
assert set(result) == set(expectation)
else:
pytest.fail(
msg=f"Unexpected features_tiles.index.table "
f"{features_idx_sch['index']['table']}",
pytrace=False,
)
def feature_tiles(
bag3d_db, features_idx_sch, features_sch
) -> tileconfig.DbTiles:
return tileconfig.DbTiles(
conn=bag3d_db,
tile_index_schema=db.Schema(features_idx_sch),
features_schema=db.Schema(features_sch),
)
def test_invalid_tiles(self, bag3d_db, features_idx_sch):
to_process = ["bla", "not_in_index"]
tiles = tileconfig.DbTiles(
bag3d_db,
tile_index_schema=db.Schema(features_idx_sch),
features_schema=None,
)
with pytest.raises(ValueError):
tiles.with_list(tiles=to_process)
def test_all_in_index(self, bag3d_db, features_idx_sch):
if features_idx_sch["index"]["table"] == "bag_index_identical":
expectation = [
"25gn1_1",
"25gn1_2",
"25gn1_3",
"25gn1_4",
"25gn1_5",
"25gn1_6",
"25gn1_7",
"25gn1_8",
"25gn1_9",
"25gn1_10",
"25gn1_11",
"25gn1_12",
"25gn1_13",
"25gn1_14",
"25gn1_15",
"25gn1_16",
]
tiles = tileconfig.DbTiles(
bag3d_db,
tile_index_schema=db.Schema(features_idx_sch),
features_schema=None,
)
result = tiles.all_in_index()
assert set(result) == set(expectation)
elif features_idx_sch["index"]["table"] == "bag_index":
expectation = ["u4", "u1", "u2", "u3", "u5", "u6"]
tiles = tileconfig.DbTiles(
bag3d_db,
tile_index_schema=db.Schema(features_idx_sch),
features_schema=None,
)
result = tiles.all_in_index()
assert set(result) == set(expectation)
else:
pytest.fail(
msg=f"Unexpected features_tiles.index.table "
f"{features_idx_sch['index']['table']}",
pytrace=False,
)
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None):
"""Configure the controller."""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
if worker_key == "Example":
tilescfg = tileconfig.FileTiles()
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {"part_A": tilescfg, "part_B": tilescfg}
else:
# For the ExampleDb worker
tilescfg = tileconfig.DbTiles(
conn=db.Db(**self.cfg["config"]["database"]),
tile_index_schema=db.Schema(
self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {
"part_A": tilescfg,
}
for part, _tilescfg in parts.items():
_tilescfg.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=_tilescfg)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None):
"""Configure the controller.
Input-specific subclasses need to implement this.
"""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
# Configure the tiles (DBTiles in this case)
tilescfg = tileconfig.DbTiles(
conn=db.Db(**self.cfg["config"]["database"]),
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic for processing different parts. Parst are required
# for example when processing a large area that needs different tile
# configurations. For instance the Netherlands with AHN2 and AHN3.
parts = {
"part_A": tilescfg,
}
# Create a processor for each part
for part, _tilescfg in parts.items():
_tilescfg.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=_tilescfg)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def test_within_extent(
self, bag3d_db, polygons, features_idx_sch, features_sch
):
"""Tile selection with a polygon should return the tiles that
intersect with the polygon."""
tiles = tileconfig.DbTiles(
bag3d_db,
tile_index_schema=db.Schema(features_idx_sch),
features_schema=db.Schema(features_sch),
)
result = tiles.within_extent(polygons["ewkb"])
if features_idx_sch["index"]["table"] == "bag_index_identical":
expectation = {"25gn1_10", "25gn1_11", "25gn1_6", "25gn1_7"}
assert set(result) == expectation
elif features_idx_sch["index"]["table"] == "bag_index":
expectation = {"u2", "u5"}
assert set(result) == expectation
else:
pytest.fail(
msg=f"Unexpected features_tiles.index.table "
f"{features_idx_sch['index']['table']}",
pytrace=False,
)
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
# Configure feature tiles with elevation from AHN3
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
elevation_tiles.configure(tiles=tiles)
ahntiles = tileconfig.DbTilesAHN(conn=None,
elevation_tiles=None,
feature_tiles=None)
ahntiles.to_process = elevation_tiles.to_process
elevation_file_paths = ahntiles.create_elevation_file_index(
directory_mapping=self.cfg["config"]["directory_mapping"])
ahn_version = 3
ahntiles.elevation_file_index = {}
for i, ahn_id in enumerate(elevation_tiles.to_process):
paths = []
if ahn_id in elevation_file_paths:
paths.extend(
(p, ahn_version) for p in elevation_file_paths[ahn_id])
else:
log.debug(
f"File matching the AHN ID {ahn_id} not found, skipping tile"
)
del ahntiles.to_process[i]
ahntiles.elevation_file_index[ahn_id] = paths
# Set up outputs
output_obj = output.Output()
if "database" in self.cfg["config"]["output"]:
output_obj.db = output.DbOutput(conn=db.Db(
**self.cfg["config"]["output"]["database"]))
elif "dir" in self.cfg["config"]["output"]:
output_obj.dir = output.DirOutput(
path=self.cfg["config"]["output"]["dir"])
for k, v in self.cfg["config"]["output"].items():
if k != "database" and k != "dir":
output_obj.kwargs[k] = v
ahntiles.output = output_obj
name = "part1"
proc = processor.factory.create(processor_key,
name=name,
tiles=ahntiles)
self.processors[proc] = name
log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
ahntiles = tileconfig.DbTilesAHN(
conn=db.Db(**self.cfg["config"]["database"]),
elevation_tiles=db.Schema(self.cfg["config"]["elevation_tiles"]),
feature_tiles=None,
)
ahntiles.configure(
tiles=tiles,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=True,
)
part = "AHN3"
ahntiles.output = output.Output(
dir=output.DirOutput(self.cfg["config"]["output"]["dir"]))
proc = processor.factory.create(processor_key,
name=part,
tiles=ahntiles)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def test_concatenate(self, relations):
index = db.Schema(relations)
result = index.schema + index.table
assert result == sql.Identifier("tile_index", "bag_index_test")
def test_init(self, relations):
index = db.Schema(relations)
assert index.schema.string == "tile_index"
assert index.schema.sqlid == sql.Identifier("tile_index")
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
# Configure the tiles
_tilecfg = {
"conn":
db.Db(**self.cfg["config"]["database"]),
"elevation_index_schema":
db.Schema(self.cfg["config"]["elevation_tiles"]),
"tile_index_schema":
db.Schema(self.cfg["config"]["features_tiles"]),
"features_schema":
db.Schema(self.cfg["config"]["features"]),
}
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
feature_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
# Configure feature tiles with elevation from AHN2
ahn_2 = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_2.configure(
tiles=tiles,
version=2,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Configure feature tiles with elevation from AHN3
ahn_3 = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_3.configure(
tiles=tiles,
version=3,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Configure feature tiles that are on the border of AHN2 and AHN3
ahn_border = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_border.configure(
tiles=tiles,
on_border=True,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {"AHN2": ahn_2, "AHN3": ahn_3, "AHN_border": ahn_border}
for part, ahntiles in parts.items():
ahntiles.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=ahntiles)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None,
restart: int = 0):
"""Configure the control logic."""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
# Configure the tiles
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
feature_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
# Configure feature tiles with elevation from AHN3
ahntiles = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahntiles.configure(
tiles=tiles,
version=None,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Set up outputs
output_obj = output.Output()
if "database" in self.cfg["config"]["output"]:
output_obj.db = output.DbOutput(conn=db.Db(
**self.cfg["config"]["output"]["database"]))
if "dir" in self.cfg["config"]["output"]:
# FIXME: Output.dir should be a dict maybe by default?
output_obj.dir = {}
if isinstance(self.cfg["config"]["output"]["dir"], str):
output_obj.dir["path"] = output.DirOutput(
path=self.cfg["config"]["output"]["dir"])
elif isinstance(self.cfg["config"]["output"]["dir"], dict):
for k, dirpath in self.cfg["config"]["output"]["dir"].items():
output_obj.dir[k] = output.DirOutput(dirpath)
else:
raise ValueError(
f'Expected str or dict in {self.cfg["config"]["output"]["dir"]}'
)
for k, v in self.cfg["config"]["output"].items():
if k != "database" and k != "dir":
output_obj.kwargs[k] = v
ahntiles.output = output_obj
name = "part1"
proc = processor.factory.create(processor_key,
name=name,
tiles=ahntiles)
self.processors[proc] = name
log.info(f"Configured {self.__class__.__name__}")
def elevation_tiles(bag3d_db, elevation_idx_sch) -> tileconfig.DbTiles:
return tileconfig.DbTiles(
conn=bag3d_db, tile_index_schema=db.Schema(elevation_idx_sch)
)
