def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
# Configure feature tiles with elevation from AHN3
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
elevation_tiles.configure(tiles=tiles)
ahntiles = tileconfig.DbTilesAHN(conn=None,
elevation_tiles=None,
feature_tiles=None)
ahntiles.to_process = elevation_tiles.to_process
elevation_file_paths = ahntiles.create_elevation_file_index(
directory_mapping=self.cfg["config"]["directory_mapping"])
ahn_version = 3
ahntiles.elevation_file_index = {}
for i, ahn_id in enumerate(elevation_tiles.to_process):
paths = []
if ahn_id in elevation_file_paths:
paths.extend(
(p, ahn_version) for p in elevation_file_paths[ahn_id])
else:
log.debug(
f"File matching the AHN ID {ahn_id} not found, skipping tile"
)
del ahntiles.to_process[i]
ahntiles.elevation_file_index[ahn_id] = paths
# Set up outputs
output_obj = output.Output()
if "database" in self.cfg["config"]["output"]:
output_obj.db = output.DbOutput(conn=db.Db(
**self.cfg["config"]["output"]["database"]))
elif "dir" in self.cfg["config"]["output"]:
output_obj.dir = output.DirOutput(
path=self.cfg["config"]["output"]["dir"])
for k, v in self.cfg["config"]["output"].items():
if k != "database" and k != "dir":
output_obj.kwargs[k] = v
ahntiles.output = output_obj
name = "part1"
proc = processor.factory.create(processor_key,
name=name,
tiles=ahntiles)
self.processors[proc] = name
log.info(f"Configured {self.__class__.__name__}")
def bag3d_db():
dbs = db.Db(
dbname="bag3d_db",
host="localhost",
port=5590,
user="******",
password="******",
)
yield dbs
dbs.close()
def test_failed_connection(self, bag3d_db):
"""Failed connection raises OperationalError"""
with pytest.raises(OperationalError) as excinfo:
# invalid dbname
db.Db(
dbname="invalid",
host=bag3d_db.host,
port=bag3d_db.port,
user=bag3d_db.user,
)
assert 'database "invalid" does not exist' in str(excinfo.value)
with pytest.raises(OperationalError) as excinfo:
# invalid host
db.Db(
dbname=bag3d_db.dbname,
host="invalid",
port=bag3d_db.port,
user=bag3d_db.user,
)
assert 'could not translate host name "invalid" to address' in str(
excinfo.value)
with pytest.raises(OperationalError) as excinfo:
# invalid port
db.Db(
dbname=bag3d_db.dbname,
host=bag3d_db.host,
port=1,
user=bag3d_db.user,
)
assert "TCP/IP connections on port 1?" in str(excinfo.value)
with pytest.raises(OperationalError) as excinfo:
# invalid user
db.Db(
dbname=bag3d_db.dbname,
host=bag3d_db.host,
port=bag3d_db.port,
user="******",
)
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None):
"""Configure the controller."""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
if worker_key == "Example":
tilescfg = tileconfig.FileTiles()
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {"part_A": tilescfg, "part_B": tilescfg}
else:
# For the ExampleDb worker
tilescfg = tileconfig.DbTiles(
conn=db.Db(**self.cfg["config"]["database"]),
tile_index_schema=db.Schema(
self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {
"part_A": tilescfg,
}
for part, _tilescfg in parts.items():
_tilescfg.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=_tilescfg)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None):
"""Configure the controller.
Input-specific subclasses need to implement this.
"""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
# Configure the tiles (DBTiles in this case)
tilescfg = tileconfig.DbTiles(
conn=db.Db(**self.cfg["config"]["database"]),
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic for processing different parts. Parst are required
# for example when processing a large area that needs different tile
# configurations. For instance the Netherlands with AHN2 and AHN3.
parts = {
"part_A": tilescfg,
}
# Create a processor for each part
for part, _tilescfg in parts.items():
_tilescfg.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=_tilescfg)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
ahntiles = tileconfig.DbTilesAHN(
conn=db.Db(**self.cfg["config"]["database"]),
elevation_tiles=db.Schema(self.cfg["config"]["elevation_tiles"]),
feature_tiles=None,
)
ahntiles.configure(
tiles=tiles,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=True,
)
part = "AHN3"
ahntiles.output = output.Output(
dir=output.DirOutput(self.cfg["config"]["output"]["dir"]))
proc = processor.factory.create(processor_key,
name=part,
tiles=ahntiles)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
# Configure the tiles
_tilecfg = {
"conn":
db.Db(**self.cfg["config"]["database"]),
"elevation_index_schema":
db.Schema(self.cfg["config"]["elevation_tiles"]),
"tile_index_schema":
db.Schema(self.cfg["config"]["features_tiles"]),
"features_schema":
db.Schema(self.cfg["config"]["features"]),
}
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
feature_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
# Configure feature tiles with elevation from AHN2
ahn_2 = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_2.configure(
tiles=tiles,
version=2,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Configure feature tiles with elevation from AHN3
ahn_3 = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_3.configure(
tiles=tiles,
version=3,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Configure feature tiles that are on the border of AHN2 and AHN3
ahn_border = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_border.configure(
tiles=tiles,
on_border=True,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {"AHN2": ahn_2, "AHN3": ahn_3, "AHN_border": ahn_border}
for part, ahntiles in parts.items():
ahntiles.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=ahntiles)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None,
restart: int = 0):
"""Configure the control logic."""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
# Configure the tiles
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
feature_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
# Configure feature tiles with elevation from AHN3
ahntiles = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahntiles.configure(
tiles=tiles,
version=None,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Set up outputs
output_obj = output.Output()
if "database" in self.cfg["config"]["output"]:
output_obj.db = output.DbOutput(conn=db.Db(
**self.cfg["config"]["output"]["database"]))
if "dir" in self.cfg["config"]["output"]:
# FIXME: Output.dir should be a dict maybe by default?
output_obj.dir = {}
if isinstance(self.cfg["config"]["output"]["dir"], str):
output_obj.dir["path"] = output.DirOutput(
path=self.cfg["config"]["output"]["dir"])
elif isinstance(self.cfg["config"]["output"]["dir"], dict):
for k, dirpath in self.cfg["config"]["output"]["dir"].items():
output_obj.dir[k] = output.DirOutput(dirpath)
else:
raise ValueError(
f'Expected str or dict in {self.cfg["config"]["output"]["dir"]}'
)
for k, v in self.cfg["config"]["output"].items():
if k != "database" and k != "dir":
output_obj.kwargs[k] = v
ahntiles.output = output_obj
name = "part1"
proc = processor.factory.create(processor_key,
name=name,
tiles=ahntiles)
self.processors[proc] = name
log.info(f"Configured {self.__class__.__name__}")