def test_output(cfg_out, bag3d_db, outdir):
dout = output.DirOutput(cfg_out["output"]["dir"])
dbout = output.DbOutput(conn=bag3d_db,
schema=cfg_out["output"]["database"]["schema"])
out = output.Output(dir=dout, db=dbout)
assert (
out.db.with_table("sometable") ==
"PG:dbname=bag3d_db host=localhost port=5590 user=bag3d_tester password=bag3d_test schemas=out_schema tables=sometable"
)
assert out.dir.join_path("AHN") == outdir / "AHN"
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None):
"""Configure the controller."""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
if worker_key == "Example":
tilescfg = tileconfig.FileTiles()
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {"part_A": tilescfg, "part_B": tilescfg}
else:
# For the ExampleDb worker
tilescfg = tileconfig.DbTiles(
conn=db.Db(**self.cfg["config"]["database"]),
tile_index_schema=db.Schema(
self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {
"part_A": tilescfg,
}
for part, _tilescfg in parts.items():
_tilescfg.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=_tilescfg)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None):
"""Configure the controller.
Input-specific subclasses need to implement this.
"""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
# Configure the tiles (DBTiles in this case)
tilescfg = tileconfig.DbTiles(
conn=db.Db(**self.cfg["config"]["database"]),
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
tilescfg.configure(tiles=tiles)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic for processing different parts. Parst are required
# for example when processing a large area that needs different tile
# configurations. For instance the Netherlands with AHN2 and AHN3.
parts = {
"part_A": tilescfg,
}
# Create a processor for each part
for part, _tilescfg in parts.items():
_tilescfg.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=_tilescfg)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
# Configure feature tiles with elevation from AHN3
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
elevation_tiles.configure(tiles=tiles)
ahntiles = tileconfig.DbTilesAHN(conn=None,
elevation_tiles=None,
feature_tiles=None)
ahntiles.to_process = elevation_tiles.to_process
elevation_file_paths = ahntiles.create_elevation_file_index(
directory_mapping=self.cfg["config"]["directory_mapping"])
ahn_version = 3
ahntiles.elevation_file_index = {}
for i, ahn_id in enumerate(elevation_tiles.to_process):
paths = []
if ahn_id in elevation_file_paths:
paths.extend(
(p, ahn_version) for p in elevation_file_paths[ahn_id])
else:
log.debug(
f"File matching the AHN ID {ahn_id} not found, skipping tile"
)
del ahntiles.to_process[i]
ahntiles.elevation_file_index[ahn_id] = paths
# Set up outputs
output_obj = output.Output()
if "database" in self.cfg["config"]["output"]:
output_obj.db = output.DbOutput(conn=db.Db(
**self.cfg["config"]["output"]["database"]))
elif "dir" in self.cfg["config"]["output"]:
output_obj.dir = output.DirOutput(
path=self.cfg["config"]["output"]["dir"])
for k, v in self.cfg["config"]["output"].items():
if k != "database" and k != "dir":
output_obj.kwargs[k] = v
ahntiles.output = output_obj
name = "part1"
proc = processor.factory.create(processor_key,
name=name,
tiles=ahntiles)
self.processors[proc] = name
log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
ahntiles = tileconfig.DbTilesAHN(
conn=db.Db(**self.cfg["config"]["database"]),
elevation_tiles=db.Schema(self.cfg["config"]["elevation_tiles"]),
feature_tiles=None,
)
ahntiles.configure(
tiles=tiles,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=True,
)
part = "AHN3"
ahntiles.output = output.Output(
dir=output.DirOutput(self.cfg["config"]["output"]["dir"]))
proc = processor.factory.create(processor_key,
name=part,
tiles=ahntiles)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def _generate(worker):
tiles = tileconfig.FileTiles(output=output.Output(
dir=output.DirOutput(Path("/tmp"))))
tiles.to_process = ["tile_1", "tile_2", "tile_3", "tile_4", "tile_5"]
args = {"arg1": "argument 1", "arg2": "argument 2"}
expectation = {
"tile_1": True,
"tile_2": True,
"tile_3": True,
"tile_4": True,
"tile_5": True,
}
threadprocessor = processor.factory.create("threadprocessor",
name="test",
tiles=tiles)
threadprocessor.configure(
threads=3,
monitor_log=None,
monitor_interval=5,
worker=worker,
config=args,
)
return threadprocessor, expectation
def output_obj(data_dir):
outdir = Path(data_dir / "output")
outdir.mkdir(exist_ok=True)
return output.DirOutput(path=outdir)
def configure(self, tiles, processor_key: str, worker_key: str):
"""Configure the control logic."""
worker_init = worker.factory.create(worker_key)
self.cfg["worker"] = worker_init.execute
# Configure the tiles
_tilecfg = {
"conn":
db.Db(**self.cfg["config"]["database"]),
"elevation_index_schema":
db.Schema(self.cfg["config"]["elevation_tiles"]),
"tile_index_schema":
db.Schema(self.cfg["config"]["features_tiles"]),
"features_schema":
db.Schema(self.cfg["config"]["features"]),
}
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
feature_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
# Configure feature tiles with elevation from AHN2
ahn_2 = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_2.configure(
tiles=tiles,
version=2,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Configure feature tiles with elevation from AHN3
ahn_3 = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_3.configure(
tiles=tiles,
version=3,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Configure feature tiles that are on the border of AHN2 and AHN3
ahn_border = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahn_border.configure(
tiles=tiles,
on_border=True,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
# Set up logic
parts = {"AHN2": ahn_2, "AHN3": ahn_3, "AHN_border": ahn_border}
for part, ahntiles in parts.items():
ahntiles.output = output.Output(
dir=output.DirOutput(out_dir.join_path(part)))
proc = processor.factory.create(processor_key,
name=part,
tiles=ahntiles)
self.processors[proc] = part
log.info(f"Configured {self.__class__.__name__}")
def configure(self,
tiles,
processor_key: str,
worker_key: str = None,
worker_class=None,
restart: int = 0):
"""Configure the control logic."""
if worker_key:
worker_init = worker.factory.create(worker_key)
else:
worker_init = worker_class
self.cfg["worker"] = worker_init.execute
# Configure the tiles
conn = db.Db(**self.cfg["config"]["database"])
elevation_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
)
feature_tiles = tileconfig.DbTiles(
conn=conn,
tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
features_schema=db.Schema(self.cfg["config"]["features"]),
)
# Configure feature tiles with elevation from AHN3
ahntiles = tileconfig.DbTilesAHN(
conn=conn,
elevation_tiles=elevation_tiles,
feature_tiles=feature_tiles,
)
ahntiles.configure(
tiles=tiles,
version=None,
directory_mapping=self.cfg["config"]["directory_mapping"],
tin=False,
)
# Set up outputs
output_obj = output.Output()
if "database" in self.cfg["config"]["output"]:
output_obj.db = output.DbOutput(conn=db.Db(
**self.cfg["config"]["output"]["database"]))
if "dir" in self.cfg["config"]["output"]:
# FIXME: Output.dir should be a dict maybe by default?
output_obj.dir = {}
if isinstance(self.cfg["config"]["output"]["dir"], str):
output_obj.dir["path"] = output.DirOutput(
path=self.cfg["config"]["output"]["dir"])
elif isinstance(self.cfg["config"]["output"]["dir"], dict):
for k, dirpath in self.cfg["config"]["output"]["dir"].items():
output_obj.dir[k] = output.DirOutput(dirpath)
else:
raise ValueError(
f'Expected str or dict in {self.cfg["config"]["output"]["dir"]}'
)
for k, v in self.cfg["config"]["output"].items():
if k != "database" and k != "dir":
output_obj.kwargs[k] = v
ahntiles.output = output_obj
name = "part1"
proc = processor.factory.create(processor_key,
name=name,
tiles=ahntiles)
self.processors[proc] = name
log.info(f"Configured {self.__class__.__name__}")
def test_diroutput(cfg_out, outdir):
dout = output.DirOutput(cfg_out["output"]["dir"])
assert dout.path == outdir