def test_tiles_in_index(self, bag3d_db, features_idx_sch): if features_idx_sch["index"]["table"] == "bag_index_identical": to_process = ["25gn1_10", "25gn1_11", "25gn1_6", "not_in_index"] expectation = ["25gn1_10", "25gn1_11", "25gn1_6"] tiles = tileconfig.DbTiles( bag3d_db, tile_index_schema=db.Schema(features_idx_sch), features_schema=None, ) result = tiles.tiles_in_index(to_process) assert set(result) == set(expectation) elif features_idx_sch["index"]["table"] == "bag_index": to_process = ["u1", "u2", "u5", "not_in_index"] expectation = ["u1", "u2", "u5"] tiles = tileconfig.DbTiles( bag3d_db, tile_index_schema=db.Schema(features_idx_sch), features_schema=None, ) result = tiles.tiles_in_index(to_process) assert set(result) == set(expectation) else: pytest.fail( msg=f"Unexpected features_tiles.index.table " f"{features_idx_sch['index']['table']}", pytrace=False, )
def test_invalid_params(self): with pytest.raises(ValueError): tiles = tileconfig.DbTiles(None, None, None) tiles.configure() with pytest.raises(ValueError): tiles = tileconfig.DbTiles(None, None, None) tiles.configure(extent="some_file", tiles=["all"])
def test_init_dbtiles(self, output_obj, bag3d_db): tiles = tileconfig.DbTiles( conn=bag3d_db, tile_index_schema=None, features_schema=None, output=output_obj, )
def feature_tiles( bag3d_db, features_idx_sch, features_sch ) -> tileconfig.DbTiles: return tileconfig.DbTiles( conn=bag3d_db, tile_index_schema=db.Schema(features_idx_sch), features_schema=db.Schema(features_sch), )
def test_read_extent(self, polygons): tiles = tileconfig.DbTiles( conn=None, tile_index_schema=None, features_schema=None ) extent = polygons["file"] poly, ewkb = tiles.read_extent(extent) assert ewkb == polygons["ewkb"] assert poly.wkt == polygons["wkt"]
def test_invalid_tiles(self, bag3d_db, features_idx_sch): to_process = ["bla", "not_in_index"] tiles = tileconfig.DbTiles( bag3d_db, tile_index_schema=db.Schema(features_idx_sch), features_schema=None, ) with pytest.raises(ValueError): tiles.with_list(tiles=to_process)
def test_all_in_index(self, bag3d_db, features_idx_sch): if features_idx_sch["index"]["table"] == "bag_index_identical": expectation = [ "25gn1_1", "25gn1_2", "25gn1_3", "25gn1_4", "25gn1_5", "25gn1_6", "25gn1_7", "25gn1_8", "25gn1_9", "25gn1_10", "25gn1_11", "25gn1_12", "25gn1_13", "25gn1_14", "25gn1_15", "25gn1_16", ] tiles = tileconfig.DbTiles( bag3d_db, tile_index_schema=db.Schema(features_idx_sch), features_schema=None, ) result = tiles.all_in_index() assert set(result) == set(expectation) elif features_idx_sch["index"]["table"] == "bag_index": expectation = ["u4", "u1", "u2", "u3", "u5", "u6"] tiles = tileconfig.DbTiles( bag3d_db, tile_index_schema=db.Schema(features_idx_sch), features_schema=None, ) result = tiles.all_in_index() assert set(result) == set(expectation) else: pytest.fail( msg=f"Unexpected features_tiles.index.table " f"{features_idx_sch['index']['table']}", pytrace=False, )
def configure(self, tiles, processor_key: str, worker_key: str): """Configure the control logic.""" worker_init = worker.factory.create(worker_key) self.cfg["worker"] = worker_init.execute # Configure feature tiles with elevation from AHN3 conn = db.Db(**self.cfg["config"]["database"]) elevation_tiles = tileconfig.DbTiles( conn=conn, tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]), ) elevation_tiles.configure(tiles=tiles) ahntiles = tileconfig.DbTilesAHN(conn=None, elevation_tiles=None, feature_tiles=None) ahntiles.to_process = elevation_tiles.to_process elevation_file_paths = ahntiles.create_elevation_file_index( directory_mapping=self.cfg["config"]["directory_mapping"]) ahn_version = 3 ahntiles.elevation_file_index = {} for i, ahn_id in enumerate(elevation_tiles.to_process): paths = [] if ahn_id in elevation_file_paths: paths.extend( (p, ahn_version) for p in elevation_file_paths[ahn_id]) else: log.debug( f"File matching the AHN ID {ahn_id} not found, skipping tile" ) del ahntiles.to_process[i] ahntiles.elevation_file_index[ahn_id] = paths # Set up outputs output_obj = output.Output() if "database" in self.cfg["config"]["output"]: output_obj.db = output.DbOutput(conn=db.Db( **self.cfg["config"]["output"]["database"])) elif "dir" in self.cfg["config"]["output"]: output_obj.dir = output.DirOutput( path=self.cfg["config"]["output"]["dir"]) for k, v in self.cfg["config"]["output"].items(): if k != "database" and k != "dir": output_obj.kwargs[k] = v ahntiles.output = output_obj name = "part1" proc = processor.factory.create(processor_key, name=name, tiles=ahntiles) self.processors[proc] = name log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str = None, worker_class=None): """Configure the controller.""" if worker_key: worker_init = worker.factory.create(worker_key) else: worker_init = worker_class self.cfg["worker"] = worker_init.execute if worker_key == "Example": tilescfg = tileconfig.FileTiles() tilescfg.configure(tiles=tiles) out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"]) # Set up logic parts = {"part_A": tilescfg, "part_B": tilescfg} else: # For the ExampleDb worker tilescfg = tileconfig.DbTiles( conn=db.Db(**self.cfg["config"]["database"]), tile_index_schema=db.Schema( self.cfg["config"]["features_tiles"]), features_schema=db.Schema(self.cfg["config"]["features"]), ) tilescfg.configure(tiles=tiles) out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"]) # Set up logic parts = { "part_A": tilescfg, } for part, _tilescfg in parts.items(): _tilescfg.output = output.Output( dir=output.DirOutput(out_dir.join_path(part))) proc = processor.factory.create(processor_key, name=part, tiles=_tilescfg) self.processors[proc] = part log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str = None, worker_class=None): """Configure the controller. Input-specific subclasses need to implement this. """ if worker_key: worker_init = worker.factory.create(worker_key) else: worker_init = worker_class self.cfg["worker"] = worker_init.execute # Configure the tiles (DBTiles in this case) tilescfg = tileconfig.DbTiles( conn=db.Db(**self.cfg["config"]["database"]), tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]), features_schema=db.Schema(self.cfg["config"]["features"]), ) tilescfg.configure(tiles=tiles) out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"]) # Set up logic for processing different parts. Parst are required # for example when processing a large area that needs different tile # configurations. For instance the Netherlands with AHN2 and AHN3. parts = { "part_A": tilescfg, } # Create a processor for each part for part, _tilescfg in parts.items(): _tilescfg.output = output.Output( dir=output.DirOutput(out_dir.join_path(part))) proc = processor.factory.create(processor_key, name=part, tiles=_tilescfg) self.processors[proc] = part log.info(f"Configured {self.__class__.__name__}")
def test_within_extent( self, bag3d_db, polygons, features_idx_sch, features_sch ): """Tile selection with a polygon should return the tiles that intersect with the polygon.""" tiles = tileconfig.DbTiles( bag3d_db, tile_index_schema=db.Schema(features_idx_sch), features_schema=db.Schema(features_sch), ) result = tiles.within_extent(polygons["ewkb"]) if features_idx_sch["index"]["table"] == "bag_index_identical": expectation = {"25gn1_10", "25gn1_11", "25gn1_6", "25gn1_7"} assert set(result) == expectation elif features_idx_sch["index"]["table"] == "bag_index": expectation = {"u2", "u5"} assert set(result) == expectation else: pytest.fail( msg=f"Unexpected features_tiles.index.table " f"{features_idx_sch['index']['table']}", pytrace=False, )
def configure(self, tiles, processor_key: str, worker_key: str): """Configure the control logic.""" worker_init = worker.factory.create(worker_key) self.cfg["worker"] = worker_init.execute # Configure the tiles _tilecfg = { "conn": db.Db(**self.cfg["config"]["database"]), "elevation_index_schema": db.Schema(self.cfg["config"]["elevation_tiles"]), "tile_index_schema": db.Schema(self.cfg["config"]["features_tiles"]), "features_schema": db.Schema(self.cfg["config"]["features"]), } conn = db.Db(**self.cfg["config"]["database"]) elevation_tiles = tileconfig.DbTiles( conn=conn, tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]), ) feature_tiles = tileconfig.DbTiles( conn=conn, tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]), features_schema=db.Schema(self.cfg["config"]["features"]), ) # Configure feature tiles with elevation from AHN2 ahn_2 = tileconfig.DbTilesAHN( conn=conn, elevation_tiles=elevation_tiles, feature_tiles=feature_tiles, ) ahn_2.configure( tiles=tiles, version=2, directory_mapping=self.cfg["config"]["directory_mapping"], tin=False, ) # Configure feature tiles with elevation from AHN3 ahn_3 = tileconfig.DbTilesAHN( conn=conn, elevation_tiles=elevation_tiles, feature_tiles=feature_tiles, ) ahn_3.configure( tiles=tiles, version=3, directory_mapping=self.cfg["config"]["directory_mapping"], tin=False, ) # Configure feature tiles that are on the border of AHN2 and AHN3 ahn_border = tileconfig.DbTilesAHN( conn=conn, elevation_tiles=elevation_tiles, feature_tiles=feature_tiles, ) ahn_border.configure( tiles=tiles, on_border=True, directory_mapping=self.cfg["config"]["directory_mapping"], tin=False, ) out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"]) # Set up logic parts = {"AHN2": ahn_2, "AHN3": ahn_3, "AHN_border": ahn_border} for part, ahntiles in parts.items(): ahntiles.output = output.Output( dir=output.DirOutput(out_dir.join_path(part))) proc = processor.factory.create(processor_key, name=part, tiles=ahntiles) self.processors[proc] = part log.info(f"Configured {self.__class__.__name__}")
def configure(self, tiles, processor_key: str, worker_key: str = None, worker_class=None, restart: int = 0): """Configure the control logic.""" if worker_key: worker_init = worker.factory.create(worker_key) else: worker_init = worker_class self.cfg["worker"] = worker_init.execute # Configure the tiles conn = db.Db(**self.cfg["config"]["database"]) elevation_tiles = tileconfig.DbTiles( conn=conn, tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]), ) feature_tiles = tileconfig.DbTiles( conn=conn, tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]), features_schema=db.Schema(self.cfg["config"]["features"]), ) # Configure feature tiles with elevation from AHN3 ahntiles = tileconfig.DbTilesAHN( conn=conn, elevation_tiles=elevation_tiles, feature_tiles=feature_tiles, ) ahntiles.configure( tiles=tiles, version=None, directory_mapping=self.cfg["config"]["directory_mapping"], tin=False, ) # Set up outputs output_obj = output.Output() if "database" in self.cfg["config"]["output"]: output_obj.db = output.DbOutput(conn=db.Db( **self.cfg["config"]["output"]["database"])) if "dir" in self.cfg["config"]["output"]: # FIXME: Output.dir should be a dict maybe by default? output_obj.dir = {} if isinstance(self.cfg["config"]["output"]["dir"], str): output_obj.dir["path"] = output.DirOutput( path=self.cfg["config"]["output"]["dir"]) elif isinstance(self.cfg["config"]["output"]["dir"], dict): for k, dirpath in self.cfg["config"]["output"]["dir"].items(): output_obj.dir[k] = output.DirOutput(dirpath) else: raise ValueError( f'Expected str or dict in {self.cfg["config"]["output"]["dir"]}' ) for k, v in self.cfg["config"]["output"].items(): if k != "database" and k != "dir": output_obj.kwargs[k] = v ahntiles.output = output_obj name = "part1" proc = processor.factory.create(processor_key, name=name, tiles=ahntiles) self.processors[proc] = name log.info(f"Configured {self.__class__.__name__}")
def elevation_tiles(bag3d_db, elevation_idx_sch) -> tileconfig.DbTiles: return tileconfig.DbTiles( conn=bag3d_db, tile_index_schema=db.Schema(elevation_idx_sch) )