コード例 #1
0
def test_output(cfg_out, bag3d_db, outdir):
    dout = output.DirOutput(cfg_out["output"]["dir"])
    dbout = output.DbOutput(conn=bag3d_db,
                            schema=cfg_out["output"]["database"]["schema"])
    out = output.Output(dir=dout, db=dbout)
    assert (
        out.db.with_table("sometable") ==
        "PG:dbname=bag3d_db host=localhost port=5590 user=bag3d_tester password=bag3d_test schemas=out_schema tables=sometable"
    )
    assert out.dir.join_path("AHN") == outdir / "AHN"
コード例 #2
0
    def configure(self, tiles, processor_key: str, worker_key: str):
        """Configure the control logic."""
        worker_init = worker.factory.create(worker_key)
        self.cfg["worker"] = worker_init.execute

        # Configure feature tiles with elevation from AHN3
        conn = db.Db(**self.cfg["config"]["database"])
        elevation_tiles = tileconfig.DbTiles(
            conn=conn,
            tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
        )
        elevation_tiles.configure(tiles=tiles)
        ahntiles = tileconfig.DbTilesAHN(conn=None,
                                         elevation_tiles=None,
                                         feature_tiles=None)
        ahntiles.to_process = elevation_tiles.to_process
        elevation_file_paths = ahntiles.create_elevation_file_index(
            directory_mapping=self.cfg["config"]["directory_mapping"])
        ahn_version = 3
        ahntiles.elevation_file_index = {}
        for i, ahn_id in enumerate(elevation_tiles.to_process):
            paths = []
            if ahn_id in elevation_file_paths:
                paths.extend(
                    (p, ahn_version) for p in elevation_file_paths[ahn_id])
            else:
                log.debug(
                    f"File matching the AHN ID {ahn_id} not found, skipping tile"
                )
                del ahntiles.to_process[i]
            ahntiles.elevation_file_index[ahn_id] = paths
        # Set up outputs
        output_obj = output.Output()
        if "database" in self.cfg["config"]["output"]:
            output_obj.db = output.DbOutput(conn=db.Db(
                **self.cfg["config"]["output"]["database"]))
        elif "dir" in self.cfg["config"]["output"]:
            output_obj.dir = output.DirOutput(
                path=self.cfg["config"]["output"]["dir"])
        for k, v in self.cfg["config"]["output"].items():
            if k != "database" and k != "dir":
                output_obj.kwargs[k] = v
        ahntiles.output = output_obj
        name = "part1"
        proc = processor.factory.create(processor_key,
                                        name=name,
                                        tiles=ahntiles)
        self.processors[proc] = name
        log.info(f"Configured {self.__class__.__name__}")
コード例 #3
0
    def configure(self,
                  tiles,
                  processor_key: str,
                  worker_key: str = None,
                  worker_class=None):
        """Configure the controller."""
        if worker_key:
            worker_init = worker.factory.create(worker_key)
        else:
            worker_init = worker_class
        self.cfg["worker"] = worker_init.execute

        if worker_key == "Example":
            tilescfg = tileconfig.FileTiles()
            tilescfg.configure(tiles=tiles)
            out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
            # Set up logic
            parts = {"part_A": tilescfg, "part_B": tilescfg}
        else:
            # For the ExampleDb worker
            tilescfg = tileconfig.DbTiles(
                conn=db.Db(**self.cfg["config"]["database"]),
                tile_index_schema=db.Schema(
                    self.cfg["config"]["features_tiles"]),
                features_schema=db.Schema(self.cfg["config"]["features"]),
            )
            tilescfg.configure(tiles=tiles)
            out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
            # Set up logic
            parts = {
                "part_A": tilescfg,
            }

        for part, _tilescfg in parts.items():
            _tilescfg.output = output.Output(
                dir=output.DirOutput(out_dir.join_path(part)))
            proc = processor.factory.create(processor_key,
                                            name=part,
                                            tiles=_tilescfg)
            self.processors[proc] = part
        log.info(f"Configured {self.__class__.__name__}")
コード例 #4
0
    def configure(self,
                  tiles,
                  processor_key: str,
                  worker_key: str = None,
                  worker_class=None):
        """Configure the controller.

        Input-specific subclasses need to implement this.
        """
        if worker_key:
            worker_init = worker.factory.create(worker_key)
        else:
            worker_init = worker_class
        self.cfg["worker"] = worker_init.execute

        # Configure the tiles (DBTiles in this case)
        tilescfg = tileconfig.DbTiles(
            conn=db.Db(**self.cfg["config"]["database"]),
            tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
            features_schema=db.Schema(self.cfg["config"]["features"]),
        )
        tilescfg.configure(tiles=tiles)
        out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
        # Set up logic for processing different parts. Parst are required
        # for example when processing a large area that needs different tile
        # configurations. For instance the Netherlands with AHN2 and AHN3.
        parts = {
            "part_A": tilescfg,
        }

        # Create a processor for each part
        for part, _tilescfg in parts.items():
            _tilescfg.output = output.Output(
                dir=output.DirOutput(out_dir.join_path(part)))
            proc = processor.factory.create(processor_key,
                                            name=part,
                                            tiles=_tilescfg)
            self.processors[proc] = part
        log.info(f"Configured {self.__class__.__name__}")
コード例 #5
0
    def configure(self, tiles, processor_key: str, worker_key: str):
        """Configure the control logic."""
        worker_init = worker.factory.create(worker_key)
        self.cfg["worker"] = worker_init.execute

        ahntiles = tileconfig.DbTilesAHN(
            conn=db.Db(**self.cfg["config"]["database"]),
            elevation_tiles=db.Schema(self.cfg["config"]["elevation_tiles"]),
            feature_tiles=None,
        )
        ahntiles.configure(
            tiles=tiles,
            directory_mapping=self.cfg["config"]["directory_mapping"],
            tin=True,
        )
        part = "AHN3"
        ahntiles.output = output.Output(
            dir=output.DirOutput(self.cfg["config"]["output"]["dir"]))
        proc = processor.factory.create(processor_key,
                                        name=part,
                                        tiles=ahntiles)
        self.processors[proc] = part
        log.info(f"Configured {self.__class__.__name__}")
コード例 #6
0
 def _generate(worker):
     tiles = tileconfig.FileTiles(output=output.Output(
         dir=output.DirOutput(Path("/tmp"))))
     tiles.to_process = ["tile_1", "tile_2", "tile_3", "tile_4", "tile_5"]
     args = {"arg1": "argument 1", "arg2": "argument 2"}
     expectation = {
         "tile_1": True,
         "tile_2": True,
         "tile_3": True,
         "tile_4": True,
         "tile_5": True,
     }
     threadprocessor = processor.factory.create("threadprocessor",
                                                name="test",
                                                tiles=tiles)
     threadprocessor.configure(
         threads=3,
         monitor_log=None,
         monitor_interval=5,
         worker=worker,
         config=args,
     )
     return threadprocessor, expectation
コード例 #7
0
    def configure(self, tiles, processor_key: str, worker_key: str):
        """Configure the control logic."""
        worker_init = worker.factory.create(worker_key)
        self.cfg["worker"] = worker_init.execute

        # Configure the tiles
        _tilecfg = {
            "conn":
            db.Db(**self.cfg["config"]["database"]),
            "elevation_index_schema":
            db.Schema(self.cfg["config"]["elevation_tiles"]),
            "tile_index_schema":
            db.Schema(self.cfg["config"]["features_tiles"]),
            "features_schema":
            db.Schema(self.cfg["config"]["features"]),
        }
        conn = db.Db(**self.cfg["config"]["database"])
        elevation_tiles = tileconfig.DbTiles(
            conn=conn,
            tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
        )
        feature_tiles = tileconfig.DbTiles(
            conn=conn,
            tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
            features_schema=db.Schema(self.cfg["config"]["features"]),
        )
        # Configure feature tiles with elevation from AHN2
        ahn_2 = tileconfig.DbTilesAHN(
            conn=conn,
            elevation_tiles=elevation_tiles,
            feature_tiles=feature_tiles,
        )
        ahn_2.configure(
            tiles=tiles,
            version=2,
            directory_mapping=self.cfg["config"]["directory_mapping"],
            tin=False,
        )
        # Configure feature tiles with elevation from AHN3
        ahn_3 = tileconfig.DbTilesAHN(
            conn=conn,
            elevation_tiles=elevation_tiles,
            feature_tiles=feature_tiles,
        )
        ahn_3.configure(
            tiles=tiles,
            version=3,
            directory_mapping=self.cfg["config"]["directory_mapping"],
            tin=False,
        )
        # Configure feature tiles that are on the border of AHN2 and AHN3
        ahn_border = tileconfig.DbTilesAHN(
            conn=conn,
            elevation_tiles=elevation_tiles,
            feature_tiles=feature_tiles,
        )
        ahn_border.configure(
            tiles=tiles,
            on_border=True,
            directory_mapping=self.cfg["config"]["directory_mapping"],
            tin=False,
        )

        out_dir = output.DirOutput(self.cfg["config"]["output"]["dir"])
        # Set up logic
        parts = {"AHN2": ahn_2, "AHN3": ahn_3, "AHN_border": ahn_border}
        for part, ahntiles in parts.items():
            ahntiles.output = output.Output(
                dir=output.DirOutput(out_dir.join_path(part)))
            proc = processor.factory.create(processor_key,
                                            name=part,
                                            tiles=ahntiles)
            self.processors[proc] = part
        log.info(f"Configured {self.__class__.__name__}")
コード例 #8
0
    def configure(self,
                  tiles,
                  processor_key: str,
                  worker_key: str = None,
                  worker_class=None,
                  restart: int = 0):
        """Configure the control logic."""
        if worker_key:
            worker_init = worker.factory.create(worker_key)
        else:
            worker_init = worker_class
        self.cfg["worker"] = worker_init.execute

        # Configure the tiles
        conn = db.Db(**self.cfg["config"]["database"])
        elevation_tiles = tileconfig.DbTiles(
            conn=conn,
            tile_index_schema=db.Schema(self.cfg["config"]["elevation_tiles"]),
        )
        feature_tiles = tileconfig.DbTiles(
            conn=conn,
            tile_index_schema=db.Schema(self.cfg["config"]["features_tiles"]),
            features_schema=db.Schema(self.cfg["config"]["features"]),
        )
        # Configure feature tiles with elevation from AHN3
        ahntiles = tileconfig.DbTilesAHN(
            conn=conn,
            elevation_tiles=elevation_tiles,
            feature_tiles=feature_tiles,
        )
        ahntiles.configure(
            tiles=tiles,
            version=None,
            directory_mapping=self.cfg["config"]["directory_mapping"],
            tin=False,
        )
        # Set up outputs
        output_obj = output.Output()
        if "database" in self.cfg["config"]["output"]:
            output_obj.db = output.DbOutput(conn=db.Db(
                **self.cfg["config"]["output"]["database"]))
        if "dir" in self.cfg["config"]["output"]:
            # FIXME: Output.dir should be a dict maybe by default?
            output_obj.dir = {}
            if isinstance(self.cfg["config"]["output"]["dir"], str):
                output_obj.dir["path"] = output.DirOutput(
                    path=self.cfg["config"]["output"]["dir"])
            elif isinstance(self.cfg["config"]["output"]["dir"], dict):
                for k, dirpath in self.cfg["config"]["output"]["dir"].items():
                    output_obj.dir[k] = output.DirOutput(dirpath)
            else:
                raise ValueError(
                    f'Expected str or dict in {self.cfg["config"]["output"]["dir"]}'
                )
        for k, v in self.cfg["config"]["output"].items():
            if k != "database" and k != "dir":
                output_obj.kwargs[k] = v
        ahntiles.output = output_obj
        name = "part1"
        proc = processor.factory.create(processor_key,
                                        name=name,
                                        tiles=ahntiles)
        self.processors[proc] = name
        log.info(f"Configured {self.__class__.__name__}")