def __init__(self, config: _cfg.StorageInstance, options: dict = None):
self._log = _util.logger_for_object(self)
self._options = options or {}
root_path_config = config.storageProps.get(
"rootPath") # TODO: Config / constants
if not root_path_config or root_path_config.isspace():
err = f"Storage root path not set"
self._log.error(err)
raise _ex.EStorageRequest(err)
supplied_root = pathlib.Path(root_path_config)
if supplied_root.is_absolute():
absolute_root = supplied_root
elif "sys_config_dir" in self._options:
absolute_root = pathlib.Path(
self._options["sys_config_dir"]).joinpath(
supplied_root).absolute()
else:
err = f"Could not resolve relative path for storage root [{supplied_root}]"
self._log.error(err)
raise _ex.EStorageConfig(err)
try:
self.__root_path = absolute_root.resolve(strict=True)
except FileNotFoundError as e:
err = f"Storage root path does not exist: [{absolute_root}]"
self._log.error(err)
raise _ex.EStorageRequest(err) from e
def __init__(self,
sys_config: tp.Union[str, pathlib.Path, _cfg.RuntimeConfig],
job_config: tp.Union[str, pathlib.Path, _cfg.JobConfig,
None] = None,
job_result_dir: tp.Union[str, pathlib.Path, None] = None,
job_result_format: tp.Optional[str] = None,
dev_mode: bool = False,
model_class: tp.Optional[_api.TracModel.__class__] = None):
trac_version = _version.__version__
python_version = sys.version.replace("\n", "")
mode = "batch" if job_config else "service"
sys_config_path = "[embedded]" if isinstance(
sys_config, _cfg.RuntimeConfig) else sys_config
job_config_path = "[embedded]" if isinstance(
job_config, _cfg.JobConfig) else job_config
print(
f">>> TRAC Python Runtime {trac_version} starting in {mode} mode at {dt.datetime.now()}"
)
print(f">>> Python installation: {python_version} ({sys.exec_prefix})")
print(f">>> System config: {sys_config_path}")
if job_config:
print(f">>> Job config: {job_config_path}")
if dev_mode:
print(
f">>> Development mode enabled (DO NOT USE THIS IN PRODUCTION)"
)
util.configure_logging()
self._log = util.logger_for_object(self)
self._log.info(f"TRAC Python Runtime {trac_version}")
self._sys_config_dir = pathlib.Path(sys_config_path).parent
self._sys_config_path = sys_config_path
self._job_config_path = job_config_path
self._job_result_dir = job_result_dir
self._job_result_format = job_result_format
self._batch_mode = bool(job_config is not None)
self._dev_mode = dev_mode
self._model_class = model_class
self._sys_config = sys_config if isinstance(
sys_config, _cfg.RuntimeConfig) else None
self._job_config = job_config if isinstance(job_config,
_cfg.JobConfig) else None
# Top level resources
self._models: tp.Optional[_models.ModelLoader] = None
self._storage: tp.Optional[_storage.StorageManager] = None
# The execution engine
self._engine: tp.Optional[_engine.TracEngine] = None
self._system: tp.Optional[_actors.ActorSystem] = None
def __init__(self, format_options: tp.Dict[str, tp.Any] = None):
self._log = _util.logger_for_object(self)
self._format_options = format_options
self._use_lenient_parser = False
if format_options:
if format_options.get(self.__LENIENT_CSV_PARSER) is True:
self._use_lenient_parser = True
def __init__(self, job_config: _cfg.JobConfig,
result_spec: _graph.JobResultSpec,
models: _models.ModelLoader,
storage: _storage.StorageManager):
super().__init__()
self.job_config = job_config
self.result_spec = result_spec
self.graph: tp.Optional[_EngineContext] = None
self._resolver = _func.FunctionResolver(models, storage)
self._log = _util.logger_for_object(self)
def __init__(self, job_key, job_config: _cfg.JobConfig,
result_spec: _graph.JobResultSpec,
models: _models.ModelLoader,
storage: _storage.StorageManager):
super().__init__()
self.job_key = job_key
self.job_config = job_config
self.result_spec = result_spec
self._models = models
self._storage = storage
self._log = _util.logger_for_object(self)
def __init__(self,
config: _cfg.StorageConfig,
file_storage: IFileStorage,
pushdown_pandas: bool = False,
pushdown_spark: bool = False):
self.__log = _util.logger_for_object(self)
self.__config = config
self.__file_storage = file_storage
self.__pushdown_pandas = pushdown_pandas
self.__pushdown_spark = pushdown_spark
def __init__(self, sys_config: _cfg.RuntimeConfig,
sys_config_dir: tp.Union[str, pathlib.Path]):
self.__log = _util.logger_for_object(self)
self.__file_storage: tp.Dict[str, IFileStorage] = dict()
self.__data_storage: tp.Dict[str, IDataStorage] = dict()
self.__settings = sys_config.storageSettings
storage_options = {"sys_config_dir": sys_config_dir}
for storage_key, storage_config in sys_config.storage.items():
self.create_storage(storage_key, storage_config, storage_options)
def __init__(self, model_def: _meta.ModelDefinition,
model_class: _api.TracModel.__class__,
local_ctx: tp.Dict[str, _data.DataView]):
self.__ctx_log = _util.logger_for_object(self)
self.__model_def = model_def
self.__model_class = model_class
self.__model_log = _util.logger_for_class(self.__model_class)
self.__parameters = local_ctx or {}
self.__data = local_ctx or {}
self.__val = TracContextValidator(self.__ctx_log, self.__parameters,
self.__data)
def __init__(self,
sys_config: _cfg.RuntimeConfig,
models: _models.ModelLoader,
storage: _storage.StorageManager,
batch_mode=False):
super().__init__()
self._log = _util.logger_for_object(self)
self._sys_config = sys_config
self._models = models
self._storage = storage
self._batch_mode = batch_mode
self._job_actors = dict()
def __init__(self, sys_config: _cfg.RuntimeConfig):
self._log = _util.logger_for_object(self)
self._loaders: tp.Dict[str, IModelRepository] = dict()
for repo_name, repo_config in sys_config.repositories.items():
if repo_config.repoType not in self.__repo_types:
msg = f"Model repository type [{repo_config.repoType}] is not recognised" \
+ " (this could indicate a missing model repository plugin)"
self._log.error(msg)
raise _ex.EModelRepoConfig(msg)
loader_class = self.__repo_types[repo_config.repoType]
loader = loader_class(repo_config)
self._loaders[repo_name] = loader
def __init__(self, main_actor: Actor, system_thread: str = "actor_system"):
super().__init__()
self._log = util.logger_for_object(self)
self.__actors: tp.Dict[ActorId, ActorNode] = {
self.__ROOT_ID: ActorNode("", self.__ROOT_ID, None)
}
self.__message_queue: tp.List[Msg] = list()
self.__system_thread = threading.Thread(name=system_thread,
target=self._actor_system_main)
self.__system_lock = threading.Lock()
self.__system_up = threading.Event()
self.__system_msg = threading.Event()
self.__system_error: tp.Optional[Exception] = None
self.__main_id = self._register_actor(self.__ROOT_ID,
main_actor,
do_start=False)
def __init__(self, config_class: _T.__class__, dev_mode_locations: tp.List[str] = None):
self._log = _util.logger_for_object(self)
self._config_class = config_class
self._dev_mode_locations = dev_mode_locations or []
self._errors = []
def __init__(self, repo_config: _cfg.RepositoryConfig):
self._repo_config = repo_config
self._log = _util.logger_for_object(self)
def __init__(self, graph: _EngineContext):
super().__init__()
self.graph = graph
self.processors: tp.Dict[NodeId, _actors.ActorId] = dict()
self._log = _util.logger_for_object(self)
def __init__(self, sys_config: _cfg.RuntimeConfig):
self.__repos = _repos.RepositoryManager(sys_config)
self.__scopes: tp.Dict[str, ModelLoader._ScopeState] = dict()
self.__log = _util.logger_for_object(self)
def __init__(self, format_options: tp.Dict[str, tp.Any] = None):
self._format_options = format_options
self._log = _util.logger_for_object(self)