def _init_store(self) -> BaseSessionStore:
store_class = _get_project_settings(self._package_name,
"SESSION_STORE_CLASS",
BaseSessionStore)
classpath = f"{store_class.__module__}.{store_class.__qualname__}"
if not issubclass(store_class, BaseSessionStore):
raise ValueError(f"Store type `{classpath}` is invalid: "
f"it must extend `BaseSessionStore`.")
store_args = deepcopy(
_get_project_settings(self._package_name, "SESSION_STORE_ARGS",
{}))
store_args.setdefault("path",
(self._project_path / "sessions").as_posix())
store_args["session_id"] = self.session_id
try:
return store_class(**store_args)
except TypeError as err:
raise ValueError(
f"\n{err}.\nStore config must only contain arguments valid "
f"for the constructor of `{classpath}`.") from err
except Exception as err:
raise ValueError(
f"\n{err}.\nFailed to instantiate session store of type `{classpath}`."
) from err
def create_pipeline(metadata: ProjectMetadata, name, skip_config, env,
**kwargs): # pylint: disable=unused-argument
"""Create a new modular pipeline by providing the new pipeline name as an argument."""
package_dir = metadata.source_dir / metadata.package_name
conf_root = _get_project_settings(metadata.package_name, "CONF_ROOT",
"conf")
project_conf_path = metadata.project_path / conf_root
env = env or "base"
if not skip_config and not (project_conf_path / env).exists():
raise KedroCliError(
f"Unable to locate environment `{env}`. "
f"Make sure it exists in the project configuration.")
result_path = _create_pipeline(name, package_dir / "pipelines")
_copy_pipeline_tests(name, result_path, package_dir)
_copy_pipeline_configs(result_path,
project_conf_path,
skip_config,
env=env)
click.secho(f"\nPipeline `{name}` was successfully created.\n", fg="green")
click.secho(
f"To be able to run the pipeline `{name}`, you will need to add it "
f"to `register_pipelines()` in `{package_dir / 'hooks.py'}`.",
fg="yellow",
)
def _get_config_loader(self) -> ConfigLoader:
"""A hook for changing the creation of a ConfigLoader instance.
Returns:
Instance of `ConfigLoader` created by `register_config_loader` hook.
Raises:
KedroContextError: Incorrect ``ConfigLoader`` registered for the project.
"""
conf_root = _get_project_settings(self.package_name, "CONF_ROOT",
self.CONF_ROOT)
conf_paths = [
str(self.project_path / conf_root / "base"),
str(self.project_path / conf_root / self.env),
]
hook_manager = get_hook_manager()
config_loader = hook_manager.hook.register_config_loader( # pylint: disable=no-member
conf_paths=conf_paths,
env=self.env,
extra_params=self._extra_params,
)
if not isinstance(config_loader, ConfigLoader):
raise KedroContextError(
f"Expected an instance of `ConfigLoader`, "
f"got `{type(config_loader).__name__}` instead.")
return config_loader
def _register_all_project_hooks(hook_manager: PluginManager,
package_name: str) -> None:
"""Register all hooks from the project settings and from installed plugins
with the global ``hook_manager``.
Args:
hook_manager: Hook manager instance to register the hooks with.
package_name: Python package name to read the settings for.
"""
# get the hooks specified in settings.py
hooks = _get_project_settings(package_name, "HOOKS", ())
# get the plugins that must be disabled
disabled_plugins = set(
_get_project_settings(package_name, "DISABLE_HOOKS_FOR_PLUGINS", ()))
_register_hooks(hook_manager, hooks)
_register_hooks_setuptools(hook_manager, disabled_plugins)
def test_property_is_missing(self, mocker):
mocker.patch(
"kedro.framework.project.settings.import_module",
return_value=TestSettings(),
)
value = _get_project_settings(self.package_name, self.property_name,
self.default_value)
assert value == self.default_value
def test_get_project_settings(self, mocker):
property_value = "property_value"
settings_obj = TestSettings()
settings_obj.property_name = property_value
mocker.patch("kedro.framework.project.settings.import_module",
return_value=settings_obj)
value = _get_project_settings(self.package_name, self.property_name,
self.default_value)
assert value == property_value
def delete_pipeline(metadata: ProjectMetadata, name, env, yes, **kwargs): # pylint: disable=unused-argument
"""Delete a modular pipeline by providing the pipeline name as an argument."""
package_dir = metadata.source_dir / metadata.package_name
conf_root = _get_project_settings(metadata.package_name, "CONF_ROOT",
"conf")
project_conf_path = metadata.project_path / conf_root
env = env or "base"
if not (project_conf_path / env).exists():
raise KedroCliError(
f"Unable to locate environment `{env}`. "
f"Make sure it exists in the project configuration.")
pipeline_artifacts = _get_pipeline_artifacts(metadata,
pipeline_name=name,
env=env)
files_to_delete = [
pipeline_artifacts.pipeline_conf / confdir / f"{name}.yml"
for confdir in ("parameters", "catalog")
if (pipeline_artifacts.pipeline_conf / confdir /
f"{name}.yml").is_file()
]
dirs_to_delete = [
path for path in (pipeline_artifacts.pipeline_dir,
pipeline_artifacts.pipeline_tests) if path.is_dir()
]
if not files_to_delete and not dirs_to_delete:
raise KedroCliError(f"Pipeline `{name}` not found.")
if not yes:
_echo_deletion_warning(
"The following paths will be removed:",
directories=dirs_to_delete,
files=files_to_delete,
)
click.echo()
yes = click.confirm(
f"Are you sure you want to delete pipeline `{name}`?")
click.echo()
if not yes:
raise KedroCliError("Deletion aborted!")
_delete_artifacts(*files_to_delete, *dirs_to_delete)
click.secho(f"\nPipeline `{name}` was successfully deleted.", fg="green")
click.secho(
f"\nIf you added the pipeline `{name}` to `register_pipelines()` in "
f"`{package_dir / 'hooks.py'}`, you will need to remove it.",
fg="yellow",
)
def _get_pipeline_artifacts(project_metadata: ProjectMetadata,
pipeline_name: str, env: str) -> PipelineArtifacts:
"""From existing project, returns in order: source_path, tests_path, config_paths"""
package_dir = project_metadata.source_dir / project_metadata.package_name
conf_root = _get_project_settings(project_metadata.package_name,
"CONF_ROOT", "conf")
project_conf_path = project_metadata.project_path / conf_root
artifacts = PipelineArtifacts(
package_dir / "pipelines" / pipeline_name,
package_dir.parent / "tests" / "pipelines" / pipeline_name,
project_conf_path / env,
)
return artifacts
def load_context(self) -> KedroContext:
"""An instance of the project context."""
env = self.store.get("env")
extra_params = self.store.get("extra_params")
context_class = _get_project_settings(self._package_name,
"CONTEXT_CLASS", KedroContext)
context = context_class(
package_name=self._package_name,
project_path=self._project_path,
env=env,
extra_params=extra_params,
)
return context
def load_context(project_path: Union[str, Path], **kwargs) -> KedroContext:
"""Loads the KedroContext object of a Kedro Project.
This is the default way to load the KedroContext object for normal workflows such as
CLI, Jupyter Notebook, Plugins, etc. It assumes the following project structure
under the given project_path::
<project_path>
|__ <src_dir>
|__ pyproject.toml
The name of the <scr_dir> is `src` by default. The `pyproject.toml` file is used
for project metadata. Kedro configuration should be under `[tool.kedro]` section.
Args:
project_path: Path to the Kedro project.
kwargs: Optional kwargs for ``KedroContext`` class.
Returns:
Instance of ``KedroContext`` class defined in Kedro project.
Raises:
KedroContextError: `pyproject.toml` was not found or the `[tool.kedro]` section
is missing, or loaded context has package conflict.
"""
warn(
"`kedro.framework.context.load_context` is now deprecated in favour of "
"`KedroSession.load_context` and will be removed in Kedro 0.18.0.",
DeprecationWarning,
)
project_path = Path(project_path).expanduser().resolve()
metadata = _get_project_metadata(project_path)
context_class = _get_project_settings(
metadata.package_name, "CONTEXT_CLASS", KedroContext
)
# update kwargs with env from the environment variable
# (defaults to None if not set)
# need to do this because some CLI command (e.g `kedro run`) defaults to
# passing in `env=None`
kwargs["env"] = kwargs.get("env") or os.getenv("KEDRO_ENV")
context = context_class(
package_name=metadata.package_name, project_path=project_path, **kwargs
)
return context
def test_register_config_loader_is_called(self, mock_session_with_hooks, caplog):
context = mock_session_with_hooks.load_context()
_ = context.config_loader
relevant_records = [
r for r in caplog.records if r.name == LoggingHooks.handler_name
]
assert len(relevant_records) == 1
record = relevant_records[0]
assert record.getMessage() == "Registering config loader"
conf_root = _get_project_settings(MOCK_PACKAGE_NAME, "CONF_ROOT", "conf")
expected_conf_paths = [
str(context.project_path / conf_root / "base"),
str(context.project_path / conf_root / "local"),
]
assert record.conf_paths == expected_conf_paths
def test_settings_py_is_missing(self):
with pytest.raises(ModuleNotFoundError):
_get_project_settings(self.package_name, self.property_name,
self.default_value)
