def build_docs(open_docs):
"""Build the project documentation."""
static_data = get_static_project_data(Path.cwd())
source_path = static_data["source_dir"]
package_name = (static_data.get("package_name")
or _load_project_context().package_name)
python_call("pip", ["install", str(source_path / "[docs]")])
python_call("pip",
["install", "-r",
str(source_path / "requirements.txt")])
python_call("ipykernel", ["install", "--user", f"--name={package_name}"])
shutil.rmtree("docs/build", ignore_errors=True)
call([
"sphinx-apidoc",
"--module-first",
"-o",
"docs/source",
str(source_path / package_name),
])
call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"])
if open_docs:
docs_page = (Path.cwd() / "docs" / "build" / "html" /
"index.html").as_uri()
secho(f"Opening {docs_page}")
webbrowser.open(docs_page)
def lint(files, check_only):
"""Run flake8, isort and black."""
static_data = get_static_project_data(Path.cwd())
source_path = static_data["source_dir"]
package_name = (static_data.get("package_name")
or _load_project_context().package_name)
files = files or (str(
source_path / "tests"), str(source_path / package_name))
if "PYTHONPATH" not in os.environ:
# isort needs the source path to be in the 'PYTHONPATH' environment
# variable to treat it as a first-party import location
os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover
for module_name in ("flake8", "isort", "black"):
try:
_check_module_importable(module_name)
except KedroCliError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module=module_name,
src=str(source_path))) from exc
python_call("black", ("--check", ) + files if check_only else files)
python_call("flake8", ("--max-line-length=88", ) + files)
check_flag = ("-c", ) if check_only else ()
python_call("isort",
(*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") +
files)
def init(force, silent):
"""Updates the template of a kedro project.
Running this command is mandatory to use kedro-mlflow.
2 actions are performed :
1. Add "conf/base/mlflow.yml": This is a configuration file
used for run parametrization when calling "kedro run" command.
See INSERT_DOC_URL for further details.
2. Modify "src/YOUR_PACKAGE_NAME/run.py" to add mlflow hooks
to the ProjectContext. This will erase your current "run.py"
script and all your modifications will be lost.
If you do not want to erase "run.py", insert the hooks manually
"""
# get constants
project_path = Path().cwd()
project_globals = get_static_project_data(project_path)
context = load_context(project_path)
conf_root = context.CONF_ROOT
# mlflow.yml is just a static file,
# but the name of the experiment is set to be the same as the project
mlflow_yml = "mlflow.yml"
write_jinja_template(
src=TEMPLATE_FOLDER_PATH / mlflow_yml,
is_cookiecutter=False,
dst=project_path / conf_root / "base" / mlflow_yml,
python_package=project_globals["package_name"],
)
if not silent:
click.secho(
click.style(
f"'{conf_root}/base/mlflow.yml' successfully updated.", fg="green"
)
)
def convert_notebook( # pylint: disable=unused-argument,too-many-locals
all_flag, overwrite_flag, filepath, env):
"""Convert selected or all notebooks found in a Kedro project
to Kedro code, by exporting code from the appropriately-tagged cells:
Cells tagged as `node` will be copied over to a Python file matching
the name of the notebook, under `<source_dir>/<package_name>/nodes`.
*Note*: Make sure your notebooks have unique names!
FILEPATH: Path(s) to exact notebook file(s) to be converted. Both
relative and absolute paths are accepted.
Should not be provided if --all flag is already present.
"""
project_path = Path.cwd()
static_data = get_static_project_data(project_path)
source_path = static_data["source_dir"]
package_name = static_data["package_name"]
_update_ipython_dir(project_path)
if not filepath and not all_flag:
secho("Please specify a notebook filepath "
"or add '--all' to convert all notebooks.")
sys.exit(1)
if all_flag:
# pathlib glob does not ignore hidden directories,
# whereas Python glob does, which is more useful in
# ensuring checkpoints will not be included
pattern = project_path / "**" / "*.ipynb"
notebooks = sorted(
Path(p) for p in iglob(str(pattern), recursive=True))
else:
notebooks = [Path(f) for f in filepath]
counter = Counter(n.stem for n in notebooks)
non_unique_names = [name for name, counts in counter.items() if counts > 1]
if non_unique_names:
names = ", ".join(non_unique_names)
raise KedroCliError(
f"Found non-unique notebook names! Please rename the following: {names}"
)
output_dir = source_path / package_name / "nodes"
if not output_dir.is_dir():
output_dir.mkdir()
(output_dir / "__init__.py").touch()
for notebook in notebooks:
secho(f"Converting notebook '{notebook}'...")
output_path = output_dir / f"{notebook.stem}.py"
if output_path.is_file():
overwrite = overwrite_flag or click.confirm(
f"Output file {output_path} already exists. Overwrite?",
default=False)
if overwrite:
_export_nodes(notebook, output_path)
else:
_export_nodes(notebook, output_path)
secho("Done!", color="green")
def _init_store(self) -> BaseSessionStore:
static_data = get_static_project_data(self._project_path)
config = deepcopy(static_data.get("session_store", {}))
config.setdefault("path", (self._project_path / "sessions").as_posix())
config["session_id"] = self.session_id
store = BaseSessionStore.from_config(config)
return store
def get_source_dir(project_path: Path) -> Path:
"""Returns project source path.
Args:
project_path: The path to the project root.
Returns:
The absolute path to the project source directory.
"""
warnings.warn(
"This function is now deprecated and will be removed in Kedro 0.17.0.",
DeprecationWarning,
)
return get_static_project_data(project_path)["source_dir"]
def init(env, force, silent):
"""Updates the template of a kedro project.
Running this command is mandatory to use kedro-mlflow. This
adds a configuration file used for run parametrization when
calling "kedro run" command.
"""
# get constants
mlflow_yml = "mlflow.yml"
project_path = Path().cwd()
project_globals = get_static_project_data(project_path)
context = load_context(project_path)
mlflow_yml_path = project_path / context.CONF_ROOT / env / mlflow_yml
# mlflow.yml is just a static file
if mlflow_yml_path.is_file() and not force:
click.secho(
click.style(
f"A 'mlflow.yml' already exists at '{mlflow_yml_path}' You can use the ``--force`` option to override it.",
fg="red",
))
return 1
else:
try:
write_jinja_template(
src=TEMPLATE_FOLDER_PATH / mlflow_yml,
is_cookiecutter=False,
dst=mlflow_yml_path,
python_package=project_globals["package_name"],
)
except FileNotFoundError:
click.secho(
click.style(
f"No env '{env}' found. Please check this folder exists inside '{context.CONF_ROOT}' folder.",
fg="red",
))
return 1
if not silent:
click.secho(
click.style(
f"'{context.CONF_ROOT}/{env}/{mlflow_yml}' successfully updated.",
fg="green",
))
return 0
def create(
cls,
project_path: Union[Path, str] = None,
save_on_close: bool = True,
env: str = None,
) -> "KedroSession":
"""Create a new instance of ``KedroSession``.
Args:
project_path: Path to the project root directory.
save_on_close: Whether or not to save the session when it's closed.
env: Environment for the KedroContext.
Returns:
A new ``KedroSession`` instance.
"""
# pylint: disable=protected-access
session = cls(
project_path=project_path,
session_id=generate_timestamp(),
save_on_close=save_on_close,
)
session_data = get_static_project_data(session._project_path)
session_data["project_path"] = session._project_path
session_data["session_id"] = session.session_id
session_data.update(_describe_git(session._project_path))
ctx = click.get_current_context(silent=True)
if ctx:
session_data["cli"] = _jsonify_cli_context(ctx)
if env:
session_data["env"] = env
session._store.update(session_data)
return session
def _get_source_path():
return get_static_project_data(Path.cwd())["source_dir"]