def build_docs(open_docs): """Build the project documentation.""" static_data = get_static_project_data(Path.cwd()) source_path = static_data["source_dir"] package_name = (static_data.get("package_name") or _load_project_context().package_name) python_call("pip", ["install", str(source_path / "[docs]")]) python_call("pip", ["install", "-r", str(source_path / "requirements.txt")]) python_call("ipykernel", ["install", "--user", f"--name={package_name}"]) shutil.rmtree("docs/build", ignore_errors=True) call([ "sphinx-apidoc", "--module-first", "-o", "docs/source", str(source_path / package_name), ]) call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"]) if open_docs: docs_page = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri() secho(f"Opening {docs_page}") webbrowser.open(docs_page)
def lint(files, check_only): """Run flake8, isort and black.""" static_data = get_static_project_data(Path.cwd()) source_path = static_data["source_dir"] package_name = (static_data.get("package_name") or _load_project_context().package_name) files = files or (str( source_path / "tests"), str(source_path / package_name)) if "PYTHONPATH" not in os.environ: # isort needs the source path to be in the 'PYTHONPATH' environment # variable to treat it as a first-party import location os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover for module_name in ("flake8", "isort", "black"): try: _check_module_importable(module_name) except KedroCliError as exc: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path))) from exc python_call("black", ("--check", ) + files if check_only else files) python_call("flake8", ("--max-line-length=88", ) + files) check_flag = ("-c", ) if check_only else () python_call("isort", (*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files)
def init(force, silent): """Updates the template of a kedro project. Running this command is mandatory to use kedro-mlflow. 2 actions are performed : 1. Add "conf/base/mlflow.yml": This is a configuration file used for run parametrization when calling "kedro run" command. See INSERT_DOC_URL for further details. 2. Modify "src/YOUR_PACKAGE_NAME/run.py" to add mlflow hooks to the ProjectContext. This will erase your current "run.py" script and all your modifications will be lost. If you do not want to erase "run.py", insert the hooks manually """ # get constants project_path = Path().cwd() project_globals = get_static_project_data(project_path) context = load_context(project_path) conf_root = context.CONF_ROOT # mlflow.yml is just a static file, # but the name of the experiment is set to be the same as the project mlflow_yml = "mlflow.yml" write_jinja_template( src=TEMPLATE_FOLDER_PATH / mlflow_yml, is_cookiecutter=False, dst=project_path / conf_root / "base" / mlflow_yml, python_package=project_globals["package_name"], ) if not silent: click.secho( click.style( f"'{conf_root}/base/mlflow.yml' successfully updated.", fg="green" ) )
def convert_notebook( # pylint: disable=unused-argument,too-many-locals all_flag, overwrite_flag, filepath, env): """Convert selected or all notebooks found in a Kedro project to Kedro code, by exporting code from the appropriately-tagged cells: Cells tagged as `node` will be copied over to a Python file matching the name of the notebook, under `<source_dir>/<package_name>/nodes`. *Note*: Make sure your notebooks have unique names! FILEPATH: Path(s) to exact notebook file(s) to be converted. Both relative and absolute paths are accepted. Should not be provided if --all flag is already present. """ project_path = Path.cwd() static_data = get_static_project_data(project_path) source_path = static_data["source_dir"] package_name = static_data["package_name"] _update_ipython_dir(project_path) if not filepath and not all_flag: secho("Please specify a notebook filepath " "or add '--all' to convert all notebooks.") sys.exit(1) if all_flag: # pathlib glob does not ignore hidden directories, # whereas Python glob does, which is more useful in # ensuring checkpoints will not be included pattern = project_path / "**" / "*.ipynb" notebooks = sorted( Path(p) for p in iglob(str(pattern), recursive=True)) else: notebooks = [Path(f) for f in filepath] counter = Counter(n.stem for n in notebooks) non_unique_names = [name for name, counts in counter.items() if counts > 1] if non_unique_names: names = ", ".join(non_unique_names) raise KedroCliError( f"Found non-unique notebook names! Please rename the following: {names}" ) output_dir = source_path / package_name / "nodes" if not output_dir.is_dir(): output_dir.mkdir() (output_dir / "__init__.py").touch() for notebook in notebooks: secho(f"Converting notebook '{notebook}'...") output_path = output_dir / f"{notebook.stem}.py" if output_path.is_file(): overwrite = overwrite_flag or click.confirm( f"Output file {output_path} already exists. Overwrite?", default=False) if overwrite: _export_nodes(notebook, output_path) else: _export_nodes(notebook, output_path) secho("Done!", color="green")
def _init_store(self) -> BaseSessionStore: static_data = get_static_project_data(self._project_path) config = deepcopy(static_data.get("session_store", {})) config.setdefault("path", (self._project_path / "sessions").as_posix()) config["session_id"] = self.session_id store = BaseSessionStore.from_config(config) return store
def get_source_dir(project_path: Path) -> Path: """Returns project source path. Args: project_path: The path to the project root. Returns: The absolute path to the project source directory. """ warnings.warn( "This function is now deprecated and will be removed in Kedro 0.17.0.", DeprecationWarning, ) return get_static_project_data(project_path)["source_dir"]
def init(env, force, silent): """Updates the template of a kedro project. Running this command is mandatory to use kedro-mlflow. This adds a configuration file used for run parametrization when calling "kedro run" command. """ # get constants mlflow_yml = "mlflow.yml" project_path = Path().cwd() project_globals = get_static_project_data(project_path) context = load_context(project_path) mlflow_yml_path = project_path / context.CONF_ROOT / env / mlflow_yml # mlflow.yml is just a static file if mlflow_yml_path.is_file() and not force: click.secho( click.style( f"A 'mlflow.yml' already exists at '{mlflow_yml_path}' You can use the ``--force`` option to override it.", fg="red", )) return 1 else: try: write_jinja_template( src=TEMPLATE_FOLDER_PATH / mlflow_yml, is_cookiecutter=False, dst=mlflow_yml_path, python_package=project_globals["package_name"], ) except FileNotFoundError: click.secho( click.style( f"No env '{env}' found. Please check this folder exists inside '{context.CONF_ROOT}' folder.", fg="red", )) return 1 if not silent: click.secho( click.style( f"'{context.CONF_ROOT}/{env}/{mlflow_yml}' successfully updated.", fg="green", )) return 0
def create( cls, project_path: Union[Path, str] = None, save_on_close: bool = True, env: str = None, ) -> "KedroSession": """Create a new instance of ``KedroSession``. Args: project_path: Path to the project root directory. save_on_close: Whether or not to save the session when it's closed. env: Environment for the KedroContext. Returns: A new ``KedroSession`` instance. """ # pylint: disable=protected-access session = cls( project_path=project_path, session_id=generate_timestamp(), save_on_close=save_on_close, ) session_data = get_static_project_data(session._project_path) session_data["project_path"] = session._project_path session_data["session_id"] = session.session_id session_data.update(_describe_git(session._project_path)) ctx = click.get_current_context(silent=True) if ctx: session_data["cli"] = _jsonify_cli_context(ctx) if env: session_data["env"] = env session._store.update(session_data) return session
def _get_source_path(): return get_static_project_data(Path.cwd())["source_dir"]