def lint(files, check_only):
"""Run flake8, isort and (on Python >=3.6) black."""
context = _load_project_context()
source_path = get_source_dir(context.project_path)
files = files or (
str(source_path / "tests"),
str(source_path / context.package_name),
)
try:
# pylint: disable=import-outside-toplevel, unused-import
import flake8 # noqa
import isort # noqa
import black # noqa
except ImportError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module=exc.name,
src=str(source_path)))
python_call("black", ("--check", ) + files if check_only else files)
python_call("flake8", ("--max-line-length=88", ) + files)
check_flag = ("-c", ) if check_only else ()
python_call("isort",
(*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") +
files)
def activate_nbstripout():
"""Install the nbstripout git hook to automatically clean notebooks."""
context = _load_project_context()
source_path = get_source_dir(context.project_path)
secho(
("Notebook output cells will be automatically cleared before committing"
" to git."),
fg="yellow",
)
try:
# pylint: disable=import-outside-toplevel, unused-import
import nbstripout # noqa
except ImportError:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="nbstripout",
src=str(source_path)))
try:
res = subprocess.run( # pylint: disable=subprocess-run-check
["git", "rev-parse", "--git-dir"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if res.returncode:
raise KedroCliError("Not a git repository. Run `git init` first.")
except FileNotFoundError:
raise KedroCliError("Git executable not found. Install Git first.")
call(["nbstripout", "--install"])
def install():
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = get_source_dir(Path.cwd())
if (source_path / "environment.yml").is_file():
call([
"conda",
"install",
"--file",
str(source_path / "environment.yml"),
"--yes",
])
pip_command = [
"install", "-U", "-r",
str(source_path / "requirements.txt")
]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def install(compile_flag):
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = get_source_dir(Path.cwd())
environment_yml = source_path / "environment.yml"
requirements_in = source_path / "requirements.in"
requirements_txt = source_path / "requirements.txt"
if environment_yml.is_file():
call(["conda", "install", "--file", str(environment_yml), "--yes"])
default_compile = bool(compile_flag is None and not requirements_in.is_file())
do_compile = compile_flag or default_compile
if do_compile:
_build_reqs(source_path)
pip_command = ["install", "-U", "-r", str(requirements_txt)]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
secho("Requirements installed!", fg="green")
def convert_notebook(all_flag, overwrite_flag, filepath, env):
"""Convert selected or all notebooks found in a Kedro project
to Kedro code, by exporting code from the appropriately-tagged cells:
Cells tagged as `node` will be copied over to a Python file matching
the name of the notebook, under `<source_dir>/<package_name>/nodes`.
*Note*: Make sure your notebooks have unique names!
FILEPATH: Path(s) to exact notebook file(s) to be converted. Both
relative and absolute paths are accepted.
Should not be provided if --all flag is already present.
"""
context = _load_project_context(env=env)
_update_ipython_dir(context.project_path)
source_path = get_source_dir(context.project_path)
if not filepath and not all_flag:
secho("Please specify a notebook filepath "
"or add '--all' to convert all notebooks.")
sys.exit(1)
if all_flag:
# pathlib glob does not ignore hidden directories,
# whereas Python glob does, which is more useful in
# ensuring checkpoints will not be included
pattern = context.project_path / "**" / "*.ipynb"
notebooks = sorted(
Path(p) for p in iglob(str(pattern), recursive=True))
else:
notebooks = [Path(f) for f in filepath]
counter = Counter(n.stem for n in notebooks)
non_unique_names = [name for name, counts in counter.items() if counts > 1]
if non_unique_names:
names = ", ".join(non_unique_names)
raise KedroCliError(
f"Found non-unique notebook names! Please rename the following: {names}"
)
output_dir = source_path / context.package_name / "nodes"
if not output_dir.is_dir():
output_dir.mkdir()
(output_dir / "__init__.py").touch()
for notebook in notebooks:
secho(f"Converting notebook '{notebook}'...")
output_path = output_dir / f"{notebook.stem}.py"
if output_path.is_file():
overwrite = overwrite_flag or click.confirm(
f"Output file {output_path} already exists. Overwrite?",
default=False,
)
if overwrite:
_export_nodes(notebook, output_path)
else:
_export_nodes(notebook, output_path)
secho("Done!", color="green")
def test_get_source_dir(tmp_path, fake_kedro_yml, expected_source_dir):
expected_source_dir = (tmp_path / expected_source_dir).resolve()
assert fake_kedro_yml.is_file()
pattern = "This function is now deprecated and will be removed in Kedro 0.17.0"
with pytest.warns(DeprecationWarning, match=re.escape(pattern)):
src_dir = get_source_dir(tmp_path)
assert src_dir == expected_source_dir
def package():
"""Package the project as a Python egg and wheel."""
context = _load_project_context()
source_path = get_source_dir(context.project_path)
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_egg"],
cwd=str(source_path),
)
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_wheel"],
cwd=str(source_path),
)
def test(args):
"""Run the test suite."""
try:
# pylint: disable=import-outside-toplevel, unused-import
import pytest # noqa
except ImportError:
context = _load_project_context()
source_path = get_source_dir(context.project_path)
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="pytest",
src=str(source_path)))
else:
python_call("pytest", args)
def build_reqs(args):
"""Build the project dependency requirements."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = get_source_dir(Path.cwd())
_build_reqs(source_path, args)
secho(
"Requirements built! Please update requirements.in "
"if you'd like to make a change in your project's dependencies, "
"and re-run build-reqs to generate the new requirements.txt.",
fg="green",
)
def build_reqs():
"""Build the project dependency requirements."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = get_source_dir(Path.cwd())
requirements_path = source_path / "requirements.in"
if not requirements_path.is_file():
secho(
"No requirements.in found. Copying contents from requirements.txt..."
)
contents = (source_path / "requirements.txt").read_text()
requirements_path.write_text(contents)
python_call("piptools", ["compile", str(requirements_path)])
secho(("Requirements built! Please update requirements.in "
"if you'd like to make a change in your project's dependencies, "
"and re-run build-reqs to generate the new requirements.txt."))
def build_docs(open_docs):
"""Build the project documentation."""
context = _load_project_context()
source_path = get_source_dir(context.project_path)
python_call("pip", ["install", str(source_path / "[docs]")])
python_call("pip", ["install", "-r", str(source_path / "requirements.txt")])
python_call("ipykernel", ["install", "--user", f"--name={context.package_name}"])
shutil.rmtree("docs/build", ignore_errors=True)
call(
[
"sphinx-apidoc",
"--module-first",
"-o",
"docs/source",
str(source_path / context.package_name),
]
)
call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"])
if open_docs:
docs_page = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri()
secho(f"Opening {docs_page}")
webbrowser.open(docs_page)
