def install():
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = get_source_dir(Path.cwd())
if (source_path / "environment.yml").is_file():
call([
"conda",
"install",
"--file",
str(source_path / "environment.yml"),
"--yes",
])
pip_command = [
"install", "-U", "-r",
str(source_path / "requirements.txt")
]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def activate_nbstripout():
"""Install the nbstripout git hook to automatically clean notebooks."""
context = _load_project_context()
source_path = get_source_dir(context.project_path)
secho(
("Notebook output cells will be automatically cleared before committing"
" to git."),
fg="yellow",
)
try:
# pylint: disable=import-outside-toplevel, unused-import
import nbstripout # noqa
except ImportError:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="nbstripout",
src=str(source_path)))
try:
res = subprocess.run( # pylint: disable=subprocess-run-check
["git", "rev-parse", "--git-dir"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if res.returncode:
raise KedroCliError("Not a git repository. Run `git init` first.")
except FileNotFoundError:
raise KedroCliError("Git executable not found. Install Git first.")
call(["nbstripout", "--install"])
def activate_nbstripout():
"""Install the nbstripout git hook to automatically clean notebooks."""
secho(
("Notebook output cells will be automatically cleared before committing"
" to git."),
fg="yellow",
)
try:
import nbstripout # pylint: disable=unused-import
except ImportError:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="nbstripout",
src=str(SOURCE_PATH)))
try:
res = subprocess.run(
["git", "rev-parse", "--git-dir"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if res.returncode:
raise KedroCliError("Not a git repository. Run `git init` first.")
except FileNotFoundError:
raise KedroCliError("Git executable not found. Install Git first.")
call(["nbstripout", "--install"])
def build_docs(open_docs):
"""Build the project documentation."""
static_data = get_static_project_data(Path.cwd())
source_path = static_data["source_dir"]
package_name = (static_data.get("package_name")
or _load_project_context().package_name)
python_call("pip", ["install", str(source_path / "[docs]")])
python_call("pip",
["install", "-r",
str(source_path / "requirements.txt")])
python_call("ipykernel", ["install", "--user", f"--name={package_name}"])
shutil.rmtree("docs/build", ignore_errors=True)
call([
"sphinx-apidoc",
"--module-first",
"-o",
"docs/source",
str(source_path / package_name),
])
call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"])
if open_docs:
docs_page = (Path.cwd() / "docs" / "build" / "html" /
"index.html").as_uri()
secho(f"Opening {docs_page}")
webbrowser.open(docs_page)
def install(compile_flag):
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = _get_source_path()
environment_yml = source_path / "environment.yml"
requirements_in = source_path / "requirements.in"
requirements_txt = source_path / "requirements.txt"
if environment_yml.is_file():
call([
"conda", "env", "update", "--file",
str(environment_yml), "--prune"
])
default_compile = bool(compile_flag is None
and not requirements_in.is_file())
do_compile = compile_flag or default_compile
if do_compile:
_build_reqs(source_path)
pip_command = ["install", "-U", "-r", str(requirements_txt)]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
secho("Requirements installed!", fg="green")
def activate_nbstripout():
"""Install the nbstripout git hook to automatically clean notebooks."""
source_path = _get_source_path()
secho(
("Notebook output cells will be automatically cleared before committing"
" to git."),
fg="yellow",
)
try:
_check_module_importable("nbstripout")
except KedroCliError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="nbstripout",
src=str(source_path))) from exc
try:
res = subprocess.run( # pylint: disable=subprocess-run-check
["git", "rev-parse", "--git-dir"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if res.returncode:
raise KedroCliError("Not a git repository. Run `git init` first.")
except FileNotFoundError as exc:
raise KedroCliError(
"Git executable not found. Install Git first.") from exc
call(["nbstripout", "--install"])
def _generate_wheel_file(package_name: str, destination: Path,
source_paths: Tuple[Path, ...], version: str) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir).resolve()
# Copy source folders
target_paths = _get_package_artifacts(temp_dir_path, package_name)
for source, target in zip(source_paths, target_paths):
if source.is_dir():
_sync_dirs(source, target)
# Build a setup.py on the fly
setup_file = _generate_setup_file(package_name, version, temp_dir_path)
package_file = destination / _get_wheel_name(name=package_name,
version=version)
if package_file.is_file():
click.secho(f"Package file {package_file} will be overwritten!",
fg="yellow")
# python setup.py bdist_wheel --dist-dir <destination>
call(
[
sys.executable,
str(setup_file.resolve()),
"bdist_wheel",
"--dist-dir",
str(destination),
],
cwd=temp_dir,
)
def _generate_wheel_file(
pipeline_name: str,
destination: Path,
source_paths: Tuple[_SourcePathType, ...],
version: str,
metadata: ProjectMetadata,
alias: str = None,
) -> None:
package_name = alias or pipeline_name
package_source, tests_source, conf_source = source_paths
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir).resolve()
project = Project(temp_dir_path) # project where to do refactoring
_refactor_code_for_package( # type: ignore
project, package_source, tests_source, alias, metadata
)
project.close()
# Copy & "refactor" config
_, _, conf_target = _get_package_artifacts(temp_dir_path, package_name)
_sync_path_list(conf_source, conf_target) # type: ignore
if conf_target.is_dir() and alias:
_rename_files(conf_target, pipeline_name, alias)
# Build a setup.py on the fly
try:
install_requires = _make_install_requires(
package_source / "requirements.txt" # type: ignore
)
except Exception as exc:
click.secho("FAILED", fg="red")
cls = exc.__class__
raise KedroCliError(f"{cls.__module__}.{cls.__qualname__}: {exc}") from exc
config_files = [str(file) for file in conf_target.rglob("*") if file.is_file()]
setup_file = _generate_setup_file(
package_name, version, install_requires, temp_dir_path, config_files
)
package_file = destination / _get_wheel_name(name=package_name, version=version)
if package_file.is_file():
click.secho(
f"Package file {package_file} will be overwritten!", fg="yellow"
)
# python setup.py bdist_wheel --dist-dir <destination>
call(
[
sys.executable,
str(setup_file.resolve()),
"bdist_wheel",
"--dist-dir",
str(destination),
],
cwd=temp_dir,
)
def ipython(env, args):
"""Open IPython with project specific variables loaded."""
context = _load_project_context(env=env)
os.environ["IPYTHONDIR"] = str(context.project_path / ".ipython")
if env:
os.environ["KEDRO_ENV"] = env
if "-h" not in args and "--help" not in args:
ipython_message()
call(["ipython"] + list(args))
def package():
"""Package the project as a Python egg and wheel."""
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_egg"],
cwd=str(SOURCE_PATH),
)
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_wheel"],
cwd=str(SOURCE_PATH),
)
def ipython(metadata: ProjectMetadata, env, args, **kwargs): # pylint: disable=unused-argument
"""Open IPython with project specific variables loaded."""
_check_module_importable("IPython")
os.environ["IPYTHONDIR"] = str(metadata.project_path / ".ipython")
if env:
os.environ["KEDRO_ENV"] = env
if "-h" not in args and "--help" not in args:
ipython_message()
call(["ipython"] + list(args))
def package(metadata: ProjectMetadata):
"""Package the project as a Python egg and wheel."""
source_path = metadata.source_dir
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_egg"],
cwd=str(source_path),
)
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_wheel"],
cwd=str(source_path),
)
def package():
"""Package the project as a Python egg and wheel."""
source_path = _get_source_path()
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_egg"],
cwd=str(source_path),
)
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_wheel"],
cwd=str(source_path),
)
def package():
"""Package the project as a Python egg and wheel."""
context = _load_project_context()
source_path = get_source_dir(context.project_path)
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_egg"],
cwd=str(source_path),
)
call(
[sys.executable, "setup.py", "clean", "--all", "bdist_wheel"],
cwd=str(source_path),
)
def _generate_wheel_file(
pipeline_name: str,
destination: Path,
source_paths: Tuple[_SourcePathType, ...],
version: str,
alias: str = None,
) -> None:
package_name = alias or pipeline_name
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir).resolve()
# Copy source folders
target_paths = _get_package_artifacts(temp_dir_path, package_name)
source_target, _, conf_target = target_paths
for source, target in zip(source_paths, target_paths):
sync_func = _sync_path_list if isinstance(source,
list) else _sync_dirs
sync_func(source, target) # type: ignore
if conf_target.is_dir() and alias:
_rename_files(conf_target, pipeline_name, alias)
# Build a setup.py on the fly
try:
install_requires = _make_install_requires(source_target /
"requirements.txt")
except Exception as exc:
click.secho("FAILED", fg="red")
cls = exc.__class__
raise KedroCliError(
f"{cls.__module__}.{cls.__qualname__}: {exc}") from exc
setup_file = _generate_setup_file(package_name, version,
install_requires, temp_dir_path)
package_file = destination / _get_wheel_name(name=package_name,
version=version)
if package_file.is_file():
click.secho(f"Package file {package_file} will be overwritten!",
fg="yellow")
# python setup.py bdist_wheel --dist-dir <destination>
call(
[
sys.executable,
str(setup_file.resolve()),
"bdist_wheel",
"--dist-dir",
str(destination),
],
cwd=temp_dir,
)
def docker_cmd(args, docker_args, image, **kwargs): # pylint: disable=unused-argument):
"""Run arbitrary command from ARGS in the Docker container.
If ARGS are not specified, this will invoke `kedro run` inside the container.
**kwargs is needed to make the global `verbose` argument work and pass it through."""
container_name = make_container_name(image, "cmd")
_docker_run_args = compose_docker_run_args(
optional_args=[("--rm", None), ("--name", container_name)],
user_args=docker_args,
**_mount_info(),
)
command = ["docker", "run"] + _docker_run_args + [image] + list(args)
call(command)
def install(metadata: ProjectMetadata, compile_flag):
"""Install project dependencies from both requirements.txt
and environment.yml (DEPRECATED)."""
deprecation_message = (
"DeprecationWarning: Command `kedro install` will be deprecated in Kedro 0.18.0. "
"In the future use `pip install -r src/requirements.txt` instead. "
"If you were running `kedro install` with the `--build-reqs` flag, "
"we recommend running `kedro build-reqs` followed by `pip install -r src/requirements.txt`"
)
click.secho(deprecation_message, fg="red")
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
# pylint: disable=consider-using-with
source_path = metadata.source_dir
environment_yml = source_path / "environment.yml"
requirements_in = source_path / "requirements.in"
requirements_txt = source_path / "requirements.txt"
if environment_yml.is_file():
call([
"conda", "env", "update", "--file",
str(environment_yml), "--prune"
])
default_compile = bool(compile_flag is None
and not requirements_in.is_file())
do_compile = compile_flag or default_compile
if do_compile:
_build_reqs(source_path)
pip_command = ["install", "-U", "-r", str(requirements_txt)]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
proc = subprocess.Popen(command,
creationflags=subprocess.CREATE_NEW_CONSOLE,
stderr=subprocess.PIPE)
_, errs = proc.communicate()
if errs:
secho(errs.decode(), fg="red")
raise click.exceptions.Exit(code=1)
secho("Requirements installed!", fg="green")
def docker_run(image, docker_args, args, **kwargs): # pylint: disable=unused-argument
"""Run the pipeline in the Docker container.
Any extra arguments unspecified in this help
are passed to `docker run` as is.
**kwargs is needed to make the global `verbose` argument work and pass it through."""
container_name = make_container_name(image, "run")
_docker_run_args = compose_docker_run_args(
optional_args=[("--rm", None), ("--name", container_name)],
user_args=docker_args,
**_mount_info(),
)
command = (["docker", "run"] + _docker_run_args + [image, "kedro", "run"] +
list(args))
call(command)
def _generate_wheel_file(
pipeline_name: str,
destination: Path,
source_paths: Tuple[_SourcePathType, ...],
version: str,
alias: str = None,
) -> None:
package_name = alias or pipeline_name
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir).resolve()
# Copy source folders
target_paths = _get_package_artifacts(temp_dir_path, package_name)
for source, target in zip(source_paths, target_paths):
sync_func = _sync_path_list if isinstance(source,
list) else _sync_dirs
sync_func(source, target) # type: ignore
conf_target = target_paths[-1]
if conf_target.is_dir() and alias:
_rename_files(conf_target, pipeline_name, alias)
# Build a setup.py on the fly
setup_file = _generate_setup_file(package_name, version, temp_dir_path)
package_file = destination / _get_wheel_name(name=package_name,
version=version)
if package_file.is_file():
click.secho(f"Package file {package_file} will be overwritten!",
fg="yellow")
# python setup.py bdist_wheel --dist-dir <destination>
call(
[
sys.executable,
str(setup_file.resolve()),
"bdist_wheel",
"--dist-dir",
str(destination),
],
cwd=temp_dir,
)
def _generate_wheel_file(package_name: str, destination: Path,
source_paths: Tuple[Path, ...]) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir).resolve()
# Copy source folders
target_paths = (
temp_dir_path / package_name,
temp_dir_path / "tests",
# package_data (non-python files) needs to live inside one of the packages
temp_dir_path / package_name / "config",
)
for source, target in zip(source_paths, target_paths):
if source.is_dir():
_sync_dirs(source, target)
# Build a setup.py on the fly
setup_file = temp_dir_path / "setup.py"
package_data = {
package_name: [
"README.md",
"config/parameters*",
"config/**/parameters*",
"config/parameters*/**",
]
}
setup_file_context = dict(name=package_name,
version="0.1",
package_data=json.dumps(package_data))
setup_file.write_text(_SETUP_PY_TEMPLATE.format(**setup_file_context))
# python setup.py bdist_wheel --dist-dir src/dist
call(
[
sys.executable,
str(setup_file.resolve()),
"bdist_wheel",
"--dist-dir",
str(destination),
],
cwd=temp_dir,
)
def docker_build(ctx, uid, gid, spark, base_image, image, docker_args):
"""Build a Docker image for the project."""
uid, gid = get_uid_gid(uid, gid)
project_path = Path.cwd()
image = image or project_path.name
ctx.invoke(docker_init, spark=spark)
combined_args = compose_docker_run_args(
required_args=[
("--build-arg", f"KEDRO_UID={uid}"),
("--build-arg", f"KEDRO_GID={gid}"),
("--build-arg", f"BASE_IMAGE={base_image}"),
],
# add image tag if only it is not already supplied by the user
optional_args=[("-t", image)],
user_args=docker_args,
)
command = ["docker", "build"] + combined_args + [str(project_path)]
call(command)
def docker_jupyter_lab(docker_args, port, image, args, **kwargs): # pylint: disable=unused-argument):
"""Run jupyter lab in the Docker container.
Any extra arguments unspecified in this help are passed to
`kedro jupyter lab` command inside the container as is.
**kwargs is needed to make the global `verbose` argument work and pass it through."""
container_name = make_container_name(image, "jupyter-lab")
_docker_run_args = compose_docker_run_args(
required_args=[("-p", f"{port}:8888")],
optional_args=[("--rm", None), ("-it", None),
("--name", container_name)],
user_args=docker_args,
**_mount_info(),
)
args = add_jupyter_args(list(args))
command = (["docker", "run"] + _docker_run_args +
[image, "kedro", "jupyter", "lab"] + args)
call(command)
def build_docs(open_docs):
"""Build the project documentation."""
python_call("pip", ["install", str(SOURCE_PATH / "[docs]")])
python_call("pip",
["install", "-r",
str(SOURCE_PATH / "requirements.txt")])
python_call("ipykernel",
["install", "--user", f"--name={KEDRO_PACKAGE_NAME}"])
shutil.rmtree("docs/build", ignore_errors=True)
call([
"sphinx-apidoc",
"--module-first",
"-o",
"docs/source",
str(SOURCE_PATH / KEDRO_PACKAGE_NAME),
])
call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"])
if open_docs:
docs_page = (Path.cwd() / "docs" / "build" / "html" /
"index.html").as_uri()
secho("Opening {}".format(docs_page))
webbrowser.open(docs_page)
def install(metadata: ProjectMetadata, compile_flag):
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
# pylint: disable=consider-using-with
source_path = metadata.source_dir
environment_yml = source_path / "environment.yml"
requirements_in = source_path / "requirements.in"
requirements_txt = source_path / "requirements.txt"
if environment_yml.is_file():
call([
"conda", "env", "update", "--file",
str(environment_yml), "--prune"
])
default_compile = bool(compile_flag is None
and not requirements_in.is_file())
do_compile = compile_flag or default_compile
if do_compile:
_build_reqs(source_path)
pip_command = ["install", "-U", "-r", str(requirements_txt)]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
proc = subprocess.Popen(command,
creationflags=subprocess.CREATE_NEW_CONSOLE,
stderr=subprocess.PIPE)
_, errs = proc.communicate()
if errs:
secho(errs.decode(), fg="red")
raise click.exceptions.Exit(code=1)
secho("Requirements installed!", fg="green")
def install():
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
if (SOURCE_PATH / "environment.yml").is_file():
call([
"conda",
"install",
"--file",
str(SOURCE_PATH / "environment.yml"),
"--yes",
])
pip_command = [
"install", "-U", "-r",
str(SOURCE_PATH / "requirements.txt")
]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def docker_dive(ci_flag, dive_ci, docker_args, image):
"""Run Dive analyzer of Docker image efficiency."""
container_name = make_container_name(image, "dive")
required_args = [("-v", "/var/run/docker.sock:/var/run/docker.sock")]
optional_args = [("--rm", None), ("--name", container_name)]
if ci_flag:
dive_ci = Path(dive_ci).absolute()
if dive_ci.is_file():
required_args.append(("-v", f"{dive_ci}:/.dive-ci"))
else:
msg = f"`{dive_ci}` file not found, using default CI config"
click.secho(msg, fg="yellow")
required_args.append(("-e", f"CI={str(ci_flag).lower()}"))
else:
optional_args.append(("-it", None))
_docker_run_args = compose_docker_run_args(required_args=required_args,
optional_args=optional_args,
user_args=docker_args)
command = ["docker", "run"] + _docker_run_args + [DIVE_IMAGE] + [image]
call(command)
def ipython(args):
"""Open IPython with project specific variables loaded."""
if "-h" not in args and "--help" not in args:
ipython_message()
call(["ipython"] + list(args))
