def install():
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = get_source_dir(Path.cwd())
if (source_path / "environment.yml").is_file():
call([
"conda",
"install",
"--file",
str(source_path / "environment.yml"),
"--yes",
])
pip_command = [
"install", "-U", "-r",
str(source_path / "requirements.txt")
]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def install(compile_flag):
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = _get_source_path()
environment_yml = source_path / "environment.yml"
requirements_in = source_path / "requirements.in"
requirements_txt = source_path / "requirements.txt"
if environment_yml.is_file():
call([
"conda", "env", "update", "--file",
str(environment_yml), "--prune"
])
default_compile = bool(compile_flag is None
and not requirements_in.is_file())
do_compile = compile_flag or default_compile
if do_compile:
_build_reqs(source_path)
pip_command = ["install", "-U", "-r", str(requirements_txt)]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
secho("Requirements installed!", fg="green")
def jupyter_lab(
metadata: ProjectMetadata,
ip_address,
all_kernels,
env,
idle_timeout,
args,
**kwargs,
): # pylint: disable=unused-argument,too-many-arguments
"""Open Jupyter Lab with project specific variables loaded."""
_check_module_importable("jupyter_core")
if "-h" not in args and "--help" not in args:
ipython_message(all_kernels)
_update_ipython_dir(metadata.project_path)
arguments = _build_jupyter_command(
"lab",
ip_address=ip_address,
all_kernels=all_kernels,
args=args,
idle_timeout=idle_timeout,
project_name=metadata.project_name,
)
python_call_kwargs = _build_jupyter_env(env)
python_call("jupyter", arguments, **python_call_kwargs)
def test(args):
"""Run the test suite."""
try:
import pytest # pylint: disable=unused-import
except ImportError:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="pytest",
src=str(SOURCE_PATH)))
else:
python_call("pytest", args)
def _build_reqs(source_path: Path, args: Sequence[str] = ()):
"""Run `pip-compile requirements.in` command.
Args:
source_path: Path to the project `src` folder.
args: Optional arguments for `pip-compile` call, e.g. `--generate-hashes`.
"""
requirements_in = _get_requirements_in(source_path)
python_call("piptools", ["compile", "-q", *args, str(requirements_in)])
def test(metadata: ProjectMetadata, args, **kwargs): # pylint: disable=unused-argument
"""Run the test suite."""
try:
_check_module_importable("pytest")
except KedroCliError as exc:
source_path = metadata.source_dir
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path))
) from exc
else:
python_call("pytest", args)
def test(args):
"""Run the test suite."""
try:
_check_module_importable("pytest")
except KedroCliError as exc:
source_path = _get_source_path()
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="pytest",
src=str(source_path))) from exc
else:
python_call("pytest", args)
def build_reqs():
"""Build the project dependency requirements."""
requirements_path = SOURCE_PATH / "requirements.in"
if not requirements_path.is_file():
secho(
"No requirements.in found. Copying contents from requirements.txt..."
)
contents = (SOURCE_PATH / "requirements.txt").read_text()
requirements_path.write_text(contents)
python_call("piptools", ["compile", str(requirements_path)])
secho(("Requirements built! Please update requirements.in "
"if you'd like to make a change in your project's dependencies, "
"and re-run build-reqs to generate the new requirements.txt."))
def test(args):
"""Run the test suite."""
try:
# pylint: disable=import-outside-toplevel, unused-import
import pytest # noqa
except ImportError:
context = _load_project_context()
source_path = get_source_dir(context.project_path)
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="pytest",
src=str(source_path)))
else:
python_call("pytest", args)
def _build_reqs(source_path: Path, args: Sequence[str] = ()):
"""Run `pip-compile requirements.in` command.
Args:
source_path: Path to the project `src` folder.
args: Optional arguments for `pip-compile` call, e.g. `--generate-hashes`.
"""
requirements_in = source_path / "requirements.in"
if not requirements_in.is_file():
secho("No requirements.in found. Copying contents from requirements.txt...")
requirements_txt = source_path / "requirements.txt"
shutil.copyfile(str(requirements_txt), str(requirements_in))
python_call("piptools", ["compile", "-q", *args, str(requirements_in)])
def install(metadata: ProjectMetadata, compile_flag):
"""Install project dependencies from both requirements.txt
and environment.yml (DEPRECATED)."""
deprecation_message = (
"DeprecationWarning: Command `kedro install` will be deprecated in Kedro 0.18.0. "
"In the future use `pip install -r src/requirements.txt` instead. "
"If you were running `kedro install` with the `--build-reqs` flag, "
"we recommend running `kedro build-reqs` followed by `pip install -r src/requirements.txt`"
)
click.secho(deprecation_message, fg="red")
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
# pylint: disable=consider-using-with
source_path = metadata.source_dir
environment_yml = source_path / "environment.yml"
requirements_in = source_path / "requirements.in"
requirements_txt = source_path / "requirements.txt"
if environment_yml.is_file():
call([
"conda", "env", "update", "--file",
str(environment_yml), "--prune"
])
default_compile = bool(compile_flag is None
and not requirements_in.is_file())
do_compile = compile_flag or default_compile
if do_compile:
_build_reqs(source_path)
pip_command = ["install", "-U", "-r", str(requirements_txt)]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
proc = subprocess.Popen(command,
creationflags=subprocess.CREATE_NEW_CONSOLE,
stderr=subprocess.PIPE)
_, errs = proc.communicate()
if errs:
secho(errs.decode(), fg="red")
raise click.exceptions.Exit(code=1)
secho("Requirements installed!", fg="green")
def build_reqs():
"""Build the project dependency requirements."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
source_path = get_source_dir(Path.cwd())
requirements_path = source_path / "requirements.in"
if not requirements_path.is_file():
secho(
"No requirements.in found. Copying contents from requirements.txt..."
)
contents = (source_path / "requirements.txt").read_text()
requirements_path.write_text(contents)
python_call("piptools", ["compile", str(requirements_path)])
secho(("Requirements built! Please update requirements.in "
"if you'd like to make a change in your project's dependencies, "
"and re-run build-reqs to generate the new requirements.txt."))
def pull_package(package_path, env, alias):
"""Pull a modular pipeline package, unpack it and install the files to corresponding
locations.
"""
# pylint: disable=import-outside-toplevel
import fsspec
from kedro.io.core import get_protocol_and_path
protocol, _ = get_protocol_and_path(package_path)
filesystem = fsspec.filesystem(protocol)
with tempfile.TemporaryDirectory() as temp_dir:
temp_dir_path = Path(temp_dir).resolve()
if package_path.endswith(".whl") and filesystem.exists(package_path):
with filesystem.open(package_path) as fs_file:
ZipFile(fs_file).extractall(temp_dir_path)
else:
python_call(
"pip",
[
"download", "--no-deps", "--dest",
str(temp_dir_path), package_path
],
)
wheel_file = list(temp_dir_path.glob("*.whl"))
# `--no-deps` should fetch only one wheel file, and CLI should fail if that's
# not the case.
if len(wheel_file) != 1:
file_names = [wf.name for wf in wheel_file]
raise KedroCliError(
f"More than 1 or no wheel files found: {str(file_names)}. "
"There has to be exactly one distribution file.")
ZipFile(wheel_file[0]).extractall(temp_dir_path)
dist_info_file = list(temp_dir_path.glob("*.dist-info"))
if len(dist_info_file) != 1:
raise KedroCliError(
f"More than 1 or no dist-info files found from {package_path}. "
"There has to be exactly one dist-info directory.")
# Extract package name, based on the naming convention for wheel files
# https://www.python.org/dev/peps/pep-0427/#file-name-convention
package_name = dist_info_file[0].stem.split("-")[0]
_clean_pycache(temp_dir_path)
_install_files(package_name, temp_dir_path, env, alias)
def jupyter_lab(ip_address, all_kernels, env, idle_timeout, args):
"""Open Jupyter Lab with project specific variables loaded."""
context = _load_project_context(env=env)
if "-h" not in args and "--help" not in args:
ipython_message(all_kernels)
_update_ipython_dir(context.project_path)
arguments = _build_jupyter_command(
"lab",
ip_address=ip_address,
all_kernels=all_kernels,
args=args,
idle_timeout=idle_timeout,
project_name=context.project_name,
)
python_call_kwargs = _build_jupyter_env(env)
python_call("jupyter", arguments, **python_call_kwargs)
def _unpack_wheel(location: str, destination: Path, fs_args: Optional[str]) -> None:
filesystem = _get_fsspec_filesystem(location, fs_args)
if location.endswith(".whl") and filesystem and filesystem.exists(location):
with filesystem.open(location) as fs_file:
ZipFile(fs_file).extractall(destination)
else:
python_call(
"pip", ["download", "--no-deps", "--dest", str(destination), location]
)
wheel_file = list(destination.glob("*.whl"))
# `--no-deps` should fetch only one wheel file, and CLI should fail if that's
# not the case.
if len(wheel_file) != 1:
file_names = [wf.name for wf in wheel_file]
raise KedroCliError(
f"More than 1 or no wheel files found: {file_names}. "
f"There has to be exactly one distribution file."
)
ZipFile(wheel_file[0]).extractall(destination)
def build_docs(open_docs):
"""Build the project documentation."""
static_data = get_static_project_data(Path.cwd())
source_path = static_data["source_dir"]
package_name = (static_data.get("package_name")
or _load_project_context().package_name)
python_call("pip", ["install", str(source_path / "[docs]")])
python_call("pip",
["install", "-r",
str(source_path / "requirements.txt")])
python_call("ipykernel", ["install", "--user", f"--name={package_name}"])
shutil.rmtree("docs/build", ignore_errors=True)
call([
"sphinx-apidoc",
"--module-first",
"-o",
"docs/source",
str(source_path / package_name),
])
call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"])
if open_docs:
docs_page = (Path.cwd() / "docs" / "build" / "html" /
"index.html").as_uri()
secho(f"Opening {docs_page}")
webbrowser.open(docs_page)
def lint(files, check_only):
"""Run flake8, isort and black."""
static_data = get_static_project_data(Path.cwd())
source_path = static_data["source_dir"]
package_name = (static_data.get("package_name")
or _load_project_context().package_name)
files = files or (str(
source_path / "tests"), str(source_path / package_name))
if "PYTHONPATH" not in os.environ:
# isort needs the source path to be in the 'PYTHONPATH' environment
# variable to treat it as a first-party import location
os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover
for module_name in ("flake8", "isort", "black"):
try:
_check_module_importable(module_name)
except KedroCliError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module=module_name,
src=str(source_path))) from exc
python_call("black", ("--check", ) + files if check_only else files)
python_call("flake8", ("--max-line-length=88", ) + files)
check_flag = ("-c", ) if check_only else ()
python_call("isort",
(*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") +
files)
def lint(
metadata: ProjectMetadata, files, check_only, **kwargs
): # pylint: disable=unused-argument
"""Run flake8, isort and black."""
source_path = metadata.source_dir
package_name = metadata.package_name
files = files or (str(source_path / "tests"), str(source_path / package_name))
if "PYTHONPATH" not in os.environ:
# isort needs the source path to be in the 'PYTHONPATH' environment
# variable to treat it as a first-party import location
os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover
for module_name in ("flake8", "isort", "black"):
try:
_check_module_importable(module_name)
except KedroCliError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path))
) from exc
python_call("black", ("--check",) + files if check_only else files)
python_call("flake8", files)
check_flag = ("-c",) if check_only else ()
python_call("isort", (*check_flag, "-rc") + files) # type: ignore
def lint(files, check_only):
"""Run flake8, isort and (on Python >=3.6) black."""
context = _load_project_context()
source_path = get_source_dir(context.project_path)
files = files or (
str(source_path / "tests"),
str(source_path / context.package_name),
)
try:
# pylint: disable=import-outside-toplevel, unused-import
import flake8 # noqa
import isort # noqa
import black # noqa
except ImportError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module=exc.name,
src=str(source_path)))
python_call("black", ("--check", ) + files if check_only else files)
python_call("flake8", ("--max-line-length=88", ) + files)
check_flag = ("-c", ) if check_only else ()
python_call("isort",
(*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") +
files)
def install(metadata: ProjectMetadata, compile_flag):
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
# we cannot use `context.project_path` as in other commands since
# context instantiation might break due to missing dependencies
# we attempt to install here
# pylint: disable=consider-using-with
source_path = metadata.source_dir
environment_yml = source_path / "environment.yml"
requirements_in = source_path / "requirements.in"
requirements_txt = source_path / "requirements.txt"
if environment_yml.is_file():
call([
"conda", "env", "update", "--file",
str(environment_yml), "--prune"
])
default_compile = bool(compile_flag is None
and not requirements_in.is_file())
do_compile = compile_flag or default_compile
if do_compile:
_build_reqs(source_path)
pip_command = ["install", "-U", "-r", str(requirements_txt)]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
proc = subprocess.Popen(command,
creationflags=subprocess.CREATE_NEW_CONSOLE,
stderr=subprocess.PIPE)
_, errs = proc.communicate()
if errs:
secho(errs.decode(), fg="red")
raise click.exceptions.Exit(code=1)
secho("Requirements installed!", fg="green")
def install():
"""Install project dependencies from both requirements.txt
and environment.yml (optional)."""
if (SOURCE_PATH / "environment.yml").is_file():
call([
"conda",
"install",
"--file",
str(SOURCE_PATH / "environment.yml"),
"--yes",
])
pip_command = [
"install", "-U", "-r",
str(SOURCE_PATH / "requirements.txt")
]
if os.name == "posix":
python_call("pip", pip_command)
else:
command = [sys.executable, "-m", "pip"] + pip_command
subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def lint(files, check_only):
"""Run flake8, isort and (on Python >=3.6) black."""
files = files or (str(
SOURCE_PATH / "tests"), str(SOURCE_PATH / KEDRO_PACKAGE_NAME))
try:
import flake8
import isort
import black
except ImportError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module=exc.name,
src=str(SOURCE_PATH)))
python_call("black", ("--check", ) + files if check_only else files)
python_call("flake8", ("--max-line-length=88", ) + files)
check_flag = ("-c", ) if check_only else ()
python_call("isort",
(*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") +
files)
def build_docs(open_docs):
"""Build the project documentation."""
python_call("pip", ["install", str(SOURCE_PATH / "[docs]")])
python_call("pip",
["install", "-r",
str(SOURCE_PATH / "requirements.txt")])
python_call("ipykernel",
["install", "--user", f"--name={KEDRO_PACKAGE_NAME}"])
shutil.rmtree("docs/build", ignore_errors=True)
call([
"sphinx-apidoc",
"--module-first",
"-o",
"docs/source",
str(SOURCE_PATH / KEDRO_PACKAGE_NAME),
])
call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"])
if open_docs:
docs_page = (Path.cwd() / "docs" / "build" / "html" /
"index.html").as_uri()
secho("Opening {}".format(docs_page))
webbrowser.open(docs_page)
