def install(): """Install project dependencies from both requirements.txt and environment.yml (optional).""" # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here source_path = get_source_dir(Path.cwd()) if (source_path / "environment.yml").is_file(): call([ "conda", "install", "--file", str(source_path / "environment.yml"), "--yes", ]) pip_command = [ "install", "-U", "-r", str(source_path / "requirements.txt") ] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def install(compile_flag): """Install project dependencies from both requirements.txt and environment.yml (optional).""" # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here source_path = _get_source_path() environment_yml = source_path / "environment.yml" requirements_in = source_path / "requirements.in" requirements_txt = source_path / "requirements.txt" if environment_yml.is_file(): call([ "conda", "env", "update", "--file", str(environment_yml), "--prune" ]) default_compile = bool(compile_flag is None and not requirements_in.is_file()) do_compile = compile_flag or default_compile if do_compile: _build_reqs(source_path) pip_command = ["install", "-U", "-r", str(requirements_txt)] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE) secho("Requirements installed!", fg="green")
def jupyter_lab( metadata: ProjectMetadata, ip_address, all_kernels, env, idle_timeout, args, **kwargs, ): # pylint: disable=unused-argument,too-many-arguments """Open Jupyter Lab with project specific variables loaded.""" _check_module_importable("jupyter_core") if "-h" not in args and "--help" not in args: ipython_message(all_kernels) _update_ipython_dir(metadata.project_path) arguments = _build_jupyter_command( "lab", ip_address=ip_address, all_kernels=all_kernels, args=args, idle_timeout=idle_timeout, project_name=metadata.project_name, ) python_call_kwargs = _build_jupyter_env(env) python_call("jupyter", arguments, **python_call_kwargs)
def test(args): """Run the test suite.""" try: import pytest # pylint: disable=unused-import except ImportError: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(SOURCE_PATH))) else: python_call("pytest", args)
def _build_reqs(source_path: Path, args: Sequence[str] = ()): """Run `pip-compile requirements.in` command. Args: source_path: Path to the project `src` folder. args: Optional arguments for `pip-compile` call, e.g. `--generate-hashes`. """ requirements_in = _get_requirements_in(source_path) python_call("piptools", ["compile", "-q", *args, str(requirements_in)])
def test(metadata: ProjectMetadata, args, **kwargs): # pylint: disable=unused-argument """Run the test suite.""" try: _check_module_importable("pytest") except KedroCliError as exc: source_path = metadata.source_dir raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path)) ) from exc else: python_call("pytest", args)
def test(args): """Run the test suite.""" try: _check_module_importable("pytest") except KedroCliError as exc: source_path = _get_source_path() raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path))) from exc else: python_call("pytest", args)
def build_reqs(): """Build the project dependency requirements.""" requirements_path = SOURCE_PATH / "requirements.in" if not requirements_path.is_file(): secho( "No requirements.in found. Copying contents from requirements.txt..." ) contents = (SOURCE_PATH / "requirements.txt").read_text() requirements_path.write_text(contents) python_call("piptools", ["compile", str(requirements_path)]) secho(("Requirements built! Please update requirements.in " "if you'd like to make a change in your project's dependencies, " "and re-run build-reqs to generate the new requirements.txt."))
def test(args): """Run the test suite.""" try: # pylint: disable=import-outside-toplevel, unused-import import pytest # noqa except ImportError: context = _load_project_context() source_path = get_source_dir(context.project_path) raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path))) else: python_call("pytest", args)
def _build_reqs(source_path: Path, args: Sequence[str] = ()): """Run `pip-compile requirements.in` command. Args: source_path: Path to the project `src` folder. args: Optional arguments for `pip-compile` call, e.g. `--generate-hashes`. """ requirements_in = source_path / "requirements.in" if not requirements_in.is_file(): secho("No requirements.in found. Copying contents from requirements.txt...") requirements_txt = source_path / "requirements.txt" shutil.copyfile(str(requirements_txt), str(requirements_in)) python_call("piptools", ["compile", "-q", *args, str(requirements_in)])
def install(metadata: ProjectMetadata, compile_flag): """Install project dependencies from both requirements.txt and environment.yml (DEPRECATED).""" deprecation_message = ( "DeprecationWarning: Command `kedro install` will be deprecated in Kedro 0.18.0. " "In the future use `pip install -r src/requirements.txt` instead. " "If you were running `kedro install` with the `--build-reqs` flag, " "we recommend running `kedro build-reqs` followed by `pip install -r src/requirements.txt`" ) click.secho(deprecation_message, fg="red") # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here # pylint: disable=consider-using-with source_path = metadata.source_dir environment_yml = source_path / "environment.yml" requirements_in = source_path / "requirements.in" requirements_txt = source_path / "requirements.txt" if environment_yml.is_file(): call([ "conda", "env", "update", "--file", str(environment_yml), "--prune" ]) default_compile = bool(compile_flag is None and not requirements_in.is_file()) do_compile = compile_flag or default_compile if do_compile: _build_reqs(source_path) pip_command = ["install", "-U", "-r", str(requirements_txt)] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command proc = subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE, stderr=subprocess.PIPE) _, errs = proc.communicate() if errs: secho(errs.decode(), fg="red") raise click.exceptions.Exit(code=1) secho("Requirements installed!", fg="green")
def build_reqs(): """Build the project dependency requirements.""" # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here source_path = get_source_dir(Path.cwd()) requirements_path = source_path / "requirements.in" if not requirements_path.is_file(): secho( "No requirements.in found. Copying contents from requirements.txt..." ) contents = (source_path / "requirements.txt").read_text() requirements_path.write_text(contents) python_call("piptools", ["compile", str(requirements_path)]) secho(("Requirements built! Please update requirements.in " "if you'd like to make a change in your project's dependencies, " "and re-run build-reqs to generate the new requirements.txt."))
def pull_package(package_path, env, alias): """Pull a modular pipeline package, unpack it and install the files to corresponding locations. """ # pylint: disable=import-outside-toplevel import fsspec from kedro.io.core import get_protocol_and_path protocol, _ = get_protocol_and_path(package_path) filesystem = fsspec.filesystem(protocol) with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir).resolve() if package_path.endswith(".whl") and filesystem.exists(package_path): with filesystem.open(package_path) as fs_file: ZipFile(fs_file).extractall(temp_dir_path) else: python_call( "pip", [ "download", "--no-deps", "--dest", str(temp_dir_path), package_path ], ) wheel_file = list(temp_dir_path.glob("*.whl")) # `--no-deps` should fetch only one wheel file, and CLI should fail if that's # not the case. if len(wheel_file) != 1: file_names = [wf.name for wf in wheel_file] raise KedroCliError( f"More than 1 or no wheel files found: {str(file_names)}. " "There has to be exactly one distribution file.") ZipFile(wheel_file[0]).extractall(temp_dir_path) dist_info_file = list(temp_dir_path.glob("*.dist-info")) if len(dist_info_file) != 1: raise KedroCliError( f"More than 1 or no dist-info files found from {package_path}. " "There has to be exactly one dist-info directory.") # Extract package name, based on the naming convention for wheel files # https://www.python.org/dev/peps/pep-0427/#file-name-convention package_name = dist_info_file[0].stem.split("-")[0] _clean_pycache(temp_dir_path) _install_files(package_name, temp_dir_path, env, alias)
def jupyter_lab(ip_address, all_kernels, env, idle_timeout, args): """Open Jupyter Lab with project specific variables loaded.""" context = _load_project_context(env=env) if "-h" not in args and "--help" not in args: ipython_message(all_kernels) _update_ipython_dir(context.project_path) arguments = _build_jupyter_command( "lab", ip_address=ip_address, all_kernels=all_kernels, args=args, idle_timeout=idle_timeout, project_name=context.project_name, ) python_call_kwargs = _build_jupyter_env(env) python_call("jupyter", arguments, **python_call_kwargs)
def _unpack_wheel(location: str, destination: Path, fs_args: Optional[str]) -> None: filesystem = _get_fsspec_filesystem(location, fs_args) if location.endswith(".whl") and filesystem and filesystem.exists(location): with filesystem.open(location) as fs_file: ZipFile(fs_file).extractall(destination) else: python_call( "pip", ["download", "--no-deps", "--dest", str(destination), location] ) wheel_file = list(destination.glob("*.whl")) # `--no-deps` should fetch only one wheel file, and CLI should fail if that's # not the case. if len(wheel_file) != 1: file_names = [wf.name for wf in wheel_file] raise KedroCliError( f"More than 1 or no wheel files found: {file_names}. " f"There has to be exactly one distribution file." ) ZipFile(wheel_file[0]).extractall(destination)
def build_docs(open_docs): """Build the project documentation.""" static_data = get_static_project_data(Path.cwd()) source_path = static_data["source_dir"] package_name = (static_data.get("package_name") or _load_project_context().package_name) python_call("pip", ["install", str(source_path / "[docs]")]) python_call("pip", ["install", "-r", str(source_path / "requirements.txt")]) python_call("ipykernel", ["install", "--user", f"--name={package_name}"]) shutil.rmtree("docs/build", ignore_errors=True) call([ "sphinx-apidoc", "--module-first", "-o", "docs/source", str(source_path / package_name), ]) call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"]) if open_docs: docs_page = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri() secho(f"Opening {docs_page}") webbrowser.open(docs_page)
def lint(files, check_only): """Run flake8, isort and black.""" static_data = get_static_project_data(Path.cwd()) source_path = static_data["source_dir"] package_name = (static_data.get("package_name") or _load_project_context().package_name) files = files or (str( source_path / "tests"), str(source_path / package_name)) if "PYTHONPATH" not in os.environ: # isort needs the source path to be in the 'PYTHONPATH' environment # variable to treat it as a first-party import location os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover for module_name in ("flake8", "isort", "black"): try: _check_module_importable(module_name) except KedroCliError as exc: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path))) from exc python_call("black", ("--check", ) + files if check_only else files) python_call("flake8", ("--max-line-length=88", ) + files) check_flag = ("-c", ) if check_only else () python_call("isort", (*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files)
def lint( metadata: ProjectMetadata, files, check_only, **kwargs ): # pylint: disable=unused-argument """Run flake8, isort and black.""" source_path = metadata.source_dir package_name = metadata.package_name files = files or (str(source_path / "tests"), str(source_path / package_name)) if "PYTHONPATH" not in os.environ: # isort needs the source path to be in the 'PYTHONPATH' environment # variable to treat it as a first-party import location os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover for module_name in ("flake8", "isort", "black"): try: _check_module_importable(module_name) except KedroCliError as exc: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path)) ) from exc python_call("black", ("--check",) + files if check_only else files) python_call("flake8", files) check_flag = ("-c",) if check_only else () python_call("isort", (*check_flag, "-rc") + files) # type: ignore
def lint(files, check_only): """Run flake8, isort and (on Python >=3.6) black.""" context = _load_project_context() source_path = get_source_dir(context.project_path) files = files or ( str(source_path / "tests"), str(source_path / context.package_name), ) try: # pylint: disable=import-outside-toplevel, unused-import import flake8 # noqa import isort # noqa import black # noqa except ImportError as exc: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module=exc.name, src=str(source_path))) python_call("black", ("--check", ) + files if check_only else files) python_call("flake8", ("--max-line-length=88", ) + files) check_flag = ("-c", ) if check_only else () python_call("isort", (*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files)
def install(metadata: ProjectMetadata, compile_flag): """Install project dependencies from both requirements.txt and environment.yml (optional).""" # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here # pylint: disable=consider-using-with source_path = metadata.source_dir environment_yml = source_path / "environment.yml" requirements_in = source_path / "requirements.in" requirements_txt = source_path / "requirements.txt" if environment_yml.is_file(): call([ "conda", "env", "update", "--file", str(environment_yml), "--prune" ]) default_compile = bool(compile_flag is None and not requirements_in.is_file()) do_compile = compile_flag or default_compile if do_compile: _build_reqs(source_path) pip_command = ["install", "-U", "-r", str(requirements_txt)] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command proc = subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE, stderr=subprocess.PIPE) _, errs = proc.communicate() if errs: secho(errs.decode(), fg="red") raise click.exceptions.Exit(code=1) secho("Requirements installed!", fg="green")
def install(): """Install project dependencies from both requirements.txt and environment.yml (optional).""" if (SOURCE_PATH / "environment.yml").is_file(): call([ "conda", "install", "--file", str(SOURCE_PATH / "environment.yml"), "--yes", ]) pip_command = [ "install", "-U", "-r", str(SOURCE_PATH / "requirements.txt") ] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def lint(files, check_only): """Run flake8, isort and (on Python >=3.6) black.""" files = files or (str( SOURCE_PATH / "tests"), str(SOURCE_PATH / KEDRO_PACKAGE_NAME)) try: import flake8 import isort import black except ImportError as exc: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module=exc.name, src=str(SOURCE_PATH))) python_call("black", ("--check", ) + files if check_only else files) python_call("flake8", ("--max-line-length=88", ) + files) check_flag = ("-c", ) if check_only else () python_call("isort", (*check_flag, "-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files)
def build_docs(open_docs): """Build the project documentation.""" python_call("pip", ["install", str(SOURCE_PATH / "[docs]")]) python_call("pip", ["install", "-r", str(SOURCE_PATH / "requirements.txt")]) python_call("ipykernel", ["install", "--user", f"--name={KEDRO_PACKAGE_NAME}"]) shutil.rmtree("docs/build", ignore_errors=True) call([ "sphinx-apidoc", "--module-first", "-o", "docs/source", str(SOURCE_PATH / KEDRO_PACKAGE_NAME), ]) call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"]) if open_docs: docs_page = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri() secho("Opening {}".format(docs_page)) webbrowser.open(docs_page)