def install(): """Install project dependencies from both requirements.txt and environment.yml (optional).""" # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here source_path = get_source_dir(Path.cwd()) if (source_path / "environment.yml").is_file(): call([ "conda", "install", "--file", str(source_path / "environment.yml"), "--yes", ]) pip_command = [ "install", "-U", "-r", str(source_path / "requirements.txt") ] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def activate_nbstripout(): """Install the nbstripout git hook to automatically clean notebooks.""" context = _load_project_context() source_path = get_source_dir(context.project_path) secho( ("Notebook output cells will be automatically cleared before committing" " to git."), fg="yellow", ) try: # pylint: disable=import-outside-toplevel, unused-import import nbstripout # noqa except ImportError: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(source_path))) try: res = subprocess.run( # pylint: disable=subprocess-run-check ["git", "rev-parse", "--git-dir"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) if res.returncode: raise KedroCliError("Not a git repository. Run `git init` first.") except FileNotFoundError: raise KedroCliError("Git executable not found. Install Git first.") call(["nbstripout", "--install"])
def activate_nbstripout(): """Install the nbstripout git hook to automatically clean notebooks.""" secho( ("Notebook output cells will be automatically cleared before committing" " to git."), fg="yellow", ) try: import nbstripout # pylint: disable=unused-import except ImportError: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(SOURCE_PATH))) try: res = subprocess.run( ["git", "rev-parse", "--git-dir"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) if res.returncode: raise KedroCliError("Not a git repository. Run `git init` first.") except FileNotFoundError: raise KedroCliError("Git executable not found. Install Git first.") call(["nbstripout", "--install"])
def build_docs(open_docs): """Build the project documentation.""" static_data = get_static_project_data(Path.cwd()) source_path = static_data["source_dir"] package_name = (static_data.get("package_name") or _load_project_context().package_name) python_call("pip", ["install", str(source_path / "[docs]")]) python_call("pip", ["install", "-r", str(source_path / "requirements.txt")]) python_call("ipykernel", ["install", "--user", f"--name={package_name}"]) shutil.rmtree("docs/build", ignore_errors=True) call([ "sphinx-apidoc", "--module-first", "-o", "docs/source", str(source_path / package_name), ]) call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"]) if open_docs: docs_page = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri() secho(f"Opening {docs_page}") webbrowser.open(docs_page)
def install(compile_flag): """Install project dependencies from both requirements.txt and environment.yml (optional).""" # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here source_path = _get_source_path() environment_yml = source_path / "environment.yml" requirements_in = source_path / "requirements.in" requirements_txt = source_path / "requirements.txt" if environment_yml.is_file(): call([ "conda", "env", "update", "--file", str(environment_yml), "--prune" ]) default_compile = bool(compile_flag is None and not requirements_in.is_file()) do_compile = compile_flag or default_compile if do_compile: _build_reqs(source_path) pip_command = ["install", "-U", "-r", str(requirements_txt)] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE) secho("Requirements installed!", fg="green")
def activate_nbstripout(): """Install the nbstripout git hook to automatically clean notebooks.""" source_path = _get_source_path() secho( ("Notebook output cells will be automatically cleared before committing" " to git."), fg="yellow", ) try: _check_module_importable("nbstripout") except KedroCliError as exc: raise KedroCliError( NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(source_path))) from exc try: res = subprocess.run( # pylint: disable=subprocess-run-check ["git", "rev-parse", "--git-dir"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) if res.returncode: raise KedroCliError("Not a git repository. Run `git init` first.") except FileNotFoundError as exc: raise KedroCliError( "Git executable not found. Install Git first.") from exc call(["nbstripout", "--install"])
def _generate_wheel_file(package_name: str, destination: Path, source_paths: Tuple[Path, ...], version: str) -> None: with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir).resolve() # Copy source folders target_paths = _get_package_artifacts(temp_dir_path, package_name) for source, target in zip(source_paths, target_paths): if source.is_dir(): _sync_dirs(source, target) # Build a setup.py on the fly setup_file = _generate_setup_file(package_name, version, temp_dir_path) package_file = destination / _get_wheel_name(name=package_name, version=version) if package_file.is_file(): click.secho(f"Package file {package_file} will be overwritten!", fg="yellow") # python setup.py bdist_wheel --dist-dir <destination> call( [ sys.executable, str(setup_file.resolve()), "bdist_wheel", "--dist-dir", str(destination), ], cwd=temp_dir, )
def _generate_wheel_file( pipeline_name: str, destination: Path, source_paths: Tuple[_SourcePathType, ...], version: str, metadata: ProjectMetadata, alias: str = None, ) -> None: package_name = alias or pipeline_name package_source, tests_source, conf_source = source_paths with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir).resolve() project = Project(temp_dir_path) # project where to do refactoring _refactor_code_for_package( # type: ignore project, package_source, tests_source, alias, metadata ) project.close() # Copy & "refactor" config _, _, conf_target = _get_package_artifacts(temp_dir_path, package_name) _sync_path_list(conf_source, conf_target) # type: ignore if conf_target.is_dir() and alias: _rename_files(conf_target, pipeline_name, alias) # Build a setup.py on the fly try: install_requires = _make_install_requires( package_source / "requirements.txt" # type: ignore ) except Exception as exc: click.secho("FAILED", fg="red") cls = exc.__class__ raise KedroCliError(f"{cls.__module__}.{cls.__qualname__}: {exc}") from exc config_files = [str(file) for file in conf_target.rglob("*") if file.is_file()] setup_file = _generate_setup_file( package_name, version, install_requires, temp_dir_path, config_files ) package_file = destination / _get_wheel_name(name=package_name, version=version) if package_file.is_file(): click.secho( f"Package file {package_file} will be overwritten!", fg="yellow" ) # python setup.py bdist_wheel --dist-dir <destination> call( [ sys.executable, str(setup_file.resolve()), "bdist_wheel", "--dist-dir", str(destination), ], cwd=temp_dir, )
def ipython(env, args): """Open IPython with project specific variables loaded.""" context = _load_project_context(env=env) os.environ["IPYTHONDIR"] = str(context.project_path / ".ipython") if env: os.environ["KEDRO_ENV"] = env if "-h" not in args and "--help" not in args: ipython_message() call(["ipython"] + list(args))
def package(): """Package the project as a Python egg and wheel.""" call( [sys.executable, "setup.py", "clean", "--all", "bdist_egg"], cwd=str(SOURCE_PATH), ) call( [sys.executable, "setup.py", "clean", "--all", "bdist_wheel"], cwd=str(SOURCE_PATH), )
def ipython(metadata: ProjectMetadata, env, args, **kwargs): # pylint: disable=unused-argument """Open IPython with project specific variables loaded.""" _check_module_importable("IPython") os.environ["IPYTHONDIR"] = str(metadata.project_path / ".ipython") if env: os.environ["KEDRO_ENV"] = env if "-h" not in args and "--help" not in args: ipython_message() call(["ipython"] + list(args))
def package(metadata: ProjectMetadata): """Package the project as a Python egg and wheel.""" source_path = metadata.source_dir call( [sys.executable, "setup.py", "clean", "--all", "bdist_egg"], cwd=str(source_path), ) call( [sys.executable, "setup.py", "clean", "--all", "bdist_wheel"], cwd=str(source_path), )
def package(): """Package the project as a Python egg and wheel.""" source_path = _get_source_path() call( [sys.executable, "setup.py", "clean", "--all", "bdist_egg"], cwd=str(source_path), ) call( [sys.executable, "setup.py", "clean", "--all", "bdist_wheel"], cwd=str(source_path), )
def package(): """Package the project as a Python egg and wheel.""" context = _load_project_context() source_path = get_source_dir(context.project_path) call( [sys.executable, "setup.py", "clean", "--all", "bdist_egg"], cwd=str(source_path), ) call( [sys.executable, "setup.py", "clean", "--all", "bdist_wheel"], cwd=str(source_path), )
def _generate_wheel_file( pipeline_name: str, destination: Path, source_paths: Tuple[_SourcePathType, ...], version: str, alias: str = None, ) -> None: package_name = alias or pipeline_name with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir).resolve() # Copy source folders target_paths = _get_package_artifacts(temp_dir_path, package_name) source_target, _, conf_target = target_paths for source, target in zip(source_paths, target_paths): sync_func = _sync_path_list if isinstance(source, list) else _sync_dirs sync_func(source, target) # type: ignore if conf_target.is_dir() and alias: _rename_files(conf_target, pipeline_name, alias) # Build a setup.py on the fly try: install_requires = _make_install_requires(source_target / "requirements.txt") except Exception as exc: click.secho("FAILED", fg="red") cls = exc.__class__ raise KedroCliError( f"{cls.__module__}.{cls.__qualname__}: {exc}") from exc setup_file = _generate_setup_file(package_name, version, install_requires, temp_dir_path) package_file = destination / _get_wheel_name(name=package_name, version=version) if package_file.is_file(): click.secho(f"Package file {package_file} will be overwritten!", fg="yellow") # python setup.py bdist_wheel --dist-dir <destination> call( [ sys.executable, str(setup_file.resolve()), "bdist_wheel", "--dist-dir", str(destination), ], cwd=temp_dir, )
def docker_cmd(args, docker_args, image, **kwargs): # pylint: disable=unused-argument): """Run arbitrary command from ARGS in the Docker container. If ARGS are not specified, this will invoke `kedro run` inside the container. **kwargs is needed to make the global `verbose` argument work and pass it through.""" container_name = make_container_name(image, "cmd") _docker_run_args = compose_docker_run_args( optional_args=[("--rm", None), ("--name", container_name)], user_args=docker_args, **_mount_info(), ) command = ["docker", "run"] + _docker_run_args + [image] + list(args) call(command)
def install(metadata: ProjectMetadata, compile_flag): """Install project dependencies from both requirements.txt and environment.yml (DEPRECATED).""" deprecation_message = ( "DeprecationWarning: Command `kedro install` will be deprecated in Kedro 0.18.0. " "In the future use `pip install -r src/requirements.txt` instead. " "If you were running `kedro install` with the `--build-reqs` flag, " "we recommend running `kedro build-reqs` followed by `pip install -r src/requirements.txt`" ) click.secho(deprecation_message, fg="red") # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here # pylint: disable=consider-using-with source_path = metadata.source_dir environment_yml = source_path / "environment.yml" requirements_in = source_path / "requirements.in" requirements_txt = source_path / "requirements.txt" if environment_yml.is_file(): call([ "conda", "env", "update", "--file", str(environment_yml), "--prune" ]) default_compile = bool(compile_flag is None and not requirements_in.is_file()) do_compile = compile_flag or default_compile if do_compile: _build_reqs(source_path) pip_command = ["install", "-U", "-r", str(requirements_txt)] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command proc = subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE, stderr=subprocess.PIPE) _, errs = proc.communicate() if errs: secho(errs.decode(), fg="red") raise click.exceptions.Exit(code=1) secho("Requirements installed!", fg="green")
def docker_run(image, docker_args, args, **kwargs): # pylint: disable=unused-argument """Run the pipeline in the Docker container. Any extra arguments unspecified in this help are passed to `docker run` as is. **kwargs is needed to make the global `verbose` argument work and pass it through.""" container_name = make_container_name(image, "run") _docker_run_args = compose_docker_run_args( optional_args=[("--rm", None), ("--name", container_name)], user_args=docker_args, **_mount_info(), ) command = (["docker", "run"] + _docker_run_args + [image, "kedro", "run"] + list(args)) call(command)
def _generate_wheel_file( pipeline_name: str, destination: Path, source_paths: Tuple[_SourcePathType, ...], version: str, alias: str = None, ) -> None: package_name = alias or pipeline_name with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir).resolve() # Copy source folders target_paths = _get_package_artifacts(temp_dir_path, package_name) for source, target in zip(source_paths, target_paths): sync_func = _sync_path_list if isinstance(source, list) else _sync_dirs sync_func(source, target) # type: ignore conf_target = target_paths[-1] if conf_target.is_dir() and alias: _rename_files(conf_target, pipeline_name, alias) # Build a setup.py on the fly setup_file = _generate_setup_file(package_name, version, temp_dir_path) package_file = destination / _get_wheel_name(name=package_name, version=version) if package_file.is_file(): click.secho(f"Package file {package_file} will be overwritten!", fg="yellow") # python setup.py bdist_wheel --dist-dir <destination> call( [ sys.executable, str(setup_file.resolve()), "bdist_wheel", "--dist-dir", str(destination), ], cwd=temp_dir, )
def _generate_wheel_file(package_name: str, destination: Path, source_paths: Tuple[Path, ...]) -> None: with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = Path(temp_dir).resolve() # Copy source folders target_paths = ( temp_dir_path / package_name, temp_dir_path / "tests", # package_data (non-python files) needs to live inside one of the packages temp_dir_path / package_name / "config", ) for source, target in zip(source_paths, target_paths): if source.is_dir(): _sync_dirs(source, target) # Build a setup.py on the fly setup_file = temp_dir_path / "setup.py" package_data = { package_name: [ "README.md", "config/parameters*", "config/**/parameters*", "config/parameters*/**", ] } setup_file_context = dict(name=package_name, version="0.1", package_data=json.dumps(package_data)) setup_file.write_text(_SETUP_PY_TEMPLATE.format(**setup_file_context)) # python setup.py bdist_wheel --dist-dir src/dist call( [ sys.executable, str(setup_file.resolve()), "bdist_wheel", "--dist-dir", str(destination), ], cwd=temp_dir, )
def docker_build(ctx, uid, gid, spark, base_image, image, docker_args): """Build a Docker image for the project.""" uid, gid = get_uid_gid(uid, gid) project_path = Path.cwd() image = image or project_path.name ctx.invoke(docker_init, spark=spark) combined_args = compose_docker_run_args( required_args=[ ("--build-arg", f"KEDRO_UID={uid}"), ("--build-arg", f"KEDRO_GID={gid}"), ("--build-arg", f"BASE_IMAGE={base_image}"), ], # add image tag if only it is not already supplied by the user optional_args=[("-t", image)], user_args=docker_args, ) command = ["docker", "build"] + combined_args + [str(project_path)] call(command)
def docker_jupyter_lab(docker_args, port, image, args, **kwargs): # pylint: disable=unused-argument): """Run jupyter lab in the Docker container. Any extra arguments unspecified in this help are passed to `kedro jupyter lab` command inside the container as is. **kwargs is needed to make the global `verbose` argument work and pass it through.""" container_name = make_container_name(image, "jupyter-lab") _docker_run_args = compose_docker_run_args( required_args=[("-p", f"{port}:8888")], optional_args=[("--rm", None), ("-it", None), ("--name", container_name)], user_args=docker_args, **_mount_info(), ) args = add_jupyter_args(list(args)) command = (["docker", "run"] + _docker_run_args + [image, "kedro", "jupyter", "lab"] + args) call(command)
def build_docs(open_docs): """Build the project documentation.""" python_call("pip", ["install", str(SOURCE_PATH / "[docs]")]) python_call("pip", ["install", "-r", str(SOURCE_PATH / "requirements.txt")]) python_call("ipykernel", ["install", "--user", f"--name={KEDRO_PACKAGE_NAME}"]) shutil.rmtree("docs/build", ignore_errors=True) call([ "sphinx-apidoc", "--module-first", "-o", "docs/source", str(SOURCE_PATH / KEDRO_PACKAGE_NAME), ]) call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"]) if open_docs: docs_page = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri() secho("Opening {}".format(docs_page)) webbrowser.open(docs_page)
def install(metadata: ProjectMetadata, compile_flag): """Install project dependencies from both requirements.txt and environment.yml (optional).""" # we cannot use `context.project_path` as in other commands since # context instantiation might break due to missing dependencies # we attempt to install here # pylint: disable=consider-using-with source_path = metadata.source_dir environment_yml = source_path / "environment.yml" requirements_in = source_path / "requirements.in" requirements_txt = source_path / "requirements.txt" if environment_yml.is_file(): call([ "conda", "env", "update", "--file", str(environment_yml), "--prune" ]) default_compile = bool(compile_flag is None and not requirements_in.is_file()) do_compile = compile_flag or default_compile if do_compile: _build_reqs(source_path) pip_command = ["install", "-U", "-r", str(requirements_txt)] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command proc = subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE, stderr=subprocess.PIPE) _, errs = proc.communicate() if errs: secho(errs.decode(), fg="red") raise click.exceptions.Exit(code=1) secho("Requirements installed!", fg="green")
def install(): """Install project dependencies from both requirements.txt and environment.yml (optional).""" if (SOURCE_PATH / "environment.yml").is_file(): call([ "conda", "install", "--file", str(SOURCE_PATH / "environment.yml"), "--yes", ]) pip_command = [ "install", "-U", "-r", str(SOURCE_PATH / "requirements.txt") ] if os.name == "posix": python_call("pip", pip_command) else: command = [sys.executable, "-m", "pip"] + pip_command subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE)
def docker_dive(ci_flag, dive_ci, docker_args, image): """Run Dive analyzer of Docker image efficiency.""" container_name = make_container_name(image, "dive") required_args = [("-v", "/var/run/docker.sock:/var/run/docker.sock")] optional_args = [("--rm", None), ("--name", container_name)] if ci_flag: dive_ci = Path(dive_ci).absolute() if dive_ci.is_file(): required_args.append(("-v", f"{dive_ci}:/.dive-ci")) else: msg = f"`{dive_ci}` file not found, using default CI config" click.secho(msg, fg="yellow") required_args.append(("-e", f"CI={str(ci_flag).lower()}")) else: optional_args.append(("-it", None)) _docker_run_args = compose_docker_run_args(required_args=required_args, optional_args=optional_args, user_args=docker_args) command = ["docker", "run"] + _docker_run_args + [DIVE_IMAGE] + [image] call(command)
def ipython(args): """Open IPython with project specific variables loaded.""" if "-h" not in args and "--help" not in args: ipython_message() call(["ipython"] + list(args))