def create_project_from_config_file(context):
"""Behave step to run kedro new given the config I previously created.
"""
res = run([context.kedro, "new", "-c",
str(context.config_file)],
env=context.env)
assert res.returncode == OK_EXIT_CODE
def _create_template_project(context):
# Sets the following fields on the context:
# - project_name (the simple project name == project-pipeline)
# - temp_dir (the directory containing the created project)
# - root_project_dir (the full path to the created project)
# - include_example (the project contains code example)
context.project_name = "project-pipeline"
context.config_file = context.temp_dir / "config"
root_project_dir = context.temp_dir / context.project_name
context.root_project_dir = root_project_dir
context.include_example = True
config = {
"project_name": context.project_name,
"repo_name": context.project_name,
"output_dir": str(context.temp_dir),
"python_package": context.project_name.replace("-", "_"),
"include_example": context.include_example,
}
with context.config_file.open("w") as config_file:
yaml.dump(config, config_file, default_flow_style=False)
res = run([context.kedro, "new", "-c",
str(context.config_file)],
env=context.env)
assert res.returncode == 0
_setup_template_files(context)
def call(cmd, print_output=False):
res = run(cmd, env=context.env)
if res.returncode or print_output:
print(">", " ".join(cmd))
print(res.stdout)
print(res.stderr)
assert res.returncode == 0
def exec_make_target(context, command):
"""Execute Makefile target."""
split_command = command.split()
make_cmd = [context.kedro] + split_command
context.result = run(make_cmd,
env=context.env,
cwd=str(context.root_project_dir))
def call(cmd, verbose=False):
res = run(cmd, env=context.env)
if res.returncode or verbose:
print(">", " ".join(cmd))
print(res.stdout)
print(res.stderr)
assert res.returncode == 0
def exec_kedro_target(context, command):
"""Execute Kedro target."""
split_command = command.split()
cmd = [context.kedro] + split_command
context.result = run(cmd,
env=context.env,
cwd=str(context.root_project_dir))
def run_template_pipeline(context):
run_cmd = [context.kedro, "run"]
context.run_result = run(run_cmd,
env=context.env,
cwd=str(context.root_project_dir))
if context.run_result.returncode == 0:
context.df_e = pd.read_csv(context.output_file1)
context.df_f = pd.read_csv(context.output_file2)
shutil.rmtree(str(context.root_project_dir))
def install_project_package(context):
"""Install the packaged project."""
cmd = [context.pip, "install", "-e", "src/"]
res = run(cmd, env=context.env, cwd=str(context.root_project_dir))
if res.returncode != OK_EXIT_CODE:
print(res.stdout)
print(res.stderr)
assert False
def create_project_without_starter(context):
"""Behave step to run kedro new given the config I previously created.
"""
res = run(
[context.kedro, "new", "-c", str(context.config_file)],
env=context.env,
cwd=context.temp_dir,
)
assert res.returncode == OK_EXIT_CODE, res
def exec_project(context):
"""Execute installed Kedro project target."""
cmd = [str(context.bin_dir / context.project_name)]
# N.B.: prior to the introduction of load_package_context, this test was passing
# accidentally because it was executing the installed project package at the
# same directory as project root, so a lot of things were available on Path.cwd().
# We take care to delete with `delete_unnecessary_assets` to simulate the behaviour
# of a installed package in a fresh environment.
context.result = run(cmd, env=context.env, cwd=str(context.root_project_dir))
def exec_kedro_target_checked(context, command):
"""Execute Kedro command and check the status."""
cmd = [context.kedro] + command.split()
res = run(cmd, env=context.env, cwd=str(context.root_project_dir))
if res.returncode != OK_EXIT_CODE:
print(res.stdout)
print(res.stderr)
assert False
def exec_make_target_checked(context, command):
"""Execute Makefile target"""
make_cmd = [context.kedro] + command.split()
res = run(make_cmd, env=context.env, cwd=str(context.root_project_dir))
if res.returncode != OK_EXIT_CODE:
print(res.stdout)
print(res.stderr)
assert False
def create_project_from_config_file(context):
"""Behave step to run Kedro new given the config I previously created."""
res = run([
context.kedro,
"new",
"--config",
str(context.config_file),
"--starter",
context.starter_path,
])
assert res.returncode == OK_EXIT_CODE, res.stdout
def create_project_without_starter(context):
"""Behave step to run kedro new given the config I previously created."""
res = run(
[context.kedro, "new", "-c", str(context.config_file)],
env=context.env,
cwd=context.temp_dir,
)
assert res.returncode == OK_EXIT_CODE, res
# prevent telemetry from prompting for input during e2e tests
telemetry_file = context.root_project_dir / ".telemetry"
telemetry_file.write_text("consent: false", encoding="utf-8")
def exec_kedro_run_with_tag(context, cmd, tags):
"""Execute `kedro run` with tags"""
kedro_args = shlex.split(cmd)
context.logfile_count = util.get_logline_count(
util.get_logfile_path(context.root_project_dir)
)
tag_list = [["--tag", t] for t in tags]
tag_args = list(itertools.chain.from_iterable(tag_list))
run_cmd = [context.kedro] + kedro_args + tag_args
context.result = run(run_cmd, env=context.env, cwd=str(context.root_project_dir))
def install_kedro(context, version):
"""Execute Kedro command and check the status."""
if version == "latest":
cmd = [context.pip, "install", "-U", "kedro"]
else:
cmd = [context.pip, "install", "kedro=={}".format(version)]
res = run(cmd, env=context.env)
if res.returncode != OK_EXIT_CODE:
print(res.stdout)
print(res.stderr)
assert False
def exec_kedro_target(context, command):
"""Execute Kedro target"""
split_command = command.split()
make_cmd = [context.kedro] + split_command
if split_command[0] == "docker" and split_command[1] in ("ipython", "jupyter"):
context.result = ChildTerminatingPopen(
make_cmd, env=context.env, cwd=str(context.root_project_dir)
)
else:
context.result = run(
make_cmd, env=context.env, cwd=str(context.root_project_dir)
)
def exec_kedro_target_checked(context, command):
"""Execute Kedro command and check the status."""
cmd = [context.kedro] + command.split()
res = run(cmd, env=context.env, cwd=str(context.root_project_dir))
if res.returncode != OK_EXIT_CODE:
print(res.stdout)
print(res.stderr)
assert False
# Wait for subprocess completion since on Windows it takes some time
# to install dependencies in a separate console
if "install" in cmd:
max_duration = 5 * 60 # 5 minutes
end_by = time() + max_duration
while time() < end_by:
result = run([context.pip, "show", "pandas"])
if result.returncode == OK_EXIT_CODE:
# package found
return
sleep(1.0)
def create_project_with_starter(context):
"""Behave step to run kedro new given the config I previously created."""
starter_dir = Path(__file__).parent / "test_starter"
res = run(
[
context.kedro,
"new",
"-c",
str(context.config_file),
"--starter",
str(starter_dir),
],
env=context.env,
cwd=context.temp_dir,
)
assert res.returncode == OK_EXIT_CODE, res
def move_package(context: behave.runner.Context, new_source_dir):
"""Move the project package to a new directory.
"""
current_src_path = context.root_project_dir / "src"
new_src_path = context.root_project_dir / "new_source_dir"
cmd = [
"mkdir",
str(new_source_dir),
";",
"mv",
str(current_src_path / context.package_name),
str(new_src_path),
]
context.result = run(cmd,
env=context.env,
cwd=str(context.root_project_dir))
def create_project_from_config_file(context):
"""Behave step to run kedro new
given the config I previously created.
"""
res = run(
[
context.kedro,
"new",
"-c",
str(context.config_file),
"--starter",
"pandas-iris",
],
env=context.env,
cwd=str(context.temp_dir),
)
assert res.returncode == 0
def create_project_with_starter(context, starter):
"""Behave step to run kedro new given the config I previously created.
"""
res = run(
[
context.kedro,
"new",
"--starter",
str(starter),
"--config",
str(context.config_file),
],
env=context.env,
cwd=str(context.temp_dir),
)
if res.returncode != OK_EXIT_CODE:
print(res.stdout)
print(res.stderr)
assert False
assert res.returncode == OK_EXIT_CODE
def create_project_from_config_file(context):
"""Behave step to run kedro new
given the config I previously created.
"""
res = run(
[
context.kedro,
"new",
"-c",
str(context.config_file),
"--starter",
"pandas-iris",
],
env=context.env,
cwd=str(context.temp_dir),
)
if res.returncode != OK_EXIT_CODE:
print(res.stdout)
print(res.stderr)
assert False
def create_project_from_config_file(context):
"""Behave step to run kedro new
given the config I previously created.
"""
res = run(
[
context.kedro,
"new",
"-c",
str(context.config_file),
"--starter",
"pandas-iris",
],
env=context.env,
cwd=str(context.temp_dir),
)
# add a consent file to prevent telemetry from prompting for input during e2e test
telemetry_file = context.root_project_dir / ".telemetry"
telemetry_file.write_text("consent: false", encoding="utf-8")
assert res.returncode == 0
def get_kedro_version_python(context):
"""Behave step to run `python -m kedro -V`."""
cmd = [context.python, "-m", "kedro", "-V"]
context.version_str = run(cmd, env=context.env).stdout
assert context.version_str # check non-empty
def get_kedro_version(context):
"""Behave step to run `kedro -V`."""
context.version_str = run([context.kedro, "-V"], env=context.env).stdout
assert context.version_str # check non-empty
def exec_project(context):
"""Execute installed Kedro project target."""
cmd = [str(context.bin_dir / context.project_name)]
context.result = run(cmd,
env=context.env,
cwd=str(context.root_project_dir))
def call(cmd, env):
res = run(cmd, env=env)
if res.returncode:
print(res.stdout)
print(res.stderr)
assert False
def uninstall_package_via_pip(context, package):
"""Uninstall a python package using pip."""
run([context.pip, "uninstall", "-y", package], env=context.env)
def install_project_package_via_pip(context):
"""Install a python package using pip."""
dist_dir = context.root_project_dir / "src" / "dist"
(whl_file, ) = dist_dir.glob("*.whl")
run([context.pip, "install", str(whl_file)], env=context.env)
