def create_project_from_config_file(context): """Behave step to run kedro new given the config I previously created. """ res = run([context.kedro, "new", "-c", str(context.config_file)], env=context.env) assert res.returncode == OK_EXIT_CODE
def _create_template_project(context): # Sets the following fields on the context: # - project_name (the simple project name == project-pipeline) # - temp_dir (the directory containing the created project) # - root_project_dir (the full path to the created project) # - include_example (the project contains code example) context.project_name = "project-pipeline" context.config_file = context.temp_dir / "config" root_project_dir = context.temp_dir / context.project_name context.root_project_dir = root_project_dir context.include_example = True config = { "project_name": context.project_name, "repo_name": context.project_name, "output_dir": str(context.temp_dir), "python_package": context.project_name.replace("-", "_"), "include_example": context.include_example, } with context.config_file.open("w") as config_file: yaml.dump(config, config_file, default_flow_style=False) res = run([context.kedro, "new", "-c", str(context.config_file)], env=context.env) assert res.returncode == 0 _setup_template_files(context)
def call(cmd, print_output=False): res = run(cmd, env=context.env) if res.returncode or print_output: print(">", " ".join(cmd)) print(res.stdout) print(res.stderr) assert res.returncode == 0
def exec_make_target(context, command): """Execute Makefile target.""" split_command = command.split() make_cmd = [context.kedro] + split_command context.result = run(make_cmd, env=context.env, cwd=str(context.root_project_dir))
def call(cmd, verbose=False): res = run(cmd, env=context.env) if res.returncode or verbose: print(">", " ".join(cmd)) print(res.stdout) print(res.stderr) assert res.returncode == 0
def exec_kedro_target(context, command): """Execute Kedro target.""" split_command = command.split() cmd = [context.kedro] + split_command context.result = run(cmd, env=context.env, cwd=str(context.root_project_dir))
def run_template_pipeline(context): run_cmd = [context.kedro, "run"] context.run_result = run(run_cmd, env=context.env, cwd=str(context.root_project_dir)) if context.run_result.returncode == 0: context.df_e = pd.read_csv(context.output_file1) context.df_f = pd.read_csv(context.output_file2) shutil.rmtree(str(context.root_project_dir))
def install_project_package(context): """Install the packaged project.""" cmd = [context.pip, "install", "-e", "src/"] res = run(cmd, env=context.env, cwd=str(context.root_project_dir)) if res.returncode != OK_EXIT_CODE: print(res.stdout) print(res.stderr) assert False
def create_project_without_starter(context): """Behave step to run kedro new given the config I previously created. """ res = run( [context.kedro, "new", "-c", str(context.config_file)], env=context.env, cwd=context.temp_dir, ) assert res.returncode == OK_EXIT_CODE, res
def exec_project(context): """Execute installed Kedro project target.""" cmd = [str(context.bin_dir / context.project_name)] # N.B.: prior to the introduction of load_package_context, this test was passing # accidentally because it was executing the installed project package at the # same directory as project root, so a lot of things were available on Path.cwd(). # We take care to delete with `delete_unnecessary_assets` to simulate the behaviour # of a installed package in a fresh environment. context.result = run(cmd, env=context.env, cwd=str(context.root_project_dir))
def exec_kedro_target_checked(context, command): """Execute Kedro command and check the status.""" cmd = [context.kedro] + command.split() res = run(cmd, env=context.env, cwd=str(context.root_project_dir)) if res.returncode != OK_EXIT_CODE: print(res.stdout) print(res.stderr) assert False
def exec_make_target_checked(context, command): """Execute Makefile target""" make_cmd = [context.kedro] + command.split() res = run(make_cmd, env=context.env, cwd=str(context.root_project_dir)) if res.returncode != OK_EXIT_CODE: print(res.stdout) print(res.stderr) assert False
def create_project_from_config_file(context): """Behave step to run Kedro new given the config I previously created.""" res = run([ context.kedro, "new", "--config", str(context.config_file), "--starter", context.starter_path, ]) assert res.returncode == OK_EXIT_CODE, res.stdout
def create_project_without_starter(context): """Behave step to run kedro new given the config I previously created.""" res = run( [context.kedro, "new", "-c", str(context.config_file)], env=context.env, cwd=context.temp_dir, ) assert res.returncode == OK_EXIT_CODE, res # prevent telemetry from prompting for input during e2e tests telemetry_file = context.root_project_dir / ".telemetry" telemetry_file.write_text("consent: false", encoding="utf-8")
def exec_kedro_run_with_tag(context, cmd, tags): """Execute `kedro run` with tags""" kedro_args = shlex.split(cmd) context.logfile_count = util.get_logline_count( util.get_logfile_path(context.root_project_dir) ) tag_list = [["--tag", t] for t in tags] tag_args = list(itertools.chain.from_iterable(tag_list)) run_cmd = [context.kedro] + kedro_args + tag_args context.result = run(run_cmd, env=context.env, cwd=str(context.root_project_dir))
def install_kedro(context, version): """Execute Kedro command and check the status.""" if version == "latest": cmd = [context.pip, "install", "-U", "kedro"] else: cmd = [context.pip, "install", "kedro=={}".format(version)] res = run(cmd, env=context.env) if res.returncode != OK_EXIT_CODE: print(res.stdout) print(res.stderr) assert False
def exec_kedro_target(context, command): """Execute Kedro target""" split_command = command.split() make_cmd = [context.kedro] + split_command if split_command[0] == "docker" and split_command[1] in ("ipython", "jupyter"): context.result = ChildTerminatingPopen( make_cmd, env=context.env, cwd=str(context.root_project_dir) ) else: context.result = run( make_cmd, env=context.env, cwd=str(context.root_project_dir) )
def exec_kedro_target_checked(context, command): """Execute Kedro command and check the status.""" cmd = [context.kedro] + command.split() res = run(cmd, env=context.env, cwd=str(context.root_project_dir)) if res.returncode != OK_EXIT_CODE: print(res.stdout) print(res.stderr) assert False # Wait for subprocess completion since on Windows it takes some time # to install dependencies in a separate console if "install" in cmd: max_duration = 5 * 60 # 5 minutes end_by = time() + max_duration while time() < end_by: result = run([context.pip, "show", "pandas"]) if result.returncode == OK_EXIT_CODE: # package found return sleep(1.0)
def create_project_with_starter(context): """Behave step to run kedro new given the config I previously created.""" starter_dir = Path(__file__).parent / "test_starter" res = run( [ context.kedro, "new", "-c", str(context.config_file), "--starter", str(starter_dir), ], env=context.env, cwd=context.temp_dir, ) assert res.returncode == OK_EXIT_CODE, res
def move_package(context: behave.runner.Context, new_source_dir): """Move the project package to a new directory. """ current_src_path = context.root_project_dir / "src" new_src_path = context.root_project_dir / "new_source_dir" cmd = [ "mkdir", str(new_source_dir), ";", "mv", str(current_src_path / context.package_name), str(new_src_path), ] context.result = run(cmd, env=context.env, cwd=str(context.root_project_dir))
def create_project_from_config_file(context): """Behave step to run kedro new given the config I previously created. """ res = run( [ context.kedro, "new", "-c", str(context.config_file), "--starter", "pandas-iris", ], env=context.env, cwd=str(context.temp_dir), ) assert res.returncode == 0
def create_project_with_starter(context, starter): """Behave step to run kedro new given the config I previously created. """ res = run( [ context.kedro, "new", "--starter", str(starter), "--config", str(context.config_file), ], env=context.env, cwd=str(context.temp_dir), ) if res.returncode != OK_EXIT_CODE: print(res.stdout) print(res.stderr) assert False assert res.returncode == OK_EXIT_CODE
def create_project_from_config_file(context): """Behave step to run kedro new given the config I previously created. """ res = run( [ context.kedro, "new", "-c", str(context.config_file), "--starter", "pandas-iris", ], env=context.env, cwd=str(context.temp_dir), ) if res.returncode != OK_EXIT_CODE: print(res.stdout) print(res.stderr) assert False
def create_project_from_config_file(context): """Behave step to run kedro new given the config I previously created. """ res = run( [ context.kedro, "new", "-c", str(context.config_file), "--starter", "pandas-iris", ], env=context.env, cwd=str(context.temp_dir), ) # add a consent file to prevent telemetry from prompting for input during e2e test telemetry_file = context.root_project_dir / ".telemetry" telemetry_file.write_text("consent: false", encoding="utf-8") assert res.returncode == 0
def get_kedro_version_python(context): """Behave step to run `python -m kedro -V`.""" cmd = [context.python, "-m", "kedro", "-V"] context.version_str = run(cmd, env=context.env).stdout assert context.version_str # check non-empty
def get_kedro_version(context): """Behave step to run `kedro -V`.""" context.version_str = run([context.kedro, "-V"], env=context.env).stdout assert context.version_str # check non-empty
def exec_project(context): """Execute installed Kedro project target.""" cmd = [str(context.bin_dir / context.project_name)] context.result = run(cmd, env=context.env, cwd=str(context.root_project_dir))
def call(cmd, env): res = run(cmd, env=env) if res.returncode: print(res.stdout) print(res.stderr) assert False
def uninstall_package_via_pip(context, package): """Uninstall a python package using pip.""" run([context.pip, "uninstall", "-y", package], env=context.env)
def install_project_package_via_pip(context): """Install a python package using pip.""" dist_dir = context.root_project_dir / "src" / "dist" (whl_file, ) = dist_dir.glob("*.whl") run([context.pip, "install", str(whl_file)], env=context.env)