def test_get_git_sha_called_process_error(mocker):
check_output_mock = mocker.patch.object(subprocess, "check_output")
check_output_mock.side_effect = subprocess.CalledProcessError(1, "")
with pytest.raises(SystemExit):
cli_utils.get_git_sha()
def _profile(subcommand, klio_config, config_meta, **kwargs):
_require_profile_input_data(kwargs["input_file"], kwargs["entity_ids"])
image_tag = kwargs.pop("image_tag", None)
if not image_tag:
image_tag = cli_utils.get_git_sha(config_meta.job_dir)
if config_meta.config_file:
image_tag = "{}-{}".format(image_tag,
os.path.basename(config_meta.config_file))
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.pop("force_build", None),
config_file_override=config_meta.config_file,
)
profile_config = ProfileConfig(
input_file=kwargs.pop("input_file", None),
output_file=kwargs.pop("output_file", None),
show_logs=kwargs.pop("show_logs", None),
entity_ids=kwargs.pop("entity_ids", None),
)
pipeline = job_commands.profile.ProfilePipeline(config_meta.job_dir,
klio_config,
runtime_config,
profile_config)
pipeline.run(what=subcommand, subcommand_flags=kwargs)
def test_job(klio_config, config_meta, force_build, image_tag, pytest_args,
**kwargs):
"""Thin wrapper around pytest. Any arguments after -- are passed through."""
pytest_args = list(pytest_args)
if pytest_args:
# test calls klioexec, which also uses click.
# click relies on the presence of -- to process option-like arguments
# such as -s
pytest_args.insert(0, "--")
if not image_tag:
image_tag = cli_utils.get_git_sha(config_meta.job_dir)
if config_meta.config_file:
image_tag = "{}-{}".format(image_tag,
os.path.basename(config_meta.config_file))
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=force_build,
config_file_override=config_meta.config_file,
)
pipeline = job_commands.test.TestPipeline(config_meta.job_dir, klio_config,
runtime_config)
rc = pipeline.run(pytest_args=pytest_args)
sys.exit(rc)
def run_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(
klio_config, kwargs.pop("direct_runner")
)
git_sha = cli_utils.get_git_sha(
config_meta.job_dir, kwargs.get("image_tag")
)
image_tag = kwargs.get("image_tag") or git_sha
if config_meta.config_file:
basename = os.path.basename(config_meta.config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
klio_pipeline = job_commands.run.RunPipeline(
config_meta.job_dir, klio_config, runtime_config, run_job_config
)
rc = klio_pipeline.run()
sys.exit(rc)
def build_image(klio_config, config_meta, **kwargs):
if not kwargs.get("image_tag"):
kwargs["image_tag"] = cli_utils.get_git_sha(config_meta.job_dir)
image_commands.build.build(
config_meta.job_dir, klio_config, config_meta.config_file, **kwargs
)
def test_get_git_sha(mocker, job_dir):
check_output_mock = mocker.patch.object(subprocess, "check_output")
check_output_mock.return_value = b""
result = cli_utils.get_git_sha(job_dir)
cmd = "git describe --match=NeVeRmAtCh --always --abbrev=8 --dirty"
check_output_mock.assert_called_once_with(cmd.split(),
cwd=job_dir,
stderr=subprocess.DEVNULL)
assert "" == result
def deploy_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(klio_config,
kwargs.pop("direct_runner"))
cli_utils.error_stackdriver_logger_metrics(klio_config, direct_runner)
git_sha = cli_utils.get_git_sha(config_meta.job_dir,
kwargs.get("image_tag"))
image_tag = kwargs.get("image_tag") or git_sha
if config_meta.config_file:
basename = os.path.basename(config_meta.config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
# TODO: make this a click option once draining is supported @lynn
if not run_job_config.update:
job_name = klio_config.job_name
gcp_project = klio_config.pipeline_options.project
region = klio_config.pipeline_options.region
strategy = "cancel"
job_commands.stop.StopJob().stop(job_name, gcp_project, region,
strategy)
if (not direct_runner and klio_config.pipeline_options.runner
== var.KlioRunner.DIRECT_GKE_RUNNER):
gke_commands = cli_utils.import_gke_commands()
run_command = gke_commands.RunPipelineGKE(config_meta.job_dir,
klio_config, runtime_config,
run_job_config)
else:
run_command = job_commands.run.RunPipeline(config_meta.job_dir,
klio_config, runtime_config,
run_job_config)
rc = run_command.run()
sys.exit(rc)
def develop_job(job_dir, config_file, **kwargs):
job_dir, config_path = core_utils.get_config_job_dir(job_dir, config_file)
config_data = core_utils.get_config_by_path(config_path)
conf = config.KlioConfig(config_data)
git_sha = cli_utils.get_git_sha(job_dir, kwargs.get("image_tag"))
image_tag = kwargs.get("image_tag") or git_sha
if config_file:
basename = os.path.basename(config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = main_cli.DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_file,
)
klio_pipeline = develop.DevelopKlioContainer(job_dir, conf, runtime_config,
kwargs["klio_path"],
kwargs["exclude"])
klio_pipeline.run()
def audit_job(klio_config, config_meta, force_build, image_tag, list_steps):
git_sha = cli_utils.get_git_sha(config_meta.job_dir, image_tag)
image_tag = image_tag or git_sha
if config_meta.config_file:
basename = os.path.basename(config_meta.config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=force_build,
config_file_override=config_meta.config_file,
)
pipeline = job_commands.audit.AuditPipeline(
job_dir=config_meta.job_dir,
klio_config=klio_config,
docker_runtime_config=runtime_config,
)
rc = pipeline.run(list_steps)
sys.exit(rc)
def deploy_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(
klio_config, kwargs.pop("direct_runner")
)
git_sha = cli_utils.get_git_sha(
config_meta.job_dir, kwargs.get("image_tag")
)
image_tag = kwargs.get("image_tag") or git_sha
if config_meta.config_file:
basename = os.path.basename(config_meta.config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
# TODO: make this a click option once draining is supported @lynn
if not run_job_config.update:
job_name = klio_config.job_name
gcp_project = klio_config.pipeline_options.project
region = klio_config.pipeline_options.region
strategy = "cancel"
job_commands.stop.StopJob().stop(
job_name, gcp_project, region, strategy
)
klio_pipeline = job_commands.run.RunPipeline(
config_meta.job_dir, klio_config, runtime_config, run_job_config
)
rc = klio_pipeline.run()
sys.exit(rc)
def run_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(klio_config,
kwargs.pop("direct_runner"))
cli_utils.error_stackdriver_logger_metrics(klio_config, direct_runner)
git_sha = cli_utils.get_git_sha(config_meta.job_dir,
kwargs.get("image_tag"))
image_tag = kwargs.get("image_tag") or git_sha
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
if (not direct_runner and klio_config.pipeline_options.runner
== var.KlioRunner.DIRECT_GKE_RUNNER):
gke_commands = cli_utils.import_gke_commands()
klio_pipeline = gke_commands.RunPipelineGKE(config_meta.job_dir,
klio_config,
runtime_config,
run_job_config)
else:
klio_pipeline = job_commands.run.RunPipeline(config_meta.job_dir,
klio_config,
runtime_config,
run_job_config)
rc = klio_pipeline.run()
sys.exit(rc)
