def test_get_git_sha_called_process_error(mocker): check_output_mock = mocker.patch.object(subprocess, "check_output") check_output_mock.side_effect = subprocess.CalledProcessError(1, "") with pytest.raises(SystemExit): cli_utils.get_git_sha()
def _profile(subcommand, klio_config, config_meta, **kwargs): _require_profile_input_data(kwargs["input_file"], kwargs["entity_ids"]) image_tag = kwargs.pop("image_tag", None) if not image_tag: image_tag = cli_utils.get_git_sha(config_meta.job_dir) if config_meta.config_file: image_tag = "{}-{}".format(image_tag, os.path.basename(config_meta.config_file)) runtime_config = DockerRuntimeConfig( image_tag=image_tag, force_build=kwargs.pop("force_build", None), config_file_override=config_meta.config_file, ) profile_config = ProfileConfig( input_file=kwargs.pop("input_file", None), output_file=kwargs.pop("output_file", None), show_logs=kwargs.pop("show_logs", None), entity_ids=kwargs.pop("entity_ids", None), ) pipeline = job_commands.profile.ProfilePipeline(config_meta.job_dir, klio_config, runtime_config, profile_config) pipeline.run(what=subcommand, subcommand_flags=kwargs)
def test_job(klio_config, config_meta, force_build, image_tag, pytest_args, **kwargs): """Thin wrapper around pytest. Any arguments after -- are passed through.""" pytest_args = list(pytest_args) if pytest_args: # test calls klioexec, which also uses click. # click relies on the presence of -- to process option-like arguments # such as -s pytest_args.insert(0, "--") if not image_tag: image_tag = cli_utils.get_git_sha(config_meta.job_dir) if config_meta.config_file: image_tag = "{}-{}".format(image_tag, os.path.basename(config_meta.config_file)) runtime_config = DockerRuntimeConfig( image_tag=image_tag, force_build=force_build, config_file_override=config_meta.config_file, ) pipeline = job_commands.test.TestPipeline(config_meta.job_dir, klio_config, runtime_config) rc = pipeline.run(pytest_args=pytest_args) sys.exit(rc)
def run_job(klio_config, config_meta, **kwargs): direct_runner = cli_utils.is_direct_runner( klio_config, kwargs.pop("direct_runner") ) git_sha = cli_utils.get_git_sha( config_meta.job_dir, kwargs.get("image_tag") ) image_tag = kwargs.get("image_tag") or git_sha if config_meta.config_file: basename = os.path.basename(config_meta.config_file) image_tag = "{}-{}".format(image_tag, basename) runtime_config = DockerRuntimeConfig( image_tag=image_tag, force_build=kwargs.get("force_build"), config_file_override=config_meta.config_file, ) run_job_config = RunJobConfig( direct_runner=direct_runner, update=kwargs.pop("update"), git_sha=git_sha, ) klio_pipeline = job_commands.run.RunPipeline( config_meta.job_dir, klio_config, runtime_config, run_job_config ) rc = klio_pipeline.run() sys.exit(rc)
def build_image(klio_config, config_meta, **kwargs): if not kwargs.get("image_tag"): kwargs["image_tag"] = cli_utils.get_git_sha(config_meta.job_dir) image_commands.build.build( config_meta.job_dir, klio_config, config_meta.config_file, **kwargs )
def test_get_git_sha(mocker, job_dir): check_output_mock = mocker.patch.object(subprocess, "check_output") check_output_mock.return_value = b"" result = cli_utils.get_git_sha(job_dir) cmd = "git describe --match=NeVeRmAtCh --always --abbrev=8 --dirty" check_output_mock.assert_called_once_with(cmd.split(), cwd=job_dir, stderr=subprocess.DEVNULL) assert "" == result
def deploy_job(klio_config, config_meta, **kwargs): direct_runner = cli_utils.is_direct_runner(klio_config, kwargs.pop("direct_runner")) cli_utils.error_stackdriver_logger_metrics(klio_config, direct_runner) git_sha = cli_utils.get_git_sha(config_meta.job_dir, kwargs.get("image_tag")) image_tag = kwargs.get("image_tag") or git_sha if config_meta.config_file: basename = os.path.basename(config_meta.config_file) image_tag = "{}-{}".format(image_tag, basename) runtime_config = DockerRuntimeConfig( image_tag=image_tag, force_build=kwargs.get("force_build"), config_file_override=config_meta.config_file, ) run_job_config = RunJobConfig( direct_runner=direct_runner, update=kwargs.pop("update"), git_sha=git_sha, ) # TODO: make this a click option once draining is supported @lynn if not run_job_config.update: job_name = klio_config.job_name gcp_project = klio_config.pipeline_options.project region = klio_config.pipeline_options.region strategy = "cancel" job_commands.stop.StopJob().stop(job_name, gcp_project, region, strategy) if (not direct_runner and klio_config.pipeline_options.runner == var.KlioRunner.DIRECT_GKE_RUNNER): gke_commands = cli_utils.import_gke_commands() run_command = gke_commands.RunPipelineGKE(config_meta.job_dir, klio_config, runtime_config, run_job_config) else: run_command = job_commands.run.RunPipeline(config_meta.job_dir, klio_config, runtime_config, run_job_config) rc = run_command.run() sys.exit(rc)
def develop_job(job_dir, config_file, **kwargs): job_dir, config_path = core_utils.get_config_job_dir(job_dir, config_file) config_data = core_utils.get_config_by_path(config_path) conf = config.KlioConfig(config_data) git_sha = cli_utils.get_git_sha(job_dir, kwargs.get("image_tag")) image_tag = kwargs.get("image_tag") or git_sha if config_file: basename = os.path.basename(config_file) image_tag = "{}-{}".format(image_tag, basename) runtime_config = main_cli.DockerRuntimeConfig( image_tag=image_tag, force_build=kwargs.get("force_build"), config_file_override=config_file, ) klio_pipeline = develop.DevelopKlioContainer(job_dir, conf, runtime_config, kwargs["klio_path"], kwargs["exclude"]) klio_pipeline.run()
def audit_job(klio_config, config_meta, force_build, image_tag, list_steps): git_sha = cli_utils.get_git_sha(config_meta.job_dir, image_tag) image_tag = image_tag or git_sha if config_meta.config_file: basename = os.path.basename(config_meta.config_file) image_tag = "{}-{}".format(image_tag, basename) runtime_config = DockerRuntimeConfig( image_tag=image_tag, force_build=force_build, config_file_override=config_meta.config_file, ) pipeline = job_commands.audit.AuditPipeline( job_dir=config_meta.job_dir, klio_config=klio_config, docker_runtime_config=runtime_config, ) rc = pipeline.run(list_steps) sys.exit(rc)
def deploy_job(klio_config, config_meta, **kwargs): direct_runner = cli_utils.is_direct_runner( klio_config, kwargs.pop("direct_runner") ) git_sha = cli_utils.get_git_sha( config_meta.job_dir, kwargs.get("image_tag") ) image_tag = kwargs.get("image_tag") or git_sha if config_meta.config_file: basename = os.path.basename(config_meta.config_file) image_tag = "{}-{}".format(image_tag, basename) runtime_config = DockerRuntimeConfig( image_tag=image_tag, force_build=kwargs.get("force_build"), config_file_override=config_meta.config_file, ) run_job_config = RunJobConfig( direct_runner=direct_runner, update=kwargs.pop("update"), git_sha=git_sha, ) # TODO: make this a click option once draining is supported @lynn if not run_job_config.update: job_name = klio_config.job_name gcp_project = klio_config.pipeline_options.project region = klio_config.pipeline_options.region strategy = "cancel" job_commands.stop.StopJob().stop( job_name, gcp_project, region, strategy ) klio_pipeline = job_commands.run.RunPipeline( config_meta.job_dir, klio_config, runtime_config, run_job_config ) rc = klio_pipeline.run() sys.exit(rc)
def run_job(klio_config, config_meta, **kwargs): direct_runner = cli_utils.is_direct_runner(klio_config, kwargs.pop("direct_runner")) cli_utils.error_stackdriver_logger_metrics(klio_config, direct_runner) git_sha = cli_utils.get_git_sha(config_meta.job_dir, kwargs.get("image_tag")) image_tag = kwargs.get("image_tag") or git_sha runtime_config = DockerRuntimeConfig( image_tag=image_tag, force_build=kwargs.get("force_build"), config_file_override=config_meta.config_file, ) run_job_config = RunJobConfig( direct_runner=direct_runner, update=kwargs.pop("update"), git_sha=git_sha, ) if (not direct_runner and klio_config.pipeline_options.runner == var.KlioRunner.DIRECT_GKE_RUNNER): gke_commands = cli_utils.import_gke_commands() klio_pipeline = gke_commands.RunPipelineGKE(config_meta.job_dir, klio_config, runtime_config, run_job_config) else: klio_pipeline = job_commands.run.RunPipeline(config_meta.job_dir, klio_config, runtime_config, run_job_config) rc = klio_pipeline.run() sys.exit(rc)