def run_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(
klio_config, kwargs.pop("direct_runner")
)
git_sha = cli_utils.get_git_sha(
config_meta.job_dir, kwargs.get("image_tag")
)
image_tag = kwargs.get("image_tag") or git_sha
if config_meta.config_file:
basename = os.path.basename(config_meta.config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
klio_pipeline = job_commands.run.RunPipeline(
config_meta.job_dir, klio_config, runtime_config, run_job_config
)
rc = klio_pipeline.run()
sys.exit(rc)
def test_is_direct_runner(mocker, monkeypatch, direct_runner):
mock_klio_cfg = mocker.Mock()
mock_validate_df_config = mocker.Mock()
monkeypatch.setattr(cli_utils, "validate_dataflow_runner_config",
mock_validate_df_config)
if not direct_runner:
assert mock_validate_df_config.called_once_with(mock_klio_cfg)
assert (cli_utils.is_direct_runner(mock_klio_cfg,
direct_runner) == direct_runner)
def deploy_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(klio_config,
kwargs.pop("direct_runner"))
cli_utils.error_stackdriver_logger_metrics(klio_config, direct_runner)
git_sha = cli_utils.get_git_sha(config_meta.job_dir,
kwargs.get("image_tag"))
image_tag = kwargs.get("image_tag") or git_sha
if config_meta.config_file:
basename = os.path.basename(config_meta.config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
# TODO: make this a click option once draining is supported @lynn
if not run_job_config.update:
job_name = klio_config.job_name
gcp_project = klio_config.pipeline_options.project
region = klio_config.pipeline_options.region
strategy = "cancel"
job_commands.stop.StopJob().stop(job_name, gcp_project, region,
strategy)
if (not direct_runner and klio_config.pipeline_options.runner
== var.KlioRunner.DIRECT_GKE_RUNNER):
gke_commands = cli_utils.import_gke_commands()
run_command = gke_commands.RunPipelineGKE(config_meta.job_dir,
klio_config, runtime_config,
run_job_config)
else:
run_command = job_commands.run.RunPipeline(config_meta.job_dir,
klio_config, runtime_config,
run_job_config)
rc = run_command.run()
sys.exit(rc)
def test_is_direct_runner(
mocker,
monkeypatch,
config_runner,
direct_runner,
exp_is_direct,
exp_mock_validate_df,
):
mock_klio_cfg = mocker.Mock()
mock_klio_cfg.pipeline_options.runner = config_runner
mock_validate_df_config = mocker.Mock()
monkeypatch.setattr(cli_utils, "validate_dataflow_runner_config",
mock_validate_df_config)
act_resp = cli_utils.is_direct_runner(mock_klio_cfg, direct_runner)
assert exp_is_direct == act_resp
if exp_mock_validate_df:
mock_validate_df_config.assert_called_once_with(mock_klio_cfg)
def deploy_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(
klio_config, kwargs.pop("direct_runner")
)
git_sha = cli_utils.get_git_sha(
config_meta.job_dir, kwargs.get("image_tag")
)
image_tag = kwargs.get("image_tag") or git_sha
if config_meta.config_file:
basename = os.path.basename(config_meta.config_file)
image_tag = "{}-{}".format(image_tag, basename)
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
# TODO: make this a click option once draining is supported @lynn
if not run_job_config.update:
job_name = klio_config.job_name
gcp_project = klio_config.pipeline_options.project
region = klio_config.pipeline_options.region
strategy = "cancel"
job_commands.stop.StopJob().stop(
job_name, gcp_project, region, strategy
)
klio_pipeline = job_commands.run.RunPipeline(
config_meta.job_dir, klio_config, runtime_config, run_job_config
)
rc = klio_pipeline.run()
sys.exit(rc)
def run_job(klio_config, config_meta, **kwargs):
direct_runner = cli_utils.is_direct_runner(klio_config,
kwargs.pop("direct_runner"))
cli_utils.error_stackdriver_logger_metrics(klio_config, direct_runner)
git_sha = cli_utils.get_git_sha(config_meta.job_dir,
kwargs.get("image_tag"))
image_tag = kwargs.get("image_tag") or git_sha
runtime_config = DockerRuntimeConfig(
image_tag=image_tag,
force_build=kwargs.get("force_build"),
config_file_override=config_meta.config_file,
)
run_job_config = RunJobConfig(
direct_runner=direct_runner,
update=kwargs.pop("update"),
git_sha=git_sha,
)
if (not direct_runner and klio_config.pipeline_options.runner
== var.KlioRunner.DIRECT_GKE_RUNNER):
gke_commands = cli_utils.import_gke_commands()
klio_pipeline = gke_commands.RunPipelineGKE(config_meta.job_dir,
klio_config,
runtime_config,
run_job_config)
else:
klio_pipeline = job_commands.run.RunPipeline(config_meta.job_dir,
klio_config,
runtime_config,
run_job_config)
rc = klio_pipeline.run()
sys.exit(rc)