def test_generic_stage_input_validation(project_repo_location: Path):
stage_name = 'stage_14_bad_executable_script'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='EXECUTABLE_SCRIPT'):
Stage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_2_bad_config'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(FileExistsError, match='Cannot find'):
Stage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_15_bad_requirements'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(FileExistsError, match='Cannot find'):
Stage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_16_bad_memory_request'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='MEMORY_REQUEST_MB'):
Stage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_17_bad_cpu_request'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='CPU_REQUEST'):
Stage(stage_name, config, path_to_stage_dir)
def test_get_workflow_stages_return_valid_stage_info(
project_repo_location: Path, ):
dag = [['stage_1_good'], ['stage_4_good', 'stage_5_good']]
path_to_stage_1_dir = project_repo_location / 'stage_1_good'
stage_1_info = BatchStage(
'stage_1_good',
BodyworkConfig(path_to_stage_1_dir / STAGE_CONFIG_FILENAME),
path_to_stage_1_dir)
path_to_stage_4_dir = project_repo_location / 'stage_4_good'
stage_4_info = BatchStage(
'stage_4_good',
BodyworkConfig(path_to_stage_4_dir / STAGE_CONFIG_FILENAME),
path_to_stage_4_dir)
path_to_stage_5_dir = project_repo_location / 'stage_5_good'
stage_5_info = ServiceStage(
'stage_5_good',
BodyworkConfig(path_to_stage_5_dir / STAGE_CONFIG_FILENAME),
path_to_stage_5_dir)
all_stage_info = _get_workflow_stages(dag, project_repo_location)
assert len(all_stage_info) == 3
assert all_stage_info['stage_1_good'] == stage_1_info
assert all_stage_info['stage_4_good'] == stage_4_info
assert all_stage_info['stage_5_good'] == stage_5_info
def test_service_stage_input_validation(project_repo_location: Path):
stage_name = 'stage_10_bad_service_data'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='REPLICAS'):
ServiceStage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_11_bad_service_data'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='PORT'):
ServiceStage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_12_bad_service_data'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='MAX_STARTUP_TIME_SECONDS'):
ServiceStage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_13_bad_service_data'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='MAX_STARTUP_TIME_SECONDS'):
ServiceStage(stage_name, config, path_to_stage_dir)
def test_that_config_file_with_missing_sections_raises_error(
bodywork_config: BodyworkConfig, ):
del bodywork_config._config["version"]
del bodywork_config._config["project"]
del bodywork_config._config["stages"]
del bodywork_config._config["logging"]
expected_exception_msg = "missing sections: version, project, stages, logging"
with raises(BodyworkConfigMissingSectionError,
match=expected_exception_msg):
bodywork_config._validate_parsed_config()
def test_that_config_file_with_mismatched_schema_version_raises_error(
bodywork_config: BodyworkConfig, ):
bodywork_config._config["version"] = "0.1"
expected_exception_msg = (
f"config file has schema version 0.1, when Bodywork "
f"version {BODYWORK_VERSION} requires schema version "
f"{BODYWORK_CONFIG_VERSION}")
with raises(BodyworkConfigVersionMismatchError,
match=expected_exception_msg):
bodywork_config._validate_parsed_config()
def test_that_config_file_with_non_list_stages_raises_error(
bodywork_config: BodyworkConfig, ):
bodywork_config._config["stages"] = "bad"
expected_exception_msg = (
"missing or invalid parameters: "
"project.workflow -> cannot find valid stage @ stages.stage_1, "
"project.workflow -> cannot find valid stage @ stages.stage_2, "
"project.workflow -> cannot find valid stage @ stages.stage_3, "
"project.workflow -> cannot find valid stage @ stages.stage_4, "
"stages._ - no stage configs provided")
with raises(BodyworkConfigValidationError, match=expected_exception_msg):
bodywork_config._validate_parsed_config()
def test_that_invalid_config_requests_raise_error(project_repo_location: Path):
config_file_path = project_repo_location / PROJECT_CONFIG_FILENAME
config = BodyworkConfig(config_file_path)
with raises(KeyError, match='not_a_real_config_section'):
config['not_a_real_config_section']
with raises(KeyError, match='not_a_real_parameter'):
config['default']['not_a_real_parameter']
def test_that_subsection_validation_feeds_through_to_validation_report(
bodywork_config: BodyworkConfig, ):
del bodywork_config._config["project"]["docker_image"]
del bodywork_config._config["logging"]["log_level"]
del bodywork_config._config["stages"]["stage_1"]["batch"]
bodywork_config._config["stages"]["stage_2"]["service"] = {"foo": "bar"}
expected_exception_msg = (
"missing or invalid parameters: "
"logging.log_level -> required field, "
"project.docker_image -> required field, "
"project.workflow -> cannot find valid stage @ stages.stage_1, "
"project.workflow -> cannot find valid stage @ stages.stage_2, "
"stages.stage_1.batch/service, "
"stages.stage_2.batch/service")
with raises(BodyworkConfigValidationError, match=expected_exception_msg):
bodywork_config._validate_parsed_config()
def test_batch_stage_input_validation(project_repo_location: Path):
stage_name = 'stage_7_bad_batch_data'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='RETRIES'):
BatchStage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_8_bad_batch_data'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='MAX_COMPLETION_TIME_SECONDS'):
BatchStage(stage_name, config, path_to_stage_dir)
stage_name = 'stage_9_bad_batch_data'
path_to_stage_dir = project_repo_location / stage_name
path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
config = BodyworkConfig(path_to_config)
with raises(BodyworkStageConfigError, match='MAX_COMPLETION_TIME_SECONDS'):
BatchStage(stage_name, config, path_to_stage_dir)
def test_failure_stage_does_not_run_for_namespace_exception(
mock_k8s: MagicMock, project_repo_location: Path
):
config_path = Path(f"{project_repo_location}/bodywork.yaml")
config = BodyworkConfig(config_path)
mock_k8s.namespace_exists.return_value = False
try:
run_workflow(config, "foo_bar_foo_993", project_repo_location)
except BodyworkWorkflowExecutionError:
pass
mock_k8s.configure_batch_stage_job.assert_not_called()
def test_failure_stage_does_not_run_for_docker_image_exception(
mock_k8s: MagicMock, mock_session: MagicMock, project_repo_location: Path
):
config_path = Path(f"{project_repo_location}/bodywork.yaml")
config = BodyworkConfig(config_path)
mock_session().get.return_value = requests.Response().status_code = 401
try:
run_workflow(config, "foo_bar_foo_993", project_repo_location)
except BodyworkWorkflowExecutionError:
pass
mock_k8s.configure_batch_stage_job.assert_not_called()
def test_usage_stats_opt_out_does_not_ping_usage_stats_server(
mock_k8s: MagicMock,
mock_git_hash: MagicMock,
mock_git_download: MagicMock,
mock_session: MagicMock,
mock_rmtree: MagicMock,
project_repo_location: Path,
):
config_path = Path(f"{project_repo_location}/bodywork.yaml")
config = BodyworkConfig(config_path)
run_workflow(config, "foo_bar_foo_993", project_repo_location)
mock_session().get.assert_called_once()
def test_run_workflow_pings_usage_stats_server(
mock_k8s: MagicMock,
mock_git_hash: MagicMock,
mock_git_download: MagicMock,
mock_session: MagicMock,
mock_rmtree: MagicMock,
project_repo_location: Path,
):
config_path = Path(f"{project_repo_location}/bodywork.yaml")
config = BodyworkConfig(config_path)
config.project.usage_stats = True
run_workflow(config, "foo_bar_foo_993", project_repo_location)
mock_session().get.assert_called_with(USAGE_STATS_SERVER_URL, params={"type": "workflow"})
def test_that_config_file_with_invalid_schema_version_raises_error(
bodywork_config: BodyworkConfig, ):
bodywork_config._config["version"] = "not the version"
expected_exception_msg = "missing or invalid parameters: version"
with raises(BodyworkConfigValidationError, match=expected_exception_msg):
bodywork_config._validate_parsed_config()
bodywork_config._config["version"] = "1.0.0"
with raises(BodyworkConfigValidationError, match=expected_exception_msg):
bodywork_config._validate_parsed_config()
def test_run_workflow_adds_git_commit_to_batch_and_service_env_vars(
mock_k8s: MagicMock,
mock_git_hash: MagicMock,
mock_git_download: MagicMock,
mock_requests: MagicMock,
mock_rmtree: MagicMock,
project_repo_location: Path,
):
commit_hash = "MY GIT COMMIT HASH"
mock_git_hash.return_value = commit_hash
expected_result = [
k8sclient.V1EnvVar(name=GIT_COMMIT_HASH_K8S_ENV_VAR, value=commit_hash)
]
mock_k8s.create_k8s_environment_variables.return_value = expected_result
mock_k8s.configure_env_vars_from_secrets.return_value = []
config_path = Path(f"{project_repo_location}/bodywork.yaml")
config = BodyworkConfig(config_path)
run_workflow(config, "foo_bar_foo_993", project_repo_location)
mock_k8s.configure_service_stage_deployment.assert_called_once_with(
ANY,
ANY,
ANY,
ANY,
ANY,
replicas=ANY,
port=ANY,
container_env_vars=expected_result,
image=ANY,
cpu_request=ANY,
memory_request=ANY,
seconds_to_be_ready_before_completing=ANY,
)
mock_k8s.configure_batch_stage_job.assert_called_with(
ANY,
ANY,
ANY,
ANY,
ANY,
retries=ANY,
container_env_vars=expected_result,
image=ANY,
cpu_request=ANY,
memory_request=ANY,
)
def test_run_workflow_runs_failure_stage_on_failure(
mock_k8s: MagicMock,
mock_git_hash: MagicMock,
mock_git_download: MagicMock,
mock_requests: MagicMock,
mock_rmtree: MagicMock,
project_repo_location: Path,
):
config_path = Path(f"{project_repo_location}/bodywork.yaml")
config = BodyworkConfig(config_path)
config.project.run_on_failure = "on_fail_stage"
error_message = "Test Error"
mock_job = MagicMock(k8sclient.V1Job)
mock_k8s.configure_batch_stage_job.side_effect = [
k8sclient.ApiException(error_message),
mock_job,
]
expected_result = [
k8sclient.V1EnvVar(name=FAILURE_EXCEPTION_K8S_ENV_VAR, value=error_message)
]
mock_k8s.create_k8s_environment_variables.return_value = expected_result
mock_k8s.configure_env_vars_from_secrets.return_value = []
try:
run_workflow(config, "foo_bar_foo_993", project_repo_location)
except BodyworkWorkflowExecutionError:
pass
mock_k8s.configure_batch_stage_job.assert_called_with(
ANY,
"on_fail_stage",
ANY,
ANY,
ANY,
retries=ANY,
container_env_vars=expected_result,
image=ANY,
cpu_request=ANY,
memory_request=ANY,
)
def test_failure_of_failure_stage_is_recorded_in_exception(
mock_k8s: MagicMock,
mock_git_hash: MagicMock,
mock_git_download: MagicMock,
mock_requests: MagicMock,
mock_rmtree: MagicMock,
project_repo_location: Path,
):
config_path = Path(f"{project_repo_location}/bodywork.yaml")
config = BodyworkConfig(config_path)
config.project.run_on_failure = "on_fail_stage"
error_message = "The run-on-failure stage experienced an error"
mock_k8s.configure_batch_stage_job.side_effect = [
k8sclient.ApiException("Original Error"),
k8sclient.ApiException(reason=error_message),
]
mock_k8s.configure_env_vars_from_secrets.return_value = []
with raises(BodyworkWorkflowExecutionError, match=f"{error_message}"):
run_workflow(config, "foo_bar_foo_993", project_repo_location)
def test_py_modules_that_cannot_be_located_raise_error(
bodywork_config: BodyworkConfig):
bodywork_config.check_py_modules_exist = True
try:
bodywork_config._validate_parsed_config()
assert True
except Exception:
assert False
stage_1 = bodywork_config._config["stages"]["stage_1"]
bodywork_config._config["stages"]["stage_one"] = stage_1
del bodywork_config._config["stages"]["stage_1"]
stage_2 = bodywork_config._config["stages"]["stage_2"]
bodywork_config._config["stages"]["stage_two"] = stage_2
del bodywork_config._config["stages"]["stage_2"]
bodywork_config._config["stages"]["stage_3"][
"executable_module_path"] = "not_dir/main.py" # noqa
expected_exception_msg = (
"missing or invalid parameters: "
"project.workflow -> cannot find valid stage @ stages.stage_1, "
"project.workflow -> cannot find valid stage @ stages.stage_2, "
"stages.stage_3.executable_module_path -> does not exist")
with raises(BodyworkConfigValidationError, match=expected_exception_msg):
bodywork_config._validate_parsed_config()
def bodywork_config(project_repo_location: Path) -> BodyworkConfig:
config_file = project_repo_location / "bodywork.yaml"
return BodyworkConfig(config_file)
def test_that_config_values_can_be_retreived(project_repo_location: Path):
config_file_path = project_repo_location / PROJECT_CONFIG_FILENAME
config = BodyworkConfig(config_file_path)
assert config['default']['PROJECT_NAME'] == 'bodywork-test-project'
assert config['logging']['LOG_LEVEL'] == 'INFO'
def test_that_invalid_config_file_path_raises_error(
project_repo_location: Path):
bad_config_file = project_repo_location / "bodywerk.yaml"
with raises(FileExistsError, match="no config file found"):
BodyworkConfig(bad_config_file)
def test_that_empty_config_file_raises_error(project_repo_location: Path):
config_file = project_repo_location / "bodywork_empty.yaml"
expected_exception_msg = f"cannot parse YAML from {config_file}"
with raises(BodyworkConfigParsingError, match=expected_exception_msg):
BodyworkConfig(config_file)
def test_check_failure_stage_is_configured(bodywork_config: BodyworkConfig, ):
bodywork_config._config["project"]["run_on_failure"] = "x"
expected_exception_msg = f"project.run_on_failure -> cannot find valid stage: x to run on workflow failure."
with raises(BodyworkConfigValidationError, match=expected_exception_msg):
bodywork_config._validate_parsed_config()
def test_that_invalid_config_file_path_raises_error():
bad_config_file_path = Path('./tests/not_a_real_directory/bodywerk.ini')
with raises(FileExistsError, match='no config file found'):
BodyworkConfig(bad_config_file_path)