Esempio n. 1
0
def test_generic_stage_input_validation(project_repo_location: Path):
    stage_name = 'stage_14_bad_executable_script'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='EXECUTABLE_SCRIPT'):
        Stage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_2_bad_config'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(FileExistsError, match='Cannot find'):
        Stage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_15_bad_requirements'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(FileExistsError, match='Cannot find'):
        Stage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_16_bad_memory_request'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='MEMORY_REQUEST_MB'):
        Stage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_17_bad_cpu_request'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='CPU_REQUEST'):
        Stage(stage_name, config, path_to_stage_dir)
Esempio n. 2
0
def test_get_workflow_stages_return_valid_stage_info(
    project_repo_location: Path, ):
    dag = [['stage_1_good'], ['stage_4_good', 'stage_5_good']]

    path_to_stage_1_dir = project_repo_location / 'stage_1_good'
    stage_1_info = BatchStage(
        'stage_1_good',
        BodyworkConfig(path_to_stage_1_dir / STAGE_CONFIG_FILENAME),
        path_to_stage_1_dir)

    path_to_stage_4_dir = project_repo_location / 'stage_4_good'
    stage_4_info = BatchStage(
        'stage_4_good',
        BodyworkConfig(path_to_stage_4_dir / STAGE_CONFIG_FILENAME),
        path_to_stage_4_dir)

    path_to_stage_5_dir = project_repo_location / 'stage_5_good'
    stage_5_info = ServiceStage(
        'stage_5_good',
        BodyworkConfig(path_to_stage_5_dir / STAGE_CONFIG_FILENAME),
        path_to_stage_5_dir)

    all_stage_info = _get_workflow_stages(dag, project_repo_location)
    assert len(all_stage_info) == 3
    assert all_stage_info['stage_1_good'] == stage_1_info
    assert all_stage_info['stage_4_good'] == stage_4_info
    assert all_stage_info['stage_5_good'] == stage_5_info
Esempio n. 3
0
def test_service_stage_input_validation(project_repo_location: Path):
    stage_name = 'stage_10_bad_service_data'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='REPLICAS'):
        ServiceStage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_11_bad_service_data'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='PORT'):
        ServiceStage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_12_bad_service_data'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='MAX_STARTUP_TIME_SECONDS'):
        ServiceStage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_13_bad_service_data'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='MAX_STARTUP_TIME_SECONDS'):
        ServiceStage(stage_name, config, path_to_stage_dir)
Esempio n. 4
0
def test_that_invalid_config_requests_raise_error(project_repo_location: Path):
    config_file_path = project_repo_location / PROJECT_CONFIG_FILENAME
    config = BodyworkConfig(config_file_path)
    with raises(KeyError, match='not_a_real_config_section'):
        config['not_a_real_config_section']
    with raises(KeyError, match='not_a_real_parameter'):
        config['default']['not_a_real_parameter']
Esempio n. 5
0
def test_failure_stage_does_not_run_for_namespace_exception(
    mock_k8s: MagicMock, project_repo_location: Path
):
    config_path = Path(f"{project_repo_location}/bodywork.yaml")
    config = BodyworkConfig(config_path)
    mock_k8s.namespace_exists.return_value = False
    try:
        run_workflow(config, "foo_bar_foo_993", project_repo_location)
    except BodyworkWorkflowExecutionError:
        pass

    mock_k8s.configure_batch_stage_job.assert_not_called()
Esempio n. 6
0
def test_batch_stage_input_validation(project_repo_location: Path):
    stage_name = 'stage_7_bad_batch_data'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='RETRIES'):
        BatchStage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_8_bad_batch_data'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='MAX_COMPLETION_TIME_SECONDS'):
        BatchStage(stage_name, config, path_to_stage_dir)

    stage_name = 'stage_9_bad_batch_data'
    path_to_stage_dir = project_repo_location / stage_name
    path_to_config = path_to_stage_dir / STAGE_CONFIG_FILENAME
    config = BodyworkConfig(path_to_config)
    with raises(BodyworkStageConfigError, match='MAX_COMPLETION_TIME_SECONDS'):
        BatchStage(stage_name, config, path_to_stage_dir)
Esempio n. 7
0
def test_failure_stage_does_not_run_for_docker_image_exception(
    mock_k8s: MagicMock, mock_session: MagicMock, project_repo_location: Path
):
    config_path = Path(f"{project_repo_location}/bodywork.yaml")
    config = BodyworkConfig(config_path)
    mock_session().get.return_value = requests.Response().status_code = 401

    try:
        run_workflow(config, "foo_bar_foo_993", project_repo_location)
    except BodyworkWorkflowExecutionError:
        pass

    mock_k8s.configure_batch_stage_job.assert_not_called()
Esempio n. 8
0
def test_usage_stats_opt_out_does_not_ping_usage_stats_server(
    mock_k8s: MagicMock,
    mock_git_hash: MagicMock,
    mock_git_download: MagicMock,
    mock_session: MagicMock,
    mock_rmtree: MagicMock,
    project_repo_location: Path,
):
    config_path = Path(f"{project_repo_location}/bodywork.yaml")
    config = BodyworkConfig(config_path)

    run_workflow(config, "foo_bar_foo_993", project_repo_location)

    mock_session().get.assert_called_once()
Esempio n. 9
0
def test_run_workflow_pings_usage_stats_server(
    mock_k8s: MagicMock,
    mock_git_hash: MagicMock,
    mock_git_download: MagicMock,
    mock_session: MagicMock,
    mock_rmtree: MagicMock,
    project_repo_location: Path,
):
    config_path = Path(f"{project_repo_location}/bodywork.yaml")
    config = BodyworkConfig(config_path)
    config.project.usage_stats = True

    run_workflow(config, "foo_bar_foo_993", project_repo_location)

    mock_session().get.assert_called_with(USAGE_STATS_SERVER_URL, params={"type": "workflow"})
Esempio n. 10
0
def test_run_workflow_adds_git_commit_to_batch_and_service_env_vars(
    mock_k8s: MagicMock,
    mock_git_hash: MagicMock,
    mock_git_download: MagicMock,
    mock_requests: MagicMock,
    mock_rmtree: MagicMock,
    project_repo_location: Path,
):
    commit_hash = "MY GIT COMMIT HASH"
    mock_git_hash.return_value = commit_hash
    expected_result = [
        k8sclient.V1EnvVar(name=GIT_COMMIT_HASH_K8S_ENV_VAR, value=commit_hash)
    ]
    mock_k8s.create_k8s_environment_variables.return_value = expected_result
    mock_k8s.configure_env_vars_from_secrets.return_value = []
    config_path = Path(f"{project_repo_location}/bodywork.yaml")
    config = BodyworkConfig(config_path)

    run_workflow(config, "foo_bar_foo_993", project_repo_location)

    mock_k8s.configure_service_stage_deployment.assert_called_once_with(
        ANY,
        ANY,
        ANY,
        ANY,
        ANY,
        replicas=ANY,
        port=ANY,
        container_env_vars=expected_result,
        image=ANY,
        cpu_request=ANY,
        memory_request=ANY,
        seconds_to_be_ready_before_completing=ANY,
    )
    mock_k8s.configure_batch_stage_job.assert_called_with(
        ANY,
        ANY,
        ANY,
        ANY,
        ANY,
        retries=ANY,
        container_env_vars=expected_result,
        image=ANY,
        cpu_request=ANY,
        memory_request=ANY,
    )
Esempio n. 11
0
def test_run_workflow_runs_failure_stage_on_failure(
    mock_k8s: MagicMock,
    mock_git_hash: MagicMock,
    mock_git_download: MagicMock,
    mock_requests: MagicMock,
    mock_rmtree: MagicMock,
    project_repo_location: Path,
):
    config_path = Path(f"{project_repo_location}/bodywork.yaml")
    config = BodyworkConfig(config_path)
    config.project.run_on_failure = "on_fail_stage"

    error_message = "Test Error"
    mock_job = MagicMock(k8sclient.V1Job)
    mock_k8s.configure_batch_stage_job.side_effect = [
        k8sclient.ApiException(error_message),
        mock_job,
    ]
    expected_result = [
        k8sclient.V1EnvVar(name=FAILURE_EXCEPTION_K8S_ENV_VAR, value=error_message)
    ]
    mock_k8s.create_k8s_environment_variables.return_value = expected_result
    mock_k8s.configure_env_vars_from_secrets.return_value = []

    try:
        run_workflow(config, "foo_bar_foo_993", project_repo_location)
    except BodyworkWorkflowExecutionError:
        pass

    mock_k8s.configure_batch_stage_job.assert_called_with(
        ANY,
        "on_fail_stage",
        ANY,
        ANY,
        ANY,
        retries=ANY,
        container_env_vars=expected_result,
        image=ANY,
        cpu_request=ANY,
        memory_request=ANY,
    )
Esempio n. 12
0
def test_failure_of_failure_stage_is_recorded_in_exception(
    mock_k8s: MagicMock,
    mock_git_hash: MagicMock,
    mock_git_download: MagicMock,
    mock_requests: MagicMock,
    mock_rmtree: MagicMock,
    project_repo_location: Path,
):
    config_path = Path(f"{project_repo_location}/bodywork.yaml")
    config = BodyworkConfig(config_path)
    config.project.run_on_failure = "on_fail_stage"

    error_message = "The run-on-failure stage experienced an error"
    mock_k8s.configure_batch_stage_job.side_effect = [
        k8sclient.ApiException("Original Error"),
        k8sclient.ApiException(reason=error_message),
    ]
    mock_k8s.configure_env_vars_from_secrets.return_value = []

    with raises(BodyworkWorkflowExecutionError, match=f"{error_message}"):
        run_workflow(config, "foo_bar_foo_993", project_repo_location)
Esempio n. 13
0
def test_that_empty_config_file_raises_error(project_repo_location: Path):
    config_file = project_repo_location / "bodywork_empty.yaml"
    expected_exception_msg = f"cannot parse YAML from {config_file}"
    with raises(BodyworkConfigParsingError, match=expected_exception_msg):
        BodyworkConfig(config_file)
Esempio n. 14
0
def bodywork_config(project_repo_location: Path) -> BodyworkConfig:
    config_file = project_repo_location / "bodywork.yaml"
    return BodyworkConfig(config_file)
Esempio n. 15
0
def test_that_config_values_can_be_retreived(project_repo_location: Path):
    config_file_path = project_repo_location / PROJECT_CONFIG_FILENAME
    config = BodyworkConfig(config_file_path)
    assert config['default']['PROJECT_NAME'] == 'bodywork-test-project'
    assert config['logging']['LOG_LEVEL'] == 'INFO'
Esempio n. 16
0
def test_that_invalid_config_file_path_raises_error(
        project_repo_location: Path):
    bad_config_file = project_repo_location / "bodywerk.yaml"
    with raises(FileExistsError, match="no config file found"):
        BodyworkConfig(bad_config_file)
Esempio n. 17
0
def test_that_invalid_config_file_path_raises_error():
    bad_config_file_path = Path('./tests/not_a_real_directory/bodywerk.ini')
    with raises(FileExistsError, match='no config file found'):
        BodyworkConfig(bad_config_file_path)