def test_deployment_of_remote_workflows(
random_test_namespace: str
):
try:
sleep(5)
process_zero = run(
['bodywork',
'setup-namespace',
random_test_namespace],
encoding='utf-8',
capture_output=True
)
assert process_zero.returncode == 0
process_one = run(
['bodywork',
'deployment',
'create',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project',
'--git-repo-url=https://github.com/bodywork-ml/bodywork-test-project'],
encoding='utf-8',
capture_output=True
)
assert process_one.returncode == 0
sleep(5)
process_two = run(
['bodywork',
'deployment',
'display',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project'],
encoding='utf-8',
capture_output=True
)
assert process_two.returncode == 0
assert 'bodywork-test-project' in process_two.stdout
process_three = run(
['bodywork',
'deployment',
'logs',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project'],
encoding='utf-8',
capture_output=True
)
assert process_three.returncode == 0
assert type(process_three.stdout) is str and len(process_three.stdout) != 0
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
rmtree(SSH_DIR_NAME, ignore_errors=True)
def test_workflow_will_cleanup_jobs_and_rollback_new_deployments_that_yield_errors(
random_test_namespace: str, docker_image: str):
try:
sleep(5)
process_zero = run(
["bodywork", "setup-namespace", random_test_namespace],
encoding="utf-8",
capture_output=True,
)
assert process_zero.returncode == 0
process_one = run(
[
"bodywork",
"workflow",
f"--namespace={random_test_namespace}",
"https://github.com/bodywork-ml/bodywork-rollback-deployment-test-project", # noqa
"master",
f"--bodywork-docker-image={docker_image}",
],
encoding="utf-8",
capture_output=True,
)
expected_output_0 = (
"deleted job=bodywork-rollback-deployment-test-project--stage-1" # noqa
)
assert expected_output_0 in process_one.stdout
assert process_one.returncode == 0
process_two = run(
[
"bodywork",
"workflow",
f"--namespace={random_test_namespace}",
"https://github.com/bodywork-ml/bodywork-rollback-deployment-test-project", # noqa
"master",
f"--bodywork-docker-image={docker_image}",
],
encoding="utf-8",
capture_output=True,
)
expected_output_1 = "deployments failed to roll-out successfully"
expected_output_2 = "rolled back deployment=bodywork-rollback-deployment-test-project--stage-2" # noqa
assert expected_output_1 in process_two.stdout
assert expected_output_2 in process_two.stdout
assert process_two.returncode == 1
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
workflow_sa_crb = workflow_cluster_role_binding_name(
random_test_namespace)
if cluster_role_binding_exists(workflow_sa_crb):
delete_cluster_role_binding(workflow_sa_crb)
def test_workflow_will_cleanup_jobs_and_rollback_new_deployments_that_yield_errors(
random_test_namespace: str,
docker_image: str
):
try:
sleep(5)
process_zero = run(
['bodywork',
'setup-namespace',
random_test_namespace],
encoding='utf-8',
capture_output=True
)
assert process_zero.returncode == 0
process_one = run(
['bodywork',
'workflow',
f'--namespace={random_test_namespace}',
'https://github.com/bodywork-ml/bodywork-rollback-deployment-test-project', # noqa
'master',
f'--bodywork-docker-image={docker_image}'],
encoding='utf-8',
capture_output=True
)
expected_output_0 = 'deleted job=bodywork-rollback-deployment-test-project--stage-1' # noqa
assert expected_output_0 in process_one.stdout
assert process_one.returncode == 0
process_two = run(
['bodywork',
'workflow',
f'--namespace={random_test_namespace}',
'https://github.com/bodywork-ml/bodywork-rollback-deployment-test-project', # noqa
'master',
f'--bodywork-docker-image={docker_image}'],
encoding='utf-8',
capture_output=True
)
expected_output_1 = 'deployments failed to roll-out successfully'
expected_output_2 = 'rolled back deployment=bodywork-rollback-deployment-test-project--stage-2' # noqa
assert expected_output_1 in process_two.stdout
assert expected_output_2 in process_two.stdout
assert process_two.returncode == 1
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
workflow_sa_crb = workflow_cluster_role_binding_name(random_test_namespace)
if cluster_role_binding_exists(workflow_sa_crb):
delete_cluster_role_binding(workflow_sa_crb)
def test_workflow_and_service_management_end_to_end_from_cli(
random_test_namespace: str,
docker_image: str
):
try:
sleep(5)
process_zero = run(
['bodywork',
'setup-namespace',
random_test_namespace],
encoding='utf-8',
capture_output=True
)
assert f'creating namespace={random_test_namespace}' in process_zero.stdout
assert 'creating service-account=bodywork-workflow-' in process_zero.stdout
assert 'creating cluster-role-binding=bodywork-workflow-' in process_zero.stdout
assert 'creating service-account=bodywork-jobs-' in process_zero.stdout
assert process_zero.returncode == 0
process_one = run(
['bodywork',
'secret',
'create',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project-credentials',
'--data',
'USERNAME=alex',
'PASSWORD=alex123'],
encoding='utf-8',
capture_output=True
)
assert process_one.stdout is not None
assert process_one.returncode == 0
process_two = run(
['bodywork',
'workflow',
f'--namespace={random_test_namespace}',
'https://github.com/bodywork-ml/bodywork-test-project',
'master',
f'--bodywork-docker-image={docker_image}'],
encoding='utf-8',
capture_output=True
)
expected_output_1 = (
'attempting to run workflow for '
'project=https://github.com/bodywork-ml/bodywork-test-project on '
f'branch=master in kubernetes namespace={random_test_namespace}')
expected_output_2 = 'successfully ran stage=stage-1'
expected_output_3 = 'attempting to run stage=stage-4'
expected_output_4 = (
'successfully ran workflow for '
'project=https://github.com/bodywork-ml/bodywork-test-project on '
f'branch=master in kubernetes namespace={random_test_namespace}')
expected_output_5 = 'successfully ran stage=stage-5'
assert expected_output_1 in process_two.stdout
assert expected_output_2 in process_two.stdout
assert expected_output_3 in process_two.stdout
assert expected_output_4 in process_two.stdout
assert expected_output_5 in process_two.stdout
assert process_two.returncode == 0
process_three = run(
['bodywork',
'workflow',
f'--namespace={random_test_namespace}',
'https://github.com/bodywork-ml/bodywork-test-project',
'master',
f'--bodywork-docker-image={docker_image}'],
encoding='utf-8',
capture_output=True
)
assert process_three.returncode == 0
process_four = run(
['bodywork',
'service',
'display',
f'--namespace={random_test_namespace}'],
encoding='utf-8',
capture_output=True
)
assert 'http://bodywork-test-project--stage-3:5000' in process_four.stdout
assert 'http://bodywork-test-project--stage-4:5000' in process_four.stdout
assert 'true' in process_four.stdout
assert process_four.returncode == 0
process_five = run(
['bodywork',
'service',
'delete',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project--stage-3'],
encoding='utf-8',
capture_output=True
)
assert 'deployment=bodywork-test-project--stage-3 deleted' in process_five.stdout
assert 'service at http://bodywork-test-project--stage-3 deleted' in process_five.stdout # noqa
assert process_five.returncode == 0
process_six = run(
['bodywork',
'service',
'delete',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project--stage-4'],
encoding='utf-8',
capture_output=True
)
assert 'deployment=bodywork-test-project--stage-4 deleted' in process_six.stdout
assert 'service at http://bodywork-test-project--stage-4 deleted' in process_six.stdout # noqa
assert process_six.returncode == 0
process_seven = run(
['bodywork',
'service',
'display',
f'--namespace={random_test_namespace}'],
encoding='utf-8',
capture_output=True
)
assert process_seven.stdout.split('\n')[1] == ''
assert process_seven.returncode == 0
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
workflow_sa_crb = workflow_cluster_role_binding_name(random_test_namespace)
if cluster_role_binding_exists(workflow_sa_crb):
delete_cluster_role_binding(workflow_sa_crb)
def test_workflow_with_ssh_github_connectivity(
random_test_namespace: str,
docker_image: str,
set_github_ssh_private_key_env_var: None
):
try:
sleep(5)
process_zero = run(
['bodywork',
'setup-namespace',
random_test_namespace],
encoding='utf-8',
capture_output=True
)
assert process_zero.returncode == 0
process_one = run(
['bodywork',
'secret',
'create',
f'--namespace={random_test_namespace}',
f'--name={SSH_GITHUB_SECRET_NAME}',
'--data',
f'{SSH_GITHUB_KEY_ENV_VAR}={os.environ[SSH_GITHUB_KEY_ENV_VAR]}'],
encoding='utf-8',
capture_output=True
)
assert process_one.stdout is not None
assert process_one.returncode == 0
process_two = run(
['bodywork',
'secret',
'create',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project-credentials',
'--data',
'USERNAME=alex',
'PASSWORD=alex123'],
encoding='utf-8',
capture_output=True
)
assert process_two.stdout is not None
assert process_two.returncode == 0
process_three = run(
['bodywork',
'workflow',
f'--namespace={random_test_namespace}',
'[email protected]:bodywork-ml/bodywork-test-project.git',
'master',
f'--bodywork-docker-image={docker_image}'],
encoding='utf-8',
capture_output=True
)
expected_output_1 = (
'attempting to run workflow for '
'[email protected]:bodywork-ml/bodywork-test-project.git on '
f'branch=master in kubernetes namespace={random_test_namespace}')
expected_output_2 = (
'successfully ran workflow for '
'[email protected]:bodywork-ml/bodywork-test-project.git on '
f'branch=master in kubernetes namespace={random_test_namespace}')
expected_output_3 = 'successfully ran stage=stage-1'
assert expected_output_1 in process_three.stdout
assert expected_output_2 in process_three.stdout
assert expected_output_3 in process_three.stdout
assert process_three.returncode == 0
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
rmtree(SSH_DIR_NAME, ignore_errors=True)
def test_deployment_of_remote_workflows(random_test_namespace: str):
try:
sleep(5)
process_zero = run(
["bodywork", "setup-namespace", random_test_namespace],
encoding="utf-8",
capture_output=True,
)
assert process_zero.returncode == 0
process_one = run(
[
"bodywork",
"deployment",
"create",
f"--namespace={random_test_namespace}",
"--name=bodywork-test-project",
"--git-repo-url=https://github.com/bodywork-ml/bodywork-test-project",
],
encoding="utf-8",
capture_output=True,
)
assert process_one.returncode == 0
sleep(5)
process_two = run(
[
"bodywork",
"deployment",
"display",
f"--namespace={random_test_namespace}",
"--name=bodywork-test-project",
],
encoding="utf-8",
capture_output=True,
)
assert process_two.returncode == 0
assert "bodywork-test-project" in process_two.stdout
process_three = run(
[
"bodywork",
"deployment",
"logs",
f"--namespace={random_test_namespace}",
"--name=bodywork-test-project",
],
encoding="utf-8",
capture_output=True,
)
assert process_three.returncode == 0
assert type(process_three.stdout) is str and len(
process_three.stdout) != 0
assert "ERROR" not in process_three.stdout
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
rmtree(SSH_DIR_NAME, ignore_errors=True)
def test_workflow_and_service_management_end_to_end_from_cli(
random_test_namespace: str, docker_image: str,
ingress_load_balancer_url: str):
try:
sleep(5)
process_zero = run(
["bodywork", "setup-namespace", random_test_namespace],
encoding="utf-8",
capture_output=True,
)
assert f"creating namespace={random_test_namespace}" in process_zero.stdout
assert "creating service-account=bodywork-workflow-" in process_zero.stdout
assert "creating cluster-role-binding=bodywork-workflow-" in process_zero.stdout
assert "creating service-account=bodywork-jobs-" in process_zero.stdout
assert process_zero.returncode == 0
process_one = run(
[
"bodywork",
"secret",
"create",
f"--namespace={random_test_namespace}",
"--name=bodywork-test-project-credentials",
"--data",
"USERNAME=alex",
"PASSWORD=alex123",
],
encoding="utf-8",
capture_output=True,
)
assert process_one.stdout is not None
assert process_one.returncode == 0
process_two = run(
[
"bodywork",
"workflow",
f"--namespace={random_test_namespace}",
"https://github.com/bodywork-ml/bodywork-test-project",
"master",
f"--bodywork-docker-image={docker_image}",
],
encoding="utf-8",
capture_output=True,
)
expected_output_1 = (
"attempting to run workflow for "
"project=https://github.com/bodywork-ml/bodywork-test-project on "
f"branch=master in kubernetes namespace={random_test_namespace}")
expected_output_2 = "successfully ran stage=stage_1"
expected_output_3 = "attempting to run stage=stage_4"
expected_output_4 = (
"successfully ran workflow for "
"project=https://github.com/bodywork-ml/bodywork-test-project on "
f"branch=master in kubernetes namespace={random_test_namespace}")
expected_output_5 = "successfully ran stage=stage_5"
assert expected_output_1 in process_two.stdout
assert expected_output_2 in process_two.stdout
assert expected_output_3 in process_two.stdout
assert expected_output_4 in process_two.stdout
assert expected_output_5 in process_two.stdout
assert process_two.returncode == 0
process_three = run(
[
"bodywork",
"workflow",
f"--namespace={random_test_namespace}",
"https://github.com/bodywork-ml/bodywork-test-project",
"master",
f"--bodywork-docker-image={docker_image}",
],
encoding="utf-8",
capture_output=True,
)
assert process_three.returncode == 0
process_four = run(
[
"bodywork", "service", "display",
f"--namespace={random_test_namespace}"
],
encoding="utf-8",
capture_output=True,
)
assert (
f"http://bodywork-test-project--stage-3.{random_test_namespace}.svc"
in process_four.stdout)
assert (
f"http://bodywork-test-project--stage-4.{random_test_namespace}.svc"
in process_four.stdout)
assert (f"/{random_test_namespace}/bodywork-test-project--stage-3"
in process_four.stdout)
assert (f"/{random_test_namespace}/bodywork-test-project--stage-4"
not in process_four.stdout)
assert "5000" in process_four.stdout
assert process_four.returncode == 0
stage_3_service_external_url = (
f"http://{ingress_load_balancer_url}/{random_test_namespace}/"
f"/bodywork-test-project--stage-3/v1/predict")
response_stage_3 = requests.get(url=stage_3_service_external_url)
assert response_stage_3.ok
assert response_stage_3.json()["y"] == "hello_world"
stage_4_service_external_url = (
f"http://{ingress_load_balancer_url}/{random_test_namespace}/"
f"/bodywork-test-project--stage-4/v2/predict")
response_stage_4 = requests.get(url=stage_4_service_external_url)
assert response_stage_4.status_code == 404
process_five = run(
[
"bodywork",
"service",
"delete",
f"--namespace={random_test_namespace}",
"--name=bodywork-test-project--stage-3",
],
encoding="utf-8",
capture_output=True,
)
assert ("deployment=bodywork-test-project--stage-3 deleted"
in process_five.stdout)
assert (
f"service at http://bodywork-test-project--stage-3.{random_test_namespace}.svc.cluster.local deleted" # noqa
in process_five.stdout) # noqa
assert (
f"ingress route /{random_test_namespace}/bodywork-test-project--stage-3 deleted" # noqa
in process_five.stdout)
assert process_five.returncode == 0
process_six = run(
[
"bodywork",
"service",
"delete",
f"--namespace={random_test_namespace}",
"--name=bodywork-test-project--stage-4",
],
encoding="utf-8",
capture_output=True,
)
assert "deployment=bodywork-test-project--stage-4 deleted" in process_six.stdout
assert (
f"service at http://bodywork-test-project--stage-4.{random_test_namespace}.svc.cluster.local deleted" # noqa
in process_six.stdout) # noqa
assert (
f"ingress route /{random_test_namespace}/bodywork-test-project--stage-4 deleted" # noqa
not in process_six.stdout) # noqa
assert process_six.returncode == 0
process_seven = run(
[
"bodywork", "service", "display",
f"--namespace={random_test_namespace}"
],
encoding="utf-8",
capture_output=True,
)
assert process_seven.stdout == ""
assert process_seven.returncode == 0
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
workflow_sa_crb = workflow_cluster_role_binding_name(
random_test_namespace)
if cluster_role_binding_exists(workflow_sa_crb):
delete_cluster_role_binding(workflow_sa_crb)
def test_workflow_with_ssh_github_connectivity(
random_test_namespace: str,
docker_image: str,
set_github_ssh_private_key_env_var: None,
):
try:
sleep(5)
process_zero = run(
["bodywork", "setup-namespace", random_test_namespace],
encoding="utf-8",
capture_output=True,
)
assert process_zero.returncode == 0
process_one = run(
[
"bodywork",
"secret",
"create",
f"--namespace={random_test_namespace}",
f"--name={SSH_SECRET_NAME}",
"--data",
f"{SSH_PRIVATE_KEY_ENV_VAR}={os.environ[SSH_PRIVATE_KEY_ENV_VAR]}",
],
encoding="utf-8",
capture_output=True,
)
assert process_one.stdout is not None
assert process_one.returncode == 0
process_two = run(
[
"bodywork",
"secret",
"create",
f"--namespace={random_test_namespace}",
"--name=bodywork-test-project-credentials",
"--data",
"USERNAME=alex",
"PASSWORD=alex123",
],
encoding="utf-8",
capture_output=True,
)
assert process_two.stdout is not None
assert process_two.returncode == 0
process_three = run(
[
"bodywork",
"workflow",
f"--namespace={random_test_namespace}",
"[email protected]:bodywork-ml/bodywork-test-project.git",
"master",
f"--bodywork-docker-image={docker_image}",
],
encoding="utf-8",
capture_output=True,
)
expected_output_1 = (
"attempting to run workflow for "
"[email protected]:bodywork-ml/bodywork-test-project.git on "
f"branch=master in kubernetes namespace={random_test_namespace}")
expected_output_2 = (
"successfully ran workflow for "
"[email protected]:bodywork-ml/bodywork-test-project.git on "
f"branch=master in kubernetes namespace={random_test_namespace}")
expected_output_3 = "successfully ran stage=stage_1"
assert expected_output_1 in process_three.stdout
assert expected_output_2 in process_three.stdout
assert expected_output_3 in process_three.stdout
assert process_three.returncode == 0
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
rmtree(SSH_DIR_NAME, ignore_errors=True)
def test_workflow_and_service_management_end_to_end_from_cli(
random_test_namespace: str,
docker_image: str,
ingress_load_balancer_url: str
):
try:
sleep(5)
process_zero = run(
['bodywork',
'setup-namespace',
random_test_namespace],
encoding='utf-8',
capture_output=True
)
assert f'creating namespace={random_test_namespace}' in process_zero.stdout
assert 'creating service-account=bodywork-workflow-' in process_zero.stdout
assert 'creating cluster-role-binding=bodywork-workflow-' in process_zero.stdout
assert 'creating service-account=bodywork-jobs-' in process_zero.stdout
assert process_zero.returncode == 0
process_one = run(
['bodywork',
'secret',
'create',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project-credentials',
'--data',
'USERNAME=alex',
'PASSWORD=alex123'],
encoding='utf-8',
capture_output=True
)
assert process_one.stdout is not None
assert process_one.returncode == 0
process_two = run(
['bodywork',
'workflow',
f'--namespace={random_test_namespace}',
'https://github.com/bodywork-ml/bodywork-test-project',
'master',
f'--bodywork-docker-image={docker_image}'],
encoding='utf-8',
capture_output=True
)
expected_output_1 = (
'attempting to run workflow for '
'project=https://github.com/bodywork-ml/bodywork-test-project on '
f'branch=master in kubernetes namespace={random_test_namespace}')
expected_output_2 = 'successfully ran stage=stage-1'
expected_output_3 = 'attempting to run stage=stage-4'
expected_output_4 = (
'successfully ran workflow for '
'project=https://github.com/bodywork-ml/bodywork-test-project on '
f'branch=master in kubernetes namespace={random_test_namespace}')
expected_output_5 = 'successfully ran stage=stage-5'
assert expected_output_1 in process_two.stdout
assert expected_output_2 in process_two.stdout
assert expected_output_3 in process_two.stdout
assert expected_output_4 in process_two.stdout
assert expected_output_5 in process_two.stdout
assert process_two.returncode == 0
process_three = run(
['bodywork',
'workflow',
f'--namespace={random_test_namespace}',
'https://github.com/bodywork-ml/bodywork-test-project',
'master',
f'--bodywork-docker-image={docker_image}'],
encoding='utf-8',
capture_output=True
)
assert process_three.returncode == 0
process_four = run(
['bodywork',
'service',
'display',
f'--namespace={random_test_namespace}'],
encoding='utf-8',
capture_output=True
)
assert (f'http://bodywork-test-project--stage-3.{random_test_namespace}.svc'
in process_four.stdout)
assert (f'http://bodywork-test-project--stage-4.{random_test_namespace}.svc'
in process_four.stdout)
assert (f'/{random_test_namespace}/bodywork-test-project--stage-3'
in process_four.stdout)
assert (f'/{random_test_namespace}/bodywork-test-project--stage-4'
not in process_four.stdout)
assert '5000' in process_four.stdout
assert process_four.returncode == 0
stage_3_service_external_url = (
f'http://{ingress_load_balancer_url}/{random_test_namespace}/'
f'/bodywork-test-project--stage-3/v1/predict'
)
response_stage_3 = requests.get(url=stage_3_service_external_url)
assert response_stage_3.ok
assert response_stage_3.json()['y'] == 'hello_world'
stage_4_service_external_url = (
f'http://{ingress_load_balancer_url}/{random_test_namespace}/'
f'/bodywork-test-project--stage-4/v2/predict'
)
response_stage_4 = requests.get(url=stage_4_service_external_url)
assert response_stage_4.status_code == 404
process_five = run(
['bodywork',
'service',
'delete',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project--stage-3'],
encoding='utf-8',
capture_output=True
)
assert 'deployment=bodywork-test-project--stage-3 deleted' in process_five.stdout
assert f'service at http://bodywork-test-project--stage-3.{random_test_namespace}.svc.cluster.local deleted' in process_five.stdout # noqa
assert f'ingress route /{random_test_namespace}/bodywork-test-project--stage-3 deleted' in process_five.stdout # noqa
assert process_five.returncode == 0
process_six = run(
['bodywork',
'service',
'delete',
f'--namespace={random_test_namespace}',
'--name=bodywork-test-project--stage-4'],
encoding='utf-8',
capture_output=True
)
assert 'deployment=bodywork-test-project--stage-4 deleted' in process_six.stdout
assert f'service at http://bodywork-test-project--stage-4.{random_test_namespace}.svc.cluster.local deleted' in process_six.stdout # noqa
assert f'ingress route /{random_test_namespace}/bodywork-test-project--stage-4 deleted' not in process_six.stdout
assert process_six.returncode == 0
process_seven = run(
['bodywork',
'service',
'display',
f'--namespace={random_test_namespace}'],
encoding='utf-8',
capture_output=True
)
assert process_seven.stdout == ''
assert process_seven.returncode == 0
except Exception:
assert False
finally:
load_kubernetes_config()
delete_namespace(random_test_namespace)
workflow_sa_crb = workflow_cluster_role_binding_name(random_test_namespace)
if cluster_role_binding_exists(workflow_sa_crb):
delete_cluster_role_binding(workflow_sa_crb)