def test_custom_decommission():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config["env"]["WORLD_COUNT"] = "1"
sdk_marathon.update_app(marathon_config)
sdk_plan.wait_for_completed_plan(foldered_name, "decommission")
decommission_plan = sdk_plan.get_decommission_plan(foldered_name)
log.info(sdk_plan.plan_string("decommission", decommission_plan))
custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"]
assert "custom_decommission_step" == custom_step_name
# scale back up
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config["env"]["WORLD_COUNT"] = "2"
sdk_marathon.update_app(marathon_config)
sdk_plan.wait_for_completed_deployment(foldered_name)
# Let's decommission again!
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config["env"]["WORLD_COUNT"] = "1"
sdk_marathon.update_app(marathon_config)
sdk_plan.wait_for_completed_deployment(foldered_name)
sdk_plan.wait_for_completed_plan(foldered_name, "decommission")
decommission_plan = sdk_plan.get_decommission_plan(foldered_name)
log.info(sdk_plan.plan_string("decommission", decommission_plan))
custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"]
assert "custom_decommission_step" == custom_step_name
def test_increase_decrease_world_nodes():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
config.check_running(foldered_name)
original_hello_ids = sdk_tasks.get_task_ids(foldered_name, 'hello')
original_world_ids = sdk_tasks.get_task_ids(foldered_name, 'world')
log.info('world ids: ' + str(original_world_ids))
# add 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', 2)
config.check_running(foldered_name)
sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids)
# check 2 world tasks added:
assert 2 + len(original_world_ids) == len(sdk_tasks.get_task_ids(foldered_name, 'world'))
# subtract 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', -2)
config.check_running(foldered_name)
# wait for the decommission plan for this subtraction to be complete
sdk_plan.wait_for_completed_plan(foldered_name, 'decommission')
# check that the total task count is back to original
sdk_tasks.check_running(
foldered_name,
len(original_hello_ids) + len(original_world_ids),
allow_more=False)
# check that original tasks weren't affected/relaunched in the process
sdk_tasks.check_tasks_not_updated(foldered_name, 'hello', original_hello_ids)
sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids)
# check that the world tasks are back to their prior state (also without changing task ids)
assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, 'world')
def test_all_tasks_are_launched():
service_options = {"service": {"yaml": "plan"}}
sdk_install.install(config.PACKAGE_NAME,
foldered_name,
0,
additional_options=service_options,
wait_for_deployment=False,
wait_for_all_conditions=True)
# after above method returns, start all plans right away.
plans = ["manual-plan-0", "manual-plan-1", "manual-plan-2"]
for plan in plans:
sdk_plan.start_plan(foldered_name, plan)
for plan in plans:
sdk_plan.wait_for_completed_plan(foldered_name, plan)
pods = ["custom-pod-A-0", "custom-pod-B-0", "custom-pod-C-0"]
for pod in pods:
# /pod/<pod-id>/info fetches data from SDK's persistence layer
pod_hello_0_info = sdk_cmd.service_request(
"GET", foldered_name, "/v1/pod/{}/info".format(pod)).json()
for taskInfoAndStatus in pod_hello_0_info:
info = taskInfoAndStatus["info"]
status = taskInfoAndStatus["status"]
# While `info` object is always present, `status` may or may not be present based
# on whether the task was launched and we received an update from mesos (or not).
if status:
assert info["taskId"]["value"] == status["taskId"]["value"]
assert len(info["taskId"]["value"]) > 0
else:
assert len(info["taskId"]["value"]) == 0
def test_custom_decommission():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config["env"]["WORLD_COUNT"] = "1"
sdk_marathon.update_app(marathon_config)
sdk_plan.wait_for_completed_plan(foldered_name, "decommission")
decommission_plan = sdk_plan.get_decommission_plan(foldered_name)
log.info(sdk_plan.plan_string("decommission", decommission_plan))
custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"]
assert "custom_decommission_step" == custom_step_name
# scale back up
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config["env"]["WORLD_COUNT"] = "2"
sdk_marathon.update_app(marathon_config)
sdk_plan.wait_for_completed_deployment(foldered_name)
# Let's decommission again!
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config["env"]["WORLD_COUNT"] = "1"
sdk_marathon.update_app(marathon_config)
sdk_plan.wait_for_completed_deployment(foldered_name)
sdk_plan.wait_for_completed_plan(foldered_name, "decommission")
decommission_plan = sdk_plan.get_decommission_plan(foldered_name)
log.info(sdk_plan.plan_string("decommission", decommission_plan))
custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"]
assert "custom_decommission_step" == custom_step_name
def test_custom_decommission():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config['env']['WORLD_COUNT'] = '1'
sdk_marathon.update_app(foldered_name, marathon_config)
sdk_plan.wait_for_completed_plan(foldered_name, 'decommission')
decommission_plan = sdk_plan.get_decommission_plan(foldered_name)
log.info("decommission plan: {}".format(decommission_plan))
custom_step_name = decommission_plan['phases'][0]['steps'][0]['name']
assert "custom_decomission_step" == custom_step_name
# scale back up
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config['env']['WORLD_COUNT'] = '2'
sdk_marathon.update_app(foldered_name, marathon_config)
sdk_plan.wait_for_completed_deployment(foldered_name)
# Let's decommission again!
marathon_config = sdk_marathon.get_config(foldered_name)
marathon_config['env']['WORLD_COUNT'] = '1'
sdk_marathon.update_app(foldered_name, marathon_config)
sdk_plan.wait_for_completed_deployment(foldered_name)
sdk_plan.wait_for_completed_plan(foldered_name, 'decommission')
decommission_plan = sdk_plan.get_decommission_plan(foldered_name)
log.info("decommission plan: {}".format(decommission_plan))
custom_step_name = decommission_plan['phases'][0]['steps'][0]['name']
assert "custom_decomission_step" == custom_step_name
def test_increase_decrease_world_nodes():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
config.check_running(foldered_name)
original_hello_ids = sdk_tasks.get_task_ids(foldered_name, 'hello')
original_world_ids = sdk_tasks.get_task_ids(foldered_name, 'world')
log.info('world ids: ' + str(original_world_ids))
# add 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', 2)
config.check_running(foldered_name)
sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids)
# check 2 world tasks added:
assert 2 + len(original_world_ids) == len(sdk_tasks.get_task_ids(foldered_name, 'world'))
# subtract 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', -2)
config.check_running(foldered_name)
# wait for the decommission plan for this subtraction to be complete
sdk_plan.wait_for_completed_plan(foldered_name, 'decommission')
# check that the total task count is back to original
sdk_tasks.check_running(
foldered_name,
len(original_hello_ids) + len(original_world_ids),
allow_more=False)
# check that original tasks weren't affected/relaunched in the process
sdk_tasks.check_tasks_not_updated(foldered_name, 'hello', original_hello_ids)
sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids)
# check that the world tasks are back to their prior state (also without changing task ids)
assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, 'world')
def test_repair_cleanup_plans_complete():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(
node_address=config.get_foldered_node_address()),
config.get_verify_data_job(
node_address=config.get_foldered_node_address())
],
after_jobs=[
config.get_delete_data_job(
node_address=config.get_foldered_node_address()),
config.get_verify_deletion_job(
node_address=config.get_foldered_node_address())
]):
sdk_plan.start_plan(config.get_foldered_service_name(),
'cleanup',
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.get_foldered_service_name(),
'cleanup')
sdk_plan.start_plan(config.get_foldered_service_name(),
'repair',
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.get_foldered_service_name(),
'repair')
def test_tls_connection(cassandra_service_tls, dcos_ca_bundle):
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext([
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)
]):
sdk_jobs.run_job(
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv('AWS_ACCESS_KEY_ID')
if not key_id:
assert False, 'AWS credentials are required for this test. ' \
'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
'AWS_ACCESS_KEY_ID':
key_id,
'AWS_SECRET_ACCESS_KEY':
os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION':
os.getenv('AWS_REGION', 'us-west-2'),
'S3_BUCKET_NAME':
os.getenv('AWS_BUCKET_NAME', 'infinity-framework-test'),
'SNAPSHOT_NAME':
str(uuid.uuid1()),
'CASSANDRA_KEYSPACES':
'"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
'backup-s3',
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'backup-s3')
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
'restore-s3',
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'restore-s3')
with sdk_jobs.InstallJobContext([
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)
]):
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def run_plan(plan_name, params=None):
sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params)
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name)
log.info("sidecar plan: " + str(started_plan))
assert(len(started_plan['phases']) == 1)
assert(started_plan['phases'][0]['name'] == plan_name + '-deploy')
assert(len(started_plan['phases'][0]['steps']) == 2)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def run_plan(plan_name, params=None):
sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params)
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name)
log.info(sdk_plan.plan_string(plan_name, started_plan))
assert len(started_plan["phases"]) == 1
assert started_plan["phases"][0]["name"] == plan_name + "-deploy"
assert len(started_plan["phases"][0]["steps"]) == 2
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def run_plan(plan_name, params=None):
plan.start_plan(PACKAGE_NAME, plan_name, params)
started_plan = plan.get_plan(PACKAGE_NAME, plan_name)
sdk_utils.out("sidecar plan: " + str(started_plan))
assert (len(started_plan['phases']) == 1)
assert (started_plan['phases'][0]['name'] == plan_name + '-deploy')
assert (len(started_plan['phases'][0]['steps']) == 2)
plan.wait_for_completed_plan(PACKAGE_NAME, plan_name)
def test_sidecar():
sdk_plan.start_plan(config.SERVICE_NAME, "sidecar")
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, "sidecar")
log.info(sdk_plan.plan_string("sidecar", started_plan))
assert len(started_plan["phases"]) == 1
assert started_plan["phases"][0]["name"] == "sidecar-deploy"
assert len(started_plan["phases"][0]["steps"]) == 2
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "sidecar")
def test_sidecar():
plan.start_plan(PACKAGE_NAME, 'sidecar')
started_plan = plan.get_plan(PACKAGE_NAME, 'sidecar')
sdk_utils.out("sidecar plan: " + str(started_plan))
assert(len(started_plan['phases']) == 1)
assert(started_plan['phases'][0]['name'] == 'sidecar-deploy')
assert(len(started_plan['phases'][0]['steps']) == 2)
plan.wait_for_completed_plan(PACKAGE_NAME, 'sidecar')
def test_sidecar():
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar')
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'sidecar')
log.info("sidecar plan: " + str(started_plan))
assert (len(started_plan['phases']) == 1)
assert (started_plan['phases'][0]['name'] == 'sidecar-deploy')
assert (len(started_plan['phases'][0]['steps']) == 2)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'sidecar')
def run_plan(plan_name, params=None):
sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params)
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name)
log.info(sdk_plan.plan_string(plan_name, started_plan))
assert len(started_plan["phases"]) == 1
assert started_plan["phases"][0]["name"] == plan_name + "-deploy"
assert len(started_plan["phases"][0]["steps"]) == 2
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def test_sidecar():
sdk_plan.start_plan(config.SERVICE_NAME, "sidecar")
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, "sidecar")
log.info(sdk_plan.plan_string("sidecar", started_plan))
assert len(started_plan["phases"]) == 1
assert started_plan["phases"][0]["name"] == "sidecar-deploy"
assert len(started_plan["phases"][0]["steps"]) == 2
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "sidecar")
def test_sidecar():
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar')
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'sidecar')
log.info("sidecar plan: " + str(started_plan))
assert(len(started_plan['phases']) == 1)
assert(started_plan['phases'][0]['name'] == 'sidecar-deploy')
assert(len(started_plan['phases'][0]['steps']) == 2)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'sidecar')
def run_plan(plan_name, params=None):
sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params)
started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name)
log.info("sidecar plan: " + str(started_plan))
assert (len(started_plan['phases']) == 1)
assert (started_plan['phases'][0]['name'] == plan_name + '-deploy')
assert (len(started_plan['phases'][0]['steps']) == 2)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def test_functionality():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with jobs.RunJobContext(before_jobs=[WRITE_DATA_JOB, VERIFY_DATA_JOB],
after_jobs=[DELETE_DATA_JOB, VERIFY_DELETION_JOB]):
plan.start_plan(PACKAGE_NAME, 'cleanup', parameters=parameters)
plan.wait_for_completed_plan(PACKAGE_NAME, 'cleanup')
plan.start_plan(PACKAGE_NAME, 'repair', parameters=parameters)
plan.wait_for_completed_plan(PACKAGE_NAME, 'repair')
def test_repair_cleanup_plans_complete():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[WRITE_DATA_JOB, VERIFY_DATA_JOB],
after_jobs=[DELETE_DATA_JOB, VERIFY_DELETION_JOB]):
sdk_plan.start_plan(FOLDERED_SERVICE_NAME, 'cleanup', parameters=parameters)
sdk_plan.wait_for_completed_plan(FOLDERED_SERVICE_NAME, 'cleanup')
sdk_plan.start_plan(FOLDERED_SERVICE_NAME, 'repair', parameters=parameters)
sdk_plan.wait_for_completed_plan(FOLDERED_SERVICE_NAME, 'repair')
def test_tls_connection(
cassandra_service: Dict[str, Any],
dcos_ca_bundle: str,
) -> None:
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext(
[
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]
):
sdk_jobs.run_job(config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv("AWS_ACCESS_KEY_ID")
if not key_id:
assert (
False
), "AWS credentials are required for this test. " 'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
"AWS_ACCESS_KEY_ID": key_id,
"AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"),
"AWS_REGION": os.getenv("AWS_REGION", "us-west-2"),
"S3_BUCKET_NAME": os.getenv("AWS_BUCKET_NAME", "infinity-framework-test"),
"SNAPSHOT_NAME": str(uuid.uuid1()),
"CASSANDRA_KEYSPACES": '"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, "backup-s3", parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "backup-s3")
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, "restore-s3", parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "restore-s3")
with sdk_jobs.InstallJobContext(
[
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]
):
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_functionality() -> None:
parameters = {"CASSANDRA_KEYSPACE": "testspace1"}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[config.get_write_data_job(), config.get_verify_data_job()],
after_jobs=[config.get_delete_data_job(), config.get_verify_deletion_job()],
):
sdk_plan.start_plan(config.SERVICE_NAME, "cleanup", parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "cleanup")
sdk_plan.start_plan(config.SERVICE_NAME, "repair", parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "repair")
def run_backup_and_restore_with_auth(
service_name: str,
backup_plan: str,
restore_plan: str,
plan_parameters: Dict[str, Optional[str]],
job_node_address: str = DEFAULT_NODE_ADDRESS,
) -> None:
write_data_job = get_write_data_job(node_address=job_node_address,
auth=True)
verify_data_job = get_verify_data_job(node_address=job_node_address,
auth=True)
delete_data_job = get_delete_data_job(node_address=job_node_address,
auth=True)
verify_deletion_job = get_verify_deletion_job(
node_address=job_node_address, auth=True)
# Ensure the keyspaces we will use aren't present. In practice this should run once and fail
# because the data isn't present. When the job is flagged as failed (due to restart=NEVER),
# the run_job() call will throw.
try:
sdk_jobs.run_job(delete_data_job)
except Exception:
log.info("Error during delete (normal if no stale data)")
log.info(traceback.format_exc())
sdk_jobs.run_job(write_data_job)
sdk_jobs.run_job(verify_data_job)
# Run backup plan, uploading snapshots and schema to the cloud
sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, backup_plan)
# Delete all keyspaces and tables with a metronome job
sdk_jobs.run_job(delete_data_job)
# Verify that the keyspaces and tables were deleted
sdk_jobs.run_job(verify_deletion_job)
# Run restore plan, retrieving snapshots and schema from S3
sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, restore_plan)
# Verify that the data we wrote and then deleted has been restored
sdk_jobs.run_job(verify_data_job)
# Delete data in preparation for any other backup tests
sdk_jobs.run_job(delete_data_job)
sdk_jobs.run_job(verify_deletion_job)
def test_plan_cli():
plan_name = 'deploy'
phase_name = 'world'
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
assert sdk_cmd.svc_cli(config.PACKAGE_NAME,
foldered_name,
'plan list',
json=True)
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
'plan show {}'.format(plan_name))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME,
foldered_name,
'plan show --json {}'.format(plan_name),
json=True)
assert sdk_cmd.svc_cli(config.PACKAGE_NAME,
foldered_name,
'plan show {} --json'.format(plan_name),
json=True)
# trigger a restart so that the plan is in a non-complete state.
# the 'interrupt' command will fail if the plan is already complete:
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
'plan force-restart {}'.format(plan_name))
assert sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name,
'plan interrupt {} {}'.format(plan_name, phase_name))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
'plan continue {} {}'.format(plan_name, phase_name))
# now wait for plan to finish before continuing to other tests:
assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def test_canary_fourth():
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'plan continue deploy world-deploy')
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3',
'world-0', 'world-1', 'world-2', 'world-3']
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod list', json=True) == expected_tasks
pl = sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'deploy')
log.info(pl)
assert pl['status'] == 'COMPLETE'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
def test_canary_fourth():
sdk_cmd.run_cli('hello-world plan continue deploy world-deploy')
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3', 'world-0', 'world-1',
'world-2', 'world-3'
]
sdk_tasks.check_running(config.PACKAGE_NAME, len(expected_tasks))
assert json.loads(
sdk_cmd.run_cli('hello-world pod list')) == expected_tasks
pl = sdk_plan.wait_for_completed_plan(config.PACKAGE_NAME, 'deploy')
log.info(pl)
assert pl['status'] == 'COMPLETE'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
def test_plan_cli():
plan_name = "deploy"
phase_name = "world"
_check_json_output(foldered_name, "plan list")
rc, _, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
"plan show {}".format(plan_name))
assert rc == 0
_check_json_output(foldered_name, "plan show --json {}".format(plan_name))
_check_json_output(foldered_name, "plan show {} --json".format(plan_name))
# trigger a restart so that the plan is in a non-complete state.
# the 'interrupt' command will fail if the plan is already complete:
rc, _, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
"plan force-restart {}".format(plan_name))
assert rc == 0
rc, _, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name,
"plan interrupt {} {}".format(plan_name, phase_name))
assert rc == 0
rc, _, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name,
"plan continue {} {}".format(plan_name, phase_name))
assert rc == 0
# now wait for plan to finish before continuing to other tests:
assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def test_plan_cli():
plan_name = "deploy"
phase_name = "world"
_check_json_output(foldered_name, "plan list")
rc, _, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, "plan show {}".format(plan_name))
assert rc == 0
_check_json_output(foldered_name, "plan show --json {}".format(plan_name))
_check_json_output(foldered_name, "plan show {} --json".format(plan_name))
# trigger a restart so that the plan is in a non-complete state.
# the 'interrupt' command will fail if the plan is already complete:
rc, _, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name, "plan force-restart {}".format(plan_name)
)
assert rc == 0
rc, _, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name, "plan interrupt {} {}".format(plan_name, phase_name)
)
assert rc == 0
rc, _, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name, "plan continue {} {}".format(plan_name, phase_name)
)
assert rc == 0
# now wait for plan to finish before continuing to other tests:
assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def run_backup_and_restore(
service_name: str,
backup_plan: str,
restore_plan: str,
plan_parameters: Dict[str, Optional[str]],
job_node_address: str = DEFAULT_NODE_ADDRESS,
) -> None:
write_data_job = get_write_data_job(node_address=job_node_address)
verify_data_job = get_verify_data_job(node_address=job_node_address)
delete_data_job = get_delete_data_job(node_address=job_node_address)
verify_deletion_job = get_verify_deletion_job(node_address=job_node_address)
# Ensure the keyspaces we will use aren't present. In practice this should run once and fail
# because the data isn't present. When the job is flagged as failed (due to restart=NEVER),
# the run_job() call will throw.
try:
sdk_jobs.run_job(delete_data_job)
except Exception:
log.info("Error during delete (normal if no stale data)")
log.info(traceback.format_exc())
# Write data to Cassandra with a metronome job, then verify it was written
# Note: Write job will fail if data already exists
sdk_jobs.run_job(write_data_job)
sdk_jobs.run_job(verify_data_job)
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, backup_plan)
# Delete all keyspaces and tables with a metronome job
sdk_jobs.run_job(delete_data_job)
# Verify that the keyspaces and tables were deleted
sdk_jobs.run_job(verify_deletion_job)
# Run restore plan, retrieving snapshots and schema from S3
sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, restore_plan)
# Verify that the data we wrote and then deleted has been restored
sdk_jobs.run_job(verify_data_job)
# Delete data in preparation for any other backup tests
sdk_jobs.run_job(delete_data_job)
sdk_jobs.run_job(verify_deletion_job)
def run_backup_and_restore(
service_name,
backup_plan,
restore_plan,
plan_parameters,
job_node_address=DEFAULT_NODE_ADDRESS):
write_data_job = get_write_data_job(node_address=job_node_address)
verify_data_job = get_verify_data_job(node_address=job_node_address)
delete_data_job = get_delete_data_job(node_address=job_node_address)
verify_deletion_job = get_verify_deletion_job(node_address=job_node_address)
# Ensure the keyspaces we will use aren't present.
try:
jobs.run_job(delete_data_job)
except:
log.info("Error during delete (normal if no stale data).")
tb = traceback.format_exc()
log.info(tb)
# Write data to Cassandra with a metronome job, then verify it was written
# Note: Write job will fail if data already exists
sdk_jobs.run_job(write_data_job)
sdk_jobs.run_job(verify_data_job)
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, backup_plan)
# Delete all keyspaces and tables with a metronome job
sdk_jobs.run_job(delete_data_job)
# Verify that the keyspaces and tables were deleted
sdk_jobs.run_job(verify_deletion_job)
# Run restore plan, retrieving snapshots and schema from S3
sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, restore_plan)
# Verify that the data we wrote and then deleted has been restored
sdk_jobs.run_job(verify_data_job)
# Delete data in preparation for any other backup tests
sdk_jobs.run_job(delete_data_job)
sdk_jobs.run_job(verify_deletion_job)
def test_toxic_sidecar_doesnt_trigger_recovery():
# 1. Run the toxic sidecar plan that will never succeed.
# 2. Restart the scheduler.
# 3. Verify that its recovery plan is empty, as a failed ONCE task should
# never trigger recovery
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert (len(recovery_plan['phases']) == 0)
log.info(recovery_plan)
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic')
wait_for_toxic_sidecar()
# Restart the scheduler and wait for it to come up.
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Now, verify that its recovery plan is empty.
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'recovery')
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert (len(recovery_plan['phases']) == 0)
def test_toxic_sidecar_doesnt_trigger_recovery():
# 1. Run the toxic sidecar plan that will never succeed.
# 2. Restart the scheduler.
# 3. Verify that its recovery plan is empty, as a failed FINISHED task should
# never trigger recovery
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert(len(recovery_plan['phases']) == 0)
log.info(recovery_plan)
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic')
shakedown.wait_for(ToxicSidecarCheck(), timeout_seconds=10 * 60)
# Restart the scheduler and wait for it to come up.
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Now, verify that its recovery plan is empty.
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'recovery')
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert(len(recovery_plan['phases']) == 0)
def test_functionality():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(),
config.get_verify_data_job()
],
after_jobs=[
config.get_delete_data_job(),
config.get_verify_deletion_job()
]):
sdk_plan.start_plan(config.SERVICE_NAME, 'cleanup', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'cleanup')
sdk_plan.start_plan(config.SERVICE_NAME, 'repair', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'repair')
def test_functionality():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(),
config.get_verify_data_job()
],
after_jobs=[
config.get_delete_data_job(),
config.get_verify_deletion_job()
]):
sdk_plan.start_plan(config.SERVICE_NAME, 'cleanup', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'cleanup')
sdk_plan.start_plan(config.SERVICE_NAME, 'repair', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'repair')
def test_tls_connection(cassandra_service_tls, dcos_ca_bundle):
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext([
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)]):
sdk_jobs.run_job(config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv('AWS_ACCESS_KEY_ID')
if not key_id:
assert False, 'AWS credentials are required for this test. ' \
'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
'AWS_ACCESS_KEY_ID': key_id,
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'S3_BUCKET_NAME': os.getenv('AWS_BUCKET_NAME', 'infinity-framework-test'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, 'backup-s3', parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'backup-s3')
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, 'restore-s3', parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'restore-s3')
with sdk_jobs.InstallJobContext([
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)]):
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_increase_decrease_world_nodes():
original_hello_ids = sdk_tasks.get_task_ids(foldered_name, "hello")
original_world_ids = sdk_tasks.get_task_ids(foldered_name, "world")
log.info("world ids: " + str(original_world_ids))
# add 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", 2)
# autodetects the correct number of nodes and waits for them to deploy:
config.check_running(foldered_name)
sdk_plan.wait_for_completed_deployment(foldered_name)
sdk_tasks.check_tasks_not_updated(foldered_name, "world",
original_world_ids)
# check 2 world tasks added:
assert 2 + len(original_world_ids) == len(
sdk_tasks.get_task_ids(foldered_name, "world"))
# subtract 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", -2)
# autodetects the correct number of nodes and waits for them to deploy:
config.check_running(foldered_name)
sdk_plan.wait_for_completed_deployment(foldered_name)
# wait for the decommission plan for this subtraction to be complete
sdk_plan.wait_for_completed_plan(foldered_name, "decommission")
# check that the total task count is back to original
sdk_tasks.check_running(foldered_name,
len(original_hello_ids) + len(original_world_ids),
allow_more=False)
# check that original tasks weren't affected/relaunched in the process
sdk_tasks.check_tasks_not_updated(foldered_name, "hello",
original_hello_ids)
sdk_tasks.check_tasks_not_updated(foldered_name, "world",
original_world_ids)
# check that the world tasks are back to their prior state (also without changing task ids)
assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, "world")
def test_increase_decrease_world_nodes():
original_hello_ids = sdk_tasks.get_task_ids(foldered_name, "hello")
original_world_ids = sdk_tasks.get_task_ids(foldered_name, "world")
log.info("world ids: " + str(original_world_ids))
# add 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", 2)
# autodetects the correct number of nodes and waits for them to deploy:
config.check_running(foldered_name)
sdk_plan.wait_for_completed_deployment(foldered_name)
sdk_tasks.check_tasks_not_updated(foldered_name, "world", original_world_ids)
# check 2 world tasks added:
assert 2 + len(original_world_ids) == len(sdk_tasks.get_task_ids(foldered_name, "world"))
# subtract 2 world nodes
sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", -2)
# autodetects the correct number of nodes and waits for them to deploy:
config.check_running(foldered_name)
sdk_plan.wait_for_completed_deployment(foldered_name)
# wait for the decommission plan for this subtraction to be complete
sdk_plan.wait_for_completed_plan(foldered_name, "decommission")
# check that the total task count is back to original
sdk_tasks.check_running(
foldered_name, len(original_hello_ids) + len(original_world_ids), allow_more=False
)
# check that original tasks weren't affected/relaunched in the process
sdk_tasks.check_tasks_not_updated(foldered_name, "hello", original_hello_ids)
sdk_tasks.check_tasks_not_updated(foldered_name, "world", original_world_ids)
# check that the world tasks are back to their prior state (also without changing task ids)
assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, "world")
def run_backup_and_restore(
service_name,
backup_plan,
restore_plan,
plan_parameters,
job_node_address=DEFAULT_NODE_ADDRESS):
write_data_job = get_write_data_job(node_address=job_node_address)
verify_data_job = get_verify_data_job(node_address=job_node_address)
delete_data_job = get_delete_data_job(node_address=job_node_address)
verify_deletion_job = get_verify_deletion_job(node_address=job_node_address)
# Write data to Cassandra with a metronome job, then verify it was written
# Note: Write job will fail if data already exists
sdk_jobs.run_job(write_data_job)
sdk_jobs.run_job(verify_data_job)
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, backup_plan)
# Delete all keyspaces and tables with a metronome job
sdk_jobs.run_job(delete_data_job)
# Verify that the keyspaces and tables were deleted
sdk_jobs.run_job(verify_deletion_job)
# Run restore plan, retrieving snapshots and schema from S3
sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(service_name, restore_plan)
# Verify that the data we wrote and then deleted has been restored
sdk_jobs.run_job(verify_data_job)
# Delete data in preparation for any other backup tests
sdk_jobs.run_job(delete_data_job)
sdk_jobs.run_job(verify_deletion_job)
def run_backup_and_restore(service_name,
backup_plan,
restore_plan,
plan_parameters,
job_node_address=DEFAULT_NODE_ADDRESS):
write_data_job = get_write_data_job(node_address=job_node_address)
verify_data_job = get_verify_data_job(node_address=job_node_address)
delete_data_job = get_delete_data_job(node_address=job_node_address)
verify_deletion_job = get_verify_deletion_job(
node_address=job_node_address)
# Write data to Cassandra with a metronome job, then verify it was written
# Note: Write job will fail if data already exists
jobs.run_job(write_data_job)
jobs.run_job(verify_data_job)
# Run backup plan, uploading snapshots and schema to the cloudddd
plan.start_plan(service_name, backup_plan, parameters=plan_parameters)
plan.wait_for_completed_plan(service_name, backup_plan)
# Delete all keyspaces and tables with a metronome job
jobs.run_job(delete_data_job)
# Verify that the keyspaces and tables were deleted
jobs.run_job(verify_deletion_job)
# Run restore plan, retrieving snapshots and schema from S3
plan.start_plan(service_name, restore_plan, parameters=plan_parameters)
plan.wait_for_completed_plan(service_name, restore_plan)
# Verify that the data we wrote and then deleted has been restored
jobs.run_job(verify_data_job)
# Delete data in preparation for any other backup tests
jobs.run_job(delete_data_job)
jobs.run_job(verify_deletion_job)
def test_functionality() -> None:
parameters = {"CASSANDRA_KEYSPACE": "testspace1"}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(),
config.get_verify_data_job()
],
after_jobs=[
config.get_delete_data_job(),
config.get_verify_deletion_job()
],
):
sdk_plan.start_plan(config.SERVICE_NAME,
"cleanup",
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "cleanup")
sdk_plan.start_plan(config.SERVICE_NAME,
"repair",
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "repair")
def test_canary_fourth():
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"plan continue deploy world-deploy")
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"world-0",
"world-1",
"world-2",
"world-3",
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
pl = sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "deploy")
log.info(pl)
assert pl["status"] == "COMPLETE"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
def test_canary_fourth():
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "plan continue deploy world-deploy")
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"world-0",
"world-1",
"world-2",
"world-3",
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
pl = sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "deploy")
log.info(pl)
assert pl["status"] == "COMPLETE"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
def test_plan_cli():
plan_name = 'deploy'
phase_name = 'world'
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan list', json=True)
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan show {}'.format(plan_name))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
'plan show --json {}'.format(plan_name), json=True)
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
'plan show {} --json'.format(plan_name), json=True)
# trigger a restart so that the plan is in a non-complete state.
# the 'interrupt' command will fail if the plan is already complete:
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan force-restart {}'.format(plan_name))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
'plan interrupt {} {}'.format(plan_name, phase_name))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
'plan continue {} {}'.format(plan_name, phase_name))
# now wait for plan to finish before continuing to other tests:
assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def test_cassandra_migration():
backup_service_name = os.getenv('CASSANDRA_BACKUP_CLUSTER_NAME')
restore_service_name = os.getenv('CASSANDRA_RESTORE_CLUSTER_NAME')
backup_node_address = os.getenv('BACKUP_NODE_ADDRESS',
DEFAULT_NODE_ADDRESS)
backup_node_port = os.getenv('BACKUP_NODE_PORT', DEFAULT_NODE_PORT)
backup_write_data_job = get_write_data_job(backup_node_address,
backup_node_port)
backup_verify_data_job = get_verify_data_job(backup_node_address,
backup_node_port)
backup_delete_data_job = get_delete_data_job(backup_node_address,
backup_node_port)
backup_verify_deletion_job = get_verify_deletion_job(
backup_node_address, backup_node_port)
plan_parameters = {
'S3_BUCKET_NAME': os.getenv('AWS_BUCKET_NAME',
'infinity-framework-test'),
'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'),
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
backup_install_job_context = jobs.InstallJobContext([
backup_write_data_job, backup_verify_data_job, backup_delete_data_job,
backup_verify_deletion_job
])
backup_run_job_context = jobs.RunJobContext(
before_jobs=[backup_write_data_job, backup_verify_data_job],
after_jobs=[backup_delete_data_job, backup_verify_deletion_job])
# Install and run the write/delete data jobs against backup cluster,
# running dcos-cassandra-service
with backup_install_job_context, backup_run_job_context:
# Back this cluster up to S3
backup_parameters = {
'backup_name':
plan_parameters['SNAPSHOT_NAME'],
's3_access_key':
plan_parameters['AWS_ACCESS_KEY_ID'],
's3_secret_key':
plan_parameters['AWS_SECRET_ACCESS_KEY'],
'external_location':
's3://{}'.format(plan_parameters['S3_BUCKET_NAME']),
}
dcos.http.put('{}v1/backup/start'.format(
shakedown.dcos_service_url(backup_service_name)),
json=backup_parameters)
plan.wait_for_completed_deployment(backup_service_name)
# Restore data to second instance:
restore_node_address = os.getenv(
'RESTORE_NODE_ADDRESS',
hosts.autoip_host('sdk-cassandra', 'node-0-server'))
restore_node_port = os.getenv('RESTORE_NODE_PORT', '9052')
restore_write_data_job = get_write_data_job(restore_node_address,
restore_node_port)
restore_verify_data_job = get_verify_data_job(restore_node_address,
restore_node_port)
restore_delete_data_job = get_delete_data_job(restore_node_address,
restore_node_port)
restore_verify_deletion_job = get_verify_deletion_job(
restore_node_address, restore_node_port)
restore_install_job_context = jobs.InstallJobContext([
restore_write_data_job, restore_verify_data_job,
restore_delete_data_job, restore_verify_deletion_job
])
restore_run_job_context = jobs.RunJobContext(after_jobs=[
restore_verify_data_job, restore_delete_data_job,
restore_verify_deletion_job
])
with restore_install_job_context, restore_run_job_context:
plan.start_plan(restore_service_name,
'restore-s3',
parameters=plan_parameters)
plan.wait_for_completed_plan(restore_service_name, 'restore-s3')
def test_cassandra_migration():
backup_service_name = os.getenv('CASSANDRA_BACKUP_CLUSTER_NAME')
restore_service_name = os.getenv('CASSANDRA_RESTORE_CLUSTER_NAME')
backup_node_address = os.getenv('BACKUP_NODE_ADDRESS', config.DEFAULT_NODE_ADDRESS)
backup_node_port = os.getenv('BACKUP_NODE_PORT', config.DEFAULT_NODE_PORT)
backup_write_data_job = config.get_write_data_job(backup_node_address, backup_node_port)
backup_verify_data_job = config.get_verify_data_job(backup_node_address, backup_node_port)
backup_delete_data_job = config.get_delete_data_job(backup_node_address, backup_node_port)
backup_verify_deletion_job = config.get_verify_deletion_job(backup_node_address, backup_node_port)
plan_parameters = {
'S3_BUCKET_NAME': os.getenv(
'AWS_BUCKET_NAME', 'infinity-framework-test'
),
'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'),
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
backup_install_job_context = sdk_jobs.InstallJobContext(
[backup_write_data_job, backup_verify_data_job,
backup_delete_data_job, backup_verify_deletion_job])
backup_run_job_context = sdk_jobs.RunJobContext(
before_jobs=[backup_write_data_job, backup_verify_data_job],
after_jobs=[backup_delete_data_job, backup_verify_deletion_job])
# Install and run the write/delete data jobs against backup cluster,
# running dcos-cassandra-service
with backup_install_job_context, backup_run_job_context:
# Back this cluster up to S3
backup_parameters = {
'backup_name': plan_parameters['SNAPSHOT_NAME'],
's3_access_key': plan_parameters['AWS_ACCESS_KEY_ID'],
's3_secret_key': plan_parameters['AWS_SECRET_ACCESS_KEY'],
'external_location': 's3://{}'.format(plan_parameters['S3_BUCKET_NAME']),
}
sdk_cmd.service_request('PUT', backup_service_name, '/v1/backup/start', json=backup_parameters)
sdk_plan.wait_for_completed_deployment(backup_service_name)
# Restore data to second instance:
restore_node_address = os.getenv(
'RESTORE_NODE_ADDRESS', sdk_hosts.autoip_host('sdk-cassandra', 'node-0-server'))
restore_node_port = os.getenv('RESTORE_NODE_PORT', '9052')
restore_write_data_job = config.get_write_data_job(restore_node_address, restore_node_port)
restore_verify_data_job = config.get_verify_data_job(restore_node_address, restore_node_port)
restore_delete_data_job = config.get_delete_data_job(restore_node_address, restore_node_port)
restore_verify_deletion_job = config.get_verify_deletion_job(restore_node_address, restore_node_port)
restore_install_job_context = sdk_jobs.InstallJobContext(
[restore_write_data_job, restore_verify_data_job,
restore_delete_data_job, restore_verify_deletion_job]
)
restore_run_job_context = sdk_jobs.RunJobContext(
after_jobs=[restore_verify_data_job, restore_delete_data_job, restore_verify_deletion_job]
)
with restore_install_job_context, restore_run_job_context:
sdk_plan.start_plan(
restore_service_name, 'restore-s3', parameters=plan_parameters
)
sdk_plan.wait_for_completed_plan(restore_service_name, 'restore-s3')
def test_tls_connection(
cassandra_service: Dict[str, Any],
dcos_ca_bundle: str,
) -> None:
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext([
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]):
sdk_jobs.run_job(
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv("AWS_ACCESS_KEY_ID")
if not key_id:
assert (
False
), "AWS credentials are required for this test. " 'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
"AWS_ACCESS_KEY_ID":
key_id,
"AWS_SECRET_ACCESS_KEY":
os.getenv("AWS_SECRET_ACCESS_KEY"),
"AWS_REGION":
os.getenv("AWS_REGION", "us-west-2"),
"S3_BUCKET_NAME":
os.getenv("AWS_BUCKET_NAME", "infinity-framework-test"),
"SNAPSHOT_NAME":
str(uuid.uuid1()),
"CASSANDRA_KEYSPACES":
'"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
"backup-s3",
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "backup-s3")
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
"restore-s3",
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "restore-s3")
with sdk_jobs.InstallJobContext([
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]):
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
