def test_custom_decommission(): foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) marathon_config = sdk_marathon.get_config(foldered_name) marathon_config["env"]["WORLD_COUNT"] = "1" sdk_marathon.update_app(marathon_config) sdk_plan.wait_for_completed_plan(foldered_name, "decommission") decommission_plan = sdk_plan.get_decommission_plan(foldered_name) log.info(sdk_plan.plan_string("decommission", decommission_plan)) custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"] assert "custom_decommission_step" == custom_step_name # scale back up marathon_config = sdk_marathon.get_config(foldered_name) marathon_config["env"]["WORLD_COUNT"] = "2" sdk_marathon.update_app(marathon_config) sdk_plan.wait_for_completed_deployment(foldered_name) # Let's decommission again! marathon_config = sdk_marathon.get_config(foldered_name) marathon_config["env"]["WORLD_COUNT"] = "1" sdk_marathon.update_app(marathon_config) sdk_plan.wait_for_completed_deployment(foldered_name) sdk_plan.wait_for_completed_plan(foldered_name, "decommission") decommission_plan = sdk_plan.get_decommission_plan(foldered_name) log.info(sdk_plan.plan_string("decommission", decommission_plan)) custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"] assert "custom_decommission_step" == custom_step_name
def test_increase_decrease_world_nodes(): foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) config.check_running(foldered_name) original_hello_ids = sdk_tasks.get_task_ids(foldered_name, 'hello') original_world_ids = sdk_tasks.get_task_ids(foldered_name, 'world') log.info('world ids: ' + str(original_world_ids)) # add 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', 2) config.check_running(foldered_name) sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids) # check 2 world tasks added: assert 2 + len(original_world_ids) == len(sdk_tasks.get_task_ids(foldered_name, 'world')) # subtract 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', -2) config.check_running(foldered_name) # wait for the decommission plan for this subtraction to be complete sdk_plan.wait_for_completed_plan(foldered_name, 'decommission') # check that the total task count is back to original sdk_tasks.check_running( foldered_name, len(original_hello_ids) + len(original_world_ids), allow_more=False) # check that original tasks weren't affected/relaunched in the process sdk_tasks.check_tasks_not_updated(foldered_name, 'hello', original_hello_ids) sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids) # check that the world tasks are back to their prior state (also without changing task ids) assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, 'world')
def test_all_tasks_are_launched(): service_options = {"service": {"yaml": "plan"}} sdk_install.install(config.PACKAGE_NAME, foldered_name, 0, additional_options=service_options, wait_for_deployment=False, wait_for_all_conditions=True) # after above method returns, start all plans right away. plans = ["manual-plan-0", "manual-plan-1", "manual-plan-2"] for plan in plans: sdk_plan.start_plan(foldered_name, plan) for plan in plans: sdk_plan.wait_for_completed_plan(foldered_name, plan) pods = ["custom-pod-A-0", "custom-pod-B-0", "custom-pod-C-0"] for pod in pods: # /pod/<pod-id>/info fetches data from SDK's persistence layer pod_hello_0_info = sdk_cmd.service_request( "GET", foldered_name, "/v1/pod/{}/info".format(pod)).json() for taskInfoAndStatus in pod_hello_0_info: info = taskInfoAndStatus["info"] status = taskInfoAndStatus["status"] # While `info` object is always present, `status` may or may not be present based # on whether the task was launched and we received an update from mesos (or not). if status: assert info["taskId"]["value"] == status["taskId"]["value"] assert len(info["taskId"]["value"]) > 0 else: assert len(info["taskId"]["value"]) == 0
def test_custom_decommission(): foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) marathon_config = sdk_marathon.get_config(foldered_name) marathon_config["env"]["WORLD_COUNT"] = "1" sdk_marathon.update_app(marathon_config) sdk_plan.wait_for_completed_plan(foldered_name, "decommission") decommission_plan = sdk_plan.get_decommission_plan(foldered_name) log.info(sdk_plan.plan_string("decommission", decommission_plan)) custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"] assert "custom_decommission_step" == custom_step_name # scale back up marathon_config = sdk_marathon.get_config(foldered_name) marathon_config["env"]["WORLD_COUNT"] = "2" sdk_marathon.update_app(marathon_config) sdk_plan.wait_for_completed_deployment(foldered_name) # Let's decommission again! marathon_config = sdk_marathon.get_config(foldered_name) marathon_config["env"]["WORLD_COUNT"] = "1" sdk_marathon.update_app(marathon_config) sdk_plan.wait_for_completed_deployment(foldered_name) sdk_plan.wait_for_completed_plan(foldered_name, "decommission") decommission_plan = sdk_plan.get_decommission_plan(foldered_name) log.info(sdk_plan.plan_string("decommission", decommission_plan)) custom_step_name = decommission_plan["phases"][0]["steps"][0]["name"] assert "custom_decommission_step" == custom_step_name
def test_custom_decommission(): foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) marathon_config = sdk_marathon.get_config(foldered_name) marathon_config['env']['WORLD_COUNT'] = '1' sdk_marathon.update_app(foldered_name, marathon_config) sdk_plan.wait_for_completed_plan(foldered_name, 'decommission') decommission_plan = sdk_plan.get_decommission_plan(foldered_name) log.info("decommission plan: {}".format(decommission_plan)) custom_step_name = decommission_plan['phases'][0]['steps'][0]['name'] assert "custom_decomission_step" == custom_step_name # scale back up marathon_config = sdk_marathon.get_config(foldered_name) marathon_config['env']['WORLD_COUNT'] = '2' sdk_marathon.update_app(foldered_name, marathon_config) sdk_plan.wait_for_completed_deployment(foldered_name) # Let's decommission again! marathon_config = sdk_marathon.get_config(foldered_name) marathon_config['env']['WORLD_COUNT'] = '1' sdk_marathon.update_app(foldered_name, marathon_config) sdk_plan.wait_for_completed_deployment(foldered_name) sdk_plan.wait_for_completed_plan(foldered_name, 'decommission') decommission_plan = sdk_plan.get_decommission_plan(foldered_name) log.info("decommission plan: {}".format(decommission_plan)) custom_step_name = decommission_plan['phases'][0]['steps'][0]['name'] assert "custom_decomission_step" == custom_step_name
def test_increase_decrease_world_nodes(): foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) config.check_running(foldered_name) original_hello_ids = sdk_tasks.get_task_ids(foldered_name, 'hello') original_world_ids = sdk_tasks.get_task_ids(foldered_name, 'world') log.info('world ids: ' + str(original_world_ids)) # add 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', 2) config.check_running(foldered_name) sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids) # check 2 world tasks added: assert 2 + len(original_world_ids) == len(sdk_tasks.get_task_ids(foldered_name, 'world')) # subtract 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, 'WORLD_COUNT', -2) config.check_running(foldered_name) # wait for the decommission plan for this subtraction to be complete sdk_plan.wait_for_completed_plan(foldered_name, 'decommission') # check that the total task count is back to original sdk_tasks.check_running( foldered_name, len(original_hello_ids) + len(original_world_ids), allow_more=False) # check that original tasks weren't affected/relaunched in the process sdk_tasks.check_tasks_not_updated(foldered_name, 'hello', original_hello_ids) sdk_tasks.check_tasks_not_updated(foldered_name, 'world', original_world_ids) # check that the world tasks are back to their prior state (also without changing task ids) assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, 'world')
def test_repair_cleanup_plans_complete(): parameters = {'CASSANDRA_KEYSPACE': 'testspace1'} # populate 'testspace1' for test, then delete afterwards: with sdk_jobs.RunJobContext( before_jobs=[ config.get_write_data_job( node_address=config.get_foldered_node_address()), config.get_verify_data_job( node_address=config.get_foldered_node_address()) ], after_jobs=[ config.get_delete_data_job( node_address=config.get_foldered_node_address()), config.get_verify_deletion_job( node_address=config.get_foldered_node_address()) ]): sdk_plan.start_plan(config.get_foldered_service_name(), 'cleanup', parameters=parameters) sdk_plan.wait_for_completed_plan(config.get_foldered_service_name(), 'cleanup') sdk_plan.start_plan(config.get_foldered_service_name(), 'repair', parameters=parameters) sdk_plan.wait_for_completed_plan(config.get_foldered_service_name(), 'repair')
def test_tls_connection(cassandra_service_tls, dcos_ca_bundle): """ Tests writing, reading and deleting data over a secure TLS connection. """ with sdk_jobs.InstallJobContext([ config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle) ]): sdk_jobs.run_job( config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job( config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) key_id = os.getenv('AWS_ACCESS_KEY_ID') if not key_id: assert False, 'AWS credentials are required for this test. ' \ 'Disable test with e.g. TEST_TYPES="sanity and not aws"' plan_parameters = { 'AWS_ACCESS_KEY_ID': key_id, 'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'), 'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'), 'S3_BUCKET_NAME': os.getenv('AWS_BUCKET_NAME', 'infinity-framework-test'), 'SNAPSHOT_NAME': str(uuid.uuid1()), 'CASSANDRA_KEYSPACES': '"testspace1 testspace2"', } # Run backup plan, uploading snapshots and schema to the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, 'backup-s3', parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'backup-s3') sdk_jobs.run_job( config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)) # Run restore plan, downloading snapshots and schema from the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, 'restore-s3', parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'restore-s3') with sdk_jobs.InstallJobContext([ config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle) ]): sdk_jobs.run_job( config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job( config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def run_plan(plan_name, params=None): sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params) started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name) log.info("sidecar plan: " + str(started_plan)) assert(len(started_plan['phases']) == 1) assert(started_plan['phases'][0]['name'] == plan_name + '-deploy') assert(len(started_plan['phases'][0]['steps']) == 2) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def run_plan(plan_name, params=None): sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params) started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name) log.info(sdk_plan.plan_string(plan_name, started_plan)) assert len(started_plan["phases"]) == 1 assert started_plan["phases"][0]["name"] == plan_name + "-deploy" assert len(started_plan["phases"][0]["steps"]) == 2 sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def run_plan(plan_name, params=None): plan.start_plan(PACKAGE_NAME, plan_name, params) started_plan = plan.get_plan(PACKAGE_NAME, plan_name) sdk_utils.out("sidecar plan: " + str(started_plan)) assert (len(started_plan['phases']) == 1) assert (started_plan['phases'][0]['name'] == plan_name + '-deploy') assert (len(started_plan['phases'][0]['steps']) == 2) plan.wait_for_completed_plan(PACKAGE_NAME, plan_name)
def test_sidecar(): sdk_plan.start_plan(config.SERVICE_NAME, "sidecar") started_plan = sdk_plan.get_plan(config.SERVICE_NAME, "sidecar") log.info(sdk_plan.plan_string("sidecar", started_plan)) assert len(started_plan["phases"]) == 1 assert started_plan["phases"][0]["name"] == "sidecar-deploy" assert len(started_plan["phases"][0]["steps"]) == 2 sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "sidecar")
def test_sidecar(): plan.start_plan(PACKAGE_NAME, 'sidecar') started_plan = plan.get_plan(PACKAGE_NAME, 'sidecar') sdk_utils.out("sidecar plan: " + str(started_plan)) assert(len(started_plan['phases']) == 1) assert(started_plan['phases'][0]['name'] == 'sidecar-deploy') assert(len(started_plan['phases'][0]['steps']) == 2) plan.wait_for_completed_plan(PACKAGE_NAME, 'sidecar')
def test_sidecar(): sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar') started_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'sidecar') log.info("sidecar plan: " + str(started_plan)) assert (len(started_plan['phases']) == 1) assert (started_plan['phases'][0]['name'] == 'sidecar-deploy') assert (len(started_plan['phases'][0]['steps']) == 2) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'sidecar')
def run_plan(plan_name, params=None): sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params) started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name) log.info(sdk_plan.plan_string(plan_name, started_plan)) assert len(started_plan["phases"]) == 1 assert started_plan["phases"][0]["name"] == plan_name + "-deploy" assert len(started_plan["phases"][0]["steps"]) == 2 sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def test_sidecar(): sdk_plan.start_plan(config.SERVICE_NAME, "sidecar") started_plan = sdk_plan.get_plan(config.SERVICE_NAME, "sidecar") log.info(sdk_plan.plan_string("sidecar", started_plan)) assert len(started_plan["phases"]) == 1 assert started_plan["phases"][0]["name"] == "sidecar-deploy" assert len(started_plan["phases"][0]["steps"]) == 2 sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "sidecar")
def test_sidecar(): sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar') started_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'sidecar') log.info("sidecar plan: " + str(started_plan)) assert(len(started_plan['phases']) == 1) assert(started_plan['phases'][0]['name'] == 'sidecar-deploy') assert(len(started_plan['phases'][0]['steps']) == 2) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'sidecar')
def run_plan(plan_name, params=None): sdk_plan.start_plan(config.SERVICE_NAME, plan_name, params) started_plan = sdk_plan.get_plan(config.SERVICE_NAME, plan_name) log.info("sidecar plan: " + str(started_plan)) assert (len(started_plan['phases']) == 1) assert (started_plan['phases'][0]['name'] == plan_name + '-deploy') assert (len(started_plan['phases'][0]['steps']) == 2) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, plan_name)
def test_functionality(): parameters = {'CASSANDRA_KEYSPACE': 'testspace1'} # populate 'testspace1' for test, then delete afterwards: with jobs.RunJobContext(before_jobs=[WRITE_DATA_JOB, VERIFY_DATA_JOB], after_jobs=[DELETE_DATA_JOB, VERIFY_DELETION_JOB]): plan.start_plan(PACKAGE_NAME, 'cleanup', parameters=parameters) plan.wait_for_completed_plan(PACKAGE_NAME, 'cleanup') plan.start_plan(PACKAGE_NAME, 'repair', parameters=parameters) plan.wait_for_completed_plan(PACKAGE_NAME, 'repair')
def test_repair_cleanup_plans_complete(): parameters = {'CASSANDRA_KEYSPACE': 'testspace1'} # populate 'testspace1' for test, then delete afterwards: with sdk_jobs.RunJobContext( before_jobs=[WRITE_DATA_JOB, VERIFY_DATA_JOB], after_jobs=[DELETE_DATA_JOB, VERIFY_DELETION_JOB]): sdk_plan.start_plan(FOLDERED_SERVICE_NAME, 'cleanup', parameters=parameters) sdk_plan.wait_for_completed_plan(FOLDERED_SERVICE_NAME, 'cleanup') sdk_plan.start_plan(FOLDERED_SERVICE_NAME, 'repair', parameters=parameters) sdk_plan.wait_for_completed_plan(FOLDERED_SERVICE_NAME, 'repair')
def test_tls_connection( cassandra_service: Dict[str, Any], dcos_ca_bundle: str, ) -> None: """ Tests writing, reading and deleting data over a secure TLS connection. """ with sdk_jobs.InstallJobContext( [ config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle), ] ): sdk_jobs.run_job(config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) key_id = os.getenv("AWS_ACCESS_KEY_ID") if not key_id: assert ( False ), "AWS credentials are required for this test. " 'Disable test with e.g. TEST_TYPES="sanity and not aws"' plan_parameters = { "AWS_ACCESS_KEY_ID": key_id, "AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"), "AWS_REGION": os.getenv("AWS_REGION", "us-west-2"), "S3_BUCKET_NAME": os.getenv("AWS_BUCKET_NAME", "infinity-framework-test"), "SNAPSHOT_NAME": str(uuid.uuid1()), "CASSANDRA_KEYSPACES": '"testspace1 testspace2"', } # Run backup plan, uploading snapshots and schema to the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, "backup-s3", parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "backup-s3") sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)) # Run restore plan, downloading snapshots and schema from the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, "restore-s3", parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "restore-s3") with sdk_jobs.InstallJobContext( [ config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle), ] ): sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_functionality() -> None: parameters = {"CASSANDRA_KEYSPACE": "testspace1"} # populate 'testspace1' for test, then delete afterwards: with sdk_jobs.RunJobContext( before_jobs=[config.get_write_data_job(), config.get_verify_data_job()], after_jobs=[config.get_delete_data_job(), config.get_verify_deletion_job()], ): sdk_plan.start_plan(config.SERVICE_NAME, "cleanup", parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "cleanup") sdk_plan.start_plan(config.SERVICE_NAME, "repair", parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "repair")
def run_backup_and_restore_with_auth( service_name: str, backup_plan: str, restore_plan: str, plan_parameters: Dict[str, Optional[str]], job_node_address: str = DEFAULT_NODE_ADDRESS, ) -> None: write_data_job = get_write_data_job(node_address=job_node_address, auth=True) verify_data_job = get_verify_data_job(node_address=job_node_address, auth=True) delete_data_job = get_delete_data_job(node_address=job_node_address, auth=True) verify_deletion_job = get_verify_deletion_job( node_address=job_node_address, auth=True) # Ensure the keyspaces we will use aren't present. In practice this should run once and fail # because the data isn't present. When the job is flagged as failed (due to restart=NEVER), # the run_job() call will throw. try: sdk_jobs.run_job(delete_data_job) except Exception: log.info("Error during delete (normal if no stale data)") log.info(traceback.format_exc()) sdk_jobs.run_job(write_data_job) sdk_jobs.run_job(verify_data_job) # Run backup plan, uploading snapshots and schema to the cloud sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, backup_plan) # Delete all keyspaces and tables with a metronome job sdk_jobs.run_job(delete_data_job) # Verify that the keyspaces and tables were deleted sdk_jobs.run_job(verify_deletion_job) # Run restore plan, retrieving snapshots and schema from S3 sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, restore_plan) # Verify that the data we wrote and then deleted has been restored sdk_jobs.run_job(verify_data_job) # Delete data in preparation for any other backup tests sdk_jobs.run_job(delete_data_job) sdk_jobs.run_job(verify_deletion_job)
def test_plan_cli(): plan_name = 'deploy' phase_name = 'world' foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan list', json=True) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan show {}'.format(plan_name)) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan show --json {}'.format(plan_name), json=True) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan show {} --json'.format(plan_name), json=True) # trigger a restart so that the plan is in a non-complete state. # the 'interrupt' command will fail if the plan is already complete: assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan force-restart {}'.format(plan_name)) assert sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, 'plan interrupt {} {}'.format(plan_name, phase_name)) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan continue {} {}'.format(plan_name, phase_name)) # now wait for plan to finish before continuing to other tests: assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def test_canary_fourth(): sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'plan continue deploy world-deploy') expected_tasks = [ 'hello-0', 'hello-1', 'hello-2', 'hello-3', 'world-0', 'world-1', 'world-2', 'world-3'] sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks)) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod list', json=True) == expected_tasks pl = sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'deploy') log.info(pl) assert pl['status'] == 'COMPLETE' assert len(pl['phases']) == 2 phase = pl['phases'][0] assert phase['status'] == 'COMPLETE' steps = phase['steps'] assert len(steps) == 4 assert steps[0]['status'] == 'COMPLETE' assert steps[1]['status'] == 'COMPLETE' assert steps[2]['status'] == 'COMPLETE' assert steps[3]['status'] == 'COMPLETE' phase = pl['phases'][1] assert phase['status'] == 'COMPLETE' steps = phase['steps'] assert len(steps) == 4 assert steps[0]['status'] == 'COMPLETE' assert steps[1]['status'] == 'COMPLETE' assert steps[2]['status'] == 'COMPLETE' assert steps[3]['status'] == 'COMPLETE'
def test_canary_fourth(): sdk_cmd.run_cli('hello-world plan continue deploy world-deploy') expected_tasks = [ 'hello-0', 'hello-1', 'hello-2', 'hello-3', 'world-0', 'world-1', 'world-2', 'world-3' ] sdk_tasks.check_running(config.PACKAGE_NAME, len(expected_tasks)) assert json.loads( sdk_cmd.run_cli('hello-world pod list')) == expected_tasks pl = sdk_plan.wait_for_completed_plan(config.PACKAGE_NAME, 'deploy') log.info(pl) assert pl['status'] == 'COMPLETE' assert len(pl['phases']) == 2 phase = pl['phases'][0] assert phase['status'] == 'COMPLETE' steps = phase['steps'] assert len(steps) == 4 assert steps[0]['status'] == 'COMPLETE' assert steps[1]['status'] == 'COMPLETE' assert steps[2]['status'] == 'COMPLETE' assert steps[3]['status'] == 'COMPLETE' phase = pl['phases'][1] assert phase['status'] == 'COMPLETE' steps = phase['steps'] assert len(steps) == 4 assert steps[0]['status'] == 'COMPLETE' assert steps[1]['status'] == 'COMPLETE' assert steps[2]['status'] == 'COMPLETE' assert steps[3]['status'] == 'COMPLETE'
def test_plan_cli(): plan_name = "deploy" phase_name = "world" _check_json_output(foldered_name, "plan list") rc, _, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, "plan show {}".format(plan_name)) assert rc == 0 _check_json_output(foldered_name, "plan show --json {}".format(plan_name)) _check_json_output(foldered_name, "plan show {} --json".format(plan_name)) # trigger a restart so that the plan is in a non-complete state. # the 'interrupt' command will fail if the plan is already complete: rc, _, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, "plan force-restart {}".format(plan_name)) assert rc == 0 rc, _, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, "plan interrupt {} {}".format(plan_name, phase_name)) assert rc == 0 rc, _, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, "plan continue {} {}".format(plan_name, phase_name)) assert rc == 0 # now wait for plan to finish before continuing to other tests: assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def test_plan_cli(): plan_name = "deploy" phase_name = "world" _check_json_output(foldered_name, "plan list") rc, _, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, "plan show {}".format(plan_name)) assert rc == 0 _check_json_output(foldered_name, "plan show --json {}".format(plan_name)) _check_json_output(foldered_name, "plan show {} --json".format(plan_name)) # trigger a restart so that the plan is in a non-complete state. # the 'interrupt' command will fail if the plan is already complete: rc, _, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, "plan force-restart {}".format(plan_name) ) assert rc == 0 rc, _, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, "plan interrupt {} {}".format(plan_name, phase_name) ) assert rc == 0 rc, _, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, "plan continue {} {}".format(plan_name, phase_name) ) assert rc == 0 # now wait for plan to finish before continuing to other tests: assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def run_backup_and_restore( service_name: str, backup_plan: str, restore_plan: str, plan_parameters: Dict[str, Optional[str]], job_node_address: str = DEFAULT_NODE_ADDRESS, ) -> None: write_data_job = get_write_data_job(node_address=job_node_address) verify_data_job = get_verify_data_job(node_address=job_node_address) delete_data_job = get_delete_data_job(node_address=job_node_address) verify_deletion_job = get_verify_deletion_job(node_address=job_node_address) # Ensure the keyspaces we will use aren't present. In practice this should run once and fail # because the data isn't present. When the job is flagged as failed (due to restart=NEVER), # the run_job() call will throw. try: sdk_jobs.run_job(delete_data_job) except Exception: log.info("Error during delete (normal if no stale data)") log.info(traceback.format_exc()) # Write data to Cassandra with a metronome job, then verify it was written # Note: Write job will fail if data already exists sdk_jobs.run_job(write_data_job) sdk_jobs.run_job(verify_data_job) # Run backup plan, uploading snapshots and schema to the cloudddd sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, backup_plan) # Delete all keyspaces and tables with a metronome job sdk_jobs.run_job(delete_data_job) # Verify that the keyspaces and tables were deleted sdk_jobs.run_job(verify_deletion_job) # Run restore plan, retrieving snapshots and schema from S3 sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, restore_plan) # Verify that the data we wrote and then deleted has been restored sdk_jobs.run_job(verify_data_job) # Delete data in preparation for any other backup tests sdk_jobs.run_job(delete_data_job) sdk_jobs.run_job(verify_deletion_job)
def run_backup_and_restore( service_name, backup_plan, restore_plan, plan_parameters, job_node_address=DEFAULT_NODE_ADDRESS): write_data_job = get_write_data_job(node_address=job_node_address) verify_data_job = get_verify_data_job(node_address=job_node_address) delete_data_job = get_delete_data_job(node_address=job_node_address) verify_deletion_job = get_verify_deletion_job(node_address=job_node_address) # Ensure the keyspaces we will use aren't present. try: jobs.run_job(delete_data_job) except: log.info("Error during delete (normal if no stale data).") tb = traceback.format_exc() log.info(tb) # Write data to Cassandra with a metronome job, then verify it was written # Note: Write job will fail if data already exists sdk_jobs.run_job(write_data_job) sdk_jobs.run_job(verify_data_job) # Run backup plan, uploading snapshots and schema to the cloudddd sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, backup_plan) # Delete all keyspaces and tables with a metronome job sdk_jobs.run_job(delete_data_job) # Verify that the keyspaces and tables were deleted sdk_jobs.run_job(verify_deletion_job) # Run restore plan, retrieving snapshots and schema from S3 sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, restore_plan) # Verify that the data we wrote and then deleted has been restored sdk_jobs.run_job(verify_data_job) # Delete data in preparation for any other backup tests sdk_jobs.run_job(delete_data_job) sdk_jobs.run_job(verify_deletion_job)
def test_toxic_sidecar_doesnt_trigger_recovery(): # 1. Run the toxic sidecar plan that will never succeed. # 2. Restart the scheduler. # 3. Verify that its recovery plan is empty, as a failed ONCE task should # never trigger recovery recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery') assert (len(recovery_plan['phases']) == 0) log.info(recovery_plan) sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic') wait_for_toxic_sidecar() # Restart the scheduler and wait for it to come up. sdk_marathon.restart_app(config.SERVICE_NAME) sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME) # Now, verify that its recovery plan is empty. sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'recovery') recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery') assert (len(recovery_plan['phases']) == 0)
def test_toxic_sidecar_doesnt_trigger_recovery(): # 1. Run the toxic sidecar plan that will never succeed. # 2. Restart the scheduler. # 3. Verify that its recovery plan is empty, as a failed FINISHED task should # never trigger recovery recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery') assert(len(recovery_plan['phases']) == 0) log.info(recovery_plan) sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic') shakedown.wait_for(ToxicSidecarCheck(), timeout_seconds=10 * 60) # Restart the scheduler and wait for it to come up. sdk_marathon.restart_app(config.SERVICE_NAME) sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME) # Now, verify that its recovery plan is empty. sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'recovery') recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery') assert(len(recovery_plan['phases']) == 0)
def test_functionality(): parameters = {'CASSANDRA_KEYSPACE': 'testspace1'} # populate 'testspace1' for test, then delete afterwards: with sdk_jobs.RunJobContext( before_jobs=[ config.get_write_data_job(), config.get_verify_data_job() ], after_jobs=[ config.get_delete_data_job(), config.get_verify_deletion_job() ]): sdk_plan.start_plan(config.SERVICE_NAME, 'cleanup', parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'cleanup') sdk_plan.start_plan(config.SERVICE_NAME, 'repair', parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'repair')
def test_functionality(): parameters = {'CASSANDRA_KEYSPACE': 'testspace1'} # populate 'testspace1' for test, then delete afterwards: with sdk_jobs.RunJobContext( before_jobs=[ config.get_write_data_job(), config.get_verify_data_job() ], after_jobs=[ config.get_delete_data_job(), config.get_verify_deletion_job() ]): sdk_plan.start_plan(config.SERVICE_NAME, 'cleanup', parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'cleanup') sdk_plan.start_plan(config.SERVICE_NAME, 'repair', parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'repair')
def test_tls_connection(cassandra_service_tls, dcos_ca_bundle): """ Tests writing, reading and deleting data over a secure TLS connection. """ with sdk_jobs.InstallJobContext([ config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)]): sdk_jobs.run_job(config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) key_id = os.getenv('AWS_ACCESS_KEY_ID') if not key_id: assert False, 'AWS credentials are required for this test. ' \ 'Disable test with e.g. TEST_TYPES="sanity and not aws"' plan_parameters = { 'AWS_ACCESS_KEY_ID': key_id, 'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'), 'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'), 'S3_BUCKET_NAME': os.getenv('AWS_BUCKET_NAME', 'infinity-framework-test'), 'SNAPSHOT_NAME': str(uuid.uuid1()), 'CASSANDRA_KEYSPACES': '"testspace1 testspace2"', } # Run backup plan, uploading snapshots and schema to the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, 'backup-s3', parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'backup-s3') sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)) # Run restore plan, downloading snapshots and schema from the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, 'restore-s3', parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'restore-s3') with sdk_jobs.InstallJobContext([ config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)]): sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_increase_decrease_world_nodes(): original_hello_ids = sdk_tasks.get_task_ids(foldered_name, "hello") original_world_ids = sdk_tasks.get_task_ids(foldered_name, "world") log.info("world ids: " + str(original_world_ids)) # add 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", 2) # autodetects the correct number of nodes and waits for them to deploy: config.check_running(foldered_name) sdk_plan.wait_for_completed_deployment(foldered_name) sdk_tasks.check_tasks_not_updated(foldered_name, "world", original_world_ids) # check 2 world tasks added: assert 2 + len(original_world_ids) == len( sdk_tasks.get_task_ids(foldered_name, "world")) # subtract 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", -2) # autodetects the correct number of nodes and waits for them to deploy: config.check_running(foldered_name) sdk_plan.wait_for_completed_deployment(foldered_name) # wait for the decommission plan for this subtraction to be complete sdk_plan.wait_for_completed_plan(foldered_name, "decommission") # check that the total task count is back to original sdk_tasks.check_running(foldered_name, len(original_hello_ids) + len(original_world_ids), allow_more=False) # check that original tasks weren't affected/relaunched in the process sdk_tasks.check_tasks_not_updated(foldered_name, "hello", original_hello_ids) sdk_tasks.check_tasks_not_updated(foldered_name, "world", original_world_ids) # check that the world tasks are back to their prior state (also without changing task ids) assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, "world")
def test_increase_decrease_world_nodes(): original_hello_ids = sdk_tasks.get_task_ids(foldered_name, "hello") original_world_ids = sdk_tasks.get_task_ids(foldered_name, "world") log.info("world ids: " + str(original_world_ids)) # add 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", 2) # autodetects the correct number of nodes and waits for them to deploy: config.check_running(foldered_name) sdk_plan.wait_for_completed_deployment(foldered_name) sdk_tasks.check_tasks_not_updated(foldered_name, "world", original_world_ids) # check 2 world tasks added: assert 2 + len(original_world_ids) == len(sdk_tasks.get_task_ids(foldered_name, "world")) # subtract 2 world nodes sdk_marathon.bump_task_count_config(foldered_name, "WORLD_COUNT", -2) # autodetects the correct number of nodes and waits for them to deploy: config.check_running(foldered_name) sdk_plan.wait_for_completed_deployment(foldered_name) # wait for the decommission plan for this subtraction to be complete sdk_plan.wait_for_completed_plan(foldered_name, "decommission") # check that the total task count is back to original sdk_tasks.check_running( foldered_name, len(original_hello_ids) + len(original_world_ids), allow_more=False ) # check that original tasks weren't affected/relaunched in the process sdk_tasks.check_tasks_not_updated(foldered_name, "hello", original_hello_ids) sdk_tasks.check_tasks_not_updated(foldered_name, "world", original_world_ids) # check that the world tasks are back to their prior state (also without changing task ids) assert original_world_ids == sdk_tasks.get_task_ids(foldered_name, "world")
def run_backup_and_restore( service_name, backup_plan, restore_plan, plan_parameters, job_node_address=DEFAULT_NODE_ADDRESS): write_data_job = get_write_data_job(node_address=job_node_address) verify_data_job = get_verify_data_job(node_address=job_node_address) delete_data_job = get_delete_data_job(node_address=job_node_address) verify_deletion_job = get_verify_deletion_job(node_address=job_node_address) # Write data to Cassandra with a metronome job, then verify it was written # Note: Write job will fail if data already exists sdk_jobs.run_job(write_data_job) sdk_jobs.run_job(verify_data_job) # Run backup plan, uploading snapshots and schema to the cloudddd sdk_plan.start_plan(service_name, backup_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, backup_plan) # Delete all keyspaces and tables with a metronome job sdk_jobs.run_job(delete_data_job) # Verify that the keyspaces and tables were deleted sdk_jobs.run_job(verify_deletion_job) # Run restore plan, retrieving snapshots and schema from S3 sdk_plan.start_plan(service_name, restore_plan, parameters=plan_parameters) sdk_plan.wait_for_completed_plan(service_name, restore_plan) # Verify that the data we wrote and then deleted has been restored sdk_jobs.run_job(verify_data_job) # Delete data in preparation for any other backup tests sdk_jobs.run_job(delete_data_job) sdk_jobs.run_job(verify_deletion_job)
def run_backup_and_restore(service_name, backup_plan, restore_plan, plan_parameters, job_node_address=DEFAULT_NODE_ADDRESS): write_data_job = get_write_data_job(node_address=job_node_address) verify_data_job = get_verify_data_job(node_address=job_node_address) delete_data_job = get_delete_data_job(node_address=job_node_address) verify_deletion_job = get_verify_deletion_job( node_address=job_node_address) # Write data to Cassandra with a metronome job, then verify it was written # Note: Write job will fail if data already exists jobs.run_job(write_data_job) jobs.run_job(verify_data_job) # Run backup plan, uploading snapshots and schema to the cloudddd plan.start_plan(service_name, backup_plan, parameters=plan_parameters) plan.wait_for_completed_plan(service_name, backup_plan) # Delete all keyspaces and tables with a metronome job jobs.run_job(delete_data_job) # Verify that the keyspaces and tables were deleted jobs.run_job(verify_deletion_job) # Run restore plan, retrieving snapshots and schema from S3 plan.start_plan(service_name, restore_plan, parameters=plan_parameters) plan.wait_for_completed_plan(service_name, restore_plan) # Verify that the data we wrote and then deleted has been restored jobs.run_job(verify_data_job) # Delete data in preparation for any other backup tests jobs.run_job(delete_data_job) jobs.run_job(verify_deletion_job)
def test_functionality() -> None: parameters = {"CASSANDRA_KEYSPACE": "testspace1"} # populate 'testspace1' for test, then delete afterwards: with sdk_jobs.RunJobContext( before_jobs=[ config.get_write_data_job(), config.get_verify_data_job() ], after_jobs=[ config.get_delete_data_job(), config.get_verify_deletion_job() ], ): sdk_plan.start_plan(config.SERVICE_NAME, "cleanup", parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "cleanup") sdk_plan.start_plan(config.SERVICE_NAME, "repair", parameters=parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "repair")
def test_canary_fourth(): sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "plan continue deploy world-deploy") expected_tasks = [ "hello-0", "hello-1", "hello-2", "hello-3", "world-0", "world-1", "world-2", "world-3", ] sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks)) rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "pod list") assert rc == 0, "Pod list failed" assert json.loads(stdout) == expected_tasks pl = sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "deploy") log.info(pl) assert pl["status"] == "COMPLETE" assert len(pl["phases"]) == 2 phase = pl["phases"][0] assert phase["status"] == "COMPLETE" steps = phase["steps"] assert len(steps) == 4 assert steps[0]["status"] == "COMPLETE" assert steps[1]["status"] == "COMPLETE" assert steps[2]["status"] == "COMPLETE" assert steps[3]["status"] == "COMPLETE" phase = pl["phases"][1] assert phase["status"] == "COMPLETE" steps = phase["steps"] assert len(steps) == 4 assert steps[0]["status"] == "COMPLETE" assert steps[1]["status"] == "COMPLETE" assert steps[2]["status"] == "COMPLETE" assert steps[3]["status"] == "COMPLETE"
def test_canary_fourth(): sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "plan continue deploy world-deploy") expected_tasks = [ "hello-0", "hello-1", "hello-2", "hello-3", "world-0", "world-1", "world-2", "world-3", ] sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks)) rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "pod list") assert rc == 0, "Pod list failed" assert json.loads(stdout) == expected_tasks pl = sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "deploy") log.info(pl) assert pl["status"] == "COMPLETE" assert len(pl["phases"]) == 2 phase = pl["phases"][0] assert phase["status"] == "COMPLETE" steps = phase["steps"] assert len(steps) == 4 assert steps[0]["status"] == "COMPLETE" assert steps[1]["status"] == "COMPLETE" assert steps[2]["status"] == "COMPLETE" assert steps[3]["status"] == "COMPLETE" phase = pl["phases"][1] assert phase["status"] == "COMPLETE" steps = phase["steps"] assert len(steps) == 4 assert steps[0]["status"] == "COMPLETE" assert steps[1]["status"] == "COMPLETE" assert steps[2]["status"] == "COMPLETE" assert steps[3]["status"] == "COMPLETE"
def test_plan_cli(): plan_name = 'deploy' phase_name = 'world' foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan list', json=True) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan show {}'.format(plan_name)) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan show --json {}'.format(plan_name), json=True) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan show {} --json'.format(plan_name), json=True) # trigger a restart so that the plan is in a non-complete state. # the 'interrupt' command will fail if the plan is already complete: assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan force-restart {}'.format(plan_name)) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan interrupt {} {}'.format(plan_name, phase_name)) assert sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, 'plan continue {} {}'.format(plan_name, phase_name)) # now wait for plan to finish before continuing to other tests: assert sdk_plan.wait_for_completed_plan(foldered_name, plan_name)
def test_cassandra_migration(): backup_service_name = os.getenv('CASSANDRA_BACKUP_CLUSTER_NAME') restore_service_name = os.getenv('CASSANDRA_RESTORE_CLUSTER_NAME') backup_node_address = os.getenv('BACKUP_NODE_ADDRESS', DEFAULT_NODE_ADDRESS) backup_node_port = os.getenv('BACKUP_NODE_PORT', DEFAULT_NODE_PORT) backup_write_data_job = get_write_data_job(backup_node_address, backup_node_port) backup_verify_data_job = get_verify_data_job(backup_node_address, backup_node_port) backup_delete_data_job = get_delete_data_job(backup_node_address, backup_node_port) backup_verify_deletion_job = get_verify_deletion_job( backup_node_address, backup_node_port) plan_parameters = { 'S3_BUCKET_NAME': os.getenv('AWS_BUCKET_NAME', 'infinity-framework-test'), 'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'), 'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'), 'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'), 'SNAPSHOT_NAME': str(uuid.uuid1()), 'CASSANDRA_KEYSPACES': '"testspace1 testspace2"', } backup_install_job_context = jobs.InstallJobContext([ backup_write_data_job, backup_verify_data_job, backup_delete_data_job, backup_verify_deletion_job ]) backup_run_job_context = jobs.RunJobContext( before_jobs=[backup_write_data_job, backup_verify_data_job], after_jobs=[backup_delete_data_job, backup_verify_deletion_job]) # Install and run the write/delete data jobs against backup cluster, # running dcos-cassandra-service with backup_install_job_context, backup_run_job_context: # Back this cluster up to S3 backup_parameters = { 'backup_name': plan_parameters['SNAPSHOT_NAME'], 's3_access_key': plan_parameters['AWS_ACCESS_KEY_ID'], 's3_secret_key': plan_parameters['AWS_SECRET_ACCESS_KEY'], 'external_location': 's3://{}'.format(plan_parameters['S3_BUCKET_NAME']), } dcos.http.put('{}v1/backup/start'.format( shakedown.dcos_service_url(backup_service_name)), json=backup_parameters) plan.wait_for_completed_deployment(backup_service_name) # Restore data to second instance: restore_node_address = os.getenv( 'RESTORE_NODE_ADDRESS', hosts.autoip_host('sdk-cassandra', 'node-0-server')) restore_node_port = os.getenv('RESTORE_NODE_PORT', '9052') restore_write_data_job = get_write_data_job(restore_node_address, restore_node_port) restore_verify_data_job = get_verify_data_job(restore_node_address, restore_node_port) restore_delete_data_job = get_delete_data_job(restore_node_address, restore_node_port) restore_verify_deletion_job = get_verify_deletion_job( restore_node_address, restore_node_port) restore_install_job_context = jobs.InstallJobContext([ restore_write_data_job, restore_verify_data_job, restore_delete_data_job, restore_verify_deletion_job ]) restore_run_job_context = jobs.RunJobContext(after_jobs=[ restore_verify_data_job, restore_delete_data_job, restore_verify_deletion_job ]) with restore_install_job_context, restore_run_job_context: plan.start_plan(restore_service_name, 'restore-s3', parameters=plan_parameters) plan.wait_for_completed_plan(restore_service_name, 'restore-s3')
def test_cassandra_migration(): backup_service_name = os.getenv('CASSANDRA_BACKUP_CLUSTER_NAME') restore_service_name = os.getenv('CASSANDRA_RESTORE_CLUSTER_NAME') backup_node_address = os.getenv('BACKUP_NODE_ADDRESS', config.DEFAULT_NODE_ADDRESS) backup_node_port = os.getenv('BACKUP_NODE_PORT', config.DEFAULT_NODE_PORT) backup_write_data_job = config.get_write_data_job(backup_node_address, backup_node_port) backup_verify_data_job = config.get_verify_data_job(backup_node_address, backup_node_port) backup_delete_data_job = config.get_delete_data_job(backup_node_address, backup_node_port) backup_verify_deletion_job = config.get_verify_deletion_job(backup_node_address, backup_node_port) plan_parameters = { 'S3_BUCKET_NAME': os.getenv( 'AWS_BUCKET_NAME', 'infinity-framework-test' ), 'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'), 'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'), 'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'), 'SNAPSHOT_NAME': str(uuid.uuid1()), 'CASSANDRA_KEYSPACES': '"testspace1 testspace2"', } backup_install_job_context = sdk_jobs.InstallJobContext( [backup_write_data_job, backup_verify_data_job, backup_delete_data_job, backup_verify_deletion_job]) backup_run_job_context = sdk_jobs.RunJobContext( before_jobs=[backup_write_data_job, backup_verify_data_job], after_jobs=[backup_delete_data_job, backup_verify_deletion_job]) # Install and run the write/delete data jobs against backup cluster, # running dcos-cassandra-service with backup_install_job_context, backup_run_job_context: # Back this cluster up to S3 backup_parameters = { 'backup_name': plan_parameters['SNAPSHOT_NAME'], 's3_access_key': plan_parameters['AWS_ACCESS_KEY_ID'], 's3_secret_key': plan_parameters['AWS_SECRET_ACCESS_KEY'], 'external_location': 's3://{}'.format(plan_parameters['S3_BUCKET_NAME']), } sdk_cmd.service_request('PUT', backup_service_name, '/v1/backup/start', json=backup_parameters) sdk_plan.wait_for_completed_deployment(backup_service_name) # Restore data to second instance: restore_node_address = os.getenv( 'RESTORE_NODE_ADDRESS', sdk_hosts.autoip_host('sdk-cassandra', 'node-0-server')) restore_node_port = os.getenv('RESTORE_NODE_PORT', '9052') restore_write_data_job = config.get_write_data_job(restore_node_address, restore_node_port) restore_verify_data_job = config.get_verify_data_job(restore_node_address, restore_node_port) restore_delete_data_job = config.get_delete_data_job(restore_node_address, restore_node_port) restore_verify_deletion_job = config.get_verify_deletion_job(restore_node_address, restore_node_port) restore_install_job_context = sdk_jobs.InstallJobContext( [restore_write_data_job, restore_verify_data_job, restore_delete_data_job, restore_verify_deletion_job] ) restore_run_job_context = sdk_jobs.RunJobContext( after_jobs=[restore_verify_data_job, restore_delete_data_job, restore_verify_deletion_job] ) with restore_install_job_context, restore_run_job_context: sdk_plan.start_plan( restore_service_name, 'restore-s3', parameters=plan_parameters ) sdk_plan.wait_for_completed_plan(restore_service_name, 'restore-s3')
def test_tls_connection( cassandra_service: Dict[str, Any], dcos_ca_bundle: str, ) -> None: """ Tests writing, reading and deleting data over a secure TLS connection. """ with sdk_jobs.InstallJobContext([ config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle), ]): sdk_jobs.run_job( config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job( config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) key_id = os.getenv("AWS_ACCESS_KEY_ID") if not key_id: assert ( False ), "AWS credentials are required for this test. " 'Disable test with e.g. TEST_TYPES="sanity and not aws"' plan_parameters = { "AWS_ACCESS_KEY_ID": key_id, "AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"), "AWS_REGION": os.getenv("AWS_REGION", "us-west-2"), "S3_BUCKET_NAME": os.getenv("AWS_BUCKET_NAME", "infinity-framework-test"), "SNAPSHOT_NAME": str(uuid.uuid1()), "CASSANDRA_KEYSPACES": '"testspace1 testspace2"', } # Run backup plan, uploading snapshots and schema to the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, "backup-s3", parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "backup-s3") sdk_jobs.run_job( config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)) # Run restore plan, downloading snapshots and schema from the cloudddd sdk_plan.start_plan(config.SERVICE_NAME, "restore-s3", parameters=plan_parameters) sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "restore-s3") with sdk_jobs.InstallJobContext([ config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle), config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle), ]): sdk_jobs.run_job( config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)) sdk_jobs.run_job( config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))