def test_tls_connection(cassandra_service_tls, dcos_ca_bundle):
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext([
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)
]):
sdk_jobs.run_job(
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv('AWS_ACCESS_KEY_ID')
if not key_id:
assert False, 'AWS credentials are required for this test. ' \
'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
'AWS_ACCESS_KEY_ID':
key_id,
'AWS_SECRET_ACCESS_KEY':
os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION':
os.getenv('AWS_REGION', 'us-west-2'),
'S3_BUCKET_NAME':
os.getenv('AWS_BUCKET_NAME', 'infinity-framework-test'),
'SNAPSHOT_NAME':
str(uuid.uuid1()),
'CASSANDRA_KEYSPACES':
'"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
'backup-s3',
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'backup-s3')
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
'restore-s3',
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'restore-s3')
with sdk_jobs.InstallJobContext([
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)
]):
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_tls_connection(
cassandra_service: Dict[str, Any],
dcos_ca_bundle: str,
) -> None:
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext(
[
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]
):
sdk_jobs.run_job(config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv("AWS_ACCESS_KEY_ID")
if not key_id:
assert (
False
), "AWS credentials are required for this test. " 'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
"AWS_ACCESS_KEY_ID": key_id,
"AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"),
"AWS_REGION": os.getenv("AWS_REGION", "us-west-2"),
"S3_BUCKET_NAME": os.getenv("AWS_BUCKET_NAME", "infinity-framework-test"),
"SNAPSHOT_NAME": str(uuid.uuid1()),
"CASSANDRA_KEYSPACES": '"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, "backup-s3", parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "backup-s3")
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, "restore-s3", parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "restore-s3")
with sdk_jobs.InstallJobContext(
[
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]
):
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_repair_cleanup_plans_complete():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(
node_address=config.get_foldered_node_address()),
config.get_verify_data_job(
node_address=config.get_foldered_node_address())
],
after_jobs=[
config.get_delete_data_job(
node_address=config.get_foldered_node_address()),
config.get_verify_deletion_job(
node_address=config.get_foldered_node_address())
]):
sdk_plan.start_plan(config.get_foldered_service_name(),
'cleanup',
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.get_foldered_service_name(),
'cleanup')
sdk_plan.start_plan(config.get_foldered_service_name(),
'repair',
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.get_foldered_service_name(),
'repair')
def test_tls_connection(cassandra_service_tls, dcos_ca_bundle):
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext([
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)]):
sdk_jobs.run_job(config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv('AWS_ACCESS_KEY_ID')
if not key_id:
assert False, 'AWS credentials are required for this test. ' \
'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
'AWS_ACCESS_KEY_ID': key_id,
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'S3_BUCKET_NAME': os.getenv('AWS_BUCKET_NAME', 'infinity-framework-test'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, 'backup-s3', parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'backup-s3')
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME, 'restore-s3', parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'restore-s3')
with sdk_jobs.InstallJobContext([
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)]):
sdk_jobs.run_job(config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_functionality() -> None:
parameters = {"CASSANDRA_KEYSPACE": "testspace1"}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[config.get_write_data_job(), config.get_verify_data_job()],
after_jobs=[config.get_delete_data_job(), config.get_verify_deletion_job()],
):
sdk_plan.start_plan(config.SERVICE_NAME, "cleanup", parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "cleanup")
sdk_plan.start_plan(config.SERVICE_NAME, "repair", parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "repair")
def verify_client_can_write_read_and_delete(dcos_ca_bundle=None):
write_data_job = config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle)
verify_data_job = config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)
delete_data_job = config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)
verify_deletion_job = config.get_verify_deletion_job(
dcos_ca_bundle=dcos_ca_bundle)
with sdk_jobs.InstallJobContext([
write_data_job, verify_data_job, delete_data_job,
verify_deletion_job
]):
sdk_jobs.run_job(write_data_job)
sdk_jobs.run_job(verify_data_job)
sdk_jobs.run_job(delete_data_job)
sdk_jobs.run_job(verify_deletion_job)
def verify_client_can_write_read_and_delete(
dcos_ca_bundle: Optional[str] = None,
) -> None:
write_data_job = config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle)
verify_data_job = config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle)
delete_data_job = config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle)
verify_deletion_job = config.get_verify_deletion_job(dcos_ca_bundle=dcos_ca_bundle)
with sdk_jobs.InstallJobContext(
[write_data_job, verify_data_job, delete_data_job, verify_deletion_job]
):
sdk_jobs.run_job(write_data_job)
sdk_jobs.run_job(verify_data_job)
sdk_jobs.run_job(delete_data_job)
sdk_jobs.run_job(verify_deletion_job)
def test_enabling_then_disabling_tls(
cassandra_service: Dict[str, Any],
dcos_ca_bundle: str,
) -> None:
# Write data.
write_data_job = config.get_write_data_job()
with sdk_jobs.InstallJobContext([write_data_job]):
sdk_jobs.run_job(write_data_job)
# Turn TLS on and off again.
update_service_transport_encryption(cassandra_service, enabled=True, allow_plaintext=True)
update_service_transport_encryption(cassandra_service, enabled=True, allow_plaintext=False)
update_service_transport_encryption(cassandra_service, enabled=False, allow_plaintext=False)
# Make sure data is still there.
verify_data_job = config.get_verify_data_job()
with sdk_jobs.InstallJobContext([verify_data_job]):
sdk_jobs.run_job(verify_data_job)
def test_functionality():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(),
config.get_verify_data_job()
],
after_jobs=[
config.get_delete_data_job(),
config.get_verify_deletion_job()
]):
sdk_plan.start_plan(config.SERVICE_NAME, 'cleanup', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'cleanup')
sdk_plan.start_plan(config.SERVICE_NAME, 'repair', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'repair')
def test_functionality():
parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(),
config.get_verify_data_job()
],
after_jobs=[
config.get_delete_data_job(),
config.get_verify_deletion_job()
]):
sdk_plan.start_plan(config.SERVICE_NAME, 'cleanup', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'cleanup')
sdk_plan.start_plan(config.SERVICE_NAME, 'repair', parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'repair')
def test_backup_and_restore():
plan_parameters = {
'S3_BUCKET_NAME': os.getenv('AWS_BUCKET_NAME',
'infinity-framework-test'),
'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'),
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
with sdk_jobs.InstallJobContext([
config.get_write_data_job(),
config.get_verify_data_job(),
config.get_delete_data_job(),
config.get_verify_deletion_job()
]):
config.run_backup_and_restore(config.SERVICE_NAME, 'backup-s3',
'restore-s3', plan_parameters)
def test_enabling_then_disabling_tls(cassandra_service, dcos_ca_bundle):
# Write data.
write_data_job = config.get_write_data_job()
with sdk_jobs.InstallJobContext([write_data_job]):
sdk_jobs.run_job(write_data_job)
# Turn TLS on and off again.
update_service_transport_encryption(cassandra_service,
enabled=True,
allow_plaintext=True)
update_service_transport_encryption(cassandra_service,
enabled=True,
allow_plaintext=False)
update_service_transport_encryption(cassandra_service,
enabled=False,
allow_plaintext=False)
# Make sure data is still there.
verify_data_job = config.get_verify_data_job()
with sdk_jobs.InstallJobContext([verify_data_job]):
sdk_jobs.run_job(verify_data_job)
def test_backup_and_restore():
plan_parameters = {
'S3_BUCKET_NAME': os.getenv(
'AWS_BUCKET_NAME', 'infinity-framework-test'
),
'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'),
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
with sdk_jobs.InstallJobContext([
config.get_write_data_job(),
config.get_verify_data_job(),
config.get_delete_data_job(),
config.get_verify_deletion_job()]):
config.run_backup_and_restore(
config.SERVICE_NAME,
'backup-s3',
'restore-s3',
plan_parameters)
def test_functionality() -> None:
parameters = {"CASSANDRA_KEYSPACE": "testspace1"}
# populate 'testspace1' for test, then delete afterwards:
with sdk_jobs.RunJobContext(
before_jobs=[
config.get_write_data_job(),
config.get_verify_data_job()
],
after_jobs=[
config.get_delete_data_job(),
config.get_verify_deletion_job()
],
):
sdk_plan.start_plan(config.SERVICE_NAME,
"cleanup",
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "cleanup")
sdk_plan.start_plan(config.SERVICE_NAME,
"repair",
parameters=parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "repair")
def test_tls_connection(
cassandra_service: Dict[str, Any],
dcos_ca_bundle: str,
) -> None:
"""
Tests writing, reading and deleting data over a secure TLS connection.
"""
with sdk_jobs.InstallJobContext([
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]):
sdk_jobs.run_job(
config.get_write_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
key_id = os.getenv("AWS_ACCESS_KEY_ID")
if not key_id:
assert (
False
), "AWS credentials are required for this test. " 'Disable test with e.g. TEST_TYPES="sanity and not aws"'
plan_parameters = {
"AWS_ACCESS_KEY_ID":
key_id,
"AWS_SECRET_ACCESS_KEY":
os.getenv("AWS_SECRET_ACCESS_KEY"),
"AWS_REGION":
os.getenv("AWS_REGION", "us-west-2"),
"S3_BUCKET_NAME":
os.getenv("AWS_BUCKET_NAME", "infinity-framework-test"),
"SNAPSHOT_NAME":
str(uuid.uuid1()),
"CASSANDRA_KEYSPACES":
'"testspace1 testspace2"',
}
# Run backup plan, uploading snapshots and schema to the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
"backup-s3",
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "backup-s3")
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
# Run restore plan, downloading snapshots and schema from the cloudddd
sdk_plan.start_plan(config.SERVICE_NAME,
"restore-s3",
parameters=plan_parameters)
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, "restore-s3")
with sdk_jobs.InstallJobContext([
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle),
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle),
]):
sdk_jobs.run_job(
config.get_verify_data_job(dcos_ca_bundle=dcos_ca_bundle))
sdk_jobs.run_job(
config.get_delete_data_job(dcos_ca_bundle=dcos_ca_bundle))
def test_cassandra_migration():
backup_service_name = os.getenv('CASSANDRA_BACKUP_CLUSTER_NAME')
restore_service_name = os.getenv('CASSANDRA_RESTORE_CLUSTER_NAME')
backup_node_address = os.getenv('BACKUP_NODE_ADDRESS', config.DEFAULT_NODE_ADDRESS)
backup_node_port = os.getenv('BACKUP_NODE_PORT', config.DEFAULT_NODE_PORT)
backup_write_data_job = config.get_write_data_job(backup_node_address, backup_node_port)
backup_verify_data_job = config.get_verify_data_job(backup_node_address, backup_node_port)
backup_delete_data_job = config.get_delete_data_job(backup_node_address, backup_node_port)
backup_verify_deletion_job = config.get_verify_deletion_job(backup_node_address, backup_node_port)
plan_parameters = {
'S3_BUCKET_NAME': os.getenv(
'AWS_BUCKET_NAME', 'infinity-framework-test'
),
'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'),
'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'),
'AWS_REGION': os.getenv('AWS_REGION', 'us-west-2'),
'SNAPSHOT_NAME': str(uuid.uuid1()),
'CASSANDRA_KEYSPACES': '"testspace1 testspace2"',
}
backup_install_job_context = sdk_jobs.InstallJobContext(
[backup_write_data_job, backup_verify_data_job,
backup_delete_data_job, backup_verify_deletion_job])
backup_run_job_context = sdk_jobs.RunJobContext(
before_jobs=[backup_write_data_job, backup_verify_data_job],
after_jobs=[backup_delete_data_job, backup_verify_deletion_job])
# Install and run the write/delete data jobs against backup cluster,
# running dcos-cassandra-service
with backup_install_job_context, backup_run_job_context:
# Back this cluster up to S3
backup_parameters = {
'backup_name': plan_parameters['SNAPSHOT_NAME'],
's3_access_key': plan_parameters['AWS_ACCESS_KEY_ID'],
's3_secret_key': plan_parameters['AWS_SECRET_ACCESS_KEY'],
'external_location': 's3://{}'.format(plan_parameters['S3_BUCKET_NAME']),
}
sdk_cmd.service_request('PUT', backup_service_name, '/v1/backup/start', json=backup_parameters)
sdk_plan.wait_for_completed_deployment(backup_service_name)
# Restore data to second instance:
restore_node_address = os.getenv(
'RESTORE_NODE_ADDRESS', sdk_hosts.autoip_host('sdk-cassandra', 'node-0-server'))
restore_node_port = os.getenv('RESTORE_NODE_PORT', '9052')
restore_write_data_job = config.get_write_data_job(restore_node_address, restore_node_port)
restore_verify_data_job = config.get_verify_data_job(restore_node_address, restore_node_port)
restore_delete_data_job = config.get_delete_data_job(restore_node_address, restore_node_port)
restore_verify_deletion_job = config.get_verify_deletion_job(restore_node_address, restore_node_port)
restore_install_job_context = sdk_jobs.InstallJobContext(
[restore_write_data_job, restore_verify_data_job,
restore_delete_data_job, restore_verify_deletion_job]
)
restore_run_job_context = sdk_jobs.RunJobContext(
after_jobs=[restore_verify_data_job, restore_delete_data_job, restore_verify_deletion_job]
)
with restore_install_job_context, restore_run_job_context:
sdk_plan.start_plan(
restore_service_name, 'restore-s3', parameters=plan_parameters
)
sdk_plan.wait_for_completed_plan(restore_service_name, 'restore-s3')
