def test_tier_2_upgrade(floating_ip_2_tier_1_clusters, tier_2_manager,
cfy, tmpdir, logger):
local_snapshot_path = str(tmpdir / 'snapshot.zip')
tier_1_cluster = floating_ip_2_tier_1_clusters[0]
tier_1_cluster.deploy_and_validate()
cfy.snapshots.create([constants.TIER_2_SNAP_ID])
tier_2_manager.wait_for_all_executions()
cfy.snapshots.download(
[constants.TIER_2_SNAP_ID, '-o', local_snapshot_path]
)
tier_2_manager.teardown()
tier_2_manager.bootstrap()
tier_2_manager.use()
_upload_resources_to_tier_2_manager(cfy, tier_2_manager, logger)
cfy.snapshots.upload([local_snapshot_path, '-s', constants.TIER_2_SNAP_ID])
restore_snapshot(tier_2_manager, constants.TIER_2_SNAP_ID, cfy, logger,
restore_certificates=True)
cfy.agents.install()
# This will only work properly if the Tier 2 manager was restored correctly
tier_1_cluster.uninstall()
def test_tier_2_upgrade(floating_ip_2_tier_1_clusters, tier_2_manager, cfy,
tmpdir, logger):
local_snapshot_path = str(tmpdir / 'snapshot.zip')
tier_1_cluster = floating_ip_2_tier_1_clusters[0]
tier_1_cluster.deploy_and_validate()
cfy.snapshots.create([constants.TIER_2_SNAP_ID])
tier_2_manager.wait_for_all_executions()
cfy.snapshots.download(
[constants.TIER_2_SNAP_ID, '-o', local_snapshot_path])
tier_2_manager.teardown()
tier_2_manager.bootstrap()
tier_2_manager.use()
_upload_resources_to_tier_2_manager(cfy, tier_2_manager, logger)
cfy.snapshots.upload([local_snapshot_path, '-s', constants.TIER_2_SNAP_ID])
restore_snapshot(tier_2_manager,
constants.TIER_2_SNAP_ID,
cfy,
logger,
restore_certificates=True)
cfy.agents.install()
# This will only work properly if the Tier 2 manager was restored correctly
tier_1_cluster.uninstall()
def test_cluster_to_upgrade_cluster(distributed_installation,
cfy,
logger,
tmpdir,
attributes,
distributed_nodecellar):
logger.info('Running Sanity check for cluster with an external database')
rabbitmq = distributed_installation.message_queue
manager1 = distributed_installation.manager
old_manager_1 = distributed_installation.old_cluster[0]
old_manager_1.use()
create_and_add_user_to_tenant(cfy, logger)
set_sanity_user(cfy, old_manager_1, logger)
# Creating secrets with 'tenant' visibility
create_secrets(cfy, logger, attributes, old_manager_1)
distributed_nodecellar.upload_and_verify_install()
set_admin_user(cfy, old_manager_1, logger)
# Create and download a snapshot
snapshot_id = 'SNAPSHOT_ID'
local_snapshot_path = str(tmpdir / 'snap.zip')
logger.info('Creating snapshot')
create_snapshot(old_manager_1, snapshot_id, attributes, logger)
download_snapshot(old_manager_1, local_snapshot_path, snapshot_id, logger)
set_admin_user(cfy, manager1, logger)
# Upload and restore snapshot to the external AIO manager
logger.info('Uploading and restoring snapshot')
upload_snapshot(manager1, local_snapshot_path, snapshot_id, logger)
restore_snapshot(manager1, snapshot_id, cfy, logger,
change_manager_password=False)
time.sleep(7)
verify_services_status(manager1, logger)
# wait for agents reconnection
time.sleep(30)
# Upgrade agents
logger.info('Upgrading agents')
copy_ssl_cert_from_manager_to_tmpdir(old_manager_1, tmpdir)
args = ['--manager-ip', rabbitmq.private_ip_address,
'--manager_certificate', str(tmpdir + 'new_manager_cert.txt'),
'--all-tenants']
cfy.agents.install(args)
set_sanity_user(cfy, manager1, logger)
# Verify `agents install` worked as expected
distributed_nodecellar.uninstall()
def test_multiple_networks(managers,
cfy,
multi_network_hello_worlds,
logger,
tmpdir,
attributes):
logger.info('Testing managers with multiple networks')
# We should have at least 3 hello world objects. We will verify the first
# one completely on the first manager.
# All the other ones will be installed on the first manager,
# then we'll create a snapshot and restore it on the second manager, and
# finally, to complete the verification, we'll uninstall the remaining
# hellos on the new manager
old_manager = managers[0]
new_manager = managers[1]
snapshot_id = 'SNAPSHOT_ID'
local_snapshot_path = str(tmpdir / 'snap.zip')
# The first hello is the one that belongs to a network that will be added
# manually post bootstrap to the new manager
post_bootstrap_hello = multi_network_hello_worlds.pop(0)
post_bootstrap_hello.manager = new_manager
for hello in multi_network_hello_worlds:
hello.upload_and_verify_install()
create_snapshot(old_manager, snapshot_id, attributes, logger)
download_snapshot(old_manager, local_snapshot_path, snapshot_id, logger)
new_manager.use()
upload_snapshot(new_manager, local_snapshot_path, snapshot_id, logger)
restore_snapshot(new_manager, snapshot_id, cfy, logger,
change_manager_password=False)
upgrade_agents(cfy, new_manager, logger)
delete_manager(old_manager, logger)
for hello in multi_network_hello_worlds:
hello.manager = new_manager
hello.uninstall()
hello.delete_deployment()
_add_new_network(new_manager, logger)
post_bootstrap_hello.verify_all()
def test_old_agent_stopped_after_agent_upgrade(
managers, nodecellar, cfy, logger, tmpdir
):
local_snapshot_path = str(tmpdir / 'snapshot.zip')
snapshot_id = 'snap'
old_manager = managers[0]
new_manager = managers[1]
old_manager.use()
nodecellar.upload_and_verify_install()
cfy.snapshots.create([snapshot_id])
old_manager.wait_for_all_executions()
cfy.snapshots.download([snapshot_id, '-o', local_snapshot_path])
new_manager.use()
cfy.snapshots.upload([local_snapshot_path, '-s', snapshot_id])
restore_snapshot(new_manager, snapshot_id, cfy, logger)
# Before upgrading the agents, the old agent should still be up
old_manager.use()
cfy.agents.validate()
# Upgrade to new agents and stop old agents
new_manager.use()
cfy.agents.install('--stop-old-agent')
# We now expect the old agent to be stopped, and thus unresponsive. So,
# calling `cfy agents validate` on the old manager should fail
try:
logger.info('Validating the old agent is indeed down')
old_manager.use()
cfy.agents.validate()
except ErrorReturnCode:
logger.info('Old agent unresponsive, as expected')
else:
raise StandardError('Old agent is still responsive')
old_manager.delete()
new_manager.use()
nodecellar.manager = new_manager
nodecellar.verify_installation()
nodecellar.uninstall()
def test_multiple_networks(managers, cfy, multi_network_hello_worlds, logger,
tmpdir, attributes):
logger.info('Testing managers with multiple networks')
# We should have at least 3 hello world objects. We will verify the first
# one completely on the first manager.
# All the other ones will be installed on the first manager,
# then we'll create a snapshot and restore it on the second manager, and
# finally, to complete the verification, we'll uninstall the remaining
# hellos on the new manager
old_manager = managers[0]
new_manager = managers[1]
snapshot_id = 'SNAPSHOT_ID'
local_snapshot_path = str(tmpdir / 'snap.zip')
# The first hello is the one that belongs to a network that will be added
# manually post bootstrap to the new manager
post_bootstrap_hello = multi_network_hello_worlds.pop(0)
post_bootstrap_hello.manager = new_manager
for hello in multi_network_hello_worlds:
hello.upload_and_verify_install()
create_snapshot(old_manager, snapshot_id, attributes, logger)
download_snapshot(old_manager, local_snapshot_path, snapshot_id, logger)
new_manager.use()
upload_snapshot(new_manager, local_snapshot_path, snapshot_id, logger)
restore_snapshot(new_manager,
snapshot_id,
cfy,
logger,
change_manager_password=False)
upgrade_agents(cfy, new_manager, logger)
delete_manager(old_manager, logger)
for hello in multi_network_hello_worlds:
hello.manager = new_manager
hello.uninstall()
hello.delete_deployment()
_add_new_network(new_manager, logger)
post_bootstrap_hello.verify_all()
def test_distributed_installation_scenario(distributed_installation, cfy,
logger, tmpdir, attributes,
distributed_nodecellar):
manager = distributed_installation.manager
_set_admin_user(cfy, manager, logger)
# Creating secrets
_create_secrets(cfy, logger, attributes, manager, visibility='global')
distributed_nodecellar.upload_and_verify_install()
snapshot_id = 'SNAPSHOT_ID'
create_snapshot(manager, snapshot_id, attributes, logger)
# Restore snapshot
logger.info('Restoring snapshot')
restore_snapshot(manager, snapshot_id, cfy, logger, force=True)
distributed_nodecellar.uninstall()
def test_distributed_installation_scenario(distributed_installation,
cfy,
logger,
tmpdir,
attributes,
distributed_nodecellar):
manager = distributed_installation.manager
set_admin_user(cfy, manager, logger)
# Creating secrets
create_secrets(cfy, logger, attributes, manager, visibility='global')
distributed_nodecellar.upload_and_verify_install()
snapshot_id = 'SNAPSHOT_ID'
create_snapshot(manager, snapshot_id, attributes, logger)
# Restore snapshot
logger.info('Restoring snapshot')
restore_snapshot(manager, snapshot_id, cfy, logger, force=True)
distributed_nodecellar.uninstall()
def test_distributed_installation_sanity(distributed_installation,
cfy,
logger,
tmpdir,
attributes,
distributed_nodecellar):
logger.info('Running Sanity check for cluster with an external database')
rabbitmq = distributed_installation.message_queue
manager1 = distributed_installation.manager
manager2, manager3 = distributed_installation.joining_managers
manager_aio = distributed_installation.sanity_manager
manager1.use()
cfy.cluster('update-profile')
logger.info('Cfy version')
cfy('--version')
logger.info('Cfy status')
result = cfy.cluster.status()
assert len(re.findall('Active', result.stdout)) == 3
create_and_add_user_to_tenant(cfy, logger)
set_sanity_user(cfy, manager1, logger)
# Creating secrets with 'tenant' visibility
create_secrets(cfy, logger, attributes, manager1)
distributed_nodecellar.upload_and_verify_install()
set_admin_user(cfy, manager1, logger)
# Simulate failover (manager2/3 will be the remaining active managers)
toggle_cluster_node(manager1, 'nginx', logger, disable=True)
# Create and download snapshots from the remaining active managers (2 or 3)
snapshot_id = 'SNAPSHOT_ID'
local_snapshot_path = str(tmpdir / 'snap.zip')
logger.info('Creating snapshot')
create_snapshot(manager2, snapshot_id, attributes, logger)
download_snapshot(manager2, local_snapshot_path, snapshot_id, logger)
set_admin_user(cfy, manager_aio, logger)
# Upload and restore snapshot to the external AIO manager
logger.info('Uploading and restoring snapshot')
upload_snapshot(manager_aio, local_snapshot_path, snapshot_id, logger)
restore_snapshot(manager_aio, snapshot_id, cfy, logger,
change_manager_password=False)
time.sleep(7)
verify_services_status(manager_aio, logger)
# wait for agents reconnection
time.sleep(30)
# Upgrade agents
logger.info('Upgrading agents')
copy_ssl_cert_from_manager_to_tmpdir(manager2, tmpdir)
args = ['--manager-ip', rabbitmq.private_ip_address,
'--manager_certificate', str(tmpdir + 'new_manager_cert.txt'),
'--all-tenants']
cfy.agents.install(args)
set_sanity_user(cfy, manager_aio, logger)
# Verify `agents install` worked as expected
distributed_nodecellar.uninstall()
def test_distributed_installation_sanity(distributed_installation, cfy, logger,
tmpdir, attributes,
distributed_nodecellar):
logger.info('Running Sanity check for cluster with an external database')
manager1 = distributed_installation.manager
manager2, manager3 = distributed_installation.joining_managers
manager_aio = distributed_installation.sanity_manager
manager1.use()
verify_nodes_status(manager1, cfy, logger)
logger.info('Cfy version')
cfy('--version')
logger.info('Cfy status')
cfy.status()
_create_and_add_user_to_tenant(cfy, logger)
_set_sanity_user(cfy, manager1, logger)
# Creating secrets with 'tenant' visibility
_create_secrets(cfy, logger, attributes, manager1)
distributed_nodecellar.upload_and_verify_install()
_set_admin_user(cfy, manager1, logger)
# Simulate failover (manager2 will be the new cluster master)
set_active(manager2, cfy, logger)
# Create and download snapshots from the new cluster master (manager2)
snapshot_id = 'SNAPSHOT_ID'
local_snapshot_path = str(tmpdir / 'snap.zip')
logger.info('Creating snapshot')
create_snapshot(manager2, snapshot_id, attributes, logger)
download_snapshot(manager2, local_snapshot_path, snapshot_id, logger)
_set_admin_user(cfy, manager_aio, logger)
# Upload and restore snapshot to manager3
logger.info('Uploading and restoring snapshot')
upload_snapshot(manager_aio, local_snapshot_path, snapshot_id, logger)
restore_snapshot(manager_aio,
snapshot_id,
cfy,
logger,
change_manager_password=False)
time.sleep(7)
verify_services_status(manager_aio, logger)
# wait for agents reconnection
time.sleep(30)
# Upgrade agents
logger.info('Upgrading agents')
_copy_ssl_cert_from_manager_to_tmpdir(manager2, tmpdir)
args = [
'--manager-ip', manager2.private_ip_address, '--manager_certificate',
str(tmpdir + 'new_manager_cert.txt'), '--all-tenants'
]
cfy.agents.install(args)
_set_sanity_user(cfy, manager_aio, logger)
# Verify `agents install` worked as expected
distributed_nodecellar.uninstall()
def test_sanity_scenario(managers,
cfy,
logger,
tmpdir,
attributes,
nodecellar):
manager1 = managers[0]
manager2 = managers[1]
manager3 = managers[2]
manager3.use()
manager2.use()
manager1.use()
logger.info('Cfy version')
cfy('--version')
logger.info('Cfy status')
cfy.status()
# Start HA cluster, add manager2 to it
_start_cluster(cfy, manager1, logger)
_set_admin_user(cfy, manager2, logger)
_join_cluster(cfy, manager1, manager2, logger)
time.sleep(30)
_create_and_add_user_to_tenant(cfy, logger)
_set_sanity_user(cfy, manager1, logger)
# Creating secrets
_create_secrets(cfy, logger, attributes, manager1)
nodecellar.upload_and_verify_install()
_set_admin_user(cfy, manager1, logger)
# Simulate failover (manager2 will be the new cluster master)
logger.info('Setting replica manager')
_set_admin_user(cfy, manager2, logger)
ha_helper.set_active(manager2, cfy, logger)
# time.sleep(30)
# Create and download snapshots from the new cluster master (manager2)
snapshot_id = 'SNAPSHOT_ID'
local_snapshot_path = str(tmpdir / 'snap.zip')
logger.info('Creating snapshot')
create_snapshot(manager2, snapshot_id, attributes, logger)
download_snapshot(manager2, local_snapshot_path, snapshot_id, logger)
_set_admin_user(cfy, manager3, logger)
# Upload and restore snapshot to manager3
logger.info('Uploading and restoring snapshot')
upload_snapshot(manager3, local_snapshot_path, snapshot_id, logger)
restore_snapshot(manager3, snapshot_id,
cfy, logger, change_manager_password=False)
verify_services_status(manager3, logger)
# Upgrade agents
logger.info('Upgrading agents')
_copy_ssl_cert_from_manager_to_tmpdir(manager2, tmpdir)
args = ['--manager-ip', manager2.private_ip_address,
'--manager_certificate', str(tmpdir + 'new_manager_cert.txt'),
'--all-tenants']
cfy.agents.install(args)
_set_sanity_user(cfy, manager3, logger)
# Verify `agents install` worked as expected
nodecellar.uninstall()