class ManagerUpgradeTest(TestCase):
def test_manager_upgrade(self):
"""Bootstrap a manager, upgrade it, rollback it, examine the results.
To test the manager in-place upgrade procedure:
- bootstrap a manager (this is part of the system under test,
does destructive changes to the manager, and need a known manager
version: so, can't use the testenv manager)
- deploy the hello world app
- upgrade the manager, changing some inputs (eg. the port that
Elasticsearch uses)
- check that everything still works (the previous deployment still
reports metrics; we can install another deployment)
- rollback the manager
- post-rollback checks: the changed inputs are now the original
values again, the installed app still reports metrics
"""
self.prepare_manager()
self.preupgrade_deployment_id = self.deploy_hello_world('pre-')
self.upgrade_manager()
self.post_upgrade_checks()
self.rollback_manager()
self.post_rollback_checks()
self.teardown_manager()
@contextmanager
def _manager_fabric_env(self):
inputs = self.manager_inputs
with fabric.context_managers.settings(
host_string=self.upgrade_manager_ip,
user=inputs['ssh_user'],
key_filename=inputs['ssh_key_filename']):
yield fabric.api
def _bootstrap_local_env(self, workdir):
storage = local.FileStorage(
os.path.join(workdir, '.cloudify', 'bootstrap'))
return local.load_env('manager', storage=storage)
def _blueprint_rpm_versions(self, blueprint_path, inputs):
"""RPM filenames that should be installed on the manager.
"""
env = local.init_env(
blueprint_path,
inputs=inputs,
ignored_modules=cli_constants.IGNORED_LOCAL_WORKFLOW_MODULES)
storage = env.storage
amqp_influx_rpm = storage.get_node('amqp_influx')['properties'][
'amqpinflux_rpm_source_url']
restservice_rpm = storage.get_node('rest_service')['properties'][
'rest_service_rpm_source_url']
mgmtworker_rpm = storage.get_node('mgmt_worker')['properties'][
'management_worker_rpm_source_url']
return {
'cloudify-amqp-influx': amqp_influx_rpm,
'cloudify-rest-service': restservice_rpm,
'cloudify-management-worker': mgmtworker_rpm
}
def _cloudify_rpm_versions(self):
with self._manager_fabric_env() as fabric:
return fabric.sudo('rpm -qa | grep cloudify')
def check_rpm_versions(self, blueprint_path, inputs):
blueprint_rpms = self._blueprint_rpm_versions(blueprint_path, inputs)
installed_rpms = self._cloudify_rpm_versions()
for service_name, rpm_filename in blueprint_rpms.items():
for line in installed_rpms.split('\n'):
line = line.strip()
if line.startswith(service_name):
self.assertIn(line.strip(), rpm_filename)
def prepare_manager(self):
# note that we're using a separate manager checkout, so we need to
# create our own utils like cfy and the rest client, rather than use
# the testenv ones
self.cfy_workdir = tempfile.mkdtemp(prefix='manager-upgrade-')
self.addCleanup(shutil.rmtree, self.cfy_workdir)
self.manager_cfy = CfyHelper(cfy_workdir=self.cfy_workdir)
self.manager_inputs = self._get_bootstrap_inputs()
self.bootstrap_manager()
self.rest_client = create_rest_client(self.upgrade_manager_ip)
self.bootstrap_manager_version = LooseVersion(
self.rest_client.manager.get_version()['version'])
def _get_bootstrap_inputs(self):
prefix = self.test_id
ssh_key_filename = os.path.join(self.workdir, 'manager.key')
self.addCleanup(self.env.handler.remove_keypair,
prefix + '-manager-key')
agent_key_path = os.path.join(self.workdir, 'agents.key')
self.addCleanup(self.env.handler.remove_keypair,
prefix + '-agents-key')
return {
'keystone_username': self.env.keystone_username,
'keystone_password': self.env.keystone_password,
'keystone_tenant_name': self.env.keystone_tenant_name,
'keystone_url': self.env.keystone_url,
'region': self.env.region,
'flavor_id': self.env.medium_flavor_id,
'image_id': self.env.centos_7_image_id,
'ssh_user': self.env.centos_7_image_user,
'external_network_name': self.env.external_network_name,
'resources_prefix': 'test-upgrade-',
'manager_server_name': prefix + '-manager',
# shared settings
'manager_public_key_name': prefix + '-manager-key',
'agent_public_key_name': prefix + '-agents-key',
'ssh_key_filename': ssh_key_filename,
'agent_private_key_path': agent_key_path,
'management_network_name': prefix + '-network',
'management_subnet_name': prefix + '-subnet',
'management_router': prefix + '-router',
'agents_user': '',
# private settings
'manager_security_group_name': prefix + '-m-sg',
'agents_security_group_name': prefix + '-a-sg',
'manager_port_name': prefix + '-port',
'management_subnet_dns_nameservers': ['8.8.8.8', '8.8.4.4']
}
def get_bootstrap_blueprint(self):
manager_repo_dir = tempfile.mkdtemp(prefix='manager-upgrade-')
self.addCleanup(shutil.rmtree, manager_repo_dir)
manager_repo = clone(BOOTSTRAP_REPO_URL,
manager_repo_dir,
branch=BOOTSTRAP_BRANCH)
yaml_path = manager_repo / 'openstack-manager-blueprint.yaml'
# allow the ports that we're going to connect to from the tests,
# when doing checks
for port in [8086, 9200, 9900]:
secgroup_cfg = [{
'port_range_min': port,
'port_range_max': port,
'remote_ip_prefix': '0.0.0.0/0'
}]
secgroup_cfg_path = 'node_templates.management_security_group'\
'.properties.rules'
with YamlPatcher(yaml_path) as patch:
patch.append_value(secgroup_cfg_path, secgroup_cfg)
return yaml_path
def _load_private_ip_from_env(self, workdir):
env = self._bootstrap_local_env(workdir)
return env.outputs()['private_ip']
def bootstrap_manager(self):
self.bootstrap_blueprint = self.get_bootstrap_blueprint()
inputs_path = self.manager_cfy._get_inputs_in_temp_file(
self.manager_inputs, self._testMethodName)
self.manager_cfy.bootstrap(self.bootstrap_blueprint,
inputs_file=inputs_path)
self.upgrade_manager_ip = self.manager_cfy.get_management_ip()
self.manager_private_ip = self._load_private_ip_from_env(
self.cfy_workdir)
# TODO: why is this needed?
self.manager_cfy.use(management_ip=self.upgrade_manager_ip)
def deploy_hello_world(self, prefix=''):
"""Install the hello world app."""
blueprint_id = prefix + self.test_id
deployment_id = prefix + self.test_id
hello_repo_dir = tempfile.mkdtemp(prefix='manager-upgrade-')
hello_repo_path = clone(
'https://github.com/cloudify-cosmo/'
'cloudify-hello-world-example.git',
hello_repo_dir
)
self.addCleanup(shutil.rmtree, hello_repo_dir)
hello_blueprint_path = hello_repo_path / 'blueprint.yaml'
self.manager_cfy.upload_blueprint(blueprint_id, hello_blueprint_path)
inputs = {
'agent_user': self.env.ubuntu_image_user,
'image': self.env.ubuntu_trusty_image_name,
'flavor': self.env.flavor_name
}
self.manager_cfy.create_deployment(blueprint_id, deployment_id,
inputs=inputs)
self.manager_cfy.execute_install(deployment_id=deployment_id)
return deployment_id
def get_upgrade_blueprint(self):
repo_dir = tempfile.mkdtemp(prefix='manager-upgrade-')
self.addCleanup(shutil.rmtree, repo_dir)
upgrade_blueprint_path = clone(UPGRADE_REPO_URL,
repo_dir,
branch=UPGRADE_BRANCH)
return upgrade_blueprint_path / 'simple-manager-blueprint.yaml'
def upgrade_manager(self):
self.upgrade_blueprint = self.get_upgrade_blueprint()
# we're changing one of the ES inputs - make sure we also re-install ES
with YamlPatcher(self.upgrade_blueprint) as patch:
patch.set_value(
('node_templates.elasticsearch.properties'
'.use_existing_on_upgrade'),
False)
self.upgrade_inputs = {
'private_ip': self.manager_private_ip,
'public_ip': self.upgrade_manager_ip,
'ssh_key_filename': self.manager_inputs['ssh_key_filename'],
'ssh_user': self.manager_inputs['ssh_user'],
'elasticsearch_endpoint_port': 9900
}
upgrade_inputs_file = self.manager_cfy._get_inputs_in_temp_file(
self.upgrade_inputs, self._testMethodName)
with self.manager_cfy.maintenance_mode():
self.manager_cfy.upgrade_manager(
blueprint_path=self.upgrade_blueprint,
inputs_file=upgrade_inputs_file)
def post_upgrade_checks(self):
"""To check if the upgrade succeeded:
- fire a request to the REST service
- check that elasticsearch is listening on the changed port
- check that the pre-existing deployment still reports to influxdb
- install a new deployment, check that it reports to influxdb,
and uninstall it: to check that the manager still allows
creating, installing and uninstalling deployments correctly
"""
upgrade_manager_version = LooseVersion(
self.rest_client.manager.get_version()['version'])
self.assertGreaterEqual(upgrade_manager_version,
self.bootstrap_manager_version)
self.check_rpm_versions(self.upgrade_blueprint, self.upgrade_inputs)
self.rest_client.blueprints.list()
self.check_elasticsearch(self.upgrade_manager_ip, 9900)
self.check_influx(self.preupgrade_deployment_id)
postupgrade_deployment_id = self.deploy_hello_world('post-')
self.check_influx(postupgrade_deployment_id)
self.uninstall_deployment(postupgrade_deployment_id)
def check_influx(self, deployment_id):
"""Check that the deployment_id continues to report metrics.
Look at the last 5 seconds worth of metrics. To avoid race conditions
(running this check before the deployment even had a chance to report
any metrics), first wait 5 seconds to allow some metrics to be
gathered.
"""
# TODO influx config should be pulled from props?
time.sleep(5)
influx_client = InfluxDBClient(self.upgrade_manager_ip, 8086,
'root', 'root', 'cloudify')
try:
result = influx_client.query('select * from /^{0}\./i '
'where time > now() - 5s'
.format(deployment_id))
except NameError as e:
self.fail('monitoring events list for deployment with ID {0} were'
' not found on influxDB. error is: {1}'
.format(deployment_id, e))
self.assertTrue(len(result) > 0)
def check_elasticsearch(self, host, port):
"""Check that elasticsearch is listening on the given host:port.
This is used for checking if the ES port changed correctly during
the upgrade.
"""
try:
response = urllib2.urlopen('http://{0}:{1}'.format(
self.upgrade_manager_ip, port))
response = json.load(response)
if response['status'] != 200:
raise ValueError('Incorrect status {0}'.format(
response['status']))
except (ValueError, urllib2.URLError):
self.fail('elasticsearch isnt listening on the changed port')
def uninstall_deployment(self, deployment_id):
self.manager_cfy.execute_uninstall(deployment_id)
def rollback_manager(self):
rollback_inputs = {
'private_ip': self.manager_private_ip,
'public_ip': self.upgrade_manager_ip,
'ssh_key_filename': self.manager_inputs['ssh_key_filename'],
'ssh_user': self.manager_inputs['ssh_user'],
}
rollback_inputs_file = self.manager_cfy._get_inputs_in_temp_file(
rollback_inputs, self._testMethodName)
with self.manager_cfy.maintenance_mode():
self.manager_cfy.rollback_manager(
blueprint_path=self.upgrade_blueprint,
inputs_file=rollback_inputs_file)
def post_rollback_checks(self):
rollback_manager_version = LooseVersion(
self.rest_client.manager.get_version()['version'])
self.assertEqual(rollback_manager_version,
self.bootstrap_manager_version)
self.check_rpm_versions(self.bootstrap_blueprint, self.manager_inputs)
self.rest_client.blueprints.list()
self.check_elasticsearch(self.upgrade_manager_ip, 9200)
self.check_influx(self.preupgrade_deployment_id)
def teardown_manager(self):
self.manager_cfy.teardown(ignore_deployments=True)
