def teardown(self):
if self._global_cleanup_context is None:
return
self.setup()
cfy = CfyHelper(cfy_workdir=self._workdir)
try:
cfy.use(self.management_ip)
cfy.teardown(verbose=True)
finally:
self._global_cleanup_context.cleanup()
self.handler.after_teardown()
if os.path.exists(self._workdir):
shutil.rmtree(self._workdir)
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.logger.info('Starting test setUp')
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}'.format(time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
def bootstrap(self):
if self._management_running:
return
self._global_cleanup_context = self.handler.CleanupContext(
'testenv', self)
cfy = CfyHelper(cfy_workdir=self._workdir)
self.handler.before_bootstrap()
if self.is_provider_bootstrap:
cfy.bootstrap_with_providers(
self.cloudify_config_path,
self.handler.provider,
keep_up_on_failure=False,
verbose=True,
dev_mode=False)
else:
install_plugins = self._get_boolean_env_var(
INSTALL_MANAGER_BLUEPRINT_DEPENDENCIES, True)
cfy.bootstrap(
self._manager_blueprint_path,
inputs_file=self.cloudify_config_path,
install_plugins=install_plugins,
keep_up_on_failure=False,
verbose=True)
self._running_env_setup(cfy.get_management_ip())
self.handler.after_bootstrap(cfy.get_provider_context())
def bootstrap(self, task_retries=5):
if self._management_running:
return
self._global_cleanup_context = self.handler.CleanupContext(
'testenv', self)
cfy = CfyHelper(cfy_workdir=self._workdir)
self.handler.before_bootstrap()
cfy.bootstrap(
self._manager_blueprint_path,
inputs_file=self.cloudify_config_path,
install_plugins=self.install_plugins,
keep_up_on_failure=False,
task_retries=task_retries,
verbose=True)
self._running_env_setup(cfy.get_management_ip())
self.handler.after_bootstrap(cfy.get_provider_context())
def setUp(self):
super(TwoManagersTest, self).setUp()
self.workdir2 = tempfile.mkdtemp(prefix='cloudify-testenv-')
self.cfy2 = CfyHelper(self.workdir2, testcase=self)
second_manager_blueprint_path = '{}_existing'.format(
self.env._manager_blueprint_path)
shutil.copy2(self.env._manager_blueprint_path,
second_manager_blueprint_path)
external_resources = [
'node_templates.management_network.properties',
'node_templates.management_subnet.properties',
'node_templates.router.properties',
'node_templates.agents_security_group.properties',
'node_templates.management_security_group.properties',
]
with YamlPatcher(second_manager_blueprint_path) as patch:
for prop in external_resources:
patch.merge_obj(prop, {'use_external_resource': True})
second_cloudify_config_path = '{}_existing'.format(
self.env.cloudify_config_path)
shutil.copy2(self.env.cloudify_config_path,
second_cloudify_config_path)
new_resources = ['manager_server_name', 'manager_port_name']
with YamlPatcher(second_cloudify_config_path) as patch:
for prop in new_resources:
patch.append_value(prop, '2')
self.cfy2.bootstrap(blueprint_path=second_manager_blueprint_path,
inputs_file=second_cloudify_config_path,
install_plugins=self.env.install_plugins,
keep_up_on_failure=False,
task_retries=5,
verbose=False)
self.client2 = create_rest_client(self.cfy2.get_management_ip())
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}'.format(time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
def prepare_manager(self):
# note that we're using a separate manager checkout, so we need to
# create our own utils like cfy and the rest client, rather than use
# the testenv ones
self.cfy_workdir = tempfile.mkdtemp(prefix='manager-upgrade-')
self.addCleanup(shutil.rmtree, self.cfy_workdir)
self.manager_cfy = CfyHelper(cfy_workdir=self.cfy_workdir)
self.manager_inputs = self._get_bootstrap_inputs()
self.bootstrap_manager()
self.rest_client = create_rest_client(self.upgrade_manager_ip)
self.bootstrap_manager_version = LooseVersion(
self.rest_client.manager.get_version()['version'])
def setUp(self):
super(TwoManagersTest, self).setUp()
self.workdir2 = tempfile.mkdtemp(prefix='cloudify-testenv-')
self.cfy2 = CfyHelper(self.workdir2, testcase=self)
second_manager_blueprint_path = '{}_existing'.format(
self.env._manager_blueprint_path)
shutil.copy2(self.env._manager_blueprint_path,
second_manager_blueprint_path)
external_resources = [
'node_templates.management_network.properties',
'node_templates.management_subnet.properties',
'node_templates.router.properties',
'node_templates.agents_security_group.properties',
'node_templates.management_security_group.properties',
]
with YamlPatcher(second_manager_blueprint_path) as patch:
for prop in external_resources:
patch.merge_obj(prop, {'use_external_resource': True})
second_cloudify_config_path = '{}_existing'.format(
self.env.cloudify_config_path)
shutil.copy2(self.env.cloudify_config_path,
second_cloudify_config_path)
new_resources = ['manager_server_name', 'manager_port_name']
with YamlPatcher(second_cloudify_config_path) as patch:
for prop in new_resources:
patch.append_value(prop, '2')
self.cfy2.bootstrap(
blueprint_path=second_manager_blueprint_path,
inputs_file=second_cloudify_config_path,
install_plugins=self.env.install_plugins,
keep_up_on_failure=False,
task_retries=5,
verbose=False
)
self.client2 = create_rest_client(self.cfy2.get_management_ip())
def teardown(self):
if self._global_cleanup_context is None:
return
self.setup()
cfy = CfyHelper(cfy_workdir=self._workdir)
try:
cfy.use(self.management_ip)
cfy.teardown(verbose=True)
finally:
self._global_cleanup_context.cleanup()
self.handler.after_teardown()
if os.path.exists(self._workdir):
shutil.rmtree(self._workdir)
def teardown(self):
if self._global_cleanup_context is None:
return
self.setup()
cfy = CfyHelper(cfy_workdir=self._workdir)
try:
cfy.use(self.management_ip, provider=self.is_provider_bootstrap)
if self.is_provider_bootstrap:
cfy.teardown_with_providers(
self.cloudify_config_path,
verbose=True)
else:
cfy.teardown(verbose=True)
finally:
self._global_cleanup_context.cleanup()
self.handler.after_teardown()
if os.path.exists(self._workdir):
shutil.rmtree(self._workdir)
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.logger.info('Starting test setUp')
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
management_user = getattr(self.env, 'management_user_name', None)
management_key_path = getattr(self.env, 'management_key_path', None)
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip,
management_user=management_user,
management_key=management_key_path)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}-{1}'.format(
self._testMethodName,
time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
self.maxDiff = 1024 * 1024 * 10
def bootstrap(self, task_retries=5):
if self._management_running:
return
self._global_cleanup_context = self.handler.CleanupContext(
'testenv', self)
cfy = CfyHelper(cfy_workdir=self._workdir)
self.handler.before_bootstrap()
cfy.bootstrap(
self._manager_blueprint_path,
inputs_file=self.cloudify_config_path,
install_plugins=self.install_plugins,
keep_up_on_failure=False,
task_retries=task_retries,
verbose=True)
self._running_env_setup(cfy.get_management_ip())
self.handler.after_bootstrap(cfy.get_provider_context())
class TestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}'.format(time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
def _cleanup(self):
self._test_cleanup_context.cleanup()
shutil.rmtree(self.workdir)
def tearDown(self):
# note that the cleanup function is registered in setUp
# because it is called regardless of whether setUp succeeded or failed
# unlike tearDown which is not called when setUp fails (which might
# happen when tests override setUp)
if self.env.management_ip:
try:
self.logger.info('Running ps aux on Cloudify manager...')
output = StringIO()
with fabric_api.settings(
user=self.env.management_user_name,
host_string=self.env.management_ip,
key_filename=get_actual_keypath(
self.env,
self.env.management_key_path),
disable_known_hosts=True):
fabric_api.run('ps aux --sort -rss', stdout=output)
self.logger.info(
'Cloudify manager ps aux output:\n{0}'.format(
output.getvalue()))
except Exception as e:
self.logger.info(
'Error running ps aux on Cloudify manager: {0}'.format(
str(e)))
def get_manager_state(self):
self.logger.info('Fetching manager current state')
blueprints = {}
for blueprint in self.client.blueprints.list():
blueprints[blueprint.id] = blueprint
deployments = {}
for deployment in self.client.deployments.list():
deployments[deployment.id] = deployment
nodes = {}
for deployment_id in deployments.keys():
for node in self.client.node_instances.list(deployment_id):
nodes[node.id] = node
deployment_nodes = {}
node_state = {}
for deployment_id in deployments.keys():
deployment_nodes[deployment_id] = self.client.node_instances.list(
deployment_id)
node_state[deployment_id] = {}
for node in deployment_nodes[deployment_id]:
node_state[deployment_id][node.id] = node
return {
'blueprints': blueprints,
'deployments': deployments,
'nodes': nodes,
'node_state': node_state,
'deployment_nodes': deployment_nodes
}
def get_manager_state_delta(self, before, after):
after = copy.deepcopy(after)
for blueprint_id in before['blueprints'].keys():
del after['blueprints'][blueprint_id]
for deployment_id in before['deployments'].keys():
del after['deployments'][deployment_id]
del after['deployment_nodes'][deployment_id]
del after['node_state'][deployment_id]
for node_id in before['nodes'].keys():
del after['nodes'][node_id]
return after
def execute_install(self,
deployment_id=None,
fetch_state=True):
return self._make_operation_with_before_after_states(
self.cfy.execute_install,
fetch_state,
deployment_id=deployment_id)
def upload_deploy_and_execute_install(self, blueprint_id=None,
deployment_id=None,
fetch_state=True,
inputs=None):
return self._make_operation_with_before_after_states(
self.cfy.upload_deploy_and_execute_install,
fetch_state,
str(self.blueprint_yaml),
blueprint_id=blueprint_id or self.test_id,
deployment_id=deployment_id or self.test_id,
inputs=inputs)
def _make_operation_with_before_after_states(self, operation, fetch_state,
*args, **kwargs):
before_state = None
after_state = None
if fetch_state:
before_state = self.get_manager_state()
operation(*args, **kwargs)
if fetch_state:
after_state = self.get_manager_state()
return before_state, after_state
def execute_uninstall(self, deployment_id=None):
self.cfy.execute_uninstall(deployment_id=deployment_id or self.test_id)
def copy_blueprint(self, blueprint_dir_name):
blueprint_path = path(self.workdir) / blueprint_dir_name
shutil.copytree(get_blueprint_path(blueprint_dir_name),
str(blueprint_path))
return blueprint_path
def wait_for_execution(self, execution, timeout):
end = time.time() + timeout
while time.time() < end:
status = self.client.executions.get(execution.id).status
if status == 'failed':
raise AssertionError('Execution "{}" failed'.format(
execution.id))
if status == 'terminated':
return
time.sleep(1)
events, _ = self.client.events.get(execution.id,
batch_size=1000,
include_logs=True)
self.logger.info('Deployment creation events & logs:')
for event in events:
self.logger.info(json.dumps(event))
raise AssertionError('Execution "{}" timed out'.format(execution.id))
def repetitive(self, func, timeout=10, exception_class=Exception,
**kwargs):
deadline = time.time() + timeout
while True:
try:
func(**kwargs)
break
except exception_class:
if time.time() > deadline:
raise
time.sleep(1)
class TestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def wait_until_deployment_ready_and_execute_install(self,
deployment_id,
inputs):
self.wait_until_all_deployment_executions_end(deployment_id)
return self.execute_install(deployment_id=deployment_id)
def wait_until_all_deployment_executions_end(
self,
deployment_id=None,
include_system_workflows=False,
verify_no_failed_execution=False,
end_status_list=Execution.END_STATES):
if deployment_id:
msg = "Waiting for executions on " \
"deployment {0} to finish".format(deployment_id)
else:
msg = "Waiting for system wide executions to finish"
if include_system_workflows:
msg = "{0}, including system workflows.".format(msg)
self.logger.info(msg)
start_time = time.time()
while len([execution for execution in self.client.executions.list(
deployment_id=deployment_id,
include_system_workflows=include_system_workflows)
if execution["status"] not in end_status_list]) > 0:
time.sleep(1)
if start_time - time.time() > DEFAULT_EXECUTE_TIMEOUT:
if deployment_id:
timeout_msg = "Timeout while waiting for " \
"executions to end " \
"on deployment {0}.".format(deployment_id)
else:
timeout_msg = "Timeout while waiting for " \
"system wide executions to end."
raise Exception(timeout_msg)
if verify_no_failed_execution:
executions = [e for e in self.client.executions.list(
deployment_id=deployment_id,
include_system_workflows=include_system_workflows)
if e["status"] == Execution.FAILED]
if executions:
self.logger.error('Failed executions found.')
for e in executions:
self.logger.error('Execution {0} logs:'.format(e.id))
self.logger.error(self.cfy.list_events(
execution_id=e.id, verbosity='-vv'))
return
def assert_outputs(self, expected_outputs, deployment_id=None):
if deployment_id is None:
deployment_id = self.test_id
outputs = self.client.deployments.outputs.get(deployment_id)
outputs = outputs['outputs']
self.assertEqual(expected_outputs, outputs)
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.logger.info('Starting test setUp')
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
management_user = getattr(self.env, 'management_user_name', None)
management_key_path = getattr(self.env, 'management_key_path', None)
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip,
management_user=management_user,
management_key=management_key_path)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}-{1}'.format(
self._testMethodName,
time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
self.maxDiff = 1024 * 1024 * 10
def _cleanup(self):
self.env.setup()
self._test_cleanup_context.cleanup()
shutil.rmtree(self.workdir)
def tearDown(self):
self.logger.info('Starting test tearDown')
# note that the cleanup function is registered in setUp
# because it is called regardless of whether setUp succeeded or failed
# unlike tearDown which is not called when setUp fails (which might
# happen when tests override setUp)
def get_manager_state(self):
self.logger.info('Fetching manager current state')
blueprints = {}
for blueprint in self.client.blueprints.list():
blueprints[blueprint.id] = blueprint
deployments = {}
for deployment in self.client.deployments.list():
deployments[deployment.id] = deployment
nodes = {}
for deployment_id in deployments.keys():
for node in self.client.node_instances.list(deployment_id):
nodes[node.id] = node
deployment_nodes = {}
node_state = {}
for deployment_id in deployments.keys():
deployment_nodes[deployment_id] = self.client.node_instances.list(
deployment_id).items
node_state[deployment_id] = {}
for node in deployment_nodes[deployment_id]:
node_state[deployment_id][node.id] = node
return {
'blueprints': blueprints,
'deployments': deployments,
'nodes': nodes,
'node_state': node_state,
'deployment_nodes': deployment_nodes
}
def get_manager_state_delta(self, before, after):
after = copy.deepcopy(after)
for blueprint_id in before['blueprints'].keys():
del after['blueprints'][blueprint_id]
for deployment_id in before['deployments'].keys():
del after['deployments'][deployment_id]
del after['deployment_nodes'][deployment_id]
del after['node_state'][deployment_id]
for node_id in before['nodes'].keys():
del after['nodes'][node_id]
return after
def execute_install(self,
deployment_id=None,
fetch_state=True):
self.logger.info("attempting to execute install on deployment {0}"
.format(deployment_id))
return self._make_operation_with_before_after_states(
self.cfy.execute_install,
fetch_state,
deployment_id=deployment_id)
def install(
self,
blueprint_id=None,
deployment_id=None,
fetch_state=True,
execute_timeout=DEFAULT_EXECUTE_TIMEOUT,
inputs=None):
return self._make_operation_with_before_after_states(
self.cfy.install,
fetch_state,
str(self.blueprint_yaml),
blueprint_id=blueprint_id or self.test_id,
deployment_id=deployment_id or self.test_id,
execute_timeout=execute_timeout,
inputs=inputs)
upload_deploy_and_execute_install = install
def upload_blueprint(
self,
blueprint_id):
self.logger.info("attempting to upload blueprint {0}"
.format(blueprint_id))
return self.cfy.upload_blueprint(
blueprint_id=blueprint_id,
blueprint_path=str(self.blueprint_yaml))
def create_deployment(
self,
blueprint_id,
deployment_id,
inputs):
self.logger.info("attempting to create_deployment deployment {0}"
.format(deployment_id))
return self.cfy.create_deployment(
blueprint_id=blueprint_id,
deployment_id=deployment_id,
inputs=inputs)
def _make_operation_with_before_after_states(self, operation, fetch_state,
*args, **kwargs):
before_state = None
after_state = None
if fetch_state:
before_state = self.get_manager_state()
operation(*args, **kwargs)
if fetch_state:
after_state = self.get_manager_state()
return before_state, after_state
def execute_uninstall(self, deployment_id=None, cfy=None,
delete_deployment_and_blueprint=False):
cfy = cfy or self.cfy
if delete_deployment_and_blueprint:
cfy.uninstall(deployment_id=deployment_id or self.test_id,
workflow_id='uninstall',
parameters=None,
allow_custom_parameters=False,
timeout=DEFAULT_EXECUTE_TIMEOUT,
include_logs=True)
else:
cfy.execute_uninstall(deployment_id=deployment_id or self.test_id)
def copy_blueprint(self, blueprint_dir_name, blueprints_dir=None):
blueprint_path = path(self.workdir) / blueprint_dir_name
shutil.copytree(get_blueprint_path(blueprint_dir_name, blueprints_dir),
str(blueprint_path))
return blueprint_path
def wait_for_execution(self, execution, timeout, client=None,
assert_success=True):
def dump_events(_client, _execution):
events, _ = _client.events.get(_execution.id,
batch_size=1000,
include_logs=True)
self.logger.info('Deployment creation events & logs:')
for event in events:
self.logger.info(json.dumps(event))
client = client or self.client
end = time.time() + timeout
while time.time() < end:
status = client.executions.get(execution.id).status
if status == 'failed':
if assert_success:
dump_events(client, execution)
raise AssertionError('Execution "{}" failed'.format(
execution.id))
else:
return
if status == 'terminated':
return
time.sleep(1)
dump_events(client, execution)
raise AssertionError('Execution "{}" timed out'.format(execution.id))
def repetitive(self, func, timeout=10, exception_class=Exception,
args=None, kwargs=None):
args = args or []
kwargs = kwargs or {}
deadline = time.time() + timeout
while True:
try:
return func(*args, **kwargs)
except exception_class:
if time.time() > deadline:
raise
time.sleep(1)
@contextmanager
def manager_env_fabric(self, **kwargs):
with fabric.context_managers.settings(
host_string=self.cfy.get_management_ip(),
user=self.env.management_user_name,
key_filename=self.env.management_key_path,
**kwargs):
yield fabric.api
def run_commands_on_agent_host(self,
user,
commands,
deployment_id=None,
compute_node_id=None,
compute_node_instance_id=None):
if not (compute_node_id or compute_node_instance_id):
self.fail('no node_id or node_instance_id')
deployment_id = deployment_id or self.test_id
filters = {'deployment_id': deployment_id}
if compute_node_id:
filters['node_id'] = compute_node_id
if compute_node_instance_id:
filters['id'] = compute_node_instance_id
computes = self.client.node_instances.list(**filters).items
if not computes:
self.fail('No compute nodes were found')
if len(computes) > 1:
self.fail('More than one instance found, please refine your query:'
' {0}'.format(computes))
compute = computes[0]
private_ip = compute.runtime_properties['ip']
with self.manager_env_fabric() as api:
api.sudo('ssh '
'-o UserKnownHostsFile=/dev/null '
'-o StrictHostKeyChecking=no '
'-t -i /root/.ssh/agent_key.pem {0}@{1} "{2}"'
.format(user, private_ip, ' && '.join(commands)))
def wait_for_resource(self, predicate_func, timeout_sec=60):
timeout = time.time() + timeout_sec
while True:
if time.time() > timeout:
raise RuntimeError('Failed waiting for resource')
try:
result = predicate_func()
if result:
break
except Exception as e:
logger.info('predicate function raised an error; {error}'
.format(error=e))
time.sleep(1)
class TestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def assert_outputs(self, expected_outputs, deployment_id=None):
if deployment_id is None:
deployment_id = self.test_id
outputs = self.client.deployments.outputs.get(deployment_id)
outputs = outputs['outputs']
self.assertEqual(expected_outputs, outputs)
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.logger.info('Starting test setUp')
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}'.format(time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
def _cleanup(self):
self._test_cleanup_context.cleanup()
shutil.rmtree(self.workdir)
def tearDown(self):
self.logger.info('Starting test tearDown')
# note that the cleanup function is registered in setUp
# because it is called regardless of whether setUp succeeded or failed
# unlike tearDown which is not called when setUp fails (which might
# happen when tests override setUp)
def get_manager_state(self):
self.logger.info('Fetching manager current state')
blueprints = {}
for blueprint in self.client.blueprints.list():
blueprints[blueprint.id] = blueprint
deployments = {}
for deployment in self.client.deployments.list():
deployments[deployment.id] = deployment
nodes = {}
for deployment_id in deployments.keys():
for node in self.client.node_instances.list(deployment_id):
nodes[node.id] = node
deployment_nodes = {}
node_state = {}
for deployment_id in deployments.keys():
deployment_nodes[deployment_id] = self.client.node_instances.list(
deployment_id)
node_state[deployment_id] = {}
for node in deployment_nodes[deployment_id]:
node_state[deployment_id][node.id] = node
return {
'blueprints': blueprints,
'deployments': deployments,
'nodes': nodes,
'node_state': node_state,
'deployment_nodes': deployment_nodes
}
def get_manager_state_delta(self, before, after):
after = copy.deepcopy(after)
for blueprint_id in before['blueprints'].keys():
del after['blueprints'][blueprint_id]
for deployment_id in before['deployments'].keys():
del after['deployments'][deployment_id]
del after['deployment_nodes'][deployment_id]
del after['node_state'][deployment_id]
for node_id in before['nodes'].keys():
del after['nodes'][node_id]
return after
def execute_install(self,
deployment_id=None,
fetch_state=True):
return self._make_operation_with_before_after_states(
self.cfy.execute_install,
fetch_state,
deployment_id=deployment_id)
def upload_deploy_and_execute_install(
self,
blueprint_id=None,
deployment_id=None,
fetch_state=True,
execute_timeout=DEFAULT_EXECUTE_TIMEOUT,
inputs=None):
return self._make_operation_with_before_after_states(
self.cfy.upload_deploy_and_execute_install,
fetch_state,
str(self.blueprint_yaml),
blueprint_id=blueprint_id or self.test_id,
deployment_id=deployment_id or self.test_id,
execute_timeout=execute_timeout,
inputs=inputs)
def _make_operation_with_before_after_states(self, operation, fetch_state,
*args, **kwargs):
before_state = None
after_state = None
if fetch_state:
before_state = self.get_manager_state()
operation(*args, **kwargs)
if fetch_state:
after_state = self.get_manager_state()
return before_state, after_state
def execute_uninstall(self, deployment_id=None):
self.cfy.execute_uninstall(deployment_id=deployment_id or self.test_id)
def copy_blueprint(self, blueprint_dir_name, blueprints_dir=None):
blueprint_path = path(self.workdir) / blueprint_dir_name
shutil.copytree(get_blueprint_path(blueprint_dir_name, blueprints_dir),
str(blueprint_path))
return blueprint_path
def wait_for_execution(self, execution, timeout):
end = time.time() + timeout
while time.time() < end:
status = self.client.executions.get(execution.id).status
if status == 'failed':
raise AssertionError('Execution "{}" failed'.format(
execution.id))
if status == 'terminated':
return
time.sleep(1)
events, _ = self.client.events.get(execution.id,
batch_size=1000,
include_logs=True)
self.logger.info('Deployment creation events & logs:')
for event in events:
self.logger.info(json.dumps(event))
raise AssertionError('Execution "{}" timed out'.format(execution.id))
def repetitive(self, func, timeout=10, exception_class=Exception,
args=None, kwargs=None):
args = args or []
kwargs = kwargs or {}
deadline = time.time() + timeout
while True:
try:
return func(*args, **kwargs)
except exception_class:
if time.time() > deadline:
raise
time.sleep(1)
@contextmanager
def manager_env_fabric(self, **kwargs):
with fabric.context_managers.settings(
host_string=self.cfy.get_management_ip(),
user=self.env.management_user_name,
key_filename=self.env.management_key_path,
**kwargs):
yield fabric.api
class TwoManagersTest(OpenStackNodeCellarTest):
"""
This test bootstraps managers, installs nodecellar using the first manager,
checks whether it has been installed correctly, creates a snapshot,
downloads it, uploads it to the second manager, uninstalls nodecellar using
the second manager, checks whether nodecellar is not running indeed and
tears down those managers.
"""
def setUp(self):
super(TwoManagersTest, self).setUp()
self.workdir2 = tempfile.mkdtemp(prefix='cloudify-testenv-')
self.cfy2 = CfyHelper(self.workdir2, testcase=self)
second_manager_blueprint_path = '{}_existing'.format(
self.env._manager_blueprint_path)
shutil.copy2(self.env._manager_blueprint_path,
second_manager_blueprint_path)
external_resources = [
'node_templates.management_network.properties',
'node_templates.management_subnet.properties',
'node_templates.router.properties',
'node_templates.agents_security_group.properties',
'node_templates.management_security_group.properties',
]
with YamlPatcher(second_manager_blueprint_path) as patch:
for prop in external_resources:
patch.merge_obj(prop, {'use_external_resource': True})
second_cloudify_config_path = '{}_existing'.format(
self.env.cloudify_config_path)
shutil.copy2(self.env.cloudify_config_path,
second_cloudify_config_path)
new_resources = ['manager_server_name', 'manager_port_name']
with YamlPatcher(second_cloudify_config_path) as patch:
for prop in new_resources:
patch.append_value(prop, '2')
self.cfy2.bootstrap(
blueprint_path=second_manager_blueprint_path,
inputs_file=second_cloudify_config_path,
install_plugins=self.env.install_plugins,
keep_up_on_failure=False,
task_retries=5,
verbose=False
)
self.client2 = create_rest_client(self.cfy2.get_management_ip())
def _start_execution_and_wait(self, client, deployment, workflow_id):
execution = client.executions.start(deployment, workflow_id)
self.wait_for_execution(execution, self.default_timeout, client)
def _create_snapshot(self, client, name):
self.wait_for_stop_dep_env_execution_to_end(self.test_id)
execution = client.snapshots.create(name, False, False)
self.wait_for_execution(execution, self.default_timeout, client)
def _restore_snapshot(self, client, name):
execution = client.snapshots.restore(name, True)
self.wait_for_execution(execution, self.default_timeout, client)
def on_nodecellar_installed(self):
self.logger.info('Creating snapshot...')
self._create_snapshot(self.client, self.test_id)
try:
self.client.snapshots.get(self.test_id)
except CloudifyClientError as e:
self.fail(e.message)
self.logger.info('Snapshot created.')
self.logger.info('Downloading snapshot...')
snapshot_file_name = ''.join(random.choice(string.ascii_letters)
for _ in xrange(10))
snapshot_file_path = os.path.join('/tmp', snapshot_file_name)
self.client.snapshots.download(self.test_id, snapshot_file_path)
self.logger.info('Snapshot downloaded.')
self.logger.info('Uploading snapshot to the second manager...')
self.client2.snapshots.upload(snapshot_file_path, self.test_id)
try:
uploaded_snapshot = self.client2.snapshots.get(
self.test_id)
self.assertEqual(
uploaded_snapshot.status,
'uploaded',
"Snapshot {} has a wrong status: '{}' instead of 'uploaded'."
.format(self.test_id, uploaded_snapshot.status)
)
except CloudifyClientError as e:
self.fail(e.message)
self.logger.info('Snapshot uploaded.')
self.logger.info('Removing snapshot file...')
if os.path.isfile(snapshot_file_path):
os.remove(snapshot_file_path)
self.logger.info('Snapshot file removed.')
self.logger.info('Restoring snapshot...')
self._restore_snapshot(self.client2, self.test_id)
try:
self.client2.deployments.get(self.test_id)
except CloudifyClientError as e:
self.fail(e.message)
self.logger.info('Snapshot restored.')
self.logger.info('Installing new agents...')
self._start_execution_and_wait(self.client2, self.test_id,
'install_new_agents')
self.logger.info('Installed new agents.')
self.wait_for_stop_dep_env_execution_to_end(self.test_id,
client=self.client2)
def execute_uninstall(self, deployment_id=None, cfy=None):
super(TwoManagersTest, self).execute_uninstall(
cfy=self.cfy2)
def post_uninstall_assertions(self, client=None):
super(TwoManagersTest, self).post_uninstall_assertions(
self.client2)
@property
def default_timeout(self):
return 1000
class TestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def wait_until_deployment_ready_and_execute_install(self,
deployment_id,
inputs):
self.wait_until_all_deployment_executions_end(deployment_id)
return self.execute_install(deployment_id=deployment_id)
def wait_until_all_deployment_executions_end(
self,
deployment_id=None,
include_system_workflows=False):
if deployment_id:
msg = "Waiting for executions on " \
"deployment {0} to finish".format(deployment_id)
else:
msg = "Waiting for system wide executions to finish"
if include_system_workflows:
msg = "{0}, including system workflows.".format(msg)
self.logger.info(msg)
start_time = time.time()
while len([execution for execution in self.client.executions.list(
deployment_id=deployment_id,
include_system_workflows=include_system_workflows)
if execution["status"] not in Execution.END_STATES]) > 0:
time.sleep(1)
if start_time - time.time() > DEFAULT_EXECUTE_TIMEOUT:
if deployment_id:
timeout_msg = "Timeout while waiting for " \
"executions to end " \
"on deployment {0}.".format(deployment_id)
else:
timeout_msg = "Timeout while waiting for " \
"system wide executions to end."
raise Exception(timeout_msg)
return
def assert_outputs(self, expected_outputs, deployment_id=None):
if deployment_id is None:
deployment_id = self.test_id
outputs = self.client.deployments.outputs.get(deployment_id)
outputs = outputs['outputs']
self.assertEqual(expected_outputs, outputs)
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.logger.info('Starting test setUp')
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip,
testcase=self)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}'.format(time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
def _cleanup(self):
self._test_cleanup_context.cleanup()
shutil.rmtree(self.workdir)
def tearDown(self):
self.logger.info('Starting test tearDown')
# note that the cleanup function is registered in setUp
# because it is called regardless of whether setUp succeeded or failed
# unlike tearDown which is not called when setUp fails (which might
# happen when tests override setUp)
def get_manager_state(self):
self.logger.info('Fetching manager current state')
blueprints = {}
for blueprint in self.client.blueprints.list():
blueprints[blueprint.id] = blueprint
deployments = {}
for deployment in self.client.deployments.list():
deployments[deployment.id] = deployment
nodes = {}
for deployment_id in deployments.keys():
for node in self.client.node_instances.list(deployment_id):
nodes[node.id] = node
deployment_nodes = {}
node_state = {}
for deployment_id in deployments.keys():
deployment_nodes[deployment_id] = self.client.node_instances.list(
deployment_id).items
node_state[deployment_id] = {}
for node in deployment_nodes[deployment_id]:
node_state[deployment_id][node.id] = node
return {
'blueprints': blueprints,
'deployments': deployments,
'nodes': nodes,
'node_state': node_state,
'deployment_nodes': deployment_nodes
}
def get_manager_state_delta(self, before, after):
after = copy.deepcopy(after)
for blueprint_id in before['blueprints'].keys():
del after['blueprints'][blueprint_id]
for deployment_id in before['deployments'].keys():
del after['deployments'][deployment_id]
del after['deployment_nodes'][deployment_id]
del after['node_state'][deployment_id]
for node_id in before['nodes'].keys():
del after['nodes'][node_id]
return after
def execute_install(self,
deployment_id=None,
fetch_state=True):
self.logger.info("attempting to execute install on deployment {0}"
.format(deployment_id))
return self._make_operation_with_before_after_states(
self.cfy.execute_install,
fetch_state,
deployment_id=deployment_id)
def upload_deploy_and_execute_install(
self,
blueprint_id=None,
deployment_id=None,
fetch_state=True,
execute_timeout=DEFAULT_EXECUTE_TIMEOUT,
inputs=None):
return self._make_operation_with_before_after_states(
self.cfy.upload_deploy_and_execute_install,
fetch_state,
str(self.blueprint_yaml),
blueprint_id=blueprint_id or self.test_id,
deployment_id=deployment_id or self.test_id,
execute_timeout=execute_timeout,
inputs=inputs)
def upload_blueprint(
self,
blueprint_id):
self.logger.info("attempting to upload blueprint {0}"
.format(blueprint_id))
return self.cfy.upload_blueprint(
blueprint_id=blueprint_id,
blueprint_path=str(self.blueprint_yaml))
def create_deployment(
self,
blueprint_id,
deployment_id,
inputs):
self.logger.info("attempting to create_deployment deployment {0}"
.format(deployment_id))
return self.cfy.create_deployment(
blueprint_id=blueprint_id,
deployment_id=deployment_id,
inputs=inputs)
def _make_operation_with_before_after_states(self, operation, fetch_state,
*args, **kwargs):
before_state = None
after_state = None
if fetch_state:
before_state = self.get_manager_state()
operation(*args, **kwargs)
if fetch_state:
after_state = self.get_manager_state()
return before_state, after_state
def execute_uninstall(self, deployment_id=None, cfy=None):
cfy = cfy or self.cfy
cfy.execute_uninstall(deployment_id=deployment_id or self.test_id)
def copy_blueprint(self, blueprint_dir_name, blueprints_dir=None):
blueprint_path = path(self.workdir) / blueprint_dir_name
shutil.copytree(get_blueprint_path(blueprint_dir_name, blueprints_dir),
str(blueprint_path))
return blueprint_path
def wait_for_execution(self, execution, timeout, client=None,
assert_success=True):
client = client or self.client
end = time.time() + timeout
while time.time() < end:
status = client.executions.get(execution.id).status
if status == 'failed':
if assert_success:
raise AssertionError('Execution "{}" failed'.format(
execution.id))
else:
return
if status == 'terminated':
return
time.sleep(1)
events, _ = client.events.get(execution.id,
batch_size=1000,
include_logs=True)
self.logger.info('Deployment creation events & logs:')
for event in events:
self.logger.info(json.dumps(event))
raise AssertionError('Execution "{}" timed out'.format(execution.id))
def wait_for_stop_dep_env_execution_to_end(self, deployment_id,
timeout_seconds=240,
client=None):
client = client or self.client
executions = client.executions.list(
deployment_id=deployment_id, include_system_workflows=True)
running_stop_executions = [e for e in executions if e.workflow_id ==
'_stop_deployment_environment' and
e.status not in Execution.END_STATES]
if not running_stop_executions:
return
if len(running_stop_executions) > 1:
raise RuntimeError('There is more than one running '
'"_stop_deployment_environment" execution: {0}'
.format(running_stop_executions))
execution = running_stop_executions[0]
return self.wait_for_execution(execution, timeout_seconds, client)
def repetitive(self, func, timeout=10, exception_class=Exception,
args=None, kwargs=None):
args = args or []
kwargs = kwargs or {}
deadline = time.time() + timeout
while True:
try:
return func(*args, **kwargs)
except exception_class:
if time.time() > deadline:
raise
time.sleep(1)
@contextmanager
def manager_env_fabric(self, **kwargs):
with fabric.context_managers.settings(
host_string=self.cfy.get_management_ip(),
user=self.env.management_user_name,
key_filename=self.env.management_key_path,
**kwargs):
yield fabric.api
class TestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def wait_until_deployment_ready_and_execute_install(self,
deployment_id,
inputs):
self.wait_until_all_deployment_executions_end(deployment_id)
return self.execute_install(deployment_id=deployment_id)
def wait_until_all_deployment_executions_end(
self,
deployment_id=None,
include_system_workflows=False):
if deployment_id:
msg = "Waiting for executions on " \
"deployment {0} to finish".format(deployment_id)
else:
msg = "Waiting for system wide executions to finish"
if include_system_workflows:
msg = "{0}, including system workflows.".format(msg)
self.logger.info(msg)
start_time = time.time()
while len([execution for execution in self.client.executions.list(
deployment_id=deployment_id,
include_system_workflows=include_system_workflows)
if execution["status"] not in Execution.END_STATES]) > 0:
time.sleep(1)
if start_time - time.time() > DEFAULT_EXECUTE_TIMEOUT:
if deployment_id:
timeout_msg = "Timeout while waiting for " \
"executions to end " \
"on deployment {0}.".format(deployment_id)
else:
timeout_msg = "Timeout while waiting for " \
"system wide executions to end."
raise Exception(timeout_msg)
return
def assert_outputs(self, expected_outputs, deployment_id=None):
if deployment_id is None:
deployment_id = self.test_id
outputs = self.client.deployments.outputs.get(deployment_id)
outputs = outputs['outputs']
self.assertEqual(expected_outputs, outputs)
def setUp(self):
global test_environment
self.env = test_environment.setup()
self.logger = logging.getLogger(self._testMethodName)
self.logger.setLevel(logging.INFO)
self.logger.info('Starting test setUp')
self.workdir = tempfile.mkdtemp(prefix='cosmo-test-')
self.cfy = CfyHelper(cfy_workdir=self.workdir,
management_ip=self.env.management_ip,
testcase=self)
self.client = self.env.rest_client
self.test_id = 'system-test-{0}'.format(time.strftime("%Y%m%d-%H%M"))
self.blueprint_yaml = None
self._test_cleanup_context = self.env.handler.CleanupContext(
self._testMethodName, self.env)
# register cleanup
self.addCleanup(self._cleanup)
self.maxDiff = 1024 * 1024 * 10
def _cleanup(self):
self._test_cleanup_context.cleanup()
shutil.rmtree(self.workdir)
def tearDown(self):
self.logger.info('Starting test tearDown')
# note that the cleanup function is registered in setUp
# because it is called regardless of whether setUp succeeded or failed
# unlike tearDown which is not called when setUp fails (which might
# happen when tests override setUp)
def get_manager_state(self):
self.logger.info('Fetching manager current state')
blueprints = {}
for blueprint in self.client.blueprints.list():
blueprints[blueprint.id] = blueprint
deployments = {}
for deployment in self.client.deployments.list():
deployments[deployment.id] = deployment
nodes = {}
for deployment_id in deployments.keys():
for node in self.client.node_instances.list(deployment_id):
nodes[node.id] = node
deployment_nodes = {}
node_state = {}
for deployment_id in deployments.keys():
deployment_nodes[deployment_id] = self.client.node_instances.list(
deployment_id).items
node_state[deployment_id] = {}
for node in deployment_nodes[deployment_id]:
node_state[deployment_id][node.id] = node
return {
'blueprints': blueprints,
'deployments': deployments,
'nodes': nodes,
'node_state': node_state,
'deployment_nodes': deployment_nodes
}
def get_manager_state_delta(self, before, after):
after = copy.deepcopy(after)
for blueprint_id in before['blueprints'].keys():
del after['blueprints'][blueprint_id]
for deployment_id in before['deployments'].keys():
del after['deployments'][deployment_id]
del after['deployment_nodes'][deployment_id]
del after['node_state'][deployment_id]
for node_id in before['nodes'].keys():
del after['nodes'][node_id]
return after
def execute_install(self,
deployment_id=None,
fetch_state=True):
self.logger.info("attempting to execute install on deployment {0}"
.format(deployment_id))
return self._make_operation_with_before_after_states(
self.cfy.execute_install,
fetch_state,
deployment_id=deployment_id)
def upload_deploy_and_execute_install(
self,
blueprint_id=None,
deployment_id=None,
fetch_state=True,
execute_timeout=DEFAULT_EXECUTE_TIMEOUT,
inputs=None):
return self._make_operation_with_before_after_states(
self.cfy.upload_deploy_and_execute_install,
fetch_state,
str(self.blueprint_yaml),
blueprint_id=blueprint_id or self.test_id,
deployment_id=deployment_id or self.test_id,
execute_timeout=execute_timeout,
inputs=inputs)
def upload_blueprint(
self,
blueprint_id):
self.logger.info("attempting to upload blueprint {0}"
.format(blueprint_id))
return self.cfy.upload_blueprint(
blueprint_id=blueprint_id,
blueprint_path=str(self.blueprint_yaml))
def create_deployment(
self,
blueprint_id,
deployment_id,
inputs):
self.logger.info("attempting to create_deployment deployment {0}"
.format(deployment_id))
return self.cfy.create_deployment(
blueprint_id=blueprint_id,
deployment_id=deployment_id,
inputs=inputs)
def _make_operation_with_before_after_states(self, operation, fetch_state,
*args, **kwargs):
before_state = None
after_state = None
if fetch_state:
before_state = self.get_manager_state()
operation(*args, **kwargs)
if fetch_state:
after_state = self.get_manager_state()
return before_state, after_state
def execute_uninstall(self, deployment_id=None, cfy=None):
cfy = cfy or self.cfy
cfy.execute_uninstall(deployment_id=deployment_id or self.test_id)
def copy_blueprint(self, blueprint_dir_name, blueprints_dir=None):
blueprint_path = path(self.workdir) / blueprint_dir_name
shutil.copytree(get_blueprint_path(blueprint_dir_name, blueprints_dir),
str(blueprint_path))
return blueprint_path
def wait_for_execution(self, execution, timeout, client=None,
assert_success=True):
client = client or self.client
end = time.time() + timeout
while time.time() < end:
status = client.executions.get(execution.id).status
if status == 'failed':
if assert_success:
raise AssertionError('Execution "{}" failed'.format(
execution.id))
else:
return
if status == 'terminated':
return
time.sleep(1)
events, _ = client.events.get(execution.id,
batch_size=1000,
include_logs=True)
self.logger.info('Deployment creation events & logs:')
for event in events:
self.logger.info(json.dumps(event))
raise AssertionError('Execution "{}" timed out'.format(execution.id))
def wait_for_stop_dep_env_execution_to_end(self, deployment_id,
timeout_seconds=240,
client=None):
client = client or self.client
executions = client.executions.list(
deployment_id=deployment_id, include_system_workflows=True)
running_stop_executions = [e for e in executions if e.workflow_id ==
'_stop_deployment_environment' and
e.status not in Execution.END_STATES]
if not running_stop_executions:
return
if len(running_stop_executions) > 1:
raise RuntimeError('There is more than one running '
'"_stop_deployment_environment" execution: {0}'
.format(running_stop_executions))
execution = running_stop_executions[0]
return self.wait_for_execution(execution, timeout_seconds, client)
def repetitive(self, func, timeout=10, exception_class=Exception,
args=None, kwargs=None):
args = args or []
kwargs = kwargs or {}
deadline = time.time() + timeout
while True:
try:
return func(*args, **kwargs)
except exception_class:
if time.time() > deadline:
raise
time.sleep(1)
@contextmanager
def manager_env_fabric(self, **kwargs):
with fabric.context_managers.settings(
host_string=self.cfy.get_management_ip(),
user=self.env.management_user_name,
key_filename=self.env.management_key_path,
**kwargs):
yield fabric.api
def run_commands_on_agent_host(self,
user,
commands,
deployment_id=None,
compute_node_id=None,
compute_node_instance_id=None):
if not (compute_node_id or compute_node_instance_id):
self.fail('no node_id or node_instance_id')
deployment_id = deployment_id or self.test_id
filters = {'deployment_id': deployment_id}
if compute_node_id:
filters['node_id'] = compute_node_id
if compute_node_instance_id:
filters['id'] = compute_node_instance_id
computes = self.client.node_instances.list(**filters).items
if not computes:
self.fail('No compute nodes were found')
if len(computes) > 1:
self.fail('More than one instance found, please refine your query:'
' {0}'.format(computes))
compute = computes[0]
private_ip = compute.runtime_properties['ip']
with self.manager_env_fabric() as api:
api.sudo('ssh '
'-o UserKnownHostsFile=/dev/null '
'-o StrictHostKeyChecking=no '
'-t -i /root/.ssh/agent_key.pem {0}@{1} "{2}"'
.format(user, private_ip, ' && '.join(commands)))
class TwoManagersTest(HelloWorldBashTest):
"""
This test bootstraps managers, installs helloworld using the first manager,
checks whether it has been installed correctly, creates a snapshot,
downloads it, uploads it to the second manager, uninstalls helloworld using
the second manager, checks whether helloworld is not running indeed and
tears down those managers.
"""
def setUp(self):
super(TwoManagersTest, self).setUp()
self.workdir2 = tempfile.mkdtemp(prefix='cloudify-testenv-')
self.cfy2 = CfyHelper(self.workdir2, testcase=self)
second_manager_blueprint_path = '{}_existing'.format(
self.env._manager_blueprint_path)
shutil.copy2(self.env._manager_blueprint_path,
second_manager_blueprint_path)
external_resources = [
'node_templates.management_network.properties',
'node_templates.management_subnet.properties',
'node_templates.router.properties',
'node_templates.agents_security_group.properties',
'node_templates.management_security_group.properties',
]
with YamlPatcher(second_manager_blueprint_path) as patch:
for prop in external_resources:
patch.merge_obj(prop, {'use_external_resource': True})
second_cloudify_config_path = '{}_existing'.format(
self.env.cloudify_config_path)
shutil.copy2(self.env.cloudify_config_path,
second_cloudify_config_path)
new_resources = ['manager_server_name', 'manager_port_name']
with YamlPatcher(second_cloudify_config_path) as patch:
for prop in new_resources:
patch.append_value(prop, '2')
self.cfy2.bootstrap(blueprint_path=second_manager_blueprint_path,
inputs_file=second_cloudify_config_path,
install_plugins=self.env.install_plugins,
keep_up_on_failure=False,
task_retries=5,
verbose=False)
self.client2 = create_rest_client(self.cfy2.get_management_ip())
def _start_execution_and_wait(self, client, deployment, workflow_id):
execution = client.executions.start(deployment, workflow_id)
self.wait_for_execution(execution, self.default_timeout, client)
def _create_snapshot(self, client, name):
self.wait_for_stop_dep_env_execution_to_end(self.test_id)
execution = client.snapshots.create(name, False, False)
self.wait_for_execution(execution, self.default_timeout, client)
def _restore_snapshot(self, client, name):
execution = client.snapshots.restore(name, True)
self.wait_for_execution(execution, self.default_timeout, client)
def _do_post_install_assertions(self):
context = super(TwoManagersTest, self)._do_post_install_assertions()
self.logger.info('Creating snapshot...')
self._create_snapshot(self.client, self.test_id)
try:
self.client.snapshots.get(self.test_id)
except CloudifyClientError as e:
self.fail(e.message)
self.logger.info('Snapshot created.')
self.logger.info('Downloading snapshot...')
snapshot_file_name = ''.join(
random.choice(string.ascii_letters) for _ in xrange(10))
snapshot_file_path = os.path.join('/tmp', snapshot_file_name)
self.client.snapshots.download(self.test_id, snapshot_file_path)
self.logger.info('Snapshot downloaded.')
self.logger.info('Uploading snapshot to the second manager...')
self.client2.snapshots.upload(snapshot_file_path, self.test_id)
try:
uploaded_snapshot = self.client2.snapshots.get(self.test_id)
self.assertEqual(
uploaded_snapshot.status, 'uploaded',
"Snapshot {} has a wrong status: '{}' instead of 'uploaded'.".
format(self.test_id, uploaded_snapshot.status))
except CloudifyClientError as e:
self.fail(e.message)
self.logger.info('Snapshot uploaded.')
self.logger.info('Removing snapshot file...')
if os.path.isfile(snapshot_file_path):
os.remove(snapshot_file_path)
self.logger.info('Snapshot file removed.')
self.logger.info('Restoring snapshot...')
self._restore_snapshot(self.client2, self.test_id)
try:
self.client2.deployments.get(self.test_id)
except CloudifyClientError as e:
self.fail(e.message)
self.logger.info('Snapshot restored.')
self.logger.info('Installing new agents...')
self._start_execution_and_wait(self.client2, self.test_id,
'install_new_agents')
self.logger.info('Installed new agents.')
self.wait_for_stop_dep_env_execution_to_end(self.test_id,
client=self.client2)
return context
def execute_uninstall(self, deployment_id=None, cfy=None):
super(TwoManagersTest, self).execute_uninstall(cfy=self.cfy2)
def _assert_nodes_deleted(self):
super(TwoManagersTest, self)._assert_nodes_deleted(self.client2)
@property
def default_timeout(self):
return 1000
class ManagerUpgradeTest(TestCase):
def test_manager_upgrade(self):
"""Bootstrap a manager, upgrade it, rollback it, examine the results.
To test the manager in-place upgrade procedure:
- bootstrap a manager (this is part of the system under test,
does destructive changes to the manager, and need a known manager
version: so, can't use the testenv manager)
- deploy the hello world app
- upgrade the manager, changing some inputs (eg. the port that
Elasticsearch uses)
- check that everything still works (the previous deployment still
reports metrics; we can install another deployment)
- rollback the manager
- post-rollback checks: the changed inputs are now the original
values again, the installed app still reports metrics
"""
self.prepare_manager()
self.preupgrade_deployment_id = self.deploy_hello_world('pre-')
self.upgrade_manager()
self.post_upgrade_checks()
self.rollback_manager()
self.post_rollback_checks()
self.teardown_manager()
@contextmanager
def _manager_fabric_env(self):
inputs = self.manager_inputs
with fabric.context_managers.settings(
host_string=self.upgrade_manager_ip,
user=inputs['ssh_user'],
key_filename=inputs['ssh_key_filename']):
yield fabric.api
def _bootstrap_local_env(self, workdir):
storage = local.FileStorage(
os.path.join(workdir, '.cloudify', 'bootstrap'))
return local.load_env('manager', storage=storage)
def _blueprint_rpm_versions(self, blueprint_path, inputs):
"""RPM filenames that should be installed on the manager.
"""
env = local.init_env(
blueprint_path,
inputs=inputs,
ignored_modules=cli_constants.IGNORED_LOCAL_WORKFLOW_MODULES)
storage = env.storage
amqp_influx_rpm = storage.get_node('amqp_influx')['properties'][
'amqpinflux_rpm_source_url']
restservice_rpm = storage.get_node('rest_service')['properties'][
'rest_service_rpm_source_url']
mgmtworker_rpm = storage.get_node('mgmt_worker')['properties'][
'management_worker_rpm_source_url']
return {
'cloudify-amqp-influx': amqp_influx_rpm,
'cloudify-rest-service': restservice_rpm,
'cloudify-management-worker': mgmtworker_rpm
}
def _cloudify_rpm_versions(self):
with self._manager_fabric_env() as fabric:
return fabric.sudo('rpm -qa | grep cloudify')
def check_rpm_versions(self, blueprint_path, inputs):
blueprint_rpms = self._blueprint_rpm_versions(blueprint_path, inputs)
installed_rpms = self._cloudify_rpm_versions()
for service_name, rpm_filename in blueprint_rpms.items():
for line in installed_rpms.split('\n'):
line = line.strip()
if line.startswith(service_name):
self.assertIn(line.strip(), rpm_filename)
def prepare_manager(self):
# note that we're using a separate manager checkout, so we need to
# create our own utils like cfy and the rest client, rather than use
# the testenv ones
self.cfy_workdir = tempfile.mkdtemp(prefix='manager-upgrade-')
self.addCleanup(shutil.rmtree, self.cfy_workdir)
self.manager_cfy = CfyHelper(cfy_workdir=self.cfy_workdir)
self.manager_inputs = self._get_bootstrap_inputs()
self.bootstrap_manager()
self.rest_client = create_rest_client(self.upgrade_manager_ip)
self.bootstrap_manager_version = LooseVersion(
self.rest_client.manager.get_version()['version'])
def _get_bootstrap_inputs(self):
prefix = self.test_id
ssh_key_filename = os.path.join(self.workdir, 'manager.key')
self.addCleanup(self.env.handler.remove_keypair,
prefix + '-manager-key')
agent_key_path = os.path.join(self.workdir, 'agents.key')
self.addCleanup(self.env.handler.remove_keypair,
prefix + '-agents-key')
return {
'keystone_username': self.env.keystone_username,
'keystone_password': self.env.keystone_password,
'keystone_tenant_name': self.env.keystone_tenant_name,
'keystone_url': self.env.keystone_url,
'region': self.env.region,
'flavor_id': self.env.medium_flavor_id,
'image_id': self.env.centos_7_image_id,
'ssh_user': self.env.centos_7_image_user,
'external_network_name': self.env.external_network_name,
'resources_prefix': 'test-upgrade-',
'manager_server_name': prefix + '-manager',
# shared settings
'manager_public_key_name': prefix + '-manager-key',
'agent_public_key_name': prefix + '-agents-key',
'ssh_key_filename': ssh_key_filename,
'agent_private_key_path': agent_key_path,
'management_network_name': prefix + '-network',
'management_subnet_name': prefix + '-subnet',
'management_router': prefix + '-router',
'agents_user': '',
# private settings
'manager_security_group_name': prefix + '-m-sg',
'agents_security_group_name': prefix + '-a-sg',
'manager_port_name': prefix + '-port',
'management_subnet_dns_nameservers': ['8.8.8.8', '8.8.4.4']
}
def get_bootstrap_blueprint(self):
manager_repo_dir = tempfile.mkdtemp(prefix='manager-upgrade-')
self.addCleanup(shutil.rmtree, manager_repo_dir)
manager_repo = clone(BOOTSTRAP_REPO_URL,
manager_repo_dir,
branch=BOOTSTRAP_BRANCH)
yaml_path = manager_repo / 'openstack-manager-blueprint.yaml'
# allow the ports that we're going to connect to from the tests,
# when doing checks
for port in [8086, 9200, 9900]:
secgroup_cfg = [{
'port_range_min': port,
'port_range_max': port,
'remote_ip_prefix': '0.0.0.0/0'
}]
secgroup_cfg_path = 'node_templates.management_security_group'\
'.properties.rules'
with YamlPatcher(yaml_path) as patch:
patch.append_value(secgroup_cfg_path, secgroup_cfg)
return yaml_path
def _load_private_ip_from_env(self, workdir):
env = self._bootstrap_local_env(workdir)
return env.outputs()['private_ip']
def bootstrap_manager(self):
self.bootstrap_blueprint = self.get_bootstrap_blueprint()
inputs_path = self.manager_cfy._get_inputs_in_temp_file(
self.manager_inputs, self._testMethodName)
self.manager_cfy.bootstrap(self.bootstrap_blueprint,
inputs_file=inputs_path)
self.upgrade_manager_ip = self.manager_cfy.get_management_ip()
self.manager_private_ip = self._load_private_ip_from_env(
self.cfy_workdir)
# TODO: why is this needed?
self.manager_cfy.use(management_ip=self.upgrade_manager_ip)
def deploy_hello_world(self, prefix=''):
"""Install the hello world app."""
blueprint_id = prefix + self.test_id
deployment_id = prefix + self.test_id
hello_repo_dir = tempfile.mkdtemp(prefix='manager-upgrade-')
hello_repo_path = clone(
'https://github.com/cloudify-cosmo/'
'cloudify-hello-world-example.git',
hello_repo_dir
)
self.addCleanup(shutil.rmtree, hello_repo_dir)
hello_blueprint_path = hello_repo_path / 'blueprint.yaml'
self.manager_cfy.upload_blueprint(blueprint_id, hello_blueprint_path)
inputs = {
'agent_user': self.env.ubuntu_image_user,
'image': self.env.ubuntu_trusty_image_name,
'flavor': self.env.flavor_name
}
self.manager_cfy.create_deployment(blueprint_id, deployment_id,
inputs=inputs)
self.manager_cfy.execute_install(deployment_id=deployment_id)
return deployment_id
def get_upgrade_blueprint(self):
repo_dir = tempfile.mkdtemp(prefix='manager-upgrade-')
self.addCleanup(shutil.rmtree, repo_dir)
upgrade_blueprint_path = clone(UPGRADE_REPO_URL,
repo_dir,
branch=UPGRADE_BRANCH)
return upgrade_blueprint_path / 'simple-manager-blueprint.yaml'
def upgrade_manager(self):
self.upgrade_blueprint = self.get_upgrade_blueprint()
# we're changing one of the ES inputs - make sure we also re-install ES
with YamlPatcher(self.upgrade_blueprint) as patch:
patch.set_value(
('node_templates.elasticsearch.properties'
'.use_existing_on_upgrade'),
False)
self.upgrade_inputs = {
'private_ip': self.manager_private_ip,
'public_ip': self.upgrade_manager_ip,
'ssh_key_filename': self.manager_inputs['ssh_key_filename'],
'ssh_user': self.manager_inputs['ssh_user'],
'elasticsearch_endpoint_port': 9900
}
upgrade_inputs_file = self.manager_cfy._get_inputs_in_temp_file(
self.upgrade_inputs, self._testMethodName)
with self.manager_cfy.maintenance_mode():
self.manager_cfy.upgrade_manager(
blueprint_path=self.upgrade_blueprint,
inputs_file=upgrade_inputs_file)
def post_upgrade_checks(self):
"""To check if the upgrade succeeded:
- fire a request to the REST service
- check that elasticsearch is listening on the changed port
- check that the pre-existing deployment still reports to influxdb
- install a new deployment, check that it reports to influxdb,
and uninstall it: to check that the manager still allows
creating, installing and uninstalling deployments correctly
"""
upgrade_manager_version = LooseVersion(
self.rest_client.manager.get_version()['version'])
self.assertGreaterEqual(upgrade_manager_version,
self.bootstrap_manager_version)
self.check_rpm_versions(self.upgrade_blueprint, self.upgrade_inputs)
self.rest_client.blueprints.list()
self.check_elasticsearch(self.upgrade_manager_ip, 9900)
self.check_influx(self.preupgrade_deployment_id)
postupgrade_deployment_id = self.deploy_hello_world('post-')
self.check_influx(postupgrade_deployment_id)
self.uninstall_deployment(postupgrade_deployment_id)
def check_influx(self, deployment_id):
"""Check that the deployment_id continues to report metrics.
Look at the last 5 seconds worth of metrics. To avoid race conditions
(running this check before the deployment even had a chance to report
any metrics), first wait 5 seconds to allow some metrics to be
gathered.
"""
# TODO influx config should be pulled from props?
time.sleep(5)
influx_client = InfluxDBClient(self.upgrade_manager_ip, 8086,
'root', 'root', 'cloudify')
try:
result = influx_client.query('select * from /^{0}\./i '
'where time > now() - 5s'
.format(deployment_id))
except NameError as e:
self.fail('monitoring events list for deployment with ID {0} were'
' not found on influxDB. error is: {1}'
.format(deployment_id, e))
self.assertTrue(len(result) > 0)
def check_elasticsearch(self, host, port):
"""Check that elasticsearch is listening on the given host:port.
This is used for checking if the ES port changed correctly during
the upgrade.
"""
try:
response = urllib2.urlopen('http://{0}:{1}'.format(
self.upgrade_manager_ip, port))
response = json.load(response)
if response['status'] != 200:
raise ValueError('Incorrect status {0}'.format(
response['status']))
except (ValueError, urllib2.URLError):
self.fail('elasticsearch isnt listening on the changed port')
def uninstall_deployment(self, deployment_id):
self.manager_cfy.execute_uninstall(deployment_id)
def rollback_manager(self):
rollback_inputs = {
'private_ip': self.manager_private_ip,
'public_ip': self.upgrade_manager_ip,
'ssh_key_filename': self.manager_inputs['ssh_key_filename'],
'ssh_user': self.manager_inputs['ssh_user'],
}
rollback_inputs_file = self.manager_cfy._get_inputs_in_temp_file(
rollback_inputs, self._testMethodName)
with self.manager_cfy.maintenance_mode():
self.manager_cfy.rollback_manager(
blueprint_path=self.upgrade_blueprint,
inputs_file=rollback_inputs_file)
def post_rollback_checks(self):
rollback_manager_version = LooseVersion(
self.rest_client.manager.get_version()['version'])
self.assertEqual(rollback_manager_version,
self.bootstrap_manager_version)
self.check_rpm_versions(self.bootstrap_blueprint, self.manager_inputs)
self.rest_client.blueprints.list()
self.check_elasticsearch(self.upgrade_manager_ip, 9200)
self.check_influx(self.preupgrade_deployment_id)
def teardown_manager(self):
self.manager_cfy.teardown(ignore_deployments=True)
