def upload_test_plugins_dev(plugins,
execute_bundle_upload=True,
bundle_path=None):
"""
Upload all plugins that we need to execute the test.
:param plugins: A list of additional plugins to upload.
(Like ones that are not in the bundle (Openstack 3, Host Pool).
:param execute_bundle_upload: Whether to install a bundle.
:param bundle_path: Path to plugins bundle.
:return:
"""
plugins = plugins or []
bundle_path = bundle_path or ''
if execute_bundle_upload:
if os.path.isfile(bundle_path):
logger.info("Using plugins bundle found at: {path}".format(
path=bundle_path))
cloudify_exec(
'cfy plugins bundle-upload --path {bundle_path}'.format(
bundle_path=copy_file_to_docker(bundle_path)),
get_json=False)
else:
cloudify_exec('cfy plugins bundle-upload', get_json=False)
for plugin in plugins:
sleep(3)
output = plugins_upload(plugin[0], plugin[1])
logger.info('Uploaded plugin: {0}'.format(output))
logger.info('Plugins list: {0}'.format(cloudify_exec('cfy plugins list')))
def upload_test_plugins(plugins,
plugin_test,
execute_bundle_upload=True,
workspace_path=None):
"""
Upload all plugins that we need to execute the test.
:param plugins: A list of additional plugins to upload.
(Like ones that are not in the bundle (Openstack 3, Host Pool).
:param plugin_test: Whether to install plugins from workspace.
:param execute_bundle_upload: Whether to install a bundle.
:return:
"""
plugins = plugins or []
if plugin_test:
for plugin_pair in get_test_plugins():
plugins.append(plugin_pair)
if execute_bundle_upload:
bundle_path = get_bundle_from_workspace(workspace_path=workspace_path)
if bundle_path:
cloudify_exec(
'cfy plugins bundle-upload --path {bundle_path}'.format(
bundle_path=copy_file_to_docker(bundle_path)),
get_json=False)
else:
cloudify_exec('cfy plugins bundle-upload', get_json=False)
for plugin in plugins:
sleep(3)
output = plugins_upload(plugin[0], plugin[1])
logger.info('Uploaded plugin: {0}'.format(output))
logger.info('Plugins list: {0}'.format(cloudify_exec('cfy plugins list')))
def secrets_create(name, is_file=False):
"""
Create a secret on the manager.
:param name: The secret key.
:param is_file: Whether to create the secret from a file.
:return:
"""
logger.info('Creating secret: {0}.'.format(name))
try:
value = (base64.b64decode(
os.environ[name].encode('utf-8'))).decode('ascii')
except KeyError:
raise EcosystemTestException(
'Secret env var not set {0}.'.format(name))
if is_file:
with NamedTemporaryFile(mode='w+', delete=True) as outfile:
outfile.write(value)
outfile.flush()
cmd = 'cfy secrets create -u {0} -f {1}'.format(
name, copy_file_to_docker(outfile.name))
return cloudify_exec(cmd, get_json=False, log=False)
return cloudify_exec('cfy secrets create -u {0} -s {1}'.format(
name, value),
get_json=False,
log=False)
def deployment_update(deployment_id, blueprint_id, inputs, timeout=None):
"""
Perform deployment update.
:param deployment_id:
:param blueprint_id: Id of the blueprint to update the deployment with,
blueprint id of a blueprint that already exists in the system.
:param inputs:
:param timeout:
"""
if not inputs:
return cloudify_exec(
'cfy deployments update {deployment_id} -b {blueprint_id}'.format(
deployment_id=deployment_id, blueprint_id=blueprint_id),
get_json=False,
timeout=timeout)
else:
with prepare_inputs(inputs) as handled_inputs:
return cloudify_exec(
'cfy deployments update {deployment_id} -b {blueprint_id} -i {'
'inputs}'.format(deployment_id=deployment_id,
blueprint_id=blueprint_id,
inputs=handled_inputs),
get_json=False,
timeout=timeout)
def _basic_blueprint_test(blueprint_file_name,
test_name,
inputs=None,
timeout=None,
endpoint_name=None,
endpoint_value=None):
"""
Simple blueprint install/uninstall test.
:param blueprint_file_name:
:param test_name:
:param inputs:
:param timeout:
:return:
"""
timeout = timeout or TIMEOUT
if inputs != '':
inputs = inputs or os.path.join(
os.path.dirname(blueprint_file_name), 'inputs/test-inputs.yaml')
logger.info('Blueprints list: {0}'.format(
cloudify_exec('cfy blueprints list')))
blueprints_upload(blueprint_file_name, test_name)
logger.info('Deployments list: {0}'.format(
cloudify_exec('cfy deployments list')))
deployments_create(test_name, inputs)
sleep(5)
logger.info(GREEN + 'Installing...' + RESET)
try:
executions_list(test_name)
executions_start('install', test_name, timeout)
except EcosystemTimeout:
# Give 5 seconds grace.
executions_list(test_name)
wait_for_execution(test_name, 'install', 10)
else:
wait_for_execution(test_name, 'install', timeout)
if endpoint_name and endpoint_value:
verify_endpoint(
get_deployment_output_by_name(
test_name,
endpoint_name
), endpoint_value)
logger.info(BLUE + 'Uninstalling...' + RESET)
executions_start('uninstall', test_name, timeout)
wait_for_execution(test_name, 'uninstall', timeout)
try:
deployment_delete(test_name)
blueprints_delete(test_name)
except Exception as e:
logger.info(RED +
'Failed to delete blueprint, {0}'.format(str(e)) +
RESET)
def cleanup_on_failure(deployment_id):
"""
Execute uninstall if a deployment failed.
:param deployment_id:
:return:
"""
try:
executions_list(deployment_id)
except EcosystemTestException:
pass
else:
cloudify_exec(
'cfy uninstall -p ignore_failure=true {0}'.format(deployment_id))
def deployments_create(blueprint_id, inputs):
"""
Create a deployment on the manager.
:param blueprint_id:
:param inputs:
:return:
"""
if not inputs:
return cloudify_exec(
'cfy deployments create -b {0}'.format(blueprint_id),
get_json=False)
with prepare_inputs(inputs) as handled_inputs:
return cloudify_exec('cfy deployments create -i {0} -b {1}'.format(
handled_inputs, blueprint_id),
get_json=False)
def executions_start(workflow_id,
deployment_id,
timeout,
params=None,
stdout_color=DEFAULT_COLOR):
"""
Start an execution on the manager.
:param workflow_id:
:param deployment_id:
:param timeout:
:param params: Valid Parameters for the workflow (Can be provided as
wildcard based paths (*.yaml, /my_inputs/,etc..) to YAML files,
a JSON string or as 'key1=value1;key2=value2')
:param stdout_color: Defines the default stdout output color.
:return:
"""
cmd = 'cfy executions start --timeout {0} -d {1} {2}'
if isinstance(params, str):
pstr = ' -p ' + '{params}'.format(params=params)
cmd = cmd + pstr
elif isinstance(params, dict) and len(params) > 0:
pstr = ''
for key, val in params.items():
pstr = pstr + ' -p {key}={val}'.format(key=key, val=val)
cmd = cmd + pstr
elif isinstance(params, list) and len(params) > 0:
cmd = cmd + ' -p ' + ' -p '.join(params)
return cloudify_exec(cmd.format(timeout, deployment_id, workflow_id),
get_json=False,
timeout=timeout,
stdout_color=stdout_color)
def plugin_already_uploaded(wagon_path):
"""
Check if a plugin is already loaded on the manager.
:param wagon_path: Path to a wagon.
:return: Bool.
"""
# It`s url
if '://' in wagon_path:
wagon_metadata = show(wagon_path)
else:
wagon_metadata = show(find_wagon_local_path(wagon_path))
plugin_name = wagon_metadata["package_name"]
plugin_version = wagon_metadata["package_version"]
plugin_distribution = \
wagon_metadata["build_server_os_properties"]["distribution"]
for plugin in cloudify_exec('cfy plugins list'):
logger.info('CHECKING if {0} {1} {2} in {3}'.format(
plugin_name, plugin_version, plugin_distribution, plugin))
compare_name = plugin['package_name']
compare_version = plugin['package_version']
compare_distro = plugin.get('distribution', '').lower() or plugin.get(
'yaml_url_path', '')
if plugin_name.replace('_', '-') in compare_name and \
plugin_version == compare_version and \
plugin_distribution.lower() in compare_distro:
return True
def executions_list(deployment_id):
"""
List executions on the manager.
:param deployment_id:
:return:
"""
return cloudify_exec('cfy executions list -d {0} '
'--include-system-workflows'.format(deployment_id))
def events_list(execution_id):
"""
List events from an execution.
:param execution_id:
:return:
"""
events = cloudify_exec('cfy events list {0}'.format(execution_id))
if not events:
return []
return [json.loads(line) for line in events.split('\n') if line]
def executions_resume(execution_id, timeout):
"""
Resume an execution on the manager.
:param execution_id:
:param timeout:
:return:
"""
return cloudify_exec('cfy executions resume {0}'.format(execution_id),
get_json=False,
timeout=timeout)
def first_invocation_test_path(blueprint_file_name,
test_name,
inputs=None,
timeout=None,
uninstall_on_success=True,
user_defined_check=None,
user_defined_check_params=None
):
logger.info('Blueprints list: {0}'.format(
cloudify_exec('cfy blueprints list')))
blueprints_upload(blueprint_file_name, test_name)
logger.info('Deployments list: {0}'.format(
cloudify_exec('cfy deployments list')))
deployments_create(test_name, inputs)
sleep(5)
start_install_workflow(test_name, timeout)
run_user_defined_check(user_defined_check, user_defined_check_params)
if uninstall_on_success:
handle_uninstall_on_success(test_name, timeout)
def create_test_secrets(secrets=None):
"""
Create secrets on the manager.
:param secrets:
:return:
"""
secrets = secrets or {}
for secret, f in secrets.items():
secrets_create(secret, f)
logger.info('Secrets list: {0}'.format(cloudify_exec('cfy secrets list')))
def executions_cancel(execution_id, timeout, force=False):
"""
Cancel an execution on the manager.
:param execution_id:
:param timeout:
:param force:
:return:
"""
cmd = 'cfy executions cancel {0}'
if force:
cmd = cmd + ' -f'
return cloudify_exec(cmd.format(execution_id),
get_json=False,
timeout=timeout)
def blueprints_upload(blueprint_file_name, blueprint_id):
"""
Upload a blueprint to the manager.
:param blueprint_file_name:
:param blueprint_id:
:return:
"""
if not os.path.isfile(blueprint_file_name):
raise EcosystemTestException(
'Cant upload blueprint {path} because the file doesn`t '
'exists.'.format(path=blueprint_file_name))
remote_dir = copy_directory_to_docker(blueprint_file_name)
blueprint_file = os.path.basename(blueprint_file_name)
return cloudify_exec('cfy blueprints upload {0} -b {1}'.format(
os.path.join(remote_dir, blueprint_file), blueprint_id),
get_json=False)
def plugins_upload(wagon_path, yaml_path):
"""
Upload a wagon and plugin YAML to the manager.
:param wagon_path: Path to the wagon on the manager.
:param yaml_path: Path to the YAML on the manager container.
:return: Command output.
"""
logger.info('Uploading plugin: {0} {1}'.format(wagon_path, yaml_path))
if not plugin_already_uploaded(wagon_path):
if os.path.exists(wagon_path):
wagon_path = copy_file_to_docker(wagon_path)
if os.path.exists(yaml_path):
yaml_path = copy_file_to_docker(yaml_path)
return cloudify_exec('cfy plugins upload {0} -y {1}'.format(
wagon_path, yaml_path),
get_json=False)
def license_upload():
"""
Upload the license to the manager.
:return: Command output.
"""
logger.info('Uploading manager license.')
try:
license = base64.b64decode(os.environ[LICENSE_ENVAR_NAME])
except KeyError:
raise EcosystemTestException('License env var not set {0}.')
file_temp = NamedTemporaryFile(delete=False)
with open(file_temp.name, 'wb') as outfile:
outfile.write(license)
return cloudify_exec('cfy license upload {0}'.format(
copy_file_to_docker(file_temp.name)),
get_json=False)
def handle_deployment_update(blueprint_file_name,
update_bp_name,
test_name,
inputs,
timeout):
logger.info('updating deployment...')
try:
logger.info('Blueprints list: {0}'.format(
cloudify_exec('cfy blueprints list')))
blueprints_upload(blueprint_file_name, update_bp_name)
deployment_update(test_name,
update_bp_name,
inputs,
timeout)
except EcosystemTimeout:
# Give 5 seconds grace.
executions_list(test_name)
wait_for_execution(test_name, 'update', 10)
else:
wait_for_execution(test_name, 'update', timeout)
def is_first_invocation(test_name):
"""
Check if this is the first invocation of the test,
by check existence of blueprint and deployment with test_name id.
param: test_name: The test name.
"""
logger.info(
'Checking if {test_name} in deployments list '.format(
test_name=test_name))
deployments_list = cloudify_exec('cfy deployments list')
def _map_func(bl_or_dep_dict):
return bl_or_dep_dict["id"]
if test_name in [_map_func(deployment) for deployment in deployments_list]:
logger.info('Not first invocation!')
return False
else:
logger.info('First invocation!')
return True
def use_cfy(timeout=60):
"""
Initialize the Cloudify CLI profile inside the container.
:param timeout:
:return: Command output.
"""
logger.info('Checking manager status.')
start = datetime.now()
# Give 10 sec of mercy for the container to boot
sleep(10)
while True:
if datetime.now() - start > timedelta(seconds=timeout):
raise EcosystemTestException('Fn use_cfy timed out.')
try:
output = cloudify_exec('cfy status', get_json=False)
logger.info(output)
except EcosystemTestException:
sleep(10)
logger.info('Manager is ready.')
break
def blueprints_get(blueprint_id):
return cloudify_exec('cfy blueprints get {0}'.format(blueprint_id))
def deployment_delete(blueprint_id):
return cloudify_exec('cfy deployments delete {0}'.format(blueprint_id),
get_json=False)
def get_deployment_outputs(deployment_id):
logger.info('Getting deployment outputs {0}'.format(deployment_id))
return cloudify_exec(
'cfy deployments outputs --json {0}'.format(deployment_id))
def get_blueprint_id_of_deployment(deployment_id):
deployments_list = cloudify_exec('cfy deployments list')
for deployment in deployments_list:
if deployment['id'] == deployment_id:
return deployment["blueprint_id"]