def _get_filters(query_params):
filters = {}
for param_pair in query_params:
k, v = param_pair
if k not in SUPPORTED_PARAMS:
LOG.warning(
_LW("Search by parameter '{name}' "
"is not supported. Skipping it.").format(name=k))
continue
if k in LIST_PARAMS:
if v.startswith('in:') and k in OPERATOR_VALUES:
in_value = v[len('in:'):]
try:
filters[k] = murano_utils.split_for_quotes(in_value)
except ValueError as err:
LOG.warning(
_LW("Search by parameter '{name}' "
"caused an {message} error."
"Skipping it.").format(name=k, message=err))
else:
filters.setdefault(k, []).append(v)
else:
filters[k] = v
order_by = filters.get('order_by', [])
for i in order_by[:]:
if ORDER_VALUES and i not in ORDER_VALUES:
filters['order_by'].remove(i)
LOG.warning(
_LW("Value of 'order_by' parameter is not valid. "
"Allowed values are: {values}. Skipping it.").format(
values=", ".join(ORDER_VALUES)))
return filters
def delete(self):
while True:
try:
if not self.current():
return
self._wait_state(lambda s: True)
self._client.stacks.delete(stack_id=self._name)
self._wait_state(
lambda status: status in ('DELETE_COMPLETE', 'NOT_FOUND'),
wait_progress=True)
except heat_exc.NotFound:
LOG.warning(_LW('Stack {stack_name} already deleted?')
.format(stack_name=self._name))
break
except heat_exc.HTTPConflict as e:
LOG.warning(_LW('Conflicting operation: {msg}').format(msg=e))
eventlet.sleep(3)
else:
break
if cache.has_key(self._name):
del cache[self._name]
if applied.has_key(self._name):
del applied[self._name]
if pushing.has_key(self._name):
del pushing[self._name]
def cleanup_duplicates(self, name_map):
for class_name, package_names in six.iteritems(name_map):
if len(package_names) >= 2:
LOG.warning(_LW("Class is defined in multiple packages!"))
for package_name in package_names:
LOG.warning(_LW("Disabling class '%(class_name)s' in "
"'%(dist)s' due to conflict") %
dict(class_name=class_name, dist=package_name))
self.packages[package_name].classes.pop(class_name)
def cleanup_duplicates(self, name_map):
for class_name, package_names in six.iteritems(name_map):
if len(package_names) >= 2:
LOG.warning(_LW("Class is defined in multiple packages!"))
for package_name in package_names:
LOG.warning(_LW("Disabling class '%(class_name)s' in "
"'%(dist)s' due to conflict") %
dict(class_name=class_name, dist=package_name))
self.packages[package_name].classes.pop(class_name)
def bind(self, req, body, instance_id, app_id):
db_service = db_cf.get_service_for_instance(instance_id)
if not db_service:
return {}
service_id = db_service.service_id
environment_id = db_service.environment_id
token = req.headers['X-Auth-Token']
m_cli = _get_muranoclient(token, req)
session_id = create_session(m_cli, environment_id)
env = m_cli.environments.get(environment_id, session_id)
LOG.debug('Got environment {0}'.format(env))
service = self._get_service(env, service_id)
LOG.debug('Got service {0}'.format(service))
# NOTE(starodubcevna): Here we need to find an action which will return
# us needed credentials. By default we will looking for getCredentials
# action.
result = {}
try:
actions = service['?']['_actions']
for action_id in list(actions):
if 'getCredentials' in action_id:
@retrying.retry(
retry_on_exception=lambda e: isinstance(e, TypeError),
wait_random_min=1000,
wait_random_max=10000,
stop_max_delay=30000)
def _get_creds(client, task_id, environment_id):
result = m_cli.actions.get_result(
environment_id, task_id)['result']
return result
task_id = m_cli.actions.call(environment_id, action_id)
result = _get_creds(m_cli, task_id, environment_id)
if not result:
LOG.warning(
_LW("This application doesn't have action "
"getCredentials"))
return response.Response(status=500)
except KeyError:
# NOTE(starodubcevna): In CF service broker API spec return
# code for failed bind is not present, so we will return 500.
LOG.warning(_LW("This application doesn't have actions at all"))
return response.Response(status=500)
if 'credentials' in list(result):
return result
else:
return {'credentials': result}
def bind(self, req, body, instance_id, app_id):
db_service = db_cf.get_service_for_instance(instance_id)
if not db_service:
return {}
service_id = db_service.service_id
environment_id = db_service.environment_id
token = req.headers['X-Auth-Token']
m_cli = _get_muranoclient(token, req)
session_id = create_session(m_cli, environment_id)
env = m_cli.environments.get(environment_id, session_id)
LOG.debug('Got environment {0}'.format(env))
service = self._get_service(env, service_id)
LOG.debug('Got service {0}'.format(service))
# NOTE(starodubcevna): Here we need to find an action which will return
# us needed credentials. By default we will looking for getCredentials
# action.
result = {}
try:
actions = service['?']['_actions']
for action_id in list(actions):
if 'getCredentials' in action_id:
@retrying.retry(retry_on_exception=lambda e: isinstance(e,
TypeError),
wait_random_min=1000,
wait_random_max=10000,
stop_max_delay=30000)
def _get_creds(client, task_id, environment_id):
result = m_cli.actions.get_result(environment_id,
task_id)['result']
return result
task_id = m_cli.actions.call(environment_id, action_id)
result = _get_creds(m_cli, task_id, environment_id)
if not result:
LOG.warning(_LW("This application doesn't have action "
"getCredentials"))
return response.Response(status=500)
except KeyError:
# NOTE(starodubcevna): In CF service broker API spec return
# code for failed bind is not present, so we will return 500.
LOG.warning(_LW("This application doesn't have actions at all"))
return response.Response(status=500)
if 'credentials' in list(result):
return result
else:
return {'credentials': result}
def deprovision(self, req, instance_id):
service = db_cf.get_service_for_instance(instance_id)
if not service:
return {}
service_id = service.service_id
environment_id = service.environment_id
tenant = service.tenant
_, _, keystone = self._check_auth(req, tenant)
# Once we get here we were authorized by keystone
token = keystone.auth_token
m_cli = muranoclient(token)
try:
session_id = create_session(m_cli, environment_id)
except exc.HTTPForbidden:
# FIXME(Kezar): this is a temporary solution, should be replaced
# with 'incomplete' response for Cloud Foudry as soon as we will
# know which is right format for it.
LOG.warning(
_LW("Can't create new session. Please remove service "
"manually in environment {0}").format(environment_id))
return {}
m_cli.services.delete(environment_id, '/' + service_id, session_id)
m_cli.sessions.deploy(environment_id, session_id)
return {}
def load_extension(self, extension, name_map):
dist_name = str(extension.entry_point.dist)
name = extension.entry_point.name
if not NAME_RE.match(name):
LOG.warning(
_LW("Entry-point 'name' {name} is invalid").format(name=name))
return
name_map.setdefault(name, []).append(dist_name)
if dist_name in self.packages:
package = self.packages[dist_name]
else:
package = PackageDefinition(extension.entry_point.dist)
self.packages[dist_name] = package
plugin = extension.plugin
try:
package.classes[name] = initialize_plugin(plugin)
except Exception:
LOG.exception(
_LE("Unable to initialize plugin for {name}").format(
name=name))
return
LOG.info(
_LI("Loaded class {class_name} from {dist}").format(
class_name=name, dist=dist_name))
def execute(self, request, environment_id, action_id, body):
policy.check("execute_action", request.context, {})
LOG.debug('Action:Execute <ActionId: {0}>'.format(action_id))
unit = db_session.get_session()
# no new session can be opened if environment has deploying status
env_status = envs.EnvironmentServices.get_status(environment_id)
if env_status in (states.EnvironmentStatus.DEPLOYING,
states.EnvironmentStatus.DELETING):
LOG.warning(_LW('Could not open session for environment '
'<EnvId: {id}>, environment has deploying '
'status.').format(id=environment_id))
raise exc.HTTPForbidden()
user_id = request.context.user
session = sessions.SessionServices.create(environment_id, user_id)
if not sessions.SessionServices.validate(session):
LOG.error(_LE('Session <SessionId {id}> '
'is invalid').format(id=session.id))
raise exc.HTTPForbidden()
task_id = actions.ActionServices.execute(
action_id, session, unit, request.context.auth_token, body or {})
return {'task_id': task_id}
def signal_destruction_dependencies(self, *objects):
if not objects:
return
elif len(objects) > 1:
return helpers.parallel_select(
objects, self.signal_destruction_dependencies)
obj = objects[0]
if obj.destroyed:
return
for dependency in obj.destruction_dependencies:
try:
handler = dependency['handler']
if handler:
subscriber = dependency['subscriber']
if subscriber:
subscriber = subscriber()
if (subscriber and subscriber.initialized
and not subscriber.destroyed):
method = subscriber.type.find_single_method(handler)
self.invoke_method(method,
subscriber,
None, [obj], {},
invoke_action=False)
except Exception as e:
LOG.warning(_LW(
'Muted exception during destruction dependency '
'execution in {0}: {1}').format(obj, e),
exc_info=True)
obj.load_dependencies(None)
def invoke_method(self, name, this, context, murano_class, *args):
external_call = False
if context is None:
external_call = True
context = self._root_context
method = this.type.find_single_method(name)
is_special_method = name in ('initialize', 'destroy')
if external_call and not is_special_method and \
method.usage != murano_method.MethodUsages.Action:
raise Exception('{0} is not an action'.format(method.name))
# TODO(slagun): check method accessibility from murano_class
if not external_call and is_special_method:
LOG.warning(_LW('initialize/destroy methods are called '
'automatically by engine. This call is no-op '
'and will become exception in the future'))
return None
# restore this from upcast object (no change if there was no upcast)
this = this.real_this
arguments_scheme = method.arguments_scheme
params = self._evaluate_parameters(
arguments_scheme, context, this, *args)
return self._invoke_method_implementation(
method, this, context, params)
def test_bind_without_get_credentials(self, mock_db_cf,
mock_get_muranoclient,
mock_get_service, mock_log):
mock_db_service = mock.Mock(
service_id=mock.sentinel.service_id,
environment_id=mock.sentinel.environment_id)
test_service = {'?': {'_actions': []}}
test_request = mock.Mock(
headers={'X-Auth-Token': mock.sentinel.auth_token})
mock_db_cf.get_service_for_instance.return_value = mock_db_service
mock_get_service.return_value = test_service
m_cli = mock_get_muranoclient.return_value
m_env = m_cli.environments.get.return_value
resp = self.controller.bind(test_request, None, None, None)
self.assertEqual(500, resp.status_code)
m_cli.environments.get.assert_called_once_with(
mock.sentinel.environment_id, mock.ANY)
mock_get_service.assert_called_once_with(self.controller, m_env,
mock.sentinel.service_id)
mock_log.warning.assert_called_once_with(
_LW("This application doesn't have action getCredentials"))
def get_last_operation(self, req, instance_id):
service = db_cf.get_service_for_instance(instance_id)
# NOTE(freerunner): Prevent code 500 if requested environment
# already doesn't exist.
if not service:
LOG.warning(_LW('Requested service for instance {} is not found'))
body = {}
resp = response.Response(status=410, json_body=body)
return resp
env_id = service.environment_id
token = req.headers["X-Auth-Token"]
m_cli = _get_muranoclient(token, req)
# NOTE(starodubcevna): we can track only environment status. it's
# murano API limitation.
m_environment = m_cli.environments.get(env_id)
if m_environment.status == 'ready':
body = {'state': 'succeeded',
'description': 'operation succeed'}
resp = response.Response(status=200, json_body=body)
elif m_environment.status in ['pending', 'deleting', 'deploying']:
body = {'state': 'in progress',
'description': 'operation in progress'}
resp = response.Response(status=202, json_body=body)
elif m_environment.status in ['deploy failure', 'delete failure']:
body = {'state': 'failed',
'description': '{0}. Please correct it manually'.format(
m_environment.status)}
resp = response.Response(status=200, json_body=body)
return resp
def get_keystone_token(self, user, password):
# TODO(starodubcevna): picking up project_name and auth_url from
# section related to Cloud Foundry service broker is probably a duct
# tape and should be rewritten as soon as we get more non-OpenStack
# services as murano recipients. The same is right for project and user
# domain names.
auth_url = CONF.cfapi.auth_url
if not (auth_url.endswith('v2.0') or auth_url.endswith('v3')):
auth_url += '/v3'
elif auth_url.endswith('v2.0'):
auth_url = auth_url.replace('v2.0', 'v3')
elif auth_url.endswith('v3'):
pass
else:
LOG.warning(_LW('Wrong format for service broker auth url'))
kwargs = {
'auth_url': auth_url,
'username': user,
'password': password,
'project_name': CONF.cfapi.tenant,
'user_domain_name': CONF.cfapi.user_domain_name,
'project_domain_name': CONF.cfapi.project_domain_name
}
password_auth = v3.Password(**kwargs)
session = ks_session.Session(auth=password_auth)
self._query_endpoints(password_auth, session)
return session.get_token()
def test_client_property(self, mock_versionutils, mock_auth_utils):
self._unpatch_loader_client()
session = mock_auth_utils.get_client_session()
session_params = mock_auth_utils.get_session_client_parameters
session.auth.get_token.return_value = 'test_token'
session.get_endpoint.return_value = 'test_endpoint/v3'
session_params.return_value = {'endpoint': 'test_endpoint/v3'}
CONF.set_override('packages_service', 'glance', group='engine',
enforce_type=True)
client = self.loader.client
mock_versionutils.report_deprecated_feature.assert_called_once_with(
package_loader.LOG,
_LW("'glance' packages_service option has been renamed "
"to 'glare', please update your configuration"))
self.assertIsNotNone(client)
self.assertIsNotNone(self.loader._glare_client)
# Test whether client is initialized using different CONF.
CONF.set_override('packages_service', 'test_service', group='engine',
enforce_type=True)
client = self.loader.client
self.assertIsNotNone(client)
def get_last_operation(self, req, instance_id):
service = db_cf.get_service_for_instance(instance_id)
# NOTE(freerunner): Prevent code 500 if requested environment
# already doesn't exist.
if not service:
LOG.warning(_LW('Requested service for instance {} is not found'))
body = {}
resp = response.Response(status=410, json_body=body)
return resp
env_id = service.environment_id
token = req.headers["X-Auth-Token"]
m_cli = _get_muranoclient(token, req)
# NOTE(starodubcevna): we can track only environment status. it's
# murano API limitation.
m_environment = m_cli.environments.get(env_id)
if m_environment.status == 'ready':
body = {'state': 'succeeded', 'description': 'operation succeed'}
resp = response.Response(status=200, json_body=body)
elif m_environment.status in ['pending', 'deleting', 'deploying']:
body = {
'state': 'in progress',
'description': 'operation in progress'
}
resp = response.Response(status=202, json_body=body)
elif m_environment.status in ['deploy failure', 'delete failure']:
body = {
'state':
'failed',
'description':
'{0}. Please correct it manually'.format(m_environment.status)
}
resp = response.Response(status=200, json_body=body)
return resp
def deprovision(self, req, instance_id):
service = db_cf.get_service_for_instance(instance_id)
if not service:
return {}
service_id = service.service_id
environment_id = service.environment_id
tenant = service.tenant
_, _, keystone = self._check_auth(req, tenant)
# Once we get here we were authorized by keystone
token = keystone.auth_token
m_cli = muranoclient(token)
try:
session_id = create_session(m_cli, environment_id)
except exc.HTTPForbidden:
# FIXME(Kezar): this is a temporary solution, should be replaced
# with 'incomplete' response for Cloud Foudry as soon as we will
# know which is right format for it.
LOG.warning(_LW("Can't create new session. Please remove service "
"manually in environment {0}")
.format(environment_id))
return {}
m_cli.services.delete(environment_id, '/' + service_id, session_id)
m_cli.sessions.deploy(environment_id, session_id)
return {}
def _create_client(region):
if not mistralcli:
LOG.warning(_LW("Mistral client is not available"))
raise mistral_import_error
mistral_settings = CONF.mistral
endpoint_type = mistral_settings.endpoint_type
service_type = mistral_settings.service_type
session = auth_utils.get_client_session()
mistral_url = mistral_settings.url or session.get_endpoint(
service_type=service_type,
endpoint_type=endpoint_type,
region_name=region)
auth_ref = session.auth.get_access(session)
return mistralcli.client(mistral_url=mistral_url,
project_id=auth_ref.project_id,
endpoint_type=endpoint_type,
service_type=service_type,
auth_token=auth_ref.auth_token,
user_id=auth_ref.user_id,
insecure=mistral_settings.insecure,
cacert=mistral_settings.ca_cert)
def execute(self):
try:
self._create_trust()
except Exception as e:
return self.exception_result(e, None, '<system>')
with package_loader.CombinedPackageLoader(self._session) as pkg_loader:
pkg_loader.import_fixation_table(
self._session.system_attributes.get('Packages', {}))
result = self._execute(pkg_loader)
self._session.system_attributes[
'Packages'] = pkg_loader.export_fixation_table()
self._model['SystemData'] = self._session.system_attributes
self._model['project_id'] = self._session.environment_owner_project_id
self._model['user_id'] = self._session.environment_owner_user_id
result['model'] = self._model
if (not self._model.get('Objects')
and not self._model.get('ObjectsCopy')):
try:
self._delete_trust()
except Exception:
LOG.warning(_LW('Cannot delete trust'), exc_info=True)
return result
def execute(self, request, environment_id, action_id, body):
policy.check("execute_action", request.context, {})
LOG.debug('Action:Execute <ActionId: {0}>'.format(action_id))
unit = db_session.get_session()
# no new session can be opened if environment has deploying status
env_status = envs.EnvironmentServices.get_status(environment_id)
if env_status in (states.EnvironmentStatus.DEPLOYING,
states.EnvironmentStatus.DELETING):
LOG.warning(
_LW('Could not open session for environment '
'<EnvId: {id}>, environment has deploying '
'status.').format(id=environment_id))
raise exc.HTTPForbidden()
user_id = request.context.user
session = sessions.SessionServices.create(environment_id, user_id)
if not sessions.SessionServices.validate(session):
LOG.error(
_LE('Session <SessionId {id}> '
'is invalid').format(id=session.id))
raise exc.HTTPForbidden()
task_id = actions.ActionServices.execute(action_id, session, unit,
request.context, body or {})
return {'task_id': task_id}
def get_keystone_token(self, user, password):
# TODO(starodubcevna): picking up project_name and auth_url from
# section related to Cloud Foundry service broker is probably a duct
# tape and should be rewritten as soon as we get more non-OpenStack
# services as murano recipients. The same is right for project and user
# domain names.
auth_url = CONF.cfapi.auth_url
if not (auth_url.endswith('v2.0') or auth_url.endswith('v3')):
auth_url += '/v3'
elif auth_url.endswith('v2.0'):
auth_url = auth_url.replace('v2.0', 'v3')
elif auth_url.endswith('v3'):
pass
else:
LOG.warning(_LW('Wrong format for service broker auth url'))
kwargs = {'auth_url': auth_url,
'username': user,
'password': password,
'project_name': CONF.cfapi.tenant,
'user_domain_name': CONF.cfapi.user_domain_name,
'project_domain_name': CONF.cfapi.project_domain_name}
password_auth = v3.Password(**kwargs)
session = ks_session.Session(auth=password_auth)
self._query_endpoints(password_auth, session)
return session.get_token()
def try_cleanup_cache(self, package_directory=None, current_id=None):
"""Attempts to cleanup cache in a given directory.
:param package_directory: directory containing cached packages
:param current_id: optional id of the package to exclude from the list
of deleted packages
"""
if not package_directory:
return
try:
pkg_ids_listed = set(os.listdir(package_directory))
except OSError:
# No directory for this package, probably someone
# already deleted everything. Anyway nothing to delete
return
# if current_id was given: ensure it's not checked for removal
pkg_ids_listed -= {current_id}
for pkg_id in pkg_ids_listed:
stale_directory = os.path.join(package_directory, pkg_id)
if not os.path.isdir(package_directory):
continue
usage_lock_path = os.path.join(self._cache_directory,
'{}_usage.lock'.format(pkg_id))
ipc_lock = m_utils.ExclusiveInterProcessLock(
path=usage_lock_path, sleep_func=eventlet.sleep)
try:
with usage_mem_locks[pkg_id].write_lock(False) as acquired:
if not acquired:
# the package is in use by other deployment in this
# process will do nothing, someone else would delete it
continue
acquired_ipc_lock = ipc_lock.acquire(blocking=False)
if not acquired_ipc_lock:
# the package is in use by other deployment in another
# process, will do nothing, someone else would delete
continue
shutil.rmtree(stale_directory, ignore_errors=True)
ipc_lock.release()
for lock_type in ('usage', 'download'):
lock_path = os.path.join(
self._cache_directory,
'{}_{}.lock'.format(pkg_id, lock_type))
try:
os.remove(lock_path)
except OSError:
LOG.warning(
_LW("Couldn't delete lock file: "
"{}").format(lock_path))
except RuntimeError:
# couldn't upgrade read lock to write-lock. go on.
continue
def _from_json(self, datastring):
value = datastring
try:
LOG.debug("Trying deserialize '{data}' to json".format(data=datastring))
value = jsonutils.loads(datastring)
except ValueError:
LOG.warning(_LW("Unable deserialize to json, using raw text"))
return value
def _from_json(self, datastring):
value = datastring
try:
LOG.debug(
"Trying deserialize '{data}' to json".format(data=datastring))
value = jsonutils.loads(datastring)
except ValueError:
LOG.warning(_LW("Unable deserialize to json, using raw text"))
return value
def test_from_json_except_value_error(self, mock_log):
data_serializer = wsgi.FormDataDeserializer()
value = data_serializer._from_json('value error')
self.assertEqual('value error', value)
mock_log.debug.assert_called_once_with(
"Trying to deserialize 'value error' to json")
mock_log.warning.assert_called_once_with(
_LW('Unable to deserialize to json, using raw text'))
def _destroy_object(self, obj):
methods = obj.type.find_methods(lambda m: m.name == '.destroy')
for method in methods:
try:
method.invoke(self, obj, (), {}, None)
except Exception as e:
LOG.warning(_LW('Muted exception during execution of .destroy '
'on {0}: {1}').format(obj, e),
exc_info=True)
def _destroy_object(self, obj):
methods = obj.type.find_methods(lambda m: m.name == '.destroy')
for method in methods:
try:
method.invoke(obj, (), {}, None)
except Exception as e:
LOG.warning(_LW(
'Muted exception during execution of .destroy '
'on {0}: {1}').format(obj, e), exc_info=True)
def test_get_last_operation_without_service(self, mock_db_cf, mock_log):
mock_db_cf.get_service_for_instance.return_value = None
resp = self.controller.get_last_operation(
None, mock.sentinel.instance_id)
self.assertEqual(410, resp.status_code)
self.assertEqual({}, resp.json_body)
mock_log.warning.assert_called_once_with(
_LW('Requested service for instance sentinel.instance_id is not '
'found'))
def get_murano_url(cls):
try:
url = cls.keystone_client().service_catalog.url_for(
service_type='application-catalog', endpoint_type='publicURL')
except ks_exceptions.EndpointNotFound:
url = CONF.murano.murano_url
LOG.warning(
_LW("Murano endpoint not found in Keystone. "
"Using CONF."))
return url if 'v1' not in url else "/".join(
url.split('/')[:url.split('/').index('v1')])
def push(self):
if self._applied or self._template is None:
return
self._tags = ','.join(CONF.heat.stack_tags)
if 'heat_template_version' not in self._template:
self._template['heat_template_version'] = HEAT_TEMPLATE_VERSION
if 'description' not in self._template and self._description:
self._template['description'] = self._description
template = copy.deepcopy(self._template)
LOG.debug('Pushing: {template}'.format(template=template))
while True:
try:
current_status = self._get_status()
resources = template.get('Resources') or template.get(
'resources')
if current_status == 'NOT_FOUND':
if resources is not None:
token_client = self._get_token_client()
token_client.stacks.create(
stack_name=self._name,
parameters=self._parameters,
template=template,
files=self._files,
environment=self._hot_environment,
disable_rollback=True,
tags=self._tags)
self._wait_state(
lambda status: status == 'CREATE_COMPLETE')
else:
if resources is not None:
self._client.stacks.update(
stack_id=self._name,
parameters=self._parameters,
files=self._files,
environment=self._hot_environment,
template=template,
disable_rollback=True,
tags=self._tags)
self._wait_state(
lambda status: status == 'UPDATE_COMPLETE', True)
else:
self.delete()
except heat_exc.HTTPConflict as e:
LOG.warning(_LW('Conflicting operation: {msg}').format(msg=e))
eventlet.sleep(3)
else:
break
self._applied = not utils.is_different(self._template, template)
def try_cleanup_cache(self, package_directory=None, current_id=None):
"""Attempts to cleanup cache in a given directory.
:param package_directory: directory containing cached packages
:param current_id: optional id of the package to exclude from the list
of deleted packages
"""
if not package_directory:
return
pkg_ids_listed = set()
try:
pkg_ids_listed = set(os.listdir(package_directory))
except OSError:
# No directory for this package, probably someone
# already deleted everything. Anyway nothing to delete
return
# if current_id was given: ensure it's not checked for removal
pkg_ids_listed -= {current_id}
for pkg_id in pkg_ids_listed:
stale_directory = os.path.join(package_directory, pkg_id)
if not os.path.isdir(package_directory):
continue
usage_lock_path = os.path.join(self._cache_directory, "{}_usage.lock".format(pkg_id))
ipc_lock = m_utils.ExclusiveInterProcessLock(path=usage_lock_path, sleep_func=eventlet.sleep)
try:
with usage_mem_locks[pkg_id].write_lock(False) as acquired:
if not acquired:
# the package is in use by other deployment in this
# process will do nothing, someone else would delete it
continue
acquired_ipc_lock = ipc_lock.acquire(blocking=False)
if not acquired_ipc_lock:
# the package is in use by other deployment in another
# process, will do nothing, someone else would delete
continue
shutil.rmtree(stale_directory, ignore_errors=True)
ipc_lock.release()
for lock_type in ("usage", "download"):
lock_path = os.path.join(self._cache_directory, "{}_{}.lock".format(pkg_id, lock_type))
try:
os.remove(lock_path)
except OSError:
LOG.warning(_LW("Couldn't delete lock file: " "{}").format(lock_path))
except RuntimeError:
# couldn't upgrade read lock to write-lock. go on.
continue
def push(self):
if self._applied or self._template is None:
return
self._tags = ','.join(CONF.heat.stack_tags)
if 'heat_template_version' not in self._template:
self._template['heat_template_version'] = HEAT_TEMPLATE_VERSION
if 'description' not in self._template and self._description:
self._template['description'] = self._description
template = copy.deepcopy(self._template)
LOG.debug('Pushing: {template}'.format(template=template))
while True:
try:
current_status = self._get_status()
resources = template.get('Resources') or template.get(
'resources')
if current_status == 'NOT_FOUND':
if resources is not None:
token_client = self._get_token_client()
token_client.stacks.create(
stack_name=self._name,
parameters=self._parameters,
template=template,
files=self._files,
environment=self._hot_environment,
disable_rollback=True,
tags=self._tags)
self._wait_state(
lambda status: status == 'CREATE_COMPLETE')
else:
if resources is not None:
self._client.stacks.update(
stack_id=self._name,
parameters=self._parameters,
files=self._files,
environment=self._hot_environment,
template=template,
disable_rollback=True,
tags=self._tags)
self._wait_state(
lambda status: status == 'UPDATE_COMPLETE', True)
else:
self.delete()
except heat_exc.HTTPConflict as e:
LOG.warning(_LW('Conflicting operation: {msg}').format(msg=e))
eventlet.sleep(3)
else:
break
self._applied = not utils.is_different(self._template, template)
def delete(self, _context):
client = self._clients.get_heat_client(_context)
try:
if not self.current(_context):
return
client.stacks.delete(stack_id=self._name)
self._wait_state(_context, lambda status: status in ("DELETE_COMPLETE", "NOT_FOUND"))
except heat_exc.NotFound:
LOG.warn(_LW("Stack {0} already deleted?").format(self._name))
self._template = {}
self._applied = True
def _get_filters(query_params):
filters = {}
for param_pair in query_params:
k, v = param_pair
if k not in SUPPORTED_PARAMS:
LOG.warning(_LW("Search by parameter '{name}' "
"is not supported. Skipping it.").format(name=k))
continue
if k in LIST_PARAMS:
filters.setdefault(k, []).append(v)
else:
filters[k] = v
order_by = filters.get('order_by', [])
for i in order_by[:]:
if ORDER_VALUES and i not in ORDER_VALUES:
filters['order_by'].remove(i)
LOG.warning(_LW("Value of 'order_by' parameter is not valid. "
"Allowed values are: {values}. Skipping it.")
.format(values=", ".join(ORDER_VALUES)))
return filters
def delete(self):
client = self._clients.get_heat_client()
try:
if not self.current():
return
client.stacks.delete(stack_id=self._name)
self._wait_state(lambda status: status in ("DELETE_COMPLETE", "NOT_FOUND"), wait_progress=True)
except heat_exc.NotFound:
LOG.warning(_LW("Stack {stack_name} already deleted?").format(stack_name=self._name))
self._template = {}
self._applied = True
def _resolve_usage_and_scope(self):
if self._usage == dsl_types.MethodUsages.Action:
runtime_version = self.declaring_type.package.runtime_version
if runtime_version > constants.RUNTIME_VERSION_1_3:
LOG.warning(
_LW('"Usage: Action" is deprecated, '
'use "Scope: Public" instead'))
if self._scope == dsl_types.MethodScopes.Session:
raise ValueError(
'Both "Usage: Action" and "Scope: Session" are '
'provided for method ' + self.name)
self._scope = dsl_types.MethodScopes.Public
def register_in_loader(self, class_loader):
for package in six.itervalues(self.packages):
for class_name, clazz in six.iteritems(package.classes):
if hasattr(clazz, "_murano_class_name"):
LOG.warning(_LW("Class '%(class_name)s' has a MuranoPL "
"name '%(name)s' defined which will be "
"ignored") %
dict(class_name=class_name,
name=getattr(clazz, "_murano_class_name")))
LOG.debug("Registering '%s' from '%s' in class loader"
% (class_name, package.name))
class_loader.import_class(clazz, name=class_name)
def register_in_loader(self, class_loader):
for package in six.itervalues(self.packages):
for class_name, clazz in six.iteritems(package.classes):
if hasattr(clazz, "_murano_class_name"):
LOG.warning(_LW("Class '%(class_name)s' has a MuranoPL "
"name '%(name)s' defined which will be "
"ignored") %
dict(class_name=class_name,
name=getattr(clazz, "_murano_class_name")))
LOG.debug("Registering '%s' from '%s' in class loader"
% (class_name, package.name))
class_loader.import_class(clazz, name=class_name)
def test_process_result_with_no_environment(self, mock_db_session,
mock_log):
test_result = {'model': None}
mock_db_session.get_session().query().get.return_value = None
result = self.result_endpoint.process_result(self.dummy_context,
test_result,
'test_env_id')
self.assertIsNone(result)
mock_log.warning.assert_called_once_with(
_LW('Environment result could not be handled, '
'specified environment not found in database'))
def _get_filters(query_params):
filters = {}
for param_pair in query_params:
k, v = param_pair
if k not in SUPPORTED_PARAMS:
LOG.warning(_LW("Search by parameter '{name}' "
"is not supported. Skipping it.").format(name=k))
continue
if k in LIST_PARAMS:
filters.setdefault(k, []).append(v)
else:
filters[k] = v
order_by = filters.get('order_by', [])
for i in order_by[:]:
if ORDER_VALUES and i not in ORDER_VALUES:
filters['order_by'].remove(i)
LOG.warning(_LW(
"Value of 'order_by' parameter is not valid. "
"Allowed values are: {0}. Skipping it.").format(
", ".join(ORDER_VALUES)))
return filters
def __init__(self, pkg_loader, package_definition):
super(MuranoPackage, self).__init__(
pkg_loader, package_definition.name, runtime_version='1.0')
for class_name, clazz in six.iteritems(package_definition.classes):
if hasattr(clazz, "_murano_class_name"):
LOG.warning(_LW("Class '%(class_name)s' has a MuranoPL "
"name '%(name)s' defined which will be "
"ignored") %
dict(class_name=class_name,
name=getattr(clazz, "_murano_class_name")))
LOG.debug("Registering '%s' from '%s' in class loader"
% (class_name, package_definition.name))
self.register_class(clazz, class_name)
def _log_report(cls, environment):
"""Used for logging reports on failures.
:param environment: Murano environment.
"""
deployment = cls.get_last_deployment(environment)
try:
details = deployment.result['result']['details']
LOG.warning(_LW('Details:\n {details}').format(details=details))
except Exception as e:
LOG.error(e)
report = cls.get_deployment_report(environment, deployment)
LOG.debug('Report:\n {report}\n'.format(report=report))
def my_process_result(context, result, environment_id):
if environment_id != scheduler.get_scheduler_id():
return old_process_result(context, result, environment_id)
model = result['model']
action_result = result.get('action', {})
unit = db_session.get_session()
# close deployment
deployment = server.get_last_deployment(unit, environment_id)
deployment.finished = timeutils.utcnow()
deployment.result = action_result
num_errors = unit.query(models.Status).filter_by(level='error', task_id=deployment.id).count()
num_warnings = unit.query(models.Status).filter_by(level='warning', task_id=deployment.id).count()
if num_errors:
final_status_text = "finished with errors"
elif num_warnings:
final_status_text = "finished with warnings"
else:
final_status_text = "finished"
status = models.Status()
status.task_id = deployment.id
status.text = final_status_text
status.level = 'info'
deployment.statuses.append(status)
deployment.save(unit)
# close session
session_id = model['SystemData']['SessionId']
conf_session = unit.query(models.Session).get(session_id)
if num_errors > 0 or result['action'].get('isException'):
conf_session.state = states.EnvironmentStatus.DEPLOY_FAILURE
else:
conf_session.state = states.EnvironmentStatus.READY
conf_session.description = model
if conf_session.description['Objects'] is not None:
conf_session.description['Objects']['services'] = conf_session.description['Objects'].pop('applications', [])
conf_session.version += 1
conf_session.save(unit)
# output application tracking information
services = []
objects = model['Objects']
if objects:
services = objects.get('services')
if num_errors + num_warnings > 0:
LOG.warning(_LW('Schedule Status: Failed Apps: {services}').format(services=services))
else:
LOG.info(_LI('Schedule Status: Successful Apps: {services}').format(services=services))
def delete(self, _context):
client = self._clients.get_heat_client(_context)
try:
if not self.current(_context):
return
client.stacks.delete(stack_id=self._name)
self._wait_state(
_context, lambda status: status in
('DELETE_COMPLETE', 'NOT_FOUND'))
except heat_exc.NotFound:
LOG.warn(_LW('Stack {0} already deleted?').format(self._name))
self._template = {}
self._applied = True
def delete(self):
while True:
try:
if not self.current():
return
self._wait_state(lambda s: True)
self._client.stacks.delete(stack_id=self._name)
self._wait_state(lambda status: status in
('DELETE_COMPLETE', 'NOT_FOUND'),
wait_progress=True)
except heat_exc.NotFound:
LOG.warning(
_LW('Stack {stack_name} already deleted?').format(
stack_name=self._name))
break
except heat_exc.HTTPConflict as e:
LOG.warning(_LW('Conflicting operation: {msg}').format(msg=e))
eventlet.sleep(3)
else:
break
self._template = {}
self._applied = True
def _do_add(package, change):
# Only categories and tags support addition
path = change["path"][0]
value = change["value"]
calculate = {"categories": _get_categories, "tags": _get_tags}
items_to_add = calculate[path](value)
for item in items_to_add:
try:
getattr(package, path).append(item)
except AssertionError:
msg = _LW("One of the specified {0} is already " "associated with a package. Doing nothing.")
LOG.warning(msg.format(path))
return package
def _do_add(package, change):
# Only categories and tags support addition
path = change['path'][0]
value = change['value']
calculate = {'categories': _get_categories, 'tags': _get_tags}
items_to_add = calculate[path](value)
for item in items_to_add:
try:
getattr(package, path).append(item)
except AssertionError:
LOG.warning(
_LW('One of the specified {path} is already associated'
' with a package. Doing nothing.').format(path=path))
return package
def delete(self):
try:
if not self.current():
return
self._wait_state(lambda s: True)
self._client.stacks.delete(stack_id=self._name)
self._wait_state(
lambda status: status in ('DELETE_COMPLETE', 'NOT_FOUND'),
wait_progress=True)
except heat_exc.NotFound:
LOG.warning(_LW('Stack {stack_name} already deleted?')
.format(stack_name=self._name))
self._template = {}
self._applied = True
def __init__(self, pkg_loader, package_definition):
super(MuranoPackage, self).__init__(pkg_loader,
package_definition.name,
runtime_version='1.0')
for class_name, clazz in six.iteritems(package_definition.classes):
if hasattr(clazz, "_murano_class_name"):
LOG.warning(
_LW("Class '%(class_name)s' has a MuranoPL "
"name '%(name)s' defined which will be "
"ignored") %
dict(class_name=class_name,
name=getattr(clazz, "_murano_class_name")))
LOG.debug("Registering '%s' from '%s' in class loader", class_name,
package_definition.name)
self.register_class(clazz, class_name)
def _do_add(package, change):
# Only categories and tags support addition
path = change['path'][0]
value = change['value']
calculate = {'categories': _get_categories,
'tags': _get_tags}
items_to_add = calculate[path](value)
for item in items_to_add:
try:
getattr(package, path).append(item)
except AssertionError:
LOG.warning(_LW('One of the specified {path} is already associated'
' with a package. Doing nothing.').format(
path=path))
return package
def execute(self):
try:
self._create_trust()
except Exception as e:
return self.exception_result(e, None, '<system>')
with package_loader.CombinedPackageLoader(self._session) as pkg_loader:
result = self._execute(pkg_loader)
self._model['SystemData'] = self._session.system_attributes
result['model'] = self._model
if (not self._model.get('Objects') and
not self._model.get('ObjectsCopy')):
try:
self._delete_trust()
except Exception:
LOG.warning(_LW('Cannot delete trust'), exc_info=True)
return result
def process_request(self, req):
"""Try to find a version first in the accept header, then the URL."""
LOG.debug(("Determining version of request:{method} {path}"
"Accept: {accept}").format(method=req.method,
path=req.path,
accept=req.accept))
LOG.debug("Using url versioning")
# Remove version in url so it doesn't conflict later
req_version = self._pop_path_info(req)
try:
version = self._match_version_string(req_version)
except ValueError:
LOG.warning(_LW("Unknown version. Returning version choices."))
return self.versions_app
req.environ['api.version'] = version
req.path_info = ''.join(('/v', str(version), req.path_info))
LOG.debug("Matched version: v{version}".format(version=version))
LOG.debug('new path {path}'.format(path=req.path_info))
return None
def load_extension(self, extension, name_map):
dist_name = str(extension.entry_point.dist)
name = extension.entry_point.name
if not NAME_RE.match(name):
LOG.warning(_LW("Entry-point 'name' %s is invalid") % name)
return
name = "%s.%s" % (self.namespace, name)
name_map.setdefault(name, []).append(dist_name)
if dist_name in self.packages:
package = self.packages[dist_name]
else:
package = PackageDefinition(extension.entry_point.dist)
self.packages[dist_name] = package
plugin = extension.plugin
try:
package.classes[name] = initialize_plugin(plugin)
except Exception:
LOG.exception(_LE("Unable to initialize plugin for %s") % name)
return
LOG.info(_LI("Loaded class '%(class_name)s' from '%(dist)s'")
% dict(class_name=name, dist=dist_name))
def cleanup(self, data):
objects_copy = data.get(constants.DM_OBJECTS_COPY)
if not objects_copy:
return
gc_object_store = object_store.ObjectStore(self)
gc_object_store.load(objects_copy, None)
objects_to_clean = []
for object_id in self._list_potential_object_ids(objects_copy):
if (gc_object_store.has(object_id) and
not self._object_store.has(object_id)):
obj = gc_object_store.get(object_id)
objects_to_clean.append(obj)
if objects_to_clean:
for obj in objects_to_clean:
methods = obj.type.find_methods(lambda m: m.name == '.destroy')
for method in methods:
try:
method.invoke(self, obj, (), {}, None)
except Exception as e:
LOG.warn(_LW(
'Muted exception during execution of .destroy '
'on {0}: {1}').format(obj, e), exc_info=True)
def client(self):
murano_settings = CONF.murano
last_glare_client = self._glare_client
if CONF.engine.packages_service in ['glance', 'glare']:
if CONF.engine.packages_service == 'glance':
versionutils.report_deprecated_feature(
LOG,
_LW("'glance' packages_service option has been renamed "
"to 'glare', please update your configuration"))
artifacts_client = self._get_glare_client()
else:
artifacts_client = None
if artifacts_client != last_glare_client:
self._murano_client = None
if not self._murano_client:
parameters = auth_utils.get_session_client_parameters(
service_type='application-catalog',
execution_session=self._execution_session,
conf=murano_settings
)
self._murano_client = muranoclient.Client(
artifacts_client=artifacts_client, **parameters)
return self._murano_client
def execute(self):
try:
self._create_trust()
except Exception as e:
return self.exception_result(e, None, '<system>')
murano_client_factory = \
lambda: self._environment.clients.get_murano_client()
with package_loader.CombinedPackageLoader(
murano_client_factory,
self._environment.tenant_id) as pkg_loader:
result = self._execute(pkg_loader)
self._model['SystemData'] = self._environment.system_attributes
result['model'] = self._model
if (not self._model.get('Objects') and
not self._model.get('ObjectsCopy')):
try:
self._delete_trust()
except Exception:
LOG.warning(_LW('Cannot delete trust'), exc_info=True)
return result
def _on_load_failure(manager, ep, exc):
LOG.warning(_LW("Error loading entry-point {ep} from package {dist}: "
"{err}").format(ep=ep.name, dist=ep.dist, err=exc))
def provision(self, req, body, instance_id):
"""Here is the example of request body given us from Cloud Foundry:
{
"service_id": "service-guid-here",
"plan_id": "plan-guid-here",
"organization_guid": "org-guid-here",
"space_guid": "space-guid-here",
"parameters": {"param1": "value1",
"param2": "value2"}
}
"""
data = json.loads(req.body)
space_guid = data['space_guid']
org_guid = data['organization_guid']
plan_id = data['plan_id']
service_id = data['service_id']
parameters = data['parameters']
self.current_session = None
# Here we'll take an entry for CF org and space from db. If we
# don't have any entries we will create it from scratch.
try:
tenant = db_cf.get_tenant_for_org(org_guid)
except AttributeError:
tenant = req.headers['X-Project-Id']
db_cf.set_tenant_for_org(org_guid, tenant)
LOG.info(_LI("Cloud Foundry {org_id} mapped to tenant "
"{tenant_name}").format(org_id=org_guid,
tenant_name=tenant))
token = req.headers['X-Auth-Token']
m_cli = _get_muranoclient(token, req)
try:
environment_id = db_cf.get_environment_for_space(space_guid)
except AttributeError:
body = {'name': 'my_{uuid}'.format(uuid=uuid.uuid4().hex)}
env = m_cli.environments.create(body)
environment_id = env.id
db_cf.set_environment_for_space(space_guid, environment_id)
LOG.info(_LI("Cloud Foundry {space_id} mapped to {environment_id}")
.format(space_id=space_guid,
environment_id=environment_id))
package = m_cli.packages.get(service_id)
LOG.debug('Adding service {name}'.format(name=package.name))
service = self._make_service(space_guid, package, plan_id)
db_cf.set_instance_for_service(instance_id, service['?']['id'],
environment_id, tenant)
# NOTE(Kezar): Here we are going through JSON and add ids where
# it's necessary. Before that we need to drop '?' key from parameters
# dictionary as far it contains murano package related info which is
# necessary in our scenario
if '?' in parameters.keys():
parameters.pop('?', None)
LOG.warning(_LW("Incorrect input parameters. Package related "
"parameters shouldn't be passed through Cloud "
"Foundry"))
params = [parameters]
while params:
a = params.pop()
for k, v in six.iteritems(a):
if isinstance(v, dict):
params.append(v)
if k == '?':
v['id'] = uuid.uuid4().hex
service.update(parameters)
# Now we need to obtain session to modify the env
session_id = create_session(m_cli, environment_id)
m_cli.services.post(environment_id,
path='/',
data=service,
session_id=session_id)
m_cli.sessions.deploy(environment_id, session_id)
self.current_session = session_id
return response.Response(status=202, json_body={})
def process_result(context, result, environment_id):
secure_result = token_sanitizer.TokenSanitizer().sanitize(result)
LOG.debug('Got result from orchestration '
'engine:\n{0}'.format(secure_result))
model = result['model']
action_result = result.get('action', {})
unit = session.get_session()
environment = unit.query(models.Environment).get(environment_id)
if not environment:
LOG.warning(_LW('Environment result could not be handled, '
'specified environment not found in database'))
return
if model['Objects'] is None and model.get('ObjectsCopy', {}) is None:
environments.EnvironmentServices.remove(environment_id)
return
environment.description = model
if environment.description['Objects'] is not None:
environment.description['Objects']['services'] = \
environment.description['Objects'].pop('applications', [])
# environment.networking = result.get('networking', {})
action_name = 'Deployment'
deleted = False
else:
action_name = 'Deletion'
deleted = True
environment.version += 1
environment.save(unit)
# close deployment
deployment = get_last_deployment(unit, environment.id)
deployment.finished = timeutils.utcnow()
deployment.result = action_result
num_errors = unit.query(models.Status)\
.filter_by(level='error', task_id=deployment.id).count()
num_warnings = unit.query(models.Status)\
.filter_by(level='warning', task_id=deployment.id).count()
final_status_text = action_name + ' finished'
if num_errors:
final_status_text += " with errors"
elif num_warnings:
final_status_text += " with warnings"
status = models.Status()
status.task_id = deployment.id
status.text = final_status_text
status.level = 'info'
deployment.statuses.append(status)
deployment.save(unit)
# close session
conf_session = unit.query(models.Session).filter_by(
**{'environment_id': environment.id,
'state': states.SessionState.DEPLOYING if not deleted
else states.SessionState.DELETING}).first()
if num_errors > 0:
conf_session.state = \
states.SessionState.DELETE_FAILURE if deleted else \
states.SessionState.DEPLOY_FAILURE
else:
conf_session.state = states.SessionState.DEPLOYED
conf_session.save(unit)
# output application tracking information
message = _LI('EnvId: {0} TenantId: {1} Status: {2} Apps: {3}').format(
environment.id,
environment.tenant_id,
_('Failed') if num_errors + num_warnings > 0 else _('Successful'),
', '.join(map(
lambda a: a['?']['type'],
model['Objects']['services']
))
)
LOG.info(message)
