def __inner(self, request, *args, **kwargs):
if hasattr(request, 'context') and not request.context.session:
LOG.info(_LI('Session is required for this call'))
raise exc.HTTPForbidden()
session_id = request.context.session
unit = db_session.get_session()
session = unit.query(models.Session).get(session_id)
if session is None:
LOG.info(
_LI('Session <SessionId {0}> '
'is not found').format(session_id))
raise exc.HTTPForbidden()
if not sessions.SessionServices.validate(session):
LOG.info(
_LI('Session <SessionId {0}> '
'is invalid').format(session_id))
raise exc.HTTPForbidden()
if session.state == states.SessionState.DEPLOYING:
LOG.info(
_LI('Session <SessionId {0}> is already in '
'deployment state').format(session_id))
raise exc.HTTPForbidden()
return func(self, request, *args, **kwargs)
def update(self, request, environment_id, body):
LOG.debug('Environments:Update <Id: {0}, '
'Body: {1}>'.format(environment_id, body))
target = {"environment_id": environment_id}
policy.check('update_environment', request.context, target)
session = db_session.get_session()
environment = session.query(models.Environment).get(environment_id)
if environment is None:
LOG.info(_LI('Environment <EnvId {0}> not '
'found').format(environment_id))
raise exc.HTTPNotFound
if environment.tenant_id != request.context.tenant:
LOG.info(_LI('User is not authorized to access '
'this tenant resources.'))
raise exc.HTTPUnauthorized
LOG.debug('ENV NAME: {0}>'.format(body['name']))
if VALID_NAME_REGEX.match(str(body['name'])):
try:
environment.update(body)
environment.save(session)
except db_exc.DBDuplicateEntry:
msg = _('Environment with specified name already exists')
LOG.exception(msg)
raise exc.HTTPConflict(explanation=msg)
else:
msg = _('Environment name must contain only alphanumeric '
'or "_-." characters, must start with alpha')
LOG.error(msg)
raise exc.HTTPClientError(explanation=msg)
return environment.to_dict()
def __inner(self, request, *args, **kwargs):
if hasattr(request, 'context') and not request.context.session:
LOG.info(_LI('Session is required for this call'))
raise exc.HTTPForbidden()
session_id = request.context.session
unit = db_session.get_session()
session = unit.query(models.Session).get(session_id)
if session is None:
LOG.info(_LI('Session <SessionId {0}> '
'is not found').format(session_id))
raise exc.HTTPForbidden()
if not sessions.SessionServices.validate(session):
LOG.info(_LI('Session <SessionId {0}> '
'is invalid').format(session_id))
raise exc.HTTPForbidden()
if session.state == states.SessionState.DEPLOYING:
LOG.info(_LI('Session <SessionId {0}> is already in '
'deployment state').format(session_id))
raise exc.HTTPForbidden()
return func(self, request, *args, **kwargs)
def show(self, request, environment_id):
LOG.debug('Environments:Show <Id: {0}>'.format(environment_id))
target = {"environment_id": environment_id}
policy.check('show_environment', request.context, target)
session = db_session.get_session()
environment = session.query(models.Environment).get(environment_id)
if environment is None:
LOG.info(_LI('Environment <EnvId {0}> is not found').format(
environment_id))
raise exc.HTTPNotFound
if environment.tenant_id != request.context.tenant:
LOG.info(_LI('User is not authorized to access '
'this tenant resources.'))
raise exc.HTTPUnauthorized
env = environment.to_dict()
env['status'] = envs.EnvironmentServices.get_status(env['id'])
session_id = None
if hasattr(request, 'context') and request.context.session:
session_id = request.context.session
# add services to env
get_data = core_services.CoreServices.get_data
env['services'] = get_data(environment_id, '/services', session_id)
return env
def show(self, request, environment_id):
LOG.debug('Environments:Show <Id: {0}>'.format(environment_id))
target = {"environment_id": environment_id}
policy.check('show_environment', request.context, target)
session = db_session.get_session()
environment = session.query(models.Environment).get(environment_id)
if environment is None:
LOG.info(_LI('Environment <EnvId {0}> is not found').format(
environment_id))
raise exc.HTTPNotFound
if environment.tenant_id != request.context.tenant:
LOG.info(_LI('User is not authorized to access '
'this tenant resources.'))
raise exc.HTTPUnauthorized
env = environment.to_dict()
env['status'] = envs.EnvironmentServices.get_status(env['id'])
session_id = None
if hasattr(request, 'context') and request.context.session:
session_id = request.context.session
# add services to env
get_data = core_services.CoreServices.get_data
env['services'] = get_data(environment_id, '/services', session_id)
return env
def update(self, request, environment_id, body):
LOG.debug('Environments:Update <Id: {0}, '
'Body: {1}>'.format(environment_id, body))
target = {"environment_id": environment_id}
policy.check('update_environment', request.context, target)
session = db_session.get_session()
environment = session.query(models.Environment).get(environment_id)
if environment is None:
LOG.info(_LI('Environment <EnvId {0}> not '
'found').format(environment_id))
raise exc.HTTPNotFound
if environment.tenant_id != request.context.tenant:
LOG.info(_LI('User is not authorized to access '
'this tenant resources.'))
raise exc.HTTPUnauthorized
LOG.debug('ENV NAME: {0}>'.format(body['name']))
if VALID_NAME_REGEX.match(str(body['name'])):
environment.update(body)
environment.save(session)
else:
msg = _('Environment name must contain only alphanumeric '
'or "_-." characters, must start with alpha')
LOG.error(msg)
raise exc.HTTPClientError(explanation=msg)
return environment.to_dict()
def validate(self, model, class_loader=None):
"""Validate model using Congress rule engine.
@type model: dict
@param model: Dictionary representation of model starting on
environment level (['Objects'])
@type class_loader: murano.dsl.class_loader.MuranoClassLoader
@param class_loader: Optional. Used for evaluating parent class types
@raises ValidationError in case validation was not successful
"""
if model is None:
return
client = self._client_manager.get_congress_client(self._environment)
if not client:
raise ValueError(_('Congress client is not configured!'))
LOG.info(_LI('Validating model'))
LOG.debug(model)
rules = congress_rules.CongressRulesManager().convert(
model, class_loader, self._environment.tenant_id)
rules_str = map(str, rules)
env_id = model['?']['id']
# cleanup of data populated by murano driver
rules_str.insert(0, 'deleteEnv("{0}")'.format(env_id))
rules_line = " ".join(rules_str)
LOG.debug('Congress rules: \n ' +
'\n '.join(rules_str))
validation_result = client.execute_policy_action(
"murano_system",
"simulate",
False,
False,
{'query': 'predeploy_errors(eid, oid, msg)',
'action_policy': 'murano_action',
'sequence': rules_line})
if validation_result["result"]:
messages = self._parse_messages(env_id,
validation_result["result"])
if messages:
result_str = "\n ".join(map(str, messages))
msg = _("Murano object model validation failed: {0}").format(
"\n " + result_str)
LOG.error(msg)
raise ValidationError(msg)
else:
LOG.info(_LI('Model valid'))
def _execute(self, pkg_loader):
class_loader = package_class_loader.PackageClassLoader(pkg_loader)
system_objects.register(class_loader, pkg_loader)
get_plugin_loader().register_in_loader(class_loader)
exc = executor.MuranoDslExecutor(class_loader, self.environment)
obj = exc.load(self.model)
self._validate_model(obj, self.action, class_loader)
action_result = None
exception = None
exception_traceback = None
try:
LOG.info(_LI('Invoking pre-cleanup hooks'))
self.environment.start()
exc.cleanup(self._model)
except Exception as e:
exception = e
exception_traceback = TaskExecutor._log_exception(e, obj, '<GC>')
finally:
LOG.info(_LI('Invoking post-cleanup hooks'))
self.environment.finish()
if exception is None and self.action:
try:
LOG.info(_LI('Invoking pre-execution hooks'))
self.environment.start()
action_result = self._invoke(exc)
except Exception as e:
exception = e
exception_traceback = TaskExecutor._log_exception(
e, obj, self.action['method'])
finally:
LOG.info(_LI('Invoking post-execution hooks'))
self.environment.finish()
model = serializer.serialize_model(obj, exc)
model['SystemData'] = self._environment.system_attributes
result = {
'model': model,
'action': {
'result': None,
'isException': False
}
}
if exception is not None:
result['action'] = TaskExecutor.exception_result(
exception, exception_traceback)
else:
result['action']['result'] = serializer.serialize_object(
action_result)
return result
def _execute(self, pkg_loader):
class_loader = package_class_loader.PackageClassLoader(pkg_loader)
system_objects.register(class_loader, pkg_loader)
get_plugin_loader().register_in_loader(class_loader)
exc = executor.MuranoDslExecutor(class_loader, self.environment)
obj = exc.load(self.model)
self._validate_model(obj, self.action, class_loader)
action_result = None
exception = None
exception_traceback = None
try:
LOG.info(_LI('Invoking pre-execution hooks'))
self.environment.start()
# Skip execution of action in case no action is provided.
# Model will be just loaded, cleaned-up and unloaded.
# Most of the time this is used for deletion of environments.
if self.action:
action_result = self._invoke(exc)
except Exception as e:
exception = e
if isinstance(e, dsl_exception.MuranoPlException):
LOG.error('\n' + e.format(prefix=' '))
else:
exception_traceback = traceback.format_exc()
LOG.exception(
_LE("Exception %(exc)s occured"
" during invocation of %(method)s"),
{'exc': e, 'method': self.action['method']})
reporter = status_reporter.StatusReporter()
reporter.initialize(obj)
reporter.report_error(obj, str(e))
finally:
LOG.info(_LI('Invoking post-execution hooks'))
self.environment.finish()
model = serializer.serialize_model(obj, exc)
model['SystemData'] = self._environment.system_attributes
result = {
'model': model,
'action': {
'result': None,
'isException': False
}
}
if exception is not None:
result['action'] = TaskExecutor.exception_result(
exception, exception_traceback)
else:
result['action']['result'] = serializer.serialize_object(
action_result)
return result
def _validate_environment(self, unit, request, environment_id):
environment = unit.query(models.Environment).get(environment_id)
if environment is None:
LOG.info(_LI('Environment <EnvId {0}> '
'is not found').format(environment_id))
raise exc.HTTPNotFound
if environment.tenant_id != request.context.tenant:
LOG.info(_LI('User is not authorized to access '
'this tenant resources.'))
raise exc.HTTPUnauthorized
def verify_and_get_env(db_session, environment_id, request):
environment = db_session.query(models.Environment).get(environment_id)
if not environment:
LOG.info(_LI(
'Environment with id {0} not found').format(environment_id))
raise exc.HTTPNotFound
if environment.tenant_id != request.context.tenant:
LOG.info(_LI(
'User is not authorized to access this tenant resources.'))
raise exc.HTTPUnauthorized
return environment
def validate(self, model, class_loader=None):
"""Validate model using Congress rule engine.
@type model: dict
@param model: Dictionary representation of model starting on
environment level (['Objects'])
@type class_loader: murano.dsl.class_loader.MuranoClassLoader
@param class_loader: Optional. Used for evaluating parent class types
@raises ValidationError in case validation was not successful
"""
if model is None:
return
client = self._client_manager.get_congress_client(self._environment)
if not client:
raise ValueError(_('Congress client is not configured!'))
LOG.info(_LI('Validating model'))
LOG.debug(model)
rules = congress_rules.CongressRulesManager().convert(
model, class_loader, self._environment.tenant_id)
rules_str = map(str, rules)
env_id = model['?']['id']
# cleanup of data populated by murano driver
rules_str.insert(0, 'deleteEnv("{0}")'.format(env_id))
rules_line = " ".join(rules_str)
LOG.debug('Congress rules: \n ' + '\n '.join(rules_str))
validation_result = client.execute_policy_action(
"murano_system", "simulate", False, False, {
'query': 'predeploy_errors(eid, oid, msg)',
'action_policy': 'murano_action',
'sequence': rules_line
})
if validation_result["result"]:
messages = self._parse_messages(env_id,
validation_result["result"])
if messages:
result_str = "\n ".join(map(str, messages))
msg = _("Murano object model validation failed: {0}").format(
"\n " + result_str)
LOG.error(msg)
raise ValidationError(msg)
else:
LOG.info(_LI('Model valid'))
def verify_and_get_deployment(db_session, environment_id, deployment_id):
deployment = db_session.query(models.Task).get(deployment_id)
if not deployment:
LOG.info(_LI('Deployment with id {0} not found').format(deployment_id))
raise exc.HTTPNotFound
if deployment.environment_id != environment_id:
LOG.info(
_LI('Deployment with id {0} not found'
' in environment {1}').format(deployment_id, environment_id))
raise exc.HTTPBadRequest
deployment.description = _patch_description(deployment.description)
return deployment
def verify_and_get_deployment(db_session, environment_id, deployment_id):
deployment = db_session.query(models.Task).get(deployment_id)
if not deployment:
LOG.info(_LI('Deployment with id {0} not found').format(deployment_id))
raise exc.HTTPNotFound
if deployment.environment_id != environment_id:
LOG.info(_LI('Deployment with id {0} not found'
' in environment {1}').format(deployment_id,
environment_id))
raise exc.HTTPBadRequest
deployment.description = _patch_description(deployment.description)
return deployment
def _validate_environment(self, unit, request, environment_id):
environment = unit.query(models.Environment).get(environment_id)
if environment is None:
LOG.info(
_LI('Environment <EnvId {0}> '
'is not found').format(environment_id))
raise exc.HTTPNotFound
if environment.tenant_id != request.context.tenant:
LOG.info(
_LI('User is not authorized to access '
'this tenant resources.'))
raise exc.HTTPUnauthorized
def __inner(self, request, env_template_id, *args, **kwargs):
unit = db_session.get_session()
template = unit.query(models.EnvironmentTemplate).get(env_template_id)
if template is None:
LOG.info(_LI("Environment Template with id '{0}' not found").
format(env_template_id))
raise exc.HTTPNotFound()
if hasattr(request, 'context'):
if template.tenant_id != request.context.tenant:
LOG.info(_LI('User is not authorized to access '
'this tenant resources'))
raise exc.HTTPUnauthorized()
return func(self, request, env_template_id, *args, **kwargs)
def call_static_action(cls, context, task):
s_task = token_sanitizer.TokenSanitizer().sanitize(task)
LOG.info(_LI('Starting execution of static action: '
'{task_desc}').format(task_desc=jsonutils.dumps(s_task)))
result = None
reporter = status_reporter.StatusReporter(task['id'])
try:
task_executor = StaticActionExecutor(task, reporter)
result = task_executor.execute()
return result
finally:
LOG.info(_LI('Finished execution of static action: '
'{task_desc}').format(task_desc=jsonutils.dumps(result)))
def execute(task):
s_task = token_sanitizer.TokenSanitizer().sanitize(task)
LOG.info(_LI('Starting processing task: {task_desc}').format(
task_desc=jsonutils.dumps(s_task)))
result = None
reporter = status_reporter.StatusReporter(task['id'])
try:
task_executor = TaskExecutor(task, reporter)
result = task_executor.execute()
return result
finally:
LOG.info(_LI('Finished processing task: {task_desc}').format(
task_desc=jsonutils.dumps(result)))
def execute(task):
s_task = token_sanitizer.TokenSanitizer().sanitize(task)
LOG.info(_LI('Starting processing task: {task_desc}').format(
task_desc=jsonutils.dumps(s_task)))
result = None
reporter = status_reporter.StatusReporter(task['id'])
try:
task_executor = TaskExecutor(task, reporter)
result = task_executor.execute()
return result
finally:
LOG.info(_LI('Finished processing task: {task_desc}').format(
task_desc=jsonutils.dumps(result)))
def get_default_router(self):
router_name = self._settings.router_name
routers = self._client.list_routers(tenant_id=self._project_id, name=router_name).get("routers")
if len(routers) == 0:
LOG.debug("Router {name} not found".format(name=router_name))
if self._settings.create_router:
LOG.debug("Attempting to create Router {router}".format(router=router_name))
external_network = self._settings.external_network
kwargs = (
{"id": external_network} if uuidutils.is_uuid_like(external_network) else {"name": external_network}
)
networks = self._client.list_networks(**kwargs).get("networks")
ext_nets = filter(lambda n: n["router:external"], networks)
if len(ext_nets) == 0:
raise KeyError("Router %s could not be created, " "no external network found" % router_name)
nid = ext_nets[0]["id"]
body_data = {
"router": {
"name": router_name,
"external_gateway_info": {"network_id": nid},
"admin_state_up": True,
}
}
router = self._client.create_router(body=body_data).get("router")
LOG.info(_LI("Created router: {id}").format(id=router["id"]))
return router["id"]
else:
raise KeyError("Router %s was not found" % router_name)
else:
if routers[0]["external_gateway_info"] is None:
raise exc.RouterInfoException("Please set external gateway for" " the router %s " % router_name)
router_id = routers[0]["id"]
return router_id
def execute(self, request, environment_id, action_id, body):
policy.check("execute_action", request.context, {})
LOG.debug('Action:Execute <ActionId: {0}>'.format(action_id))
unit = db_session.get_session()
# no new session can be opened if environment has deploying status
env_status = envs.EnvironmentServices.get_status(environment_id)
if env_status in (states.EnvironmentStatus.DEPLOYING,
states.EnvironmentStatus.DELETING):
LOG.info(_LI('Could not open session for environment <EnvId: {0}>,'
'environment has deploying '
'status.').format(environment_id))
raise exc.HTTPForbidden()
user_id = request.context.user
session = sessions.SessionServices.create(environment_id, user_id)
if not sessions.SessionServices.validate(session):
LOG.error(_LE('Session <SessionId {0}> '
'is invalid').format(session.id))
raise exc.HTTPForbidden()
task_id = actions.ActionServices.execute(
action_id, session, unit, request.context.auth_token, body or {})
return {'task_id': task_id}
def _build_index(self):
for folder in self.search_package_folders(self._base_path):
try:
package = load_utils.load_from_dir(folder)
dsl_package = murano_package.MuranoPackage(self._root_loader, package)
for class_name in package.classes:
dsl_package.register_class(
(lambda pkg, cls: lambda: get_class(pkg, cls))(package, class_name), class_name
)
if dsl_package.name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
self.register_package(dsl_package)
except pkg_exc.PackageLoadError:
LOG.info(_LI("Unable to load package from path: {0}").format(folder))
continue
LOG.info(_LI("Loaded package from path {0}").format(folder))
def execute(self, request, environment_id, action_id, body):
policy.check("execute_action", request.context, {})
LOG.debug("Action:Execute <ActionId: {0}>".format(action_id))
unit = db_session.get_session()
# no new session can be opened if environment has deploying status
env_status = envs.EnvironmentServices.get_status(environment_id)
if env_status in (states.EnvironmentStatus.DEPLOYING, states.EnvironmentStatus.DELETING):
LOG.info(
_LI(
"Could not open session for environment <EnvId: {0}>," "environment has deploying " "status."
).format(environment_id)
)
raise exc.HTTPForbidden()
user_id = request.context.user
session = sessions.SessionServices.create(environment_id, user_id)
if not sessions.SessionServices.validate(session):
LOG.error(_LE("Session <SessionId {0}> " "is invalid").format(session.id))
raise exc.HTTPForbidden()
task_id = actions.ActionServices.execute(action_id, session, unit, request.context.auth_token, body or {})
return {"task_id": task_id}
def load_extension(self, extension, name_map):
dist_name = str(extension.entry_point.dist)
name = extension.entry_point.name
if not NAME_RE.match(name):
LOG.warning(
_LW("Entry-point 'name' {name} is invalid").format(name=name))
return
name_map.setdefault(name, []).append(dist_name)
if dist_name in self.packages:
package = self.packages[dist_name]
else:
package = PackageDefinition(extension.entry_point.dist)
self.packages[dist_name] = package
plugin = extension.plugin
try:
package.classes[name] = initialize_plugin(plugin)
except Exception:
LOG.exception(
_LE("Unable to initialize plugin for {name}").format(
name=name))
return
LOG.info(
_LI("Loaded class {class_name} from {dist}").format(
class_name=name, dist=dist_name))
def validate(self, model, package_loader=None):
"""Validate model using Congress rule engine.
@type model: dict
@param model: Dictionary representation of model starting on
environment level (['Objects'])
@type package_loader: murano.dsl.package_loader.MuranoPackageLoader
@param package_loader: Optional. Used for evaluating parent class types
@raises ValidationError in case validation was not successful
"""
if model is None:
return
env_id = model['?']['id']
validation_result = self._execute_simulation(
package_loader, env_id, model,
'predeploy_errors(eid, oid, msg)')
if validation_result["result"]:
messages = self._parse_simulation_result(
'predeploy_errors', env_id,
validation_result["result"])
if messages:
result_str = "\n ".join(map(str, messages))
msg = _("Murano object model validation failed: {0}").format(
"\n " + result_str)
LOG.error(msg)
raise ValidationError(msg)
else:
LOG.info(_LI('Model valid'))
def validate(self, model, package_loader=None):
"""Validate model using Congress rule engine.
@type model: dict
@param model: Dictionary representation of model starting on
environment level (['Objects'])
@type package_loader: murano.dsl.package_loader.MuranoPackageLoader
@param package_loader: Optional. Used for evaluating parent class types
@raises ValidationError in case validation was not successful
"""
if model is None:
return
env_id = model['?']['id']
validation_result = self._execute_simulation(
package_loader, env_id, model,
'predeploy_errors(eid, oid, msg)')
if validation_result["result"]:
messages = self._parse_simulation_result(
'predeploy_errors', env_id,
validation_result["result"])
if messages:
result_str = "\n ".join(map(str, messages))
msg = _("Murano object model validation failed: {0}").format(
"\n " + result_str)
LOG.error(msg)
raise ValidationError(msg)
else:
LOG.info(_LI('Model valid'))
def call_static_action(cls, context, task):
s_task = token_sanitizer.TokenSanitizer().sanitize(task)
LOG.info(
_LI('Starting execution of static action: '
'{task_desc}').format(task_desc=jsonutils.dumps(s_task)))
result = None
reporter = status_reporter.StatusReporter(task['id'])
try:
task_executor = StaticActionExecutor(task, reporter)
result = task_executor.execute()
return result
finally:
LOG.info(
_LI('Finished execution of static action: '
'{task_desc}').format(task_desc=jsonutils.dumps(result)))
def __inner(self, request, env_template_id, *args, **kwargs):
unit = db_session.get_session()
template = unit.query(models.EnvironmentTemplate).get(env_template_id)
if template is None:
LOG.info(
_LI("Environment Template with id '{0}' not found").format(
env_template_id))
raise exc.HTTPNotFound()
if hasattr(request, 'context'):
if template.tenant_id != request.context.tenant:
LOG.info(
_LI('User is not authorized to access '
'this tenant resources'))
raise exc.HTTPUnauthorized()
return func(self, request, env_template_id, *args, **kwargs)
def __init__(self):
LOG.info(_LI('Loading package type plugins'))
extension_manager = dispatch.EnabledExtensionManager(
NAMESPACE,
self._is_plugin_enabled,
on_load_failure_callback=self._on_load_failure)
self.formats = {}
for ext in extension_manager.extensions:
self._load_plugin(ext)
def _do_import_package(_dir, categories, update=False):
LOG.info(_LI("Going to import Murano package from {0}").format(_dir))
pkg = load_utils.load_from_dir(_dir)
LOG.info(_LI("Checking for existing"))
existing = db_catalog_api.package_search(
{'fqn': pkg.full_name},
AdminContext())
if existing:
existing_pkg = existing[0]
if update:
LOG.info(_LI(
"Deleting existing package {0}").format(existing_pkg.id))
db_catalog_api.package_delete(existing_pkg.id, AdminContext())
else:
LOG.error(_LE("Package '{0}' exists ({1}). Use --update.").format(
pkg.full_name,
existing_pkg.id))
return
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
# note: we explicitly mark all the imported packages as public,
# until a parameter added to control visibility scope of a package
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.raw_ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': categories or []
}
# note(ruhe): the second parameter is tenant_id
# it is a required field in the DB, that's why we pass an empty string
result = db_catalog_api.package_upload(package, '')
LOG.info(_LI("Finished import of package {0}").format(result.id))
def _build_index(self):
for folder in self.search_package_folders(self._base_path):
try:
package = load_utils.load_from_dir(folder)
dsl_package = murano_package.MuranoPackage(
self._root_loader, package)
for class_name in package.classes:
dsl_package.register_class(
(lambda pkg, cls: lambda: get_class(pkg, cls))(
package, class_name), class_name)
if dsl_package.name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
self.register_package(dsl_package)
except pkg_exc.PackageLoadError:
LOG.info(
_LI('Unable to load package from path: {0}').format(
folder))
continue
LOG.info(_LI('Loaded package from path {0}').format(folder))
def register_format(self, format_name, package_class):
try:
name, version = self._parse_format_string(format_name)
except ValueError:
return
else:
self._initialize_plugin(package_class)
self.formats.setdefault(name, {})[version] = package_class
LOG.info(
_LI('Plugin for "{0}" package type was loaded').format(
format_name))
def _build_index(self):
for entry in os.listdir(self._base_path):
folder = os.path.join(self._base_path, entry)
if not os.path.isdir(folder) or entry in self._processed_entries:
continue
try:
package = load_utils.load_from_dir(
folder, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.info(_LI('Unable to load package from path: {0}').format(
os.path.join(self._base_path, entry)))
continue
LOG.info(_LI('Loaded package from path {0}').format(
os.path.join(self._base_path, entry)))
for c in package.classes:
self._packages_by_class[c] = package
self._packages_by_name[package.full_name] = package
self._processed_entries.add(entry)
def my_process_result(context, result, environment_id):
if environment_id != scheduler.get_scheduler_id():
return old_process_result(context, result, environment_id)
model = result['model']
action_result = result.get('action', {})
unit = db_session.get_session()
# close deployment
deployment = server.get_last_deployment(unit, environment_id)
deployment.finished = timeutils.utcnow()
deployment.result = action_result
num_errors = unit.query(models.Status).filter_by(level='error', task_id=deployment.id).count()
num_warnings = unit.query(models.Status).filter_by(level='warning', task_id=deployment.id).count()
if num_errors:
final_status_text = "finished with errors"
elif num_warnings:
final_status_text = "finished with warnings"
else:
final_status_text = "finished"
status = models.Status()
status.task_id = deployment.id
status.text = final_status_text
status.level = 'info'
deployment.statuses.append(status)
deployment.save(unit)
# close session
session_id = model['SystemData']['SessionId']
conf_session = unit.query(models.Session).get(session_id)
if num_errors > 0 or result['action'].get('isException'):
conf_session.state = states.EnvironmentStatus.DEPLOY_FAILURE
else:
conf_session.state = states.EnvironmentStatus.READY
conf_session.description = model
if conf_session.description['Objects'] is not None:
conf_session.description['Objects']['services'] = conf_session.description['Objects'].pop('applications', [])
conf_session.version += 1
conf_session.save(unit)
# output application tracking information
services = []
objects = model['Objects']
if objects:
services = objects.get('services')
if num_errors + num_warnings > 0:
LOG.warning(_LW('Schedule Status: Failed Apps: {services}').format(services=services))
else:
LOG.info(_LI('Schedule Status: Successful Apps: {services}').format(services=services))
def test_provision_except_http_not_found(self, mock_get_muranoclient,
mock_db_cf, mock_log):
test_body = {
'space_guid': 'foo_space_guid',
'organization_guid': 'foo_organization_guid',
'plan_id': 'foo_plan_id',
'service_id': 'foo_service_id',
'parameters': {}
}
test_headers = {
'X-Auth-Token': mock.sentinel.token
}
mock_request = mock.Mock(body=json.dumps(test_body),
headers=test_headers)
mock_muranoclient = mock.Mock()
mock_muranoclient.environments.get.side_effect = \
exceptions.HTTPNotFound
mock_muranoclient.environments.create.return_value = \
mock.Mock(id=mock.sentinel.alt_environment_id)
mock_get_muranoclient.return_value = mock_muranoclient
mock_db_cf.get_environment_for_space.return_value = \
mock.sentinel.environment_id
resp = self.controller.provision(mock_request, None, None)
self.assertEqual(202, resp.status_code)
self.assertEqual({}, resp.json_body)
mock_db_cf.get_environment_for_space.assert_called_once_with(
'foo_space_guid')
mock_muranoclient.environments.get.assert_called_once_with(
mock.sentinel.environment_id)
mock_log.info.assert_has_calls([
mock.call(_LI("Can not find environment_id sentinel.environment_id"
", will create a new one.")),
mock.call(_LI("Cloud Foundry foo_space_guid remapped to "
"sentinel.alt_environment_id"))
])
def _build_index(self):
for entry in os.listdir(self._base_path):
folder = os.path.join(self._base_path, entry)
if not os.path.isdir(folder) or entry in self._processed_entries:
continue
try:
package = load_utils.load_from_dir(
folder,
preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.info(
_LI('Unable to load package from path: {0}').format(
os.path.join(self._base_path, entry)))
continue
LOG.info(
_LI('Loaded package from path {0}').format(
os.path.join(self._base_path, entry)))
for c in package.classes:
self._packages_by_class[c] = package
self._packages_by_name[package.full_name] = package
self._processed_entries.add(entry)
def test_process_result(self, mock_db_session, mock_models,
mock_last_deployment, mock_log):
test_result = {
'model': {
'Objects': {
'applications': ['app1', 'app2'],
'services': ['service1', 'service2']
}
},
'action': {
'isException': False
}
}
mock_env = mock.MagicMock(id='test_env_id',
tenant_id='test_tenant_id',
description=None,
version=1)
mock_db_session.get_session().query().get.return_value = mock_env
mock_db_session.get_session().query().filter_by().count.\
return_value = 0
self.result_endpoint.process_result(self.dummy_context, test_result,
'test_env_id')
self.assertEqual(mock_env.description, test_result['model'])
self.assertEqual(2, mock_env.version)
self.assertEqual(test_result['action'], mock_last_deployment().result)
self.assertEqual('Deployment finished', mock_models.Status().text)
self.assertEqual('info', mock_models.Status().level)
mock_last_deployment().statuses.append.assert_called_once_with(
mock_models.Status())
mock_db_session.get_session().query().filter_by.assert_any_call(
**{
'environment_id': mock_env.id,
'state': states.SessionState.DEPLOYING
})
self.assertEqual(
states.SessionState.DEPLOYED,
mock_db_session.get_session().query().filter_by().first().state)
mock_log.info.assert_called_once_with(
_LI('EnvId: {env_id} TenantId: {tenant_id} Status: '
'Successful Apps: {services}').format(
env_id=mock_env.id,
tenant_id=mock_env.tenant_id,
services=test_result['model']['Objects']['services']))
def push(self, _context):
if self._applied or self._template is None:
return
if 'heat_template_version' not in self._template:
self._template['heat_template_version'] = HEAT_TEMPLATE_VERSION
if 'description' not in self._template and self._description:
self._template['description'] = self._description
template = copy.deepcopy(self._template)
LOG.info(_LI('Pushing: {0}').format(template))
current_status = self._get_status(_context)
resources = template.get('Resources') or template.get('resources')
if current_status == 'NOT_FOUND':
if resources is not None:
token_client = self._clients.get_heat_client(_context, False)
token_client.stacks.create(
stack_name=self._name,
parameters=self._parameters,
template=template,
files=self._files,
disable_rollback=True)
self._wait_state(
_context,
lambda status: status == 'CREATE_COMPLETE')
else:
if resources is not None:
trust_client = self._clients.get_heat_client(_context)
trust_client.stacks.update(
stack_id=self._name,
parameters=self._parameters,
files=self._files,
template=template,
disable_rollback=True)
self._wait_state(
_context,
lambda status: status == 'UPDATE_COMPLETE', True)
else:
self.delete(_context)
self._applied = not utils.is_different(self._template, template)
def get_default_router(self):
router_name = self._settings.router_name
routers = self._client.list_routers(
tenant_id=self._project_id, name=router_name).get('routers')
if len(routers) == 0:
LOG.debug('Router {name} not found'.format(name=router_name))
if self._settings.create_router:
LOG.debug('Attempting to create Router {router}'.
format(router=router_name))
external_network = self._settings.external_network
kwargs = {'id': external_network} \
if uuidutils.is_uuid_like(external_network) \
else {'name': external_network}
networks = self._client.list_networks(**kwargs).get('networks')
ext_nets = list(filter(lambda n: n['router:external'],
networks))
if len(ext_nets) == 0:
raise KeyError('Router %s could not be created, '
'no external network found' % router_name)
nid = ext_nets[0]['id']
body_data = {
'router': {
'name': router_name,
'external_gateway_info': {
'network_id': nid
},
'admin_state_up': True,
}
}
router = self._client.create_router(
body=body_data).get('router')
LOG.info(_LI('Created router: {id}').format(id=router['id']))
return router['id']
else:
raise KeyError('Router %s was not found' % router_name)
else:
if routers[0]['external_gateway_info'] is None:
raise exc.RouterInfoException('Please set external gateway for'
' the router %s ' % router_name)
router_id = routers[0]['id']
return router_id
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
forever = mtries == -1
while forever or mtries > 1:
try:
return f(*args, **kwargs)
except ExceptionToCheck as e:
LOG.exception(e)
LOG.info(_LI("Retrying in {0} seconds...").format(mdelay))
eventlet.sleep(mdelay)
if not forever:
mtries -= 1
if mdelay < 60:
mdelay *= backoff
return f(*args, **kwargs)
def handle_task(context, task):
s_task = token_sanitizer.TokenSanitizer().sanitize(task)
LOG.info(_LI('Starting processing task: {task_desc}').format(
task_desc=jsonutils.dumps(s_task)))
result = {'model': task['model']}
try:
task_executor = TaskExecutor(task)
result = task_executor.execute()
except Exception as e:
LOG.exception(_LE('Error during task execution for tenant %s'),
task['tenant_id'])
result['action'] = TaskExecutor.exception_result(e)
msg_env = Environment(task['id'])
reporter = status_reporter.StatusReporter()
reporter.initialize(msg_env)
reporter.report_error(msg_env, str(e))
finally:
rpc.api().process_result(result, task['id'])
def get_default_router(self):
router_name = self._settings.router_name
routers = self._client.list_routers(tenant_id=self._project_id,
name=router_name).get('routers')
if len(routers) == 0:
LOG.debug('Router {name} not found'.format(name=router_name))
if self._settings.create_router:
LOG.debug('Attempting to create Router {router}'.format(
router=router_name))
external_network = self._settings.external_network
kwargs = {'id': external_network} \
if uuidutils.is_uuid_like(external_network) \
else {'name': external_network}
networks = self._client.list_networks(**kwargs).get('networks')
ext_nets = list(
filter(lambda n: n['router:external'], networks))
if len(ext_nets) == 0:
raise KeyError('Router %s could not be created, '
'no external network found' % router_name)
nid = ext_nets[0]['id']
body_data = {
'router': {
'name': router_name,
'external_gateway_info': {
'network_id': nid
},
'admin_state_up': True,
}
}
router = self._client.create_router(
body=body_data).get('router')
LOG.info(_LI('Created router: {id}').format(id=router['id']))
return router['id']
else:
raise KeyError('Router %s was not found' % router_name)
else:
if routers[0]['external_gateway_info'] is None:
raise exc.RouterInfoException('Please set external gateway for'
' the router %s ' % router_name)
router_id = routers[0]['id']
return router_id
def push(self, _context):
if self._applied or self._template is None:
return
if "heat_template_version" not in self._template:
self._template["heat_template_version"] = HEAT_TEMPLATE_VERSION
if "description" not in self._template and self._description:
self._template["description"] = self._description
template = copy.deepcopy(self._template)
LOG.info(_LI("Pushing: {0}").format(template))
current_status = self._get_status(_context)
resources = template.get("Resources") or template.get("resources")
if current_status == "NOT_FOUND":
if resources is not None:
token_client = self._clients.get_heat_client(_context, False)
token_client.stacks.create(
stack_name=self._name,
parameters=self._parameters,
template=template,
files=self._files,
disable_rollback=True,
)
self._wait_state(_context, lambda status: status == "CREATE_COMPLETE")
else:
if resources is not None:
trust_client = self._clients.get_heat_client(_context)
trust_client.stacks.update(
stack_id=self._name,
parameters=self._parameters,
files=self._files,
template=template,
disable_rollback=True,
)
self._wait_state(_context, lambda status: status == "UPDATE_COMPLETE", True)
else:
self.delete(_context)
self._applied = not utils.is_different(self._template, template)
def push(self, _context):
if self._applied or self._template is None:
return
if 'heat_template_version' not in self._template:
self._template['heat_template_version'] = HEAT_TEMPLATE_VERSION
if 'description' not in self._template and self._description:
self._template['description'] = self._description
template = copy.deepcopy(self._template)
LOG.info(_LI('Pushing: {0}').format(template))
current_status = self._get_status(_context)
resources = template.get('Resources') or template.get('resources')
if current_status == 'NOT_FOUND':
if resources is not None:
token_client = self._clients.get_heat_client(_context, False)
token_client.stacks.create(
stack_name=self._name,
parameters=self._parameters,
template=template,
disable_rollback=True)
self._wait_state(
_context,
lambda status: status == 'CREATE_COMPLETE')
else:
if resources is not None:
trust_client = self._clients.get_heat_client(_context)
trust_client.stacks.update(
stack_id=self._name,
parameters=self._parameters,
template=template,
disable_rollback=True)
self._wait_state(
_context,
lambda status: status == 'UPDATE_COMPLETE')
else:
self.delete(_context)
self._applied = not utils.is_different(self._template, template)
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
forever = mtries == -1
while forever or mtries > 1:
try:
return f(*args, **kwargs)
except ExceptionToCheck as e:
LOG.exception(e)
LOG.info(_LI("Retrying in {0} seconds...").format(mdelay))
eventlet.sleep(mdelay)
if not forever:
mtries -= 1
if mdelay < 60:
mdelay *= backoff
return f(*args, **kwargs)
def test_process_result_with_warnings(self, mock_db_session, mock_models,
mock_last_deployment, mock_log):
test_result = {
'model': {
'Objects': None,
'ObjectsCopy': ['object1', 'object2']
},
'action': {
'isException': True
}
}
mock_env = mock.MagicMock(id='test_env_id',
tenant_id='test_tenant_id',
description=None,
version=1)
mock_db_session.get_session().query().get.return_value = mock_env
# num_errors will be initialized to 0, num_warnings to 1
mock_db_session.get_session().query().filter_by().count.\
side_effect = [0, 1]
self.result_endpoint.process_result(self.dummy_context, test_result,
'test_env_id')
self.assertEqual(mock_env.description, test_result['model'])
self.assertEqual(test_result['action'], mock_last_deployment().result)
self.assertEqual('Deletion finished with warnings',
mock_models.Status().text)
mock_last_deployment().statuses.append.assert_called_once_with(
mock_models.Status())
mock_db_session.get_session().query().filter_by.assert_any_call(
**{
'environment_id': mock_env.id,
'state': states.SessionState.DELETING
})
self.assertEqual(
states.SessionState.DELETE_FAILURE,
mock_db_session.get_session().query().filter_by().first().state)
mock_log.warning.assert_called_once_with(
_LI('EnvId: {env_id} TenantId: {tenant_id} Status: '
'Failed Apps: {services}').format(env_id=mock_env.id,
tenant_id=mock_env.tenant_id,
services=[]))
def handle_task(context, task):
s_task = token_sanitizer.TokenSanitizer().sanitize(task)
LOG.info(_LI('Starting processing task: {task_desc}').format(
task_desc=jsonutils.dumps(s_task)))
result = {'model': task['model']}
try:
task_executor = TaskExecutor(task)
result = task_executor.execute()
except Exception as e:
LOG.exception(_LE('Error during task execution for tenant %s'),
task['tenant_id'])
result['action'] = TaskExecutor.exception_result(
e, traceback.format_exc())
msg_env = Environment(task['id'])
reporter = status_reporter.StatusReporter()
reporter.initialize(msg_env)
reporter.report_error(msg_env, str(e))
finally:
rpc.api().process_result(result, task['id'])
def load_extension(self, extension, name_map):
dist_name = str(extension.entry_point.dist)
name = extension.entry_point.name
if not NAME_RE.match(name):
LOG.warning(_LW("Entry-point 'name' %s is invalid") % name)
return
name = "%s.%s" % (self.namespace, name)
name_map.setdefault(name, []).append(dist_name)
if dist_name in self.packages:
package = self.packages[dist_name]
else:
package = PackageDefinition(extension.entry_point.dist)
self.packages[dist_name] = package
plugin = extension.plugin
try:
package.classes[name] = initialize_plugin(plugin)
except Exception:
LOG.exception(_LE("Unable to initialize plugin for %s") % name)
return
LOG.info(_LI("Loaded class '%(class_name)s' from '%(dist)s'")
% dict(class_name=name, dist=dist_name))
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory,
package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'),
package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try and download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(),\
download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e)
)
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError(msg), None, exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info(_LI(
"Successfully downloaded and unpacked package {} {}")
.format(package_def.fully_qualified_name, package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(
os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def provision(self, req, body, instance_id):
"""Here is the example of request body given us from Cloud Foundry:
{
"service_id": "service-guid-here",
"plan_id": "plan-guid-here",
"organization_guid": "org-guid-here",
"space_guid": "space-guid-here",
"parameters": {"param1": "value1",
"param2": "value2"}
}
"""
data = json.loads(req.body)
space_guid = data['space_guid']
org_guid = data['organization_guid']
plan_id = data['plan_id']
service_id = data['service_id']
parameters = data['parameters']
self.current_session = None
# Here we'll take an entry for CF org and space from db. If we
# don't have any entries we will create it from scratch.
try:
tenant = db_cf.get_tenant_for_org(org_guid)
except AttributeError:
# FIXME(Kezar): need to find better way to get tenant
tenant = CONF.cfapi.tenant
db_cf.set_tenant_for_org(org_guid, tenant)
LOG.info(
_LI("Cloud Foundry {org_id} mapped to tenant "
"{tenant_name}").format(org_id=org_guid,
tenant_name=tenant))
# Now as we have all parameters we can try to auth user in actual
# tenant
user, _, keystone = self._check_auth(req, tenant)
# Once we get here we were authorized by keystone
token = keystone.auth_token
m_cli = muranoclient(token)
try:
environment_id = db_cf.get_environment_for_space(space_guid)
except AttributeError:
body = {'name': 'my_{uuid}'.format(uuid=uuid.uuid4().hex)}
env = m_cli.environments.create(body)
environment_id = env.id
db_cf.set_environment_for_space(space_guid, environment_id)
LOG.info(
_LI("Cloud Foundry {space_id} mapped to {environment_id}").
format(space_id=space_guid, environment_id=environment_id))
LOG.debug('Keystone endpoint: {0}'.format(keystone.auth_ref))
tenant_id = keystone.project_id
ctx = context.RequestContext(user=user, tenant=tenant_id)
package = db_api.package_get(service_id, ctx)
LOG.debug('Adding service {name}'.format(name=package.name))
service = self._make_service(space_guid, package, plan_id)
db_cf.set_instance_for_service(instance_id, service['?']['id'],
environment_id, tenant)
# NOTE(Kezar): Here we are going through JSON and add ids where
# it's necessary
params = [parameters]
while params:
a = params.pop()
for k, v in a.iteritems():
if isinstance(v, dict):
params.append(v)
if k == '?':
v['id'] = uuid.uuid4().hex
service.update(parameters)
# Now we need to obtain session to modify the env
session_id = create_session(m_cli, environment_id)
m_cli.services.post(environment_id,
path='/',
data=service,
session_id=session_id)
m_cli.sessions.deploy(environment_id, session_id)
self.current_session = session_id
return {}
def provision(self, req, body, instance_id):
"""Here is the example of request body given us from Cloud Foundry:
{
"service_id": "service-guid-here",
"plan_id": "plan-guid-here",
"organization_guid": "org-guid-here",
"space_guid": "space-guid-here",
"parameters": {"param1": "value1",
"param2": "value2"}
}
"""
data = json.loads(req.body)
space_guid = data['space_guid']
org_guid = data['organization_guid']
plan_id = data['plan_id']
service_id = data['service_id']
parameters = data['parameters']
self.current_session = None
# Here we'll take an entry for CF org and space from db. If we
# don't have any entries we will create it from scratch.
try:
tenant = db_cf.get_tenant_for_org(org_guid)
except AttributeError:
# FIXME(Kezar): need to find better way to get tenant
tenant = CONF.cfapi.tenant
db_cf.set_tenant_for_org(org_guid, tenant)
LOG.info(_LI("Cloud Foundry {org_id} mapped to tenant "
"{tenant_name}").format(org_id=org_guid,
tenant_name=tenant))
# Now as we have all parameters we can try to auth user in actual
# tenant
user, _, keystone = self._check_auth(req, tenant)
# Once we get here we were authorized by keystone
token = keystone.auth_token
m_cli = muranoclient(token)
try:
environment_id = db_cf.get_environment_for_space(space_guid)
except AttributeError:
body = {'name': 'my_{uuid}'.format(uuid=uuid.uuid4().hex)}
env = m_cli.environments.create(body)
environment_id = env.id
db_cf.set_environment_for_space(space_guid, environment_id)
LOG.info(_LI("Cloud Foundry {space_id} mapped to {environment_id}")
.format(space_id=space_guid,
environment_id=environment_id))
LOG.debug('Keystone endpoint: {0}'.format(keystone.auth_ref))
tenant_id = keystone.project_id
ctx = context.RequestContext(user=user, tenant=tenant_id)
package = db_api.package_get(service_id, ctx)
LOG.debug('Adding service {name}'.format(name=package.name))
service = self._make_service(space_guid, package, plan_id)
db_cf.set_instance_for_service(instance_id, service['?']['id'],
environment_id, tenant)
# NOTE(Kezar): Here we are going through JSON and add ids where
# it's necessary
params = [parameters]
while params:
a = params.pop()
for k, v in a.iteritems():
if isinstance(v, dict):
params.append(v)
if k == '?':
v['id'] = uuid.uuid4().hex
service.update(parameters)
# Now we need to obtain session to modify the env
session_id = create_session(m_cli, environment_id)
m_cli.services.post(environment_id,
path='/',
data=service,
session_id=session_id)
m_cli.sessions.deploy(environment_id, session_id)
self.current_session = session_id
return response.Response(status=202, json_body={})
def _execute(self, pkg_loader):
class_loader = package_class_loader.PackageClassLoader(pkg_loader)
system_objects.register(class_loader, pkg_loader)
get_plugin_loader().register_in_loader(class_loader)
exc = executor.MuranoDslExecutor(class_loader, self.environment)
obj = exc.load(self.model)
self._validate_model(obj, self.action, class_loader)
action_result = None
exception = None
exception_traceback = None
try:
LOG.info(_LI('Invoking pre-cleanup hooks'))
self.environment.start()
exc.cleanup(self._model)
except Exception as e:
exception = e
exception_traceback = TaskExecutor._log_exception(e, obj, '<GC>')
finally:
LOG.info(_LI('Invoking post-cleanup hooks'))
self.environment.finish()
if exception is None and self.action:
try:
LOG.info(_LI('Invoking pre-execution hooks'))
self.environment.start()
action_result = self._invoke(exc)
except Exception as e:
exception = e
exception_traceback = TaskExecutor._log_exception(
e, obj, self.action['method'])
finally:
LOG.info(_LI('Invoking post-execution hooks'))
self.environment.finish()
model = serializer.serialize_model(obj, exc)
model['SystemData'] = self._environment.system_attributes
result = {
'model': model,
'action': {
'result': None,
'isException': False
}
}
if exception is not None:
result['action'] = TaskExecutor.exception_result(
exception, exception_traceback)
# NOTE(kzaitsev): Exception here means that it happened during
# cleanup. ObjectsCopy and Attributes would be empty if obj
# is empty. This would cause failed env to be deleted.
# Therefore restore these attrs from self._model
for attr in ['ObjectsCopy', 'Attributes']:
if not model.get(attr):
model[attr] = self._model[attr]
else:
result['action']['result'] = serializer.serialize_object(
action_result)
return result
def provision(self, req, body, instance_id):
"""Here is the example of request body given us from Cloud Foundry:
{
"service_id": "service-guid-here",
"plan_id": "plan-guid-here",
"organization_guid": "org-guid-here",
"space_guid": "space-guid-here",
"parameters": {"param1": "value1",
"param2": "value2"}
}
"""
data = json.loads(req.body)
space_guid = data['space_guid']
org_guid = data['organization_guid']
plan_id = data['plan_id']
service_id = data['service_id']
parameters = data['parameters']
self.current_session = None
# Here we'll take an entry for CF org and space from db. If we
# don't have any entries we will create it from scratch.
try:
tenant = db_cf.get_tenant_for_org(org_guid)
except AttributeError:
tenant = req.headers['X-Project-Id']
db_cf.set_tenant_for_org(org_guid, tenant)
LOG.info(_LI("Cloud Foundry {org_id} mapped to tenant "
"{tenant_name}").format(org_id=org_guid,
tenant_name=tenant))
token = req.headers['X-Auth-Token']
m_cli = _get_muranoclient(token, req)
try:
environment_id = db_cf.get_environment_for_space(space_guid)
except AttributeError:
body = {'name': 'my_{uuid}'.format(uuid=uuid.uuid4().hex)}
env = m_cli.environments.create(body)
environment_id = env.id
db_cf.set_environment_for_space(space_guid, environment_id)
LOG.info(_LI("Cloud Foundry {space_id} mapped to {environment_id}")
.format(space_id=space_guid,
environment_id=environment_id))
package = m_cli.packages.get(service_id)
LOG.debug('Adding service {name}'.format(name=package.name))
service = self._make_service(space_guid, package, plan_id)
db_cf.set_instance_for_service(instance_id, service['?']['id'],
environment_id, tenant)
# NOTE(Kezar): Here we are going through JSON and add ids where
# it's necessary. Before that we need to drop '?' key from parameters
# dictionary as far it contains murano package related info which is
# necessary in our scenario
if '?' in parameters.keys():
parameters.pop('?', None)
LOG.warning(_LW("Incorrect input parameters. Package related "
"parameters shouldn't be passed through Cloud "
"Foundry"))
params = [parameters]
while params:
a = params.pop()
for k, v in six.iteritems(a):
if isinstance(v, dict):
params.append(v)
if k == '?':
v['id'] = uuid.uuid4().hex
service.update(parameters)
# Now we need to obtain session to modify the env
session_id = create_session(m_cli, environment_id)
m_cli.services.post(environment_id,
path='/',
data=service,
session_id=session_id)
m_cli.sessions.deploy(environment_id, session_id)
self.current_session = session_id
return response.Response(status=202, json_body={})
def provision(self, req, body, instance_id):
"""Here is the example of request body given us from Cloud Foundry:
{
"service_id": "service-guid-here",
"plan_id": "plan-guid-here",
"organization_guid": "org-guid-here",
"space_guid": "space-guid-here",
"parameters": {"param1": "value1",
"param2": "value2"}
}
"""
data = json.loads(req.body)
space_guid = data['space_guid']
org_guid = data['organization_guid']
plan_id = data['plan_id']
service_id = data['service_id']
parameters = data['parameters']
self.current_session = None
# Here we'll take an entry for CF org and space from db. If we
# don't have any entries we will create it from scratch.
try:
tenant = db_cf.get_tenant_for_org(org_guid)
except AttributeError:
tenant = req.headers['X-Project-Id']
db_cf.set_tenant_for_org(org_guid, tenant)
LOG.info(
_LI("Cloud Foundry {org_id} mapped to tenant "
"{tenant_name}").format(org_id=org_guid,
tenant_name=tenant))
token = req.headers['X-Auth-Token']
m_cli = _get_muranoclient(token, req)
def _set_new_environment_for_space(space_guid, log_msg):
body = {'name': 'my_{uuid}'.format(uuid=uuid.uuid4().hex)}
env = m_cli.environments.create(body)
db_cf.set_environment_for_space(space_guid, env.id)
LOG.info(log_msg.format(space_id=space_guid,
environment_id=env.id))
return env.id
try:
environment_id = db_cf.get_environment_for_space(space_guid)
# NOTE: Check that environment which was previously linked with
# CF space still exist, reset a new environment for space.
try:
env = m_cli.environments.get(environment_id)
except exceptions.HTTPNotFound:
msg = (_LI("Can not find environment_id {environment_id}, "
"will create a new one.").format(
environment_id=environment_id))
LOG.info(msg)
env = {}
if not env:
log_msg = (_LI("Cloud Foundry {space_id} remapped to "
"{environment_id}"))
environment_id = _set_new_environment_for_space(
space_guid, log_msg)
except AttributeError:
log_msg = (_LI("Cloud Foundry {space_id} mapped to "
"{environment_id}"))
environment_id = _set_new_environment_for_space(
space_guid, log_msg)
package = m_cli.packages.get(service_id)
LOG.debug('Adding service {name}'.format(name=package.name))
service = self._make_service(space_guid, package, plan_id)
db_cf.set_instance_for_service(instance_id, service['?']['id'],
environment_id, tenant)
# NOTE(Kezar): Here we are going through JSON and add ids where
# it's necessary. Before that we need to drop '?' key from parameters
# dictionary as far it contains murano package related info which is
# necessary in our scenario
if '?' in parameters.keys():
parameters.pop('?', None)
LOG.warning(
_LW("Incorrect input parameters. Package related "
"parameters shouldn't be passed through Cloud "
"Foundry"))
params = [parameters]
while params:
a = params.pop()
for k, v in six.iteritems(a):
if isinstance(v, dict):
params.append(v)
if k == '?':
v['id'] = uuid.uuid4().hex
service.update(parameters)
# Now we need to obtain session to modify the env
session_id = create_session(m_cli, environment_id)
m_cli.services.post(environment_id,
path='/',
data=service,
session_id=session_id)
m_cli.sessions.deploy(environment_id, session_id)
self.current_session = session_id
return response.Response(status=202, json_body={})
def process_result(context, result, environment_id):
secure_result = token_sanitizer.TokenSanitizer().sanitize(result)
LOG.debug('Got result from orchestration '
'engine:\n{0}'.format(secure_result))
model = result['model']
action_result = result.get('action', {})
unit = session.get_session()
environment = unit.query(models.Environment).get(environment_id)
if not environment:
LOG.warning(_LW('Environment result could not be handled, '
'specified environment not found in database'))
return
if model['Objects'] is None and model.get('ObjectsCopy', {}) is None:
environments.EnvironmentServices.remove(environment_id)
return
environment.description = model
if environment.description['Objects'] is not None:
environment.description['Objects']['services'] = \
environment.description['Objects'].pop('applications', [])
# environment.networking = result.get('networking', {})
action_name = 'Deployment'
deleted = False
else:
action_name = 'Deletion'
deleted = True
environment.version += 1
environment.save(unit)
# close deployment
deployment = get_last_deployment(unit, environment.id)
deployment.finished = timeutils.utcnow()
deployment.result = action_result
num_errors = unit.query(models.Status)\
.filter_by(level='error', task_id=deployment.id).count()
num_warnings = unit.query(models.Status)\
.filter_by(level='warning', task_id=deployment.id).count()
final_status_text = action_name + ' finished'
if num_errors:
final_status_text += " with errors"
elif num_warnings:
final_status_text += " with warnings"
status = models.Status()
status.task_id = deployment.id
status.text = final_status_text
status.level = 'info'
deployment.statuses.append(status)
deployment.save(unit)
# close session
conf_session = unit.query(models.Session).filter_by(
**{'environment_id': environment.id,
'state': states.SessionState.DEPLOYING if not deleted
else states.SessionState.DELETING}).first()
if num_errors > 0:
conf_session.state = \
states.SessionState.DELETE_FAILURE if deleted else \
states.SessionState.DEPLOY_FAILURE
else:
conf_session.state = states.SessionState.DEPLOYED
conf_session.save(unit)
# output application tracking information
message = _LI('EnvId: {0} TenantId: {1} Status: {2} Apps: {3}').format(
environment.id,
environment.tenant_id,
_('Failed') if num_errors + num_warnings > 0 else _('Successful'),
', '.join(map(
lambda a: a['?']['type'],
model['Objects']['services']
))
)
LOG.info(message)
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory, package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'), package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try and download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(), download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self.client.packages.download(package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info(
_LI("Successfully downloaded and unpacked package {} {}"
).format(package_def.fully_qualified_name,
package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
