def main():
test_runner = MuranoTestRunner()
try:
if test_runner.args.config_file:
default_config_files = [test_runner.args.config_file]
else:
default_config_files = cfg.find_config_files('murano')
if not default_config_files:
murano_conf = os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir,
'etc', 'murano',
'murano.conf')
if os.path.exists(murano_conf):
default_config_files = [murano_conf]
sys.argv = [sys.argv[0]]
config.parse_args(default_config_files=default_config_files)
logging.setup(CONF, 'murano')
except RuntimeError as e:
LOG.exception(_LE("Failed to initialize murano-test-runner: %s") % e)
sys.exit("ERROR: %s" % e)
try:
exit_code = test_runner.run_tests()
sys.exit(exit_code)
except Exception as e:
tb = traceback.format_exc()
err_msg = _LE("Command failed: {0}\n{1}").format(e, tb)
LOG.error(err_msg)
sys.exit(err_msg)
def main():
test_runner = MuranoTestRunner()
try:
if test_runner.args.config_file:
default_config_files = [test_runner.args.config_file]
else:
default_config_files = cfg.find_config_files('murano')
if not default_config_files:
murano_conf = os.path.join(os.path.dirname(__file__),
os.pardir, os.pardir, 'etc',
'murano', 'murano.conf')
if os.path.exists(murano_conf):
default_config_files = [murano_conf]
sys.argv = [sys.argv[0]]
config.parse_args(default_config_files=default_config_files)
logging.setup(CONF, 'murano')
except RuntimeError as e:
LOG.exception(_LE("Failed to initialize murano-test-runner: %s") % e)
sys.exit("ERROR: %s" % e)
try:
exit_code = test_runner.run_tests()
sys.exit(exit_code)
except Exception as e:
tb = traceback.format_exc()
err_msg = _LE("Command failed: {0}\n{1}").format(e, tb)
LOG.error(err_msg)
sys.exit(err_msg)
def test_get_muranoclient_without_urls(self, mock_glare_client,
mock_muranoclient, mock_log):
self._override_conf(without_urls=True)
m_artifacts_client = mock.Mock()
m_muranoclient = mock.Mock()
mock_glare_client.Client.return_value = m_artifacts_client
mock_muranoclient.Client.return_value = m_muranoclient
mock_request = mock.Mock(endpoints={'murano': None})
client = api._get_muranoclient(mock.sentinel.token_id, mock_request)
self.assertEqual(m_muranoclient, client)
mock_glare_client.Client.assert_called_once_with(
endpoint=None, token=mock.sentinel.token_id,
insecure=True, key_file='foo_key_file', ca_file='foo_ca_file',
cert_file='foo_cert_file', type_name='murano', type_version=1)
mock_muranoclient.Client.assert_called_once_with(
1, None, token=mock.sentinel.token_id,
artifacts_client=m_artifacts_client)
mock_log.error.assert_has_calls([
mock.call(_LE('No glare url is specified and no "artifact" '
'service is registered in keystone.')),
mock.call(_LE('No murano url is specified and no '
'"application-catalog" service is registered in '
'keystone.'))
])
def verify_and_get_deployment(db_session, environment_id, deployment_id):
deployment = db_session.query(models.Task).get(deployment_id)
if not deployment:
LOG.error(_LE('Deployment with id {id} not found')
.format(id=deployment_id))
raise exc.HTTPNotFound
if deployment.environment_id != environment_id:
LOG.error(_LE('Deployment with id {d_id} not found in environment '
'{env_id}').format(d_id=deployment_id,
env_id=environment_id))
raise exc.HTTPBadRequest
deployment.description = _patch_description(deployment.description)
return deployment
def verify_and_get_deployment(db_session, environment_id, deployment_id):
deployment = db_session.query(models.Task).get(deployment_id)
if not deployment:
LOG.error(_LE('Deployment with id {id} not found')
.format(id=deployment_id))
raise exc.HTTPNotFound
if deployment.environment_id != environment_id:
LOG.error(_LE('Deployment with id {d_id} not found in environment '
'{env_id}').format(d_id=deployment_id,
env_id=environment_id))
raise exc.HTTPBadRequest
deployment.description = _patch_description(deployment.description)
return deployment
def __inner(self, request, env_template_id, *args, **kwargs):
unit = db_session.get_session()
template = unit.query(models.EnvironmentTemplate).get(env_template_id)
if template is None:
LOG.error(_LE("Environment Template with id '{id}' not found").
format(id=env_template_id))
raise exc.HTTPNotFound()
if hasattr(request, 'context'):
if template.tenant_id != request.context.tenant:
LOG.error(_LE('User is not authorized to access '
'this tenant resources'))
raise exc.HTTPUnauthorized()
return func(self, request, env_template_id, *args, **kwargs)
def _migrate_up(self, engine, version, with_data=False):
"""Migrate up to a new version of the db.
We allow for data insertion and post checks at every
migration version with special _pre_upgrade_### and
_check_### functions in the main test.
"""
# NOTE(sdague): try block is here because it's impossible to debug
# where a failed data migration happens otherwise
check_version = version
try:
if with_data:
data = None
pre_upgrade = getattr(
self, "_pre_upgrade_%s" % check_version, None)
if pre_upgrade:
data = pre_upgrade(engine)
self._migrate(engine, version, 'upgrade')
self.assertEqual(version, self._get_version_from_db(engine))
if with_data:
check = getattr(self, "_check_%s" % check_version, None)
if check:
check(engine, data)
except Exception:
LOG.error(_LE("Failed to migrate to version {ver} on engine {eng}")
.format(ver=version, eng=engine))
raise
def clone(self, request, env_template_id, body):
"""Clones env template from another tenant
It clones the env template from another env template
from other tenant.
:param request: the operation request.
:param env_template_id: the env template ID.
:param body: the request body.
:return: the description of the created template.
"""
LOG.debug('EnvTemplates:Clone <Env Template {0} for body {1}>'.format(
body, env_template_id))
policy.check('clone_env_template', request.context)
old_env_template = self._validate_exists(env_template_id)
if not old_env_template.get('is_public'):
msg = _LE('User has no access to these resources.')
LOG.error(msg)
raise exc.HTTPForbidden(explanation=msg)
self._validate_body_name(body)
LOG.debug('ENV TEMP NAME: {0}'.format(body['name']))
try:
is_public = body.get('is_public', False)
template = env_temps.EnvTemplateServices.clone(
env_template_id, request.context.tenant, body['name'],
is_public)
except db_exc.DBDuplicateEntry:
msg = _('Environment with specified name already exists')
LOG.error(msg)
raise exc.HTTPConflict(explanation=msg)
return template.to_dict()
def _parse_format_string(format_string):
parts = format_string.rsplit('/', 1)
if len(parts) != 2:
LOG.error(
_LE("Incorrect format name {name}").format(name=format_string))
raise ValueError(format_string)
return (parts[0].strip(), semantic_version.Version.coerce(parts[1]))
def load_extension(self, extension, name_map):
dist_name = str(extension.entry_point.dist)
name = extension.entry_point.name
if not NAME_RE.match(name):
LOG.warning(
_LW("Entry-point 'name' {name} is invalid").format(name=name))
return
name_map.setdefault(name, []).append(dist_name)
if dist_name in self.packages:
package = self.packages[dist_name]
else:
package = PackageDefinition(extension.entry_point.dist)
self.packages[dist_name] = package
plugin = extension.plugin
try:
package.classes[name] = initialize_plugin(plugin)
except Exception:
LOG.exception(
_LE("Unable to initialize plugin for {name}").format(
name=name))
return
LOG.info(
_LI("Loaded class {class_name} from {dist}").format(
class_name=name, dist=dist_name))
def _validate_keystone_opts(self, args):
ks_opts_to_config = {
'auth_url': 'auth_uri',
'username': '******',
'password': '******',
'project_name': 'admin_tenant_name'
}
ks_opts = {
'auth_url': getattr(args, 'os_auth_url', None),
'username': getattr(args, 'os_username', None),
'password': getattr(args, 'os_password', None),
'project_name': getattr(args, 'os_project_name', None)
}
if None in ks_opts.values() and not CONF.default_config_files:
msg = _LE('Please provide murano config file or credentials for '
'authorization: {0}').format(', '.join([
'--os-auth-url', '--os-username', '--os-password',
'--os-project-name', '--os-tenant-id'
]))
LOG.error(msg)
self.error(msg)
# Load keystone configuration parameters from config
importutils.import_module('keystonemiddleware.auth_token')
for param, value in six.iteritems(ks_opts):
if not value:
ks_opts[param] = getattr(CONF.keystone_authtoken,
ks_opts_to_config[param])
if param == 'auth_url':
ks_opts[param] = ks_opts[param].replace('v2.0', 'v3')
return ks_opts
def _validate_change(self, change):
change_path = change['path'][0]
change_op = change['op']
allowed_methods = self.allowed_operations.get(change_path)
if not allowed_methods:
msg = _("Attribute '{0}' is invalid").format(change_path)
raise webob.exc.HTTPForbidden(explanation=six.text_type(msg))
if change_op not in allowed_methods:
msg = _("Method '{method}' is not allowed for a path with name "
"'{name}'. Allowed operations are: "
"'{ops}'").format(method=change_op,
name=change_path,
ops=', '.join(allowed_methods))
raise webob.exc.HTTPForbidden(explanation=six.text_type(msg))
property_to_update = {change_path: change['value']}
try:
jsonschema.validate(property_to_update, schemas.PKG_UPDATE_SCHEMA)
except jsonschema.ValidationError as e:
LOG.error(
_LE("Schema validation error occured: {error}").format(
error=e))
raise webob.exc.HTTPBadRequest(explanation=e.message)
def _validate_keystone_opts(self, args):
ks_opts_to_config = {
'auth_url': 'auth_uri',
'username': '******',
'password': '******',
'project_name': 'admin_tenant_name'}
ks_opts = {'auth_url': getattr(args, 'os_auth_url', None),
'username': getattr(args, 'os_username', None),
'password': getattr(args, 'os_password', None),
'project_name': getattr(args, 'os_project_name', None)}
if None in ks_opts.values() and not CONF.default_config_files:
msg = _LE('Please provide murano config file or credentials for '
'authorization: {0}').format(
', '.join(['--os-auth-url', '--os-username', '--os-password',
'--os-project-name', '--os-tenant-id']))
LOG.error(msg)
self.error(msg)
# Load keystone configuration parameters from config
importutils.import_module('keystonemiddleware.auth_token')
for param, value in six.iteritems(ks_opts):
if not value:
ks_opts[param] = getattr(CONF.keystone_authtoken,
ks_opts_to_config[param])
if param == 'auth_url':
ks_opts[param] = ks_opts[param].replace('v2.0', 'v3')
return ks_opts
def test_main_except_general_exception(self, mock_conf):
mock_conf.command.func.side_effect = Exception
expected_err_msg = _LE("murano-manage command failed:")
with self.assertRaisesRegexp(SystemExit, expected_err_msg):
manage.main()
def _validate_change(self, change):
change_path = change['path'][0]
change_op = change['op']
allowed_methods = self.allowed_operations.get(change_path)
if not allowed_methods:
msg = _("Attribute '{0}' is invalid").format(change_path)
raise webob.exc.HTTPForbidden(explanation=six.text_type(msg))
if change_op not in allowed_methods:
msg = _("Method '{method}' is not allowed for a path with name "
"'{name}'. Allowed operations are: "
"'{ops}'").format(method=change_op,
name=change_path,
ops=', '.join(allowed_methods))
raise webob.exc.HTTPForbidden(explanation=six.text_type(msg))
property_to_update = {change_path: change['value']}
try:
jsonschema.validate(property_to_update, schemas.PKG_UPDATE_SCHEMA)
except jsonschema.ValidationError as e:
LOG.error(_LE("Schema validation error occured: {error}")
.format(error=e))
raise webob.exc.HTTPBadRequest(explanation=e.message)
def execute(self, request, environment_id, action_id, body):
policy.check("execute_action", request.context, {})
LOG.debug('Action:Execute <ActionId: {0}>'.format(action_id))
unit = db_session.get_session()
# no new session can be opened if environment has deploying status
env_status = envs.EnvironmentServices.get_status(environment_id)
if env_status in (states.EnvironmentStatus.DEPLOYING,
states.EnvironmentStatus.DELETING):
LOG.info(_LI('Could not open session for environment <EnvId: {0}>,'
'environment has deploying '
'status.').format(environment_id))
raise exc.HTTPForbidden()
user_id = request.context.user
session = sessions.SessionServices.create(environment_id, user_id)
if not sessions.SessionServices.validate(session):
LOG.error(_LE('Session <SessionId {0}> '
'is invalid').format(session.id))
raise exc.HTTPForbidden()
task_id = actions.ActionServices.execute(
action_id, session, unit, request.context.auth_token, body or {})
return {'task_id': task_id}
def clone(self, request, env_template_id, body):
"""Clones env template from another tenant
It clones the env template from another env template
from other tenant.
:param request: the operation request.
:param env_template_id: the env template ID.
:param body: the request body.
:return: the description of the created template.
"""
LOG.debug('EnvTemplates:Clone <Env Template {0} for body {1}>'.
format(body, env_template_id))
policy.check('clone_env_template', request.context)
old_env_template = self._validate_exists(env_template_id)
if not old_env_template.get('is_public'):
msg = _LE('User has no access to these resources.')
LOG.error(msg)
raise exc.HTTPForbidden(explanation=msg)
self._validate_body_name(body)
LOG.debug('ENV TEMP NAME: {0}'.format(body['name']))
try:
is_public = body.get('is_public', False)
template = env_temps.EnvTemplateServices.clone(
env_template_id, request.context.tenant, body['name'],
is_public)
except db_exc.DBDuplicateEntry:
msg = _('Environment with specified name already exists')
LOG.error(msg)
raise exc.HTTPConflict(explanation=msg)
return template.to_dict()
def execute(self, request, environment_id, action_id, body):
policy.check("execute_action", request.context, {})
LOG.debug("Action:Execute <ActionId: {0}>".format(action_id))
unit = db_session.get_session()
# no new session can be opened if environment has deploying status
env_status = envs.EnvironmentServices.get_status(environment_id)
if env_status in (states.EnvironmentStatus.DEPLOYING, states.EnvironmentStatus.DELETING):
LOG.info(
_LI(
"Could not open session for environment <EnvId: {0}>," "environment has deploying " "status."
).format(environment_id)
)
raise exc.HTTPForbidden()
user_id = request.context.user
session = sessions.SessionServices.create(environment_id, user_id)
if not sessions.SessionServices.validate(session):
LOG.error(_LE("Session <SessionId {0}> " "is invalid").format(session.id))
raise exc.HTTPForbidden()
task_id = actions.ActionServices.execute(action_id, session, unit, request.context.auth_token, body or {})
return {"task_id": task_id}
def _migrate_up(self, engine, version, with_data=False):
"""migrate up to a new version of the db.
We allow for data insertion and post checks at every
migration version with special _pre_upgrade_### and
_check_### functions in the main test.
"""
# NOTE(sdague): try block is here because it's impossible to debug
# where a failed data migration happens otherwise
check_version = version
try:
if with_data:
data = None
pre_upgrade = getattr(self, "_pre_upgrade_%s" % check_version,
None)
if pre_upgrade:
data = pre_upgrade(engine)
self._migrate(engine, version, 'upgrade')
self.assertEqual(version, self._get_version_from_db(engine))
if with_data:
check = getattr(self, "_check_%s" % check_version, None)
if check:
check(engine, data)
except Exception:
LOG.error(
_LE("Failed to migrate to version {ver} on engine {eng}").
format(ver=version, eng=engine))
raise
def finish(self):
for delegate in self._tear_down_list:
try:
delegate()
except Exception:
LOG.exception(_LE("Unhandled exception on invocation of " "post-execution hook"))
self._tear_down_list = []
def _check_enabled(self):
if CONF.engine.disable_murano_agent:
LOG.error(_LE("Use of murano-agent is disallowed " "by the server configuration"))
raise exceptions.PolicyViolationException(
"Use of murano-agent is disallowed " "by the server configuration"
)
def start(self):
for delegate in self._set_up_list:
try:
delegate()
except Exception:
LOG.exception(_LE("Unhandled exception on invocation of " "pre-execution hook"))
self._set_up_list = []
def __inner(self, request, env_template_id, *args, **kwargs):
unit = db_session.get_session()
template = unit.query(models.EnvironmentTemplate).get(env_template_id)
if template is None:
LOG.error(
_LE("Environment Template with id '{id}' not found").format(
id=env_template_id))
raise exc.HTTPNotFound()
if hasattr(request, 'context'):
if template.tenant_id != request.context.tenant:
LOG.error(
_LE('User is not authorized to access '
'this tenant resources'))
raise exc.HTTPUnauthorized()
return func(self, request, env_template_id, *args, **kwargs)
def _check_enabled(self):
if CONF.engine.disable_murano_agent:
LOG.error(_LE('Use of murano-agent is disallowed '
'by the server configuration'))
raise exceptions.PolicyViolationException(
'Use of murano-agent is disallowed '
'by the server configuration')
def start(self):
for delegate in self._set_up_list:
try:
delegate()
except Exception:
LOG.exception(_LE('Unhandled exception on invocation of '
'pre-execution hook'))
self._set_up_list = []
def finish(self):
for delegate in self._tear_down_list:
try:
delegate()
except Exception:
LOG.exception(_LE('Unhandled exception on invocation of '
'post-execution hook'))
self._tear_down_list = []
def test_update_stats_handle_exception(self, mock_log, _):
self.service._stats_db.update.side_effect =\
Exception('test_error_code')
self.service.update_stats()
mock_log.exception.assert_called_once_with(
_LE("Failed to get statistics object from a "
"database. {error_code}").format(error_code='test_error_code'))
def execute(self, request, body):
policy.check("execute_action", request.context, {})
class_name = body.get('className')
method_name = body.get('methodName')
if not class_name or not method_name:
msg = _('Class name and method name must be specified for '
'static action')
LOG.error(msg)
raise exc.HTTPBadRequest(msg)
args = body.get('parameters')
pkg_name = body.get('packageName')
class_version = body.get('classVersion', '=0')
LOG.debug('StaticAction:Execute <MethodName: {0}, '
'ClassName: {1}>'.format(method_name, class_name))
credentials = {
'token': request.context.auth_token,
'project_id': request.context.tenant,
'user_id': request.context.user
}
try:
return static_actions.StaticActionServices.execute(
method_name, class_name, pkg_name, class_version, args,
credentials)
except client.RemoteError as e:
LOG.error(
_LE('Exception during call of the method {method_name}: '
'{exc}').format(method_name=method_name, exc=str(e)))
if e.exc_type in ('NoClassFound', 'NoMethodFound',
'NoPackageFound', 'NoPackageForClassFound',
'MethodNotExposed', 'NoMatchingMethodException'):
raise exc.HTTPNotFound(e.value)
elif e.exc_type == 'ContractViolationException':
raise exc.HTTPBadRequest(e.value)
raise exc.HTTPServiceUnavailable(e.value)
except ValueError as e:
LOG.error(
_LE('Exception during call of the method {method_name}: '
'{exc}').format(method_name=method_name, exc=str(e)))
raise exc.HTTPBadRequest(e.message)
def main():
CONF.register_cli_opt(command_opt)
try:
default_config_files = cfg.find_config_files('murano', 'murano')
CONF(sys.argv[1:], project='murano', prog='murano-manage',
version=version.version_string,
default_config_files=default_config_files)
except RuntimeError as e:
LOG.error(_LE("failed to initialize murano-manage: %s") % e)
sys.exit("ERROR: %s" % e)
try:
CONF.command.func()
except Exception as e:
tb = traceback.format_exc()
err_msg = _LE("murano-manage command failed: {0}\n{1}").format(e, tb)
LOG.error(err_msg)
sys.exit(err_msg)
def _validate_request(self, request, env_template_id):
self._validate_exists(env_template_id)
get_env_template = env_temps.EnvTemplateServices.get_env_template
env_template = get_env_template(env_template_id)
if env_template.is_public or request.context.is_admin:
return
if env_template.tenant_id != request.context.tenant:
msg = _LE('User has no access to these resources.')
LOG.error(msg)
raise exc.HTTPForbidden(explanation=msg)
def _validate_request(self, request, env_template_id):
self._validate_exists(env_template_id)
get_env_template = env_temps.EnvTemplateServices.get_env_template
env_template = get_env_template(env_template_id)
if env_template.is_public or request.context.is_admin:
return
if env_template.tenant_id != request.context.tenant:
msg = _LE('User has no access to these resources.')
LOG.error(msg)
raise exc.HTTPForbidden(explanation=msg)
def _parse_format_string(format_string):
parts = format_string.rsplit('/', 1)
if len(parts) != 2:
LOG.error(_LE("Incorrect format name {name}").format(
name=format_string))
raise ValueError(format_string)
return (
parts[0].strip(),
semantic_version.Version.coerce(parts[1])
)
def execute(self, request, body):
policy.check("execute_action", request.context, {})
class_name = body.get('className')
method_name = body.get('methodName')
if not class_name or not method_name:
msg = _('Class name and method name must be specified for '
'static action')
LOG.error(msg)
raise exc.HTTPBadRequest(msg)
args = body.get('parameters')
pkg_name = body.get('packageName')
class_version = body.get('classVersion', '=0')
LOG.debug('StaticAction:Execute <MethodName: {0}, '
'ClassName: {1}>'.format(method_name, class_name))
credentials = {
'token': request.context.auth_token,
'tenant_id': request.context.tenant
}
try:
return static_actions.StaticActionServices.execute(
method_name, class_name, pkg_name, class_version, args,
credentials)
except client.RemoteError as e:
LOG.error(_LE('Exception during call of the method {method_name}: '
'{exc}').format(method_name=method_name, exc=str(e)))
if e.exc_type in (
'NoClassFound', 'NoMethodFound', 'NoPackageFound',
'NoPackageForClassFound', 'MethodNotExposed',
'NoMatchingMethodException'):
raise exc.HTTPNotFound(e.value)
elif e.exc_type == 'ContractViolationException':
raise exc.HTTPBadRequest(e.value)
raise exc.HTTPServiceUnavailable(e.value)
except ValueError as e:
LOG.error(_LE('Exception during call of the method {method_name}: '
'{exc}').format(method_name=method_name, exc=str(e)))
raise exc.HTTPBadRequest(e.message)
def _execute(self, pkg_loader):
class_loader = package_class_loader.PackageClassLoader(pkg_loader)
system_objects.register(class_loader, pkg_loader)
get_plugin_loader().register_in_loader(class_loader)
exc = executor.MuranoDslExecutor(class_loader, self.environment)
obj = exc.load(self.model)
self._validate_model(obj, self.action, class_loader)
action_result = None
exception = None
exception_traceback = None
try:
LOG.info(_LI('Invoking pre-execution hooks'))
self.environment.start()
# Skip execution of action in case no action is provided.
# Model will be just loaded, cleaned-up and unloaded.
# Most of the time this is used for deletion of environments.
if self.action:
action_result = self._invoke(exc)
except Exception as e:
exception = e
if isinstance(e, dsl_exception.MuranoPlException):
LOG.error('\n' + e.format(prefix=' '))
else:
exception_traceback = traceback.format_exc()
LOG.exception(
_LE("Exception %(exc)s occured"
" during invocation of %(method)s"),
{'exc': e, 'method': self.action['method']})
reporter = status_reporter.StatusReporter()
reporter.initialize(obj)
reporter.report_error(obj, str(e))
finally:
LOG.info(_LI('Invoking post-execution hooks'))
self.environment.finish()
model = serializer.serialize_model(obj, exc)
model['SystemData'] = self._environment.system_attributes
result = {
'model': model,
'action': {
'result': None,
'isException': False
}
}
if exception is not None:
result['action'] = TaskExecutor.exception_result(
exception, exception_traceback)
else:
result['action']['result'] = serializer.serialize_object(
action_result)
return result
def _validate_request(self, request, env_template_id):
env_template_exists = env_temps.EnvTemplateServices.env_template_exist
if not env_template_exists(env_template_id):
msg = _('EnvTemplate <TempId {temp_id}> is not found').format(
temp_id=env_template_id)
LOG.exception(msg)
raise exc.HTTPNotFound(explanation=msg)
get_env_template = env_temps.EnvTemplateServices.get_env_template
env_template = get_env_template(env_template_id)
if env_template.tenant_id != request.context.tenant:
LOG.exception(_LE('User is not authorized to access this tenant '
'resources.'))
raise exc.HTTPUnauthorized
def _validate_request(self, request, env_template_id):
env_template_exists = env_temps.EnvTemplateServices.env_template_exist
if not env_template_exists(env_template_id):
msg = _('EnvTemplate <TempId {temp_id}> is not found').format(
temp_id=env_template_id)
LOG.exception(msg)
raise exc.HTTPNotFound(explanation=msg)
get_env_template = env_temps.EnvTemplateServices.get_env_template
env_template = get_env_template(env_template_id)
if env_template.tenant_id != request.context.tenant:
LOG.exception(
_LE('User is not authorized to access this tenant '
'resources.'))
raise exc.HTTPUnauthorized
def _get_muranoclient(token_id, req):
artifacts_client = None
if CONF.engine.packages_service in ['glance', 'glare']:
artifacts_client = _get_glareclient(token_id, req)
murano_url = CONF.murano.url or req.endpoints.get('murano')
if not murano_url:
LOG.error(_LE('No murano url is specified and no '
'"application-catalog" '
'service is registered in keystone.'))
return muranoclient.Client(1, murano_url, token=token_id,
artifacts_client=artifacts_client)
def test_get_package_by_definition_except_package_load_error(
self, mock_load_utils, mock_os, mock_log):
# Test that the first instance of the exception is caught.
temp_directory = tempfile.mkdtemp(prefix='test-package-loader-',
dir=tempfile.tempdir)
mock_os.path.isdir.return_value = True
mock_os.path.join.return_value = temp_directory
mock_load_utils.load_from_dir.side_effect = pkg_exc.PackageLoadError
fqn = 'io.murano.apps.test'
path, _ = utils.compose_package(
'test', self.location, archive_dir=self.location)
with open(path, 'rb') as f:
package_data = f.read()
package = mock.MagicMock()
package.fully_qualified_name = fqn
package.id = '123'
package.version = '0.0.1'
self.murano_client.packages.download = mock.MagicMock(
return_value=package_data)
self.loader._get_package_by_definition(package)
mock_log.exception.assert_called_once_with(
_LE('Unable to load package from cache. Clean-up.'))
mock_log.exception.reset_mock()
# Test that the second instance of the exception is caught.
mock_os.path.isdir.return_value = [False, True]
self.loader._get_package_by_definition(package)
mock_log.exception.assert_called_once_with(
_LE('Unable to load package from cache. Clean-up.'))
os.remove(temp_directory)
def __exit__(self, ex_type, ex_value, ex_traceback):
if not ex_value:
return True
# TODO(lin.a.yang): current only handle TypeError here, we should
# process other kind of internal exceptions generated by API and
# convert to webob exceptions.
if isinstance(ex_value, TypeError):
exc_info = (ex_type, ex_value, ex_traceback)
LOG.error(_LE("Exception handling resource: {0}").format(ex_value),
exc_info=exc_info)
raise webob.exc.HTTPBadRequest()
# We didn't handle this kind of exception
return False
def _log_exception(e, root, method_name):
if isinstance(e, dsl_exception.MuranoPlException):
LOG.error('\n' + e.format(prefix=' '))
exception_traceback = e.format()
else:
exception_traceback = traceback.format_exc()
LOG.exception(
_LE("Exception %(exc)s occurred"
" during invocation of %(method)s"),
{'exc': e, 'method': method_name})
if root is not None:
reporter = status_reporter.StatusReporter()
reporter.initialize(root)
reporter.report_error(root, str(e))
return exception_traceback
def wrap(*args, **kwargs):
try:
ts = time.time()
result = func(*args, **kwargs)
te = time.time()
tenant = args[1].context.tenant
update_count(api, method, te - ts,
tenant)
return result
except Exception:
te = time.time()
tenant = args[1].context.tenant
LOG.exception(_LE('API {api} method {method} raised an '
'exception').format(api=api, method=method))
update_error_count(api, method, te - te, tenant)
raise
def wrap(*args, **kwargs):
try:
ts = time.time()
result = func(*args, **kwargs)
te = time.time()
tenant = args[1].context.tenant
update_count(api, method, te - ts, tenant)
return result
except Exception:
te = time.time()
tenant = args[1].context.tenant
LOG.exception(
_LE('API {0} method {1} raised an exception').format(
api, method))
update_error_count(api, method, te - te, tenant)
raise
def test_do_import_package_without_update(self, mock_db_catalog_api,
mock_load_utils, mock_log):
mock_db_catalog_api.package_search.return_value =\
[self.test_package]
mock_load_utils.load_from_dir.return_value =\
mock.MagicMock(full_name='test_full_name')
manage.CONF = mock.MagicMock()
manage.CONF.command = mock.MagicMock(directory='test_dir',
categories=[],
update=False)
manage.do_import_package()
mock_log.error.assert_called_once_with(
_LE("Package '{name}' exists ({pkg_id}). Use --update.").format(
name='test_full_name', pkg_id=self.test_package.id))
def _get_glareclient(token_id, req):
glare_settings = CONF.glare
url = glare_settings.url or req.endpoints.get('glare')
if not url:
LOG.error(_LE('No glare url is specified and no "artifact" '
'service is registered in keystone.'))
return glare_client.Client(
endpoint=url, token=token_id,
insecure=glare_settings.insecure,
key_file=glare_settings.key_file or None,
ca_file=glare_settings.ca_file or None,
cert_file=glare_settings.cert_file or None,
type_name='murano',
type_version=1)
def _do_import_package(_dir, categories, update=False):
LOG.debug("Going to import Murano package from {source}".format(
source=_dir))
pkg = load_utils.load_from_dir(_dir)
LOG.debug("Checking for existing packages")
existing = db_catalog_api.package_search(
{'fqn': pkg.full_name},
AdminContext())
if existing:
existing_pkg = existing[0]
if update:
LOG.debug('Deleting existing package {exst_pkg_id}').format(
exst_pkg_id=existing_pkg.id)
db_catalog_api.package_delete(existing_pkg.id, AdminContext())
else:
LOG.error(_LE("Package '{name}' exists ({pkg_id}). Use --update.")
.format(name=pkg.full_name, pkg_id=existing_pkg.id))
return
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
# note: we explicitly mark all the imported packages as public,
# until a parameter added to control visibility scope of a package
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': categories or []
}
# note(ruhe): the second parameter is tenant_id
# it is a required field in the DB, that's why we pass an empty string
result = db_catalog_api.package_upload(package, '')
LOG.info(_LI("Finished import of package {res_id}").format(
res_id=result.id))
def _do_import_package(_dir, categories, update=False):
LOG.debug(
"Going to import Murano package from {source}".format(source=_dir))
pkg = load_utils.load_from_dir(_dir)
LOG.debug("Checking for existing packages")
existing = db_catalog_api.package_search({'fqn': pkg.full_name},
AdminContext())
if existing:
existing_pkg = existing[0]
if update:
LOG.debug('Deleting existing package {exst_pkg_id}').format(
exst_pkg_id=existing_pkg.id)
db_catalog_api.package_delete(existing_pkg.id, AdminContext())
else:
LOG.error(
_LE("Package '{name}' exists ({pkg_id}). Use --update.").
format(name=pkg.full_name, pkg_id=existing_pkg.id))
return
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
# note: we explicitly mark all the imported packages as public,
# until a parameter added to control visibility scope of a package
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': categories or []
}
# note(ruhe): the second parameter is tenant_id
# it is a required field in the DB, that's why we pass an empty string
result = db_catalog_api.package_upload(package, '')
LOG.info(
_LI("Finished import of package {res_id}").format(res_id=result.id))
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
forever = mtries == -1
while forever or mtries > 1:
try:
return f(*args, **kwargs)
except ExceptionToCheck as e:
LOG.exception(_LE("An exception occurred {exc}. Retrying "
"in {time} seconds").format(exc=e,
time=mdelay))
eventlet.sleep(mdelay)
if not forever:
mtries -= 1
if mdelay < 60:
mdelay *= backoff
return f(*args, **kwargs)
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(_LE(
'Unable to load package from cache. Clean-up...'))
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e)
)
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader
)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory,
preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up...'))
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def handle_task(context, task):
s_task = token_sanitizer.TokenSanitizer().sanitize(task)
LOG.info(_LI('Starting processing task: {task_desc}').format(
task_desc=jsonutils.dumps(s_task)))
result = {'model': task['model']}
try:
task_executor = TaskExecutor(task)
result = task_executor.execute()
except Exception as e:
LOG.exception(_LE('Error during task execution for tenant %s'),
task['tenant_id'])
result['action'] = TaskExecutor.exception_result(e)
msg_env = Environment(task['id'])
reporter = status_reporter.StatusReporter()
reporter.initialize(msg_env)
reporter.report_error(msg_env, str(e))
finally:
rpc.api().process_result(result, task['id'])
def wrap(*args, **kwargs):
try:
ts = time.time()
result = None
if (api == 'Environments' and method == 'Show'):
environment_id = kwargs.get('environment_id')
if environment_id in specs:
request = args[1]
unit = db_session.get_session()
environment = unit.query(models.Environment).get(environment_id)
result = environment.to_dict()
result['status'] = states.EnvironmentStatus.READY
result['acquired_by'] = None
schedules = unit.query(models.Session).filter_by(environment_id=environment_id, user_id=request.context.user)
result['services'] = []
for session in schedules:
if session is not None and session.description['Objects'].has_key('services'):
services = session.description['Objects']['services']
for service in services:
service['?']['status'] = session.state
service['?']['id'] = session.id
result['services'].extend(services)
elif (api == 'Sessions' and method == 'Create'):
environment_id = kwargs.get('environment_id')
if environment_id in specs:
result = my_session_create(environment_id, args[1]).to_dict()
elif (api == 'Services' and method == 'Create'):
environment_id = kwargs.get('environment_id')
if environment_id in specs:
request = args[1]
result = my_post_data(environment_id, request.context.session, kwargs.get('body'), kwargs.get('path'), request)
if result is None:
result = func(*args, **kwargs)
te = time.time()
tenant = args[1].context.tenant
request_statistics.update_count(api, method, te - ts, tenant)
return result
except Exception:
te = time.time()
tenant = args[1].context.tenant
LOG.exception(_LE('API {api} method {method} raised an exception').format(api=api, method=method))
request_statistics.update_error_count(api, method, te - te, tenant)
raise
def _build_index(self):
for entry in os.listdir(self._base_path):
folder = os.path.join(self._base_path, entry)
if not os.path.isdir(folder) or entry in self._processed_entries:
continue
try:
package = load_utils.load_from_dir(
folder, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(_LE('Unable to load package from path: '
'{0}').format(entry))
continue
for c in package.classes:
self._packages_by_class[c] = package
self._packages_by_name[package.full_name] = package
self._processed_entries.add(entry)
def exception_result(self, exception, root, method_name):
if isinstance(exception, dsl_exception.MuranoPlException):
LOG.error('\n' + exception.format(prefix=' '))
exception_traceback = exception.format()
else:
exception_traceback = traceback.format_exc()
LOG.exception(
_LE("Exception %(exc)s occurred"
" during invocation of %(method)s"),
{'exc': exception, 'method': method_name})
self._reporter.report_error(root, str(exception))
return {
'action': {
'isException': True,
'result': {
'message': str(exception),
'details': exception_traceback
}
}
}
def update_stats(self):
LOG.debug("Updating statistic information.")
LOG.debug("Stats object: {stats}".format(stats=v1.stats))
LOG.debug("Stats: Requests:{amount} Errors: {error}"
"Ave.Res.Time {time:2.4f}\n Per tenant: {req_count}".format(
amount=v1.stats.request_count,
error=v1.stats.error_count,
time=v1.stats.average_time,
req_count=v1.stats.requests_per_tenant))
try:
stats = self._stats_db.get_stats_by_host(self._hostname)
if stats is None:
self._stats_db.create(self._hostname,
v1.stats.request_count,
v1.stats.error_count,
v1.stats.average_time,
v1.stats.requests_per_tenant,
psutil.NUM_CPUS,
psutil.cpu_percent())
return
now = time.time()
t_delta = now - self._prev_time
requests_per_second = (v1.stats.request_count -
stats.request_count) / t_delta
errors_per_second = (v1.stats.error_count -
stats.error_count) / t_delta
self._prev_time = now
stats.request_count = v1.stats.request_count
stats.error_count = v1.stats.error_count
stats.average_response_time = v1.stats.average_time
stats.requests_per_tenant = json.dumps(v1.stats.
requests_per_tenant)
stats.requests_per_second = requests_per_second
stats.errors_per_second = errors_per_second
stats.cpu_percent = psutil.cpu_percent()
self._stats_db.update(self._hostname, stats)
except Exception as e:
LOG.exception(_LE("Failed to get statistics object from a "
"database. {error_code}").format(error_code=e))
def update_stats(self):
LOG.debug("Updating statistic information.")
LOG.debug("Stats object: %s" % v1.stats)
LOG.debug("Stats: Requests:%s Errors: %s Ave.Res.Time %2.4f\n"
"Per tenant: %s" %
(v1.stats.request_count,
v1.stats.error_count,
v1.stats.average_time,
v1.stats.requests_per_tenant))
try:
stats = self._stats_db.get_stats_by_host(self._hostname)
if stats is None:
self._stats_db.create(self._hostname,
v1.stats.request_count,
v1.stats.error_count,
v1.stats.average_time,
v1.stats.requests_per_tenant,
psutil.NUM_CPUS,
psutil.cpu_percent())
return
now = time.time()
t_delta = now - self._prev_time
requests_per_second = (v1.stats.request_count -
stats.request_count) / t_delta
errors_per_second = (v1.stats.error_count -
stats.error_count) / t_delta
self._prev_time = now
stats.request_count = v1.stats.request_count
stats.error_count = v1.stats.error_count
stats.average_response_time = v1.stats.average_time
stats.requests_per_tenant = json.dumps(v1.stats.
requests_per_tenant)
stats.requests_per_second = requests_per_second
stats.errors_per_second = errors_per_second
stats.cpu_percent = psutil.cpu_percent()
self._stats_db.update(self._hostname, stats)
except Exception as e:
LOG.exception(_LE("Failed to get statistics object "
"form a database. %s"), e)
def load_extension(self, extension, name_map):
dist_name = str(extension.entry_point.dist)
name = extension.entry_point.name
if not NAME_RE.match(name):
LOG.warning(_LW("Entry-point 'name' %s is invalid") % name)
return
name = "%s.%s" % (self.namespace, name)
name_map.setdefault(name, []).append(dist_name)
if dist_name in self.packages:
package = self.packages[dist_name]
else:
package = PackageDefinition(extension.entry_point.dist)
self.packages[dist_name] = package
plugin = extension.plugin
try:
package.classes[name] = initialize_plugin(plugin)
except Exception:
LOG.exception(_LE("Unable to initialize plugin for %s") % name)
return
LOG.info(_LI("Loaded class '%(class_name)s' from '%(dist)s'")
% dict(class_name=name, dist=dist_name))
def deserialize_body(self, request, action):
if not len(request.body) > 0:
LOG.debug("Empty body provided in request")
return {}
try:
content_type = request.get_content_type()
except exceptions.UnsupportedContentType as e:
LOG.error(_LE("Unrecognized Content-Type provided in request: " "{error}").format(error=str(e)))
raise
if content_type is None:
LOG.debug("No Content-Type provided in request")
return {}
try:
deserializer = self.get_body_deserializer(content_type)
except exceptions.UnsupportedContentType:
LOG.debug("Unable to deserialize body as provided Content-Type")
raise
return deserializer.deserialize(request, action)
