def main():
"""Parse options and call the appropriate class/method."""
script_name = sys.argv[0]
if len(sys.argv) < 2:
print(_("\nOpenStack masakari version: %(version)s\n") %
{'version': version.version_string()})
print(script_name + " category action [<args>]")
print(_("Available categories:"))
for category in CATEGORIES:
print(_("\t%s") % category)
sys.exit(2)
try:
CONF(sys.argv[1:], project='masakari',
version=version.version_string())
logging.setup(CONF, "masakari")
python_logging.captureWarnings(True)
except cfg.ConfigDirNotFoundError as details:
print(_("Invalid directory: %s") % details)
sys.exit(2)
except cfg.ConfigFilesNotFoundError as e:
cfg_files = ', '.join(e.config_files)
print(_("Failed to read configuration file(s): %s") % cfg_files)
sys.exit(2)
fn = CONF.category.action_fn
fn_args = fetch_func_args(fn)
fn(*fn_args)
def index(self, req, segment_id):
"""Returns a list a hosts."""
context = req.environ['masakari.context']
authorize(context)
try:
filters = {}
limit, marker = common.get_limit_and_marker(req)
sort_keys, sort_dirs = common.get_sort_params(req.params)
segment = objects.FailoverSegment.get_by_uuid(context,
segment_id)
filters['failover_segment_id'] = segment.uuid
if 'name' in req.params:
filters['name'] = req.params['name']
if 'type' in req.params:
filters['type'] = req.params['type']
if 'control_attributes' in req.params:
filters['control_attributes'] = req.params[
'control_attributes']
if 'on_maintenance' in req.params:
try:
filters['on_maintenance'] = strutils.bool_from_string(
req.params['on_maintenance'], strict=True)
except ValueError as ex:
msg = _("Invalid value for on_maintenance: "
"%s") % encodeutils.exception_to_unicode(ex)
raise exc.HTTPBadRequest(explanation=msg)
if 'reserved' in req.params:
try:
filters['reserved'] = strutils.bool_from_string(
req.params['reserved'], strict=True)
except ValueError as ex:
msg = _("Invalid value for reserved: "
"%s") % encodeutils.exception_to_unicode(ex)
raise exc.HTTPBadRequest(explanation=msg)
hosts = self.api.get_all(context, filters=filters,
sort_keys=sort_keys, sort_dirs=sort_dirs,
limit=limit, marker=marker)
except exception.MarkerNotFound as ex:
raise exc.HTTPBadRequest(explanation=ex.format_message())
except exception.Invalid as e:
raise exc.HTTPBadRequest(explanation=e.format_message())
except exception.FailoverSegmentNotFound as ex:
raise exc.HTTPNotFound(explanation=ex.format_message())
return {'hosts': hosts}
def execute(self, context, instance_uuid):
"""Stop the instance for recovery."""
instance = self.novaclient.get_server(context, instance_uuid)
# If an instance is not HA_Enabled and "process_all_instances" config
# option is also disabled, then there is no need to take any recovery
# action.
if not CONF.instance_failure.process_all_instances and not (
strutils.bool_from_string(
instance.metadata.get('HA_Enabled', False))):
LOG.info("Skipping recovery for instance: %s as it is "
"not Ha_Enabled.", instance_uuid)
raise exception.SkipInstanceRecoveryException()
vm_state = getattr(instance, 'OS-EXT-STS:vm_state')
if vm_state in ['paused', 'rescued']:
msg = _("Recovery of instance '%(instance_uuid)s' is ignored as"
" it is in '%(vm_state)s' state.") % {
'instance_uuid': instance_uuid, 'vm_state': vm_state}
LOG.warning(msg)
raise exception.IgnoreInstanceRecoveryException(msg)
if vm_state != 'stopped':
if vm_state == 'resized':
self.novaclient.reset_instance_state(
context, instance.id, 'active')
self.novaclient.stop_server(context, instance.id)
def _wait_for_power_off():
new_instance = self.novaclient.get_server(context, instance_uuid)
vm_state = getattr(new_instance, 'OS-EXT-STS:vm_state')
if vm_state == 'stopped':
raise loopingcall.LoopingCallDone()
periodic_call = loopingcall.FixedIntervalLoopingCall(
_wait_for_power_off)
try:
# add a timeout to the periodic call.
periodic_call.start(interval=CONF.verify_interval)
etimeout.with_timeout(CONF.wait_period_after_power_off,
periodic_call.wait)
except etimeout.Timeout:
msg = _("Failed to stop instance %(instance)s") % {
'instance': instance.id
}
raise exception.InstanceRecoveryFailureException(message=msg)
finally:
# stop the periodic call, in case of exceptions or Timeout.
periodic_call.stop()
def model_query(context, model, args=None, read_deleted=None):
"""Query helper that accounts for context's `read_deleted` field.
:param context: MasakariContext of the query.
:param model: Model to query. Must be a subclass of ModelBase.
:param args: Arguments to query. If None - model is used.
:param read_deleted: If not None, overrides context's read_deleted field.
Permitted values are 'no', which does not return
deleted values; 'only', which only returns deleted
values; and 'yes', which does not filter deleted
values.
"""
if read_deleted is None:
read_deleted = context.read_deleted
query_kwargs = {}
if 'no' == read_deleted:
query_kwargs['deleted'] = False
elif 'only' == read_deleted:
query_kwargs['deleted'] = True
elif 'yes' == read_deleted:
pass
else:
raise ValueError(_("Unrecognized read_deleted value '%s'")
% read_deleted)
query = sqlalchemyutils.model_query(
model, context.session, args, **query_kwargs)
return query
def __call__(self, request):
"""WSGI method that controls (de)serialization and method dispatch."""
if self.support_api_request_version:
# Set the version of the API requested based on the header
try:
request.set_api_version_request()
except exception.InvalidAPIVersionString as e:
return Fault(webob.exc.HTTPBadRequest(
explanation=e.format_message()))
except exception.InvalidGlobalAPIVersion as e:
return Fault(webob.exc.HTTPNotAcceptable(
explanation=e.format_message()))
# Identify the action, its arguments, and the requested
# content type
action_args = self.get_action_args(request.environ)
action = action_args.pop('action', None)
# NOTE: we filter out InvalidContentTypes early so we
# know everything is good from here on out.
try:
content_type, body = self.get_body(request)
accept = request.best_match_content_type()
except exception.InvalidContentType:
msg = _("Unsupported Content-Type")
return Fault(webob.exc.HTTPUnsupportedMediaType(explanation=msg))
# NOTE: Splitting the function up this way allows for
# auditing by external tools that wrap the existing
# function. If we try to audit __call__(), we can
# run into troubles due to the @webob.dec.wsgify()
# decorator.
return self._process_stack(request, action, action_args,
content_type, body, accept)
def update_host(self, context, segment_uuid, id, host_data):
"""Update the host"""
segment = objects.FailoverSegment.get_by_uuid(context, segment_uuid)
host = objects.Host.get_by_uuid(context, id)
if is_failover_segment_under_recovery(segment):
msg = _("Host %s can't be updated as "
"it is in-use to process notifications.") % host.uuid
LOG.error(msg)
raise exception.HostInUse(msg)
if 'name' in host_data:
self._is_valid_host_name(context, host_data.get('name'))
if 'on_maintenance' in host_data:
host_data['on_maintenance'] = strutils.bool_from_string(
host_data['on_maintenance'], strict=True)
if 'reserved' in host_data:
host_data['reserved'] = strutils.bool_from_string(
host_data['reserved'], strict=True)
try:
host.update(host_data)
host.save()
except Exception as e:
with excutils.save_and_reraise_exception():
tb = traceback.format_exc()
api_utils.notify_about_host_api(context, host,
action=fields.EventNotificationAction.HOST_UPDATE,
phase=fields.EventNotificationPhase.ERROR, exception=e,
tb=tb)
return host
def execute_host_failure(self, context, host_name, recovery_method,
notification_uuid, reserved_host_list=None):
novaclient = nova.API()
# get flow for host failure
process_what = {
'context': context,
'host_name': host_name
}
try:
if recovery_method == fields.FailoverSegmentRecoveryMethod.AUTO:
self._execute_auto_workflow(novaclient, process_what)
elif recovery_method == (
fields.FailoverSegmentRecoveryMethod.RESERVED_HOST):
self._execute_rh_workflow(novaclient, process_what,
reserved_host_list)
elif recovery_method == (
fields.FailoverSegmentRecoveryMethod.AUTO_PRIORITY):
self._execute_auto_priority_workflow(novaclient, process_what,
reserved_host_list)
else:
self._execute_rh_priority_workflow(novaclient, process_what,
reserved_host_list)
except Exception as exc:
with excutils.save_and_reraise_exception(reraise=False) as ctxt:
if isinstance(exc, (exception.SkipHostRecoveryException,
exception.HostRecoveryFailureException,
exception.ReservedHostsUnavailable)):
ctxt.reraise = True
return
msg = _("Failed to execute host failure flow for "
"notification '%s'.") % notification_uuid
raise exception.MasakariException(msg)
def execute_process_failure(self, context, process_name, host_name,
notification_uuid):
novaclient = nova.API()
# get flow for process failure
process_what = {
'context': context,
'process_name': process_name,
'host_name': host_name
}
# TODO(abhishekk) We need to create a map for process_name and
# respective python-client so that we can pass appropriate client
# as a input to the process.
if process_name == "nova-compute":
recovery_flow = process_failure.get_compute_process_recovery_flow
else:
LOG.warning("Skipping recovery for process: %s.",
process_name)
raise exception.SkipProcessRecoveryException()
try:
flow_engine = recovery_flow(novaclient, process_what)
except Exception:
msg = (_('Failed to create process failure flow.'),
notification_uuid)
LOG.exception(msg)
raise exception.MasakariException(msg)
# Attaching this listener will capture all of the notifications that
# taskflow sends out and redirect them to a more useful log for
# masakari's debugging (or error reporting) usage.
with base.DynamicLogListener(flow_engine, logger=LOG):
flow_engine.run()
def index(self, req):
"""Returns a summary list of notifications."""
context = req.environ['masakari.context']
authorize(context)
try:
limit, marker = common.get_limit_and_marker(req)
sort_keys, sort_dirs = common.get_sort_params(req.params)
filters = {}
if 'status' in req.params:
filters['status'] = req.params['status']
if 'source_host_uuid' in req.params:
filters['source_host_uuid'] = req.params['source_host_uuid']
if 'type' in req.params:
filters['type'] = req.params['type']
if 'generated-since' in req.params:
try:
parsed = timeutils.parse_isotime(
req.params['generated-since'])
except ValueError:
msg = _('Invalid generated-since value')
raise exc.HTTPBadRequest(explanation=msg)
filters['generated-since'] = parsed
notifications = self.api.get_all(context, filters, sort_keys,
sort_dirs, limit, marker)
except exception.MarkerNotFound as err:
raise exc.HTTPBadRequest(explanation=err.format_message())
except exception.Invalid as err:
raise exc.HTTPBadRequest(explanation=err.format_message())
return {'notifications': notifications}
def serve(server, workers=None):
global _launcher
if _launcher:
raise RuntimeError(_('serve() can only be called once'))
_launcher = service.launch(CONF, server, workers=workers,
restart_method='mutate')
def __init__(self, name, loader=None, use_ssl=False, max_url_len=None):
"""Initialize, but do not start the WSGI server.
:param name: The name of the WSGI server given to the loader.
:param loader: Loads the WSGI application using the given name.
:returns: None
"""
self.name = name
self.binary = 'masakari-%s' % name
self.topic = None
self.loader = loader or wsgi.Loader()
self.app = self.loader.load_app(name)
self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
self.port = getattr(CONF, '%s_listen_port' % name, 0)
self.workers = (getattr(CONF, '%s_workers' % name, None) or
processutils.get_worker_count())
if self.workers and self.workers < 1:
worker_name = '%s_workers' % name
msg = (_("%(worker_name)s value of %(workers)s is invalid, "
"must be greater than 0") %
{'worker_name': worker_name,
'workers': str(self.workers)})
raise exception.InvalidInput(msg)
self.use_ssl = use_ssl
self.server = wsgi.Server(name,
self.app,
host=self.host,
port=self.port,
use_ssl=self.use_ssl,
max_url_len=max_url_len)
def __call__(self, environ, start_response):
r"""Subclasses will probably want to implement __call__ like this:
@webob.dec.wsgify(RequestClass=Request)
def __call__(self, req):
# Any of the following objects work as responses:
# Option 1: simple string
res = 'message\n'
# Option 2: a nicely formatted HTTP exception page
res = exc.HTTPForbidden(explanation='Nice try')
# Option 3: a webob Response object (in case you need to play with
# headers, or you want to be treated like an iterable, or ...)
res = Response()
res.app_iter = open('somefile')
# Option 4: any wsgi app to be run next
res = self.application
# Option 5: you can get a Response object for a wsgi app, too, to
# play with headers etc
res = req.get_response(self.application)
# You can then just return your response...
return res
# ... or set req.response and return None.
req.response = res
See the end of http://pythonpaste.org/webob/modules/dec.html
for more info.
"""
raise NotImplementedError(_('You must implement __call__'))
def create_notification(self, context, notification_data):
"""Create notification"""
# Check whether host from which the notification came is already
# present in failover segment or not
host_name = notification_data.get('hostname')
host_object = objects.Host.get_by_name(context, host_name)
host_on_maintenance = host_object.on_maintenance
if host_on_maintenance:
message = (_("Notification received from host %(host)s of type "
"'%(type)s' is ignored as the host is already under "
"maintenance.") % {
'host': host_name,
'type': notification_data.get('type')
})
raise exception.HostOnMaintenanceError(message=message)
notification = objects.Notification(context=context)
# Populate notification object for create
notification.type = notification_data.get('type')
notification.generated_time = notification_data.get('generated_time')
notification.source_host_uuid = host_object.uuid
notification.payload = notification_data.get('payload')
notification.status = fields.NotificationStatus.NEW
if self._is_duplicate_notification(context, notification):
message = (_("Notification received from host %(host)s of "
" type '%(type)s' is duplicate.") %
{'host': host_name, 'type': notification.type})
raise exception.DuplicateNotification(message=message)
try:
notification.create()
self.engine_rpcapi.process_notification(context, notification)
except Exception as e:
with excutils.save_and_reraise_exception():
tb = traceback.format_exc()
api_utils.notify_about_notification_api(context, notification,
action=fields.EventNotificationAction.NOTIFICATION_CREATE,
phase=fields.EventNotificationPhase.ERROR, exception=e,
tb=tb)
return notification
def __call__(self, req):
user_id = req.headers.get('X_USER')
user_id = req.headers.get('X_USER_ID', user_id)
if user_id is None:
LOG.debug("Neither X_USER_ID nor X_USER found in request")
return webob.exc.HTTPUnauthorized()
roles = self._get_roles(req)
if 'X_TENANT_ID' in req.headers:
# This is the new header since Keystone went to ID/Name
project_id = req.headers['X_TENANT_ID']
else:
# This is for legacy compatibility
project_id = req.headers['X_TENANT']
project_name = req.headers.get('X_TENANT_NAME')
user_name = req.headers.get('X_USER_NAME')
req_id = req.environ.get(request_id.ENV_REQUEST_ID)
# Get the auth token
auth_token = req.headers.get('X_AUTH_TOKEN',
req.headers.get('X_STORAGE_TOKEN'))
# Build a context, including the auth_token...
remote_address = req.remote_addr
if CONF.use_forwarded_for:
remote_address = req.headers.get('X-Forwarded-For', remote_address)
service_catalog = None
if req.headers.get('X_SERVICE_CATALOG') is not None:
try:
catalog_header = req.headers.get('X_SERVICE_CATALOG')
service_catalog = jsonutils.loads(catalog_header)
except ValueError:
raise webob.exc.HTTPInternalServerError(
_('Invalid service catalog json.'))
# NOTE: This is a full auth plugin set by auth_token
# middleware in newer versions.
user_auth_plugin = req.environ.get('keystone.token_auth')
ctx = context.RequestContext(user_id,
project_id,
user_name=user_name,
project_name=project_name,
roles=roles,
auth_token=auth_token,
remote_address=remote_address,
service_catalog=service_catalog,
request_id=req_id,
user_auth_plugin=user_auth_plugin)
req.environ['masakari.context'] = ctx
return self.application
def action_peek(body):
"""Determine action to invoke.
This looks inside the json body and fetches out the action method
name.
"""
try:
decoded = jsonutils.loads(body)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
# Make sure there's exactly one key...
if len(decoded) != 1:
msg = _("too many body keys")
raise exception.MalformedRequestBody(reason=msg)
# Return the action name
return list(decoded.keys())[0]
def __init__(self, name, app, host='0.0.0.0', port=0, pool_size=None,
protocol=eventlet.wsgi.HttpProtocol, backlog=128,
use_ssl=False, max_url_len=None):
"""Initialize, but do not start, a WSGI server.
:param name: Pretty name for logging.
:param app: The WSGI application to serve.
:param host: IP address to serve the application.
:param port: Port number to server the application.
:param pool_size: Maximum number of eventlets to spawn concurrently.
:param backlog: Maximum number of queued connections.
:param max_url_len: Maximum length of permitted URLs.
:returns: None
:raises: masakari.exception.InvalidInput
"""
# Allow operators to customize http requests max header line size.
eventlet.wsgi.MAX_HEADER_LINE = CONF.wsgi.max_header_line
self.name = name
self.app = app
self._server = None
self._protocol = protocol
self.pool_size = pool_size or self.default_pool_size
self._pool = eventlet.GreenPool(self.pool_size)
self._logger = logging.getLogger("masakari.%s.wsgi.server" % self.name)
self._use_ssl = use_ssl
self._max_url_len = max_url_len
self.client_socket_timeout = CONF.wsgi.client_socket_timeout or None
if backlog < 1:
raise exception.InvalidInput(
reason=_('The backlog must be more than 0'))
bind_addr = (host, port)
try:
info = socket.getaddrinfo(bind_addr[0],
bind_addr[1],
socket.AF_UNSPEC,
socket.SOCK_STREAM)[0]
family = info[0]
bind_addr = info[-1]
except Exception:
family = socket.AF_INET
try:
self._socket = eventlet.listen(bind_addr, family, backlog=backlog)
except EnvironmentError:
LOG.error("Could not bind to %(host)s:%(port)d",
{'host': host, 'port': port})
raise
(self.host, self.port) = self._socket.getsockname()[0:2]
LOG.info("%(name)s listening on %(host)s:%(port)d",
{'name': self.name, 'host': self.host, 'port': self.port})
def execute(self, context, instance_uuid):
"""Start the instance."""
instance = self.novaclient.get_server(context, instance_uuid)
vm_state = getattr(instance, 'OS-EXT-STS:vm_state')
if vm_state == 'stopped':
self.novaclient.start_server(context, instance.id)
else:
msg = _("Invalid state for Instance %(instance)s. Expected state: "
"'STOPPED', Actual state: '%(actual_state)s'") % {
'instance': instance_uuid,
'actual_state': vm_state
}
raise exception.InstanceRecoveryFailureException(message=msg)
def db_sync(version=None, init_version=INIT_VERSION, engine=None):
if engine is None:
engine = db_api.get_engine()
current_db_version = get_backend().db_version(engine,
MIGRATE_REPO_PATH,
init_version)
if version and int(version) < current_db_version:
msg = _('Database schema downgrade is not allowed.')
raise exception.InvalidInput(reason=msg)
if version and int(version) > db.MAX_INT:
message = _('Version should be less than or equal to %(max_version)d.'
) % {'max_version': db.MAX_INT}
raise exception.InvalidInput(reason=message)
try:
return get_backend().db_sync(engine=engine,
abs_path=MIGRATE_REPO_PATH,
version=version,
init_version=init_version)
except oslo_exception.DbMigrationError as exc:
raise exception.InvalidInput(reason=exc)
def _execute_rh_workflow(self, novaclient, process_what,
reserved_host_list):
if not reserved_host_list:
msg = _('No reserved_hosts available for evacuation.')
LOG.info(msg)
raise exception.ReservedHostsUnavailable(message=msg)
process_what['reserved_host_list'] = reserved_host_list
flow_engine = host_failure.get_rh_flow(novaclient, process_what)
with base.DynamicLogListener(flow_engine, logger=LOG):
try:
flow_engine.run()
except exception.LockAlreadyAcquired as ex:
raise exception.HostRecoveryFailureException(ex.message)
def validate_integer(value, name, min_value=None, max_value=None):
"""Make sure that value is a valid integer, potentially within range."""
try:
value = int(str(value))
except (ValueError, UnicodeEncodeError):
msg = _('%(value_name)s must be an integer')
raise exception.InvalidInput(reason=(
msg % {'value_name': name}))
if min_value is not None:
if value < min_value:
msg = _('%(value_name)s must be >= %(min_value)d')
raise exception.InvalidInput(
reason=(msg % {'value_name': name,
'min_value': min_value}))
if max_value is not None:
if value > max_value:
msg = _('%(value_name)s must be <= %(max_value)d')
raise exception.InvalidInput(
reason=(
msg % {'value_name': name,
'max_value': max_value})
)
return value
def _do_evacuate(context, host_name, instance_list,
reserved_host=None):
failed_evacuation_instances = []
if reserved_host:
if CONF.host_failure.add_reserved_host_to_aggregate:
# Assign reserved_host to an aggregate to which the failed
# compute host belongs to.
aggregates = self.novaclient.get_aggregate_list(context)
for aggregate in aggregates:
if host_name in aggregate.hosts:
self.novaclient.add_host_to_aggregate(
context, reserved_host.name, aggregate)
# A failed compute host can be associated with
# multiple aggregates but operators will not
# associate it with multiple aggregates in real
# deployment so adding reserved_host to the very
# first aggregate from the list.
break
self.novaclient.enable_disable_service(
context, reserved_host.name, enable=True)
# Sleep until nova-compute service is marked as enabled.
msg = ("Sleeping %(wait)s sec before starting recovery "
"thread until nova recognizes the node up.")
LOG.info(msg, {
'wait': CONF.wait_period_after_service_update})
eventlet.sleep(CONF.wait_period_after_service_update)
# Set reserved property of reserved_host to False
reserved_host.reserved = False
reserved_host.save()
thread_pool = greenpool.GreenPool(
CONF.host_failure_recovery_threads)
for instance in instance_list:
thread_pool.spawn_n(self._evacuate_and_confirm, context,
instance, host_name,
failed_evacuation_instances, reserved_host)
thread_pool.waitall()
if failed_evacuation_instances:
msg = _("Failed to evacuate instances %(instances)s from "
"host %(host_name)s.") % {
'instances': failed_evacuation_instances,
'host_name': host_name
}
raise exception.HostRecoveryFailureException(message=msg)
def delete_segment(self, context, uuid):
"""Deletes the segment."""
segment = objects.FailoverSegment.get_by_uuid(context, uuid)
if is_failover_segment_under_recovery(segment):
msg = _("Failover segment (%s) can't be deleted as "
"it is in-use to process notifications.") % uuid
LOG.error(msg)
raise exception.FailoverSegmentInUse(msg)
try:
segment.destroy()
except Exception as e:
with excutils.save_and_reraise_exception():
tb = traceback.format_exc()
api_utils.notify_about_segment_api(context, segment,
action=fields.EventNotificationAction.SEGMENT_DELETE,
phase=fields.EventNotificationPhase.ERROR, exception=e,
tb=tb)
def get_notification_recovery_workflow_details(self, context,
notification):
"""Retrieve recovery workflow details of the notification"""
try:
host_obj = objects.Host.get_by_uuid(
context, notification.source_host_uuid)
recovery_method = host_obj.failover_segment.recovery_method
progress_details = (
self.driver.get_notification_recovery_workflow_details(
context, recovery_method, notification))
notification['recovery_workflow_details'] = progress_details
except Exception:
msg = (_('Failed to fetch notification recovery workflow details '
'for %s') % notification.notification_uuid)
LOG.exception(msg)
raise exception.MasakariException(msg)
return notification
def update_segment(self, context, uuid, segment_data):
"""Update the properties of a failover segment."""
segment = objects.FailoverSegment.get_by_uuid(context, uuid)
if is_failover_segment_under_recovery(segment):
msg = _("Failover segment %s can't be updated as "
"it is in-use to process notifications.") % uuid
LOG.error(msg)
raise exception.FailoverSegmentInUse(msg)
try:
segment.update(segment_data)
segment.save()
except Exception as e:
with excutils.save_and_reraise_exception():
tb = traceback.format_exc()
api_utils.notify_about_segment_api(context, segment,
action=fields.EventNotificationAction.SEGMENT_UPDATE,
phase=fields.EventNotificationPhase.ERROR, exception=e,
tb=tb)
return segment
def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as exc:
if isinstance(exc, webob.exc.WSGIHTTPException):
if isinstance(errors, int):
t_errors = (errors,)
else:
t_errors = errors
if exc.code in t_errors:
raise
elif isinstance(exc, exception.Forbidden):
raise
elif isinstance(exc, exception.ValidationError):
raise
LOG.exception("Unexpected exception in API method")
msg = _('Unexpected API Error. Please report this at '
'https://bugs.launchpad.net/masakari/ and attach the '
'Masakari API log if possible.\n%s') % type(exc)
raise webob.exc.HTTPInternalServerError(explanation=msg)
def delete_host(self, context, segment_uuid, id):
"""Delete the host"""
segment = objects.FailoverSegment.get_by_uuid(context, segment_uuid)
host = objects.Host.get_by_uuid(context, id, segment_uuid=segment_uuid)
if is_failover_segment_under_recovery(segment):
msg = _("Host %s can't be deleted as "
"it is in-use to process notifications.") % host.uuid
LOG.error(msg)
raise exception.HostInUse(msg)
try:
host.destroy()
except Exception as e:
with excutils.save_and_reraise_exception():
tb = traceback.format_exc()
api_utils.notify_about_host_api(context, host,
action=fields.EventNotificationAction.HOST_DELETE,
phase=fields.EventNotificationPhase.ERROR, exception=e,
tb=tb)
def validate(self, *args, **kwargs):
try:
self.validator.validate(*args, **kwargs)
except jsonschema.ValidationError as ex:
if isinstance(ex.cause, exception.InvalidName):
detail = ex.cause.format_message()
elif len(ex.path) > 0:
detail = _("Invalid input for field/attribute %(path)s."
" Value: %(value)s. %(message)s") % {
'path': ex.path.pop(), 'value': ex.instance,
'message': ex.message
}
else:
detail = ex.message
raise exception.ValidationError(detail=detail)
except TypeError as ex:
# NOTE: If passing non string value to patternProperties parameter,
# TypeError happens. Here is for catching the TypeError.
detail = six.text_type(ex)
raise exception.ValidationError(detail=detail)
def execute(self, context, instance_uuid):
def _wait_for_active():
new_instance = self.novaclient.get_server(context, instance_uuid)
vm_state = getattr(new_instance, 'OS-EXT-STS:vm_state')
if vm_state == 'active':
raise loopingcall.LoopingCallDone()
periodic_call = loopingcall.FixedIntervalLoopingCall(
_wait_for_active)
try:
# add a timeout to the periodic call.
periodic_call.start(interval=CONF.verify_interval)
etimeout.with_timeout(CONF.wait_period_after_power_on,
periodic_call.wait)
except etimeout.Timeout:
msg = _("Failed to start instance %(instance)s") % {
'instance': instance_uuid
}
raise exception.InstanceRecoveryFailureException(message=msg)
finally:
# stop the periodic call, in case of exceptions or Timeout.
periodic_call.stop()
def execute(self, context, host_name):
def _filter_instances(instance_list):
ha_enabled_instances = []
non_ha_enabled_instances = []
for instance in instance_list:
is_instance_ha_enabled = strutils.bool_from_string(
instance.metadata.get('HA_Enabled', False))
if CONF.host_failure.ignore_instances_in_error_state and (
getattr(instance, "OS-EXT-STS:vm_state") == "error"):
if is_instance_ha_enabled:
msg = ("Ignoring recovery of HA_Enabled instance "
"'%(instance_id)s' as it is in 'error' state.")
LOG.info(msg, {'instance_id': instance.id})
continue
if is_instance_ha_enabled:
ha_enabled_instances.append(instance)
else:
non_ha_enabled_instances.append(instance)
if CONF.host_failure.evacuate_all_instances:
ha_enabled_instances.extend(non_ha_enabled_instances)
return ha_enabled_instances
instance_list = self.novaclient.get_servers(context, host_name)
instance_list = _filter_instances(instance_list)
if not instance_list:
msg = _('No instances to evacuate on host: %s.') % host_name
LOG.info(msg)
raise exception.SkipHostRecoveryException(message=msg)
return {
"instance_list": instance_list,
}
def execute_instance_failure(self, context, instance_uuid,
notification_uuid):
novaclient = nova.API()
# get flow for instance failure
process_what = {
'context': context,
'instance_uuid': instance_uuid
}
try:
flow_engine = instance_failure.get_instance_recovery_flow(
novaclient, process_what)
except Exception:
msg = (_('Failed to create instance failure flow.'),
notification_uuid)
LOG.exception(msg)
raise exception.MasakariException(msg)
# Attaching this listener will capture all of the notifications that
# taskflow sends out and redirect them to a more useful log for
# masakari's debugging (or error reporting) usage.
with base.DynamicLogListener(flow_engine, logger=LOG):
flow_engine.run()
class DBNotAllowed(MasakariException):
msg_fmt = _('%(binary)s attempted direct database access which is '
'not allowed by policy')
class HostRecoveryFailureException(MasakariException):
msg_fmt = _('Failed to execute host recovery.')
class FailoverSegmentInUse(Conflict):
msg_fmt = _("Failover segment %(uuid)s can't be updated as it is in-use "
"to process notifications.")
class HostExists(MasakariException):
msg_fmt = _("Host with name %(name)s already exists.")
class FailoverSegmentExists(MasakariException):
msg_fmt = _("Failover segment with name %(name)s already exists.")
class HostNotFoundByName(HostNotFound):
msg_fmt = _("Host with name %(host_name)s could not be found.")
class LockAlreadyAcquired(MasakariException):
msg_fmt = _('Lock is already acquired on %(resource)s.')
class NotificationNotFound(NotFound):
msg_fmt = _("No notification with id %(id)s.")
class HostOnMaintenanceError(Invalid):
msg_fmt = _('Host %(host_name)s is already under maintenance.')
code = http.CONFLICT
class Unauthorized(MasakariException):
msg_fmt = _("Not authorized.")
code = http.UNAUTHORIZED
def _process_sort_params(sort_keys,
sort_dirs,
default_keys=['created_at', 'id'],
default_dir='desc'):
"""Process the sort parameters to include default keys.
Creates a list of sort keys and a list of sort directions. Adds the default
keys to the end of the list if they are not already included.
When adding the default keys to the sort keys list, the associated
direction is:
1) The first element in the 'sort_dirs' list (if specified), else
2) 'default_dir' value (Note that 'asc' is the default value since this is
the default in sqlalchemy.utils.paginate_query)
:param sort_keys: List of sort keys to include in the processed list
:param sort_dirs: List of sort directions to include in the processed list
:param default_keys: List of sort keys that need to be included in the
processed list, they are added at the end of the list
if not already specified.
:param default_dir: Sort direction associated with each of the default
keys that are not supplied, used when they are added
to the processed list
:returns: list of sort keys, list of sort directions
:raise exception.InvalidInput: If more sort directions than sort keys
are specified or if an invalid sort
direction is specified
"""
# Determine direction to use for when adding default keys
default_dir_value = default_dir
if sort_dirs and len(sort_dirs) != 0:
default_dir_value = sort_dirs[0]
# Create list of keys (do not modify the input list)
result_keys = []
if sort_keys:
result_keys = list(sort_keys)
# If a list of directions is not provided, use the default sort direction
# for all provided keys
if sort_dirs:
result_dirs = []
# Verify sort direction
for sort_dir in sort_dirs:
if sort_dir not in ('asc', 'desc'):
msg = _("Unknown sort direction, must be 'asc' or 'desc'")
raise exception.InvalidInput(reason=msg)
result_dirs.append(sort_dir)
else:
result_dirs = [default_dir_value for _sort_key in result_keys]
# Ensure that the key and direction length match
while len(result_dirs) < len(result_keys):
result_dirs.append(default_dir_value)
# Unless more direction are specified, which is an error
if len(result_dirs) > len(result_keys):
msg = _("Sort direction size exceeds sort key size")
raise exception.InvalidInput(reason=msg)
# Ensure defaults are included
for key in default_keys:
if key not in result_keys:
result_keys.append(key)
result_dirs.append(default_dir_value)
return result_keys, result_dirs
class ObjectActionError(MasakariException):
msg_fmt = _('Object action %(action)s failed because: %(reason)s')
class HostNotFoundUnderFailoverSegment(HostNotFound):
msg_fmt = _("Host '%(host_uuid)s' under failover_segment "
"'%(segment_uuid)s' could not be found.")
class IgnoreInstanceRecoveryException(MasakariException):
msg_fmt = _('Instance recovery is ignored.')
class HypervisorNotFoundByName(NotFound):
msg_fmt = _("Hypervisor with name %(hypervisor_name)s could not be found.")
class HostInUse(Conflict):
msg_fmt = _("Host %(uuid)s can't be updated as it is in-use to process "
"notifications.")
class ProcessRecoveryFailureException(MasakariException):
msg_fmt = _('Failed to execute process recovery workflow.')
class SkipHostRecoveryException(MasakariException):
msg_fmt = _('Skipping execution of host recovery workflow.')
class FailoverSegmentNotFoundByName(FailoverSegmentNotFound):
msg_fmt = _("Failover segment with name %(segment_name)s could not "
"be found.")
class SkipInstanceRecoveryException(MasakariException):
msg_fmt = _('Skipping execution of instance recovery workflow.')
class HostNotFound(NotFound):
msg_fmt = _("No host with id %(id)s.")
class InstanceRecoveryFailureException(MasakariException):
msg_fmt = _('Failed to execute instance recovery workflow.')
class FailoverSegmentNotFound(NotFound):
msg_fmt = _("No failover segment with id %(id)s.")
class DuplicateNotification(Invalid):
msg_fmt = _('Duplicate notification received for type: %(type)s')
code = http.CONFLICT
def _set_read_deleted(self, read_deleted):
if read_deleted not in ('no', 'yes', 'only'):
raise ValueError(
_("read_deleted can only be one of 'no', "
"'yes' or 'only', not %r") % read_deleted)
self._read_deleted = read_deleted
class SkipProcessRecoveryException(MasakariException):
msg_fmt = _('Skipping execution of process recovery workflow.')
def _from_json(self, datastring):
try:
return jsonutils.loads(datastring)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
def _process_stack(self, request, action, action_args, content_type, body,
accept):
"""Implement the processing stack."""
# Get the implementing method
try:
meth, extensions = self.get_method(request, action, content_type,
body)
except (AttributeError, TypeError):
return Fault(webob.exc.HTTPNotFound())
except KeyError as ex:
msg = _("There is no such action: %s") % ex.args[0]
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
except exception.MalformedRequestBody:
msg = _("Malformed request body")
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
except webob.exc.HTTPMethodNotAllowed as e:
return Fault(e)
if body:
msg = _("Action: '%(action)s', calling method: %(meth)s, body: "
"%(body)s") % {
'action': action,
'body': six.text_type(body, 'utf-8'),
'meth': str(meth)
}
LOG.debug(strutils.mask_password(msg))
else:
LOG.debug("Calling method '%(meth)s'", {'meth': str(meth)})
# Now, deserialize the request body...
try:
contents = {}
if self._should_have_body(request):
# allow empty body with PUT and POST
if request.content_length == 0:
contents = {'body': None}
else:
contents = self.deserialize(body)
except exception.MalformedRequestBody:
msg = _("Malformed request body")
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
# Update the action args
action_args.update(contents)
project_id = action_args.pop("project_id", None)
context = request.environ.get('masakari.context')
if (context and project_id and (project_id != context.project_id)):
msg = _("Malformed request URL: URL's project_id '%(project_id)s'"
" doesn't match Context's project_id"
" '%(context_project_id)s'") % {
'project_id': project_id,
'context_project_id': context.project_id
}
return Fault(webob.exc.HTTPBadRequest(explanation=msg))
# Run pre-processing extensions
response, post = self.pre_process_extensions(extensions, request,
action_args)
if not response:
try:
with ResourceExceptionHandler():
action_result = self.dispatch(meth, request, action_args)
except Fault as ex:
response = ex
if not response:
# No exceptions; convert action_result into a
# ResponseObject
resp_obj = None
if type(action_result) is dict or action_result is None:
resp_obj = ResponseObject(action_result)
elif isinstance(action_result, ResponseObject):
resp_obj = action_result
else:
response = action_result
# Run post-processing extensions
if resp_obj:
# Do a preserialize to set up the response object
if hasattr(meth, 'wsgi_code'):
resp_obj._default_code = meth.wsgi_code
# Process post-processing extensions
response = self.post_process_extensions(
post, resp_obj, request, action_args)
if resp_obj and not response:
response = resp_obj.serialize(request, accept)
if hasattr(response, 'headers'):
for hdr, val in list(response.headers.items()):
# Headers must be utf-8 strings
response.headers[hdr] = utils.utf8(val)
if not request.api_version_request.is_null():
response.headers[API_VERSION_REQUEST_HEADER] = \
'masakari ' + request.api_version_request.get_string()
response.headers.add('Vary', API_VERSION_REQUEST_HEADER)
return response
class ReservedHostsUnavailable(MasakariException):
msg_fmt = _('No reserved_hosts available for evacuation.')
class OrphanedObjectError(MasakariException):
msg_fmt = _('Cannot call %(method)s on orphaned %(objtype)s object')
