def test_is_cell_failure_sentinel(self):
record = context.did_not_respond_sentinel
self.assertTrue(context.is_cell_failure_sentinel(record))
record = TypeError()
self.assertTrue(context.is_cell_failure_sentinel(record))
record = objects.Instance()
self.assertFalse(context.is_cell_failure_sentinel(record))
def test_is_cell_failure_sentinel(self):
record = context.did_not_respond_sentinel
self.assertTrue(context.is_cell_failure_sentinel(record))
record = TypeError()
self.assertTrue(context.is_cell_failure_sentinel(record))
record = objects.Instance()
self.assertFalse(context.is_cell_failure_sentinel(record))
def __lt__(self, other):
# NOTE(danms): This makes us always sort failure sentinels
# higher than actual results. We do this so that they bubble
# up in the get_records_sorted() feeder loop ahead of anything
# else, and so that the implementation of RecordSortContext
# never sees or has to handle the sentinels. If we did not
# sort these to the top then we could potentially return
# $limit results from good cells before we noticed the failed
# cells, and would not properly report them as failed for
# fix-up in the higher layers.
if context.is_cell_failure_sentinel(self._db_record):
return True
elif context.is_cell_failure_sentinel(other._db_record):
return False
r = self._sort_ctx.compare_records(self._db_record, other._db_record)
# cmp(x, y) returns -1 if x < y
return r == -1
def __lt__(self, other):
# NOTE(danms): This makes us always sort failure sentinels
# higher than actual results. We do this so that they bubble
# up in the get_records_sorted() feeder loop ahead of anything
# else, and so that the implementation of RecordSortContext
# never sees or has to handle the sentinels. If we did not
# sort these to the top then we could potentially return
# $limit results from good cells before we noticed the failed
# cells, and would not properly report them as failed for
# fix-up in the higher layers.
if context.is_cell_failure_sentinel(self._db_record):
return True
elif context.is_cell_failure_sentinel(other._db_record):
return False
r = self._sort_ctx.compare_records(self._db_record,
other._db_record)
# cmp(x, y) returns -1 if x < y
return r == -1
def get_compute_nodes_by_host_or_node(self, ctxt, host, node, cell=None):
'''Get compute nodes from given host or node'''
def return_empty_list_for_not_found(func):
def wrapper(*args, **kwargs):
try:
ret = func(*args, **kwargs)
except exception.NotFound:
ret = objects.ComputeNodeList()
return ret
return wrapper
@return_empty_list_for_not_found
def _get_by_host_and_node(ctxt):
compute_node = objects.ComputeNode.get_by_host_and_nodename(
ctxt, host, node)
return objects.ComputeNodeList(objects=[compute_node])
@return_empty_list_for_not_found
def _get_by_host(ctxt):
return objects.ComputeNodeList.get_all_by_host(ctxt, host)
@return_empty_list_for_not_found
def _get_by_node(ctxt):
compute_node = objects.ComputeNode.get_by_nodename(ctxt, node)
return objects.ComputeNodeList(objects=[compute_node])
if host and node:
target_fnc = _get_by_host_and_node
elif host:
target_fnc = _get_by_host
else:
target_fnc = _get_by_node
if host and not cell:
# optimization not to issue queries to every cell DB
cell = self._get_cell_by_host(ctxt, host)
cells = [cell] if cell else self.enabled_cells
timeout = context_module.CELL_TIMEOUT
nodes_by_cell = context_module.scatter_gather_cells(
ctxt, cells, timeout, target_fnc)
# Only one cell should have values for the compute nodes
# so we get them here, or return an empty list if no cell
# has a value; be sure to filter out cell failures.
nodes = next(
(nodes for nodes in nodes_by_cell.values()
if nodes and not context_module.is_cell_failure_sentinel(nodes)),
objects.ComputeNodeList())
return nodes
def get_marker_record(self, ctx, marker):
"""Get the marker migration from its cell.
This returns the marker migration from the cell in which it lives
"""
results = context.scatter_gather_skip_cell0(
ctx, db.migration_get_by_uuid, marker)
db_migration = None
for result_cell_uuid, result in results.items():
if not context.is_cell_failure_sentinel(result):
db_migration = result
cell_uuid = result_cell_uuid
break
if not db_migration:
raise exception.MarkerNotFound(marker=marker)
return cell_uuid, db_migration
def get_marker_record(self, ctx, marker):
"""Get the marker migration from its cell.
This returns the marker migration from the cell in which it lives
"""
results = context.scatter_gather_skip_cell0(ctx,
db.migration_get_by_uuid,
marker)
db_migration = None
for result_cell_uuid, result in results.items():
if not context.is_cell_failure_sentinel(result):
db_migration = result
cell_uuid = result_cell_uuid
break
if not db_migration:
raise exception.MarkerNotFound(marker=marker)
return cell_uuid, db_migration
def _show(self, req, id, rdp_only):
"""Checks a console auth token and returns the related connect info."""
context = req.environ['nova.context']
context.can(cat_policies.BASE_POLICY_NAME)
token = id
if not token:
msg = _("token not provided")
raise webob.exc.HTTPBadRequest(explanation=msg)
connect_info = None
if CONF.workarounds.enable_consoleauth:
connect_info = self._consoleauth_rpcapi.check_token(context, token)
else:
results = nova_context.scatter_gather_skip_cell0(
context, objects.ConsoleAuthToken.validate, token)
# NOTE(melwitt): Console token auths are stored in cell databases,
# but with only the token as a request param, we can't know which
# cell database contains the token's corresponding connection info.
# So, we must query all cells for the info and we can break the
# loop as soon as we find a result because the token is associated
# with one instance, which can only be in one cell.
for result in results.values():
if not nova_context.is_cell_failure_sentinel(result):
connect_info = result.to_dict()
break
if not connect_info:
raise webob.exc.HTTPNotFound(explanation=_("Token not found"))
console_type = connect_info.get('console_type')
if rdp_only and console_type != "rdp-html5":
raise webob.exc.HTTPUnauthorized(
explanation=_("The requested console type details are not "
"accessible"))
return {
'console': {
i: connect_info[i]
for i in
['instance_uuid', 'host', 'port', 'internal_access_path']
if i in connect_info
}
}
def _show(self, req, id, rdp_only):
"""Checks a console auth token and returns the related connect info."""
context = req.environ['nova.context']
context.can(cat_policies.BASE_POLICY_NAME)
token = id
if not token:
msg = _("token not provided")
raise webob.exc.HTTPBadRequest(explanation=msg)
connect_info = None
if CONF.workarounds.enable_consoleauth:
connect_info = self._consoleauth_rpcapi.check_token(context, token)
else:
results = nova_context.scatter_gather_skip_cell0(
context, objects.ConsoleAuthToken.validate, token)
# NOTE(melwitt): Console token auths are stored in cell databases,
# but with only the token as a request param, we can't know which
# cell database contains the token's corresponding connection info.
# So, we must query all cells for the info and we can break the
# loop as soon as we find a result because the token is associated
# with one instance, which can only be in one cell.
for result in results.values():
if not nova_context.is_cell_failure_sentinel(result):
connect_info = result.to_dict()
break
if not connect_info:
raise webob.exc.HTTPNotFound(explanation=_("Token not found"))
console_type = connect_info.get('console_type')
if rdp_only and console_type != "rdp-html5":
raise webob.exc.HTTPUnauthorized(
explanation=_("The requested console type details are not "
"accessible"))
return {'console':
{i: connect_info[i]
for i in ['instance_uuid', 'host', 'port',
'internal_access_path']
if i in connect_info}}
def _get_instance_group_hosts_all_cells(context, instance_group):
def get_hosts_in_cell(cell_context):
# NOTE(melwitt): The obj_alternate_context is going to mutate the
# cell_instance_group._context and to do this in a scatter-gather
# with multiple parallel greenthreads, we need the instance groups
# to be separate object copies.
cell_instance_group = instance_group.obj_clone()
with cell_instance_group.obj_alternate_context(cell_context):
return cell_instance_group.get_hosts()
results = nova_context.scatter_gather_skip_cell0(context,
get_hosts_in_cell)
hosts = []
for result in results.values():
# TODO(melwitt): We will need to handle scenarios where an exception
# is raised while targeting a cell and when a cell does not respond
# as part of the "handling of a down cell" spec:
# https://blueprints.launchpad.net/nova/+spec/handling-down-cell
if not nova_context.is_cell_failure_sentinel(result):
hosts.extend(result)
return hosts
def _get_instance_group_hosts_all_cells(context, instance_group):
def get_hosts_in_cell(cell_context):
# NOTE(melwitt): The obj_alternate_context is going to mutate the
# cell_instance_group._context and to do this in a scatter-gather
# with multiple parallel greenthreads, we need the instance groups
# to be separate object copies.
cell_instance_group = instance_group.obj_clone()
with cell_instance_group.obj_alternate_context(cell_context):
return cell_instance_group.get_hosts()
results = nova_context.scatter_gather_skip_cell0(context,
get_hosts_in_cell)
hosts = []
for result in results.values():
# TODO(melwitt): We will need to handle scenarios where an exception
# is raised while targeting a cell and when a cell does not respond
# as part of the "handling of a down cell" spec:
# https://blueprints.launchpad.net/nova/+spec/handling-down-cell
if not nova_context.is_cell_failure_sentinel(result):
hosts.extend(result)
return hosts
def get_records_sorted(self, ctx, filters, limit, marker, **kwargs):
"""Get a cross-cell list of records matching filters.
This iterates cells in parallel generating a unified and sorted
list of records as efficiently as possible. It takes care to
iterate the list as infrequently as possible. We wrap the results
in RecordWrapper objects so that they are sortable by
heapq.merge(), which requires that the '<' operator just works.
Our sorting requirements are encapsulated into the
RecordSortContext provided to the constructor for this object.
This function is a generator of records from the database like what you
would get from instance_get_all_by_filters_sort() in the DB API.
NOTE: Since we do these in parallel, a nonzero limit will be passed
to each database query, although the limit will be enforced in the
output of this function. Meaning, we will still query $limit from each
database, but only return $limit total results.
"""
if marker:
# A marker identifier was provided from the API. Call this
# the 'global' marker as it determines where we start the
# process across all cells. Look up the record in
# whatever cell it is in and record the values for the
# sort keys so we can find the marker instance in each
# cell (called the 'local' marker).
global_marker_cell, global_marker_record = self.get_marker_record(
ctx, marker)
global_marker_values = [global_marker_record[key]
for key in self.sort_ctx.sort_keys]
def do_query(cctx):
"""Generate RecordWrapper(record) objects from a cell.
We do this inside the thread (created by
scatter_gather_all_cells()) so that we return wrappers and
avoid having to iterate the combined result list in the
caller again. This is run against each cell by the
scatter_gather routine.
"""
# The local marker is an identifier of a record in a cell
# that is found by the special method
# get_marker_by_values(). It should be the next record
# in order according to the sort provided, but after the
# marker instance which may have been in another cell.
local_marker = None
# Since the regular DB query routines take a marker and assume that
# the marked record was the last entry of the previous page, we
# may need to prefix it to our result query if we're not the cell
# that had the actual marker record.
local_marker_prefix = []
marker_id = self.marker_identifier
if marker:
if cctx.cell_uuid == global_marker_cell:
local_marker = marker
else:
local_marker = self.get_marker_by_values(
cctx, global_marker_values)
if local_marker:
if local_marker != marker:
# We did find a marker in our cell, but it wasn't
# the global marker. Thus, we will use it as our
# marker in the main query below, but we also need
# to prefix that result with this marker instance
# since the result below will not return it and it
# has not been returned to the user yet. Note that
# we do _not_ prefix the marker instance if our
# marker was the global one since that has already
# been sent to the user.
local_marker_filters = copy.copy(filters)
if marker_id not in local_marker_filters:
# If an $id filter was provided, it will
# have included our marker already if this
# instance is desired in the output
# set. If it wasn't, we specifically query
# for it. If the other filters would have
# excluded it, then we'll get an empty set
# here and not include it in the output as
# expected.
local_marker_filters[marker_id] = [local_marker]
local_marker_prefix = self.get_by_filters(
cctx, local_marker_filters, limit=1, marker=None,
**kwargs)
else:
# There was a global marker but everything in our
# cell is _before_ that marker, so we return
# nothing. If we didn't have this clause, we'd
# pass marker=None to the query below and return a
# full unpaginated set for our cell.
return
if local_marker_prefix:
# Per above, if we had a matching marker object, that is
# the first result we should generate.
yield RecordWrapper(cctx, self.sort_ctx,
local_marker_prefix[0])
# If a batch size was provided, use that as the limit per
# batch. If not, then ask for the entire $limit in a single
# batch.
batch_size = self.batch_size or limit
# Keep track of how many we have returned in all batches
return_count = 0
# If limit was unlimited then keep querying batches until
# we run out of results. Otherwise, query until the total count
# we have returned exceeds the limit.
while limit is None or return_count < limit:
batch_count = 0
# Do not query a full batch if it would cause our total
# to exceed the limit
if limit:
query_size = min(batch_size, limit - return_count)
else:
query_size = batch_size
# Get one batch
query_result = self.get_by_filters(
cctx, filters,
limit=query_size or None, marker=local_marker,
**kwargs)
# Yield wrapped results from the batch, counting as we go
# (to avoid traversing the list to count). Also, update our
# local_marker each time so that local_marker is the end of
# this batch in order to find the next batch.
for item in query_result:
local_marker = item[self.marker_identifier]
yield RecordWrapper(cctx, self.sort_ctx, item)
batch_count += 1
# No results means we are done for this cell
if not batch_count:
break
return_count += batch_count
LOG.debug(('Listed batch of %(batch)i results from cell '
'out of %(limit)s limit. Returned %(total)i '
'total so far.'),
{'batch': batch_count,
'total': return_count,
'limit': limit or 'no'})
# NOTE(danms): The calls to do_query() will return immediately
# with a generator. There is no point in us checking the
# results for failure or timeout since we have not actually
# run any code in do_query() until the first iteration
# below. The query_wrapper() utility handles inline
# translation of failures and timeouts to sentinels which will
# be generated and consumed just like any normal result below.
if self.cells:
results = context.scatter_gather_cells(ctx, self.cells,
context.CELL_TIMEOUT,
query_wrapper, do_query)
else:
results = context.scatter_gather_all_cells(ctx,
query_wrapper, do_query)
# If a limit was provided, it was passed to the per-cell query
# routines. That means we have NUM_CELLS * limit items across
# results. So, we need to consume from that limit below and
# stop returning results. Call that total_limit since we will
# modify it in the loop below, but do_query() above also looks
# at the original provided limit.
total_limit = limit or 0
# Generate results from heapq so we can return the inner
# instance instead of the wrapper. This is basically free
# as it works as our caller iterates the results.
feeder = heapq.merge(*results.values())
while True:
try:
item = next(feeder)
except StopIteration:
return
if context.is_cell_failure_sentinel(item._db_record):
if not CONF.api.list_records_by_skipping_down_cells:
raise exception.NovaException(
_('Cell %s is not responding but configuration '
'indicates that we should fail.') % item.cell_uuid)
LOG.warning('Cell %s is not responding and hence is '
'being omitted from the results',
item.cell_uuid)
if item._db_record == context.did_not_respond_sentinel:
self._cells_timed_out.add(item.cell_uuid)
elif isinstance(item._db_record, Exception):
self._cells_failed.add(item.cell_uuid)
# We might have received one batch but timed out or failed
# on a later one, so be sure we fix the accounting.
if item.cell_uuid in self._cells_responded:
self._cells_responded.remove(item.cell_uuid)
continue
yield item._db_record
self._cells_responded.add(item.cell_uuid)
total_limit -= 1
if total_limit == 0:
# We'll only hit this if limit was nonzero and we just
# generated our last one
return
