def test_fixture(self):
# The test setup creates two cell mappings (cell0 and cell1) by
# default. Let's first list servers across all cells while they are
# "up" to make sure that works as expected. We'll create a single
# instance in cell1.
ctxt = context.get_admin_context()
cell1 = self.cell_mappings[test.CELL1_NAME]
with context.target_cell(ctxt, cell1) as cctxt:
inst = fake_instance.fake_instance_obj(cctxt)
if 'id' in inst:
delattr(inst, 'id')
inst.create()
# Now list all instances from all cells (should get one back).
results = context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_all)
self.assertEqual(2, len(results))
self.assertEqual(0, len(results[objects.CellMapping.CELL0_UUID]))
self.assertEqual(1, len(results[cell1.uuid]))
# Now do the same but with the DownCellFixture which should result
# in exception results from both cells.
with fixtures.DownCellFixture():
results = context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_all)
self.assertEqual(2, len(results))
for result in results.values():
self.assertIsInstance(result, db_exc.DBError)
def test_fixture(self):
# The test setup creates two cell mappings (cell0 and cell1) by
# default. Let's first list servers across all cells while they are
# "up" to make sure that works as expected. We'll create a single
# instance in cell1.
ctxt = context.get_admin_context()
cell1 = self.cell_mappings[test.CELL1_NAME]
with context.target_cell(ctxt, cell1) as cctxt:
inst = fake_instance.fake_instance_obj(cctxt)
if 'id' in inst:
delattr(inst, 'id')
inst.create()
# Now list all instances from all cells (should get one back).
results = context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_all)
self.assertEqual(2, len(results))
self.assertEqual(0, len(results[objects.CellMapping.CELL0_UUID]))
self.assertEqual(1, len(results[cell1.uuid]))
# Now do the same but with the DownCellFixture which should result
# in exception results from both cells.
with fixtures.DownCellFixture():
results = context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_all)
self.assertEqual(2, len(results))
for result in results.values():
self.assertIsInstance(result, db_exc.DBError)
def test_fixture_when_explicitly_passing_down_cell_mappings(self):
# The test setup creates two cell mappings (cell0 and cell1) by
# default. We'll create one instance per cell and pass cell0 as
# the down cell. We should thus get db_exc.DBError for cell0 and
# correct InstanceList object from cell1.
ctxt = context.get_admin_context()
cell0 = self.cell_mappings['cell0']
cell1 = self.cell_mappings['cell1']
with context.target_cell(ctxt, cell0) as cctxt:
inst1 = fake_instance.fake_instance_obj(cctxt)
if 'id' in inst1:
delattr(inst1, 'id')
inst1.create()
with context.target_cell(ctxt, cell1) as cctxt:
inst2 = fake_instance.fake_instance_obj(cctxt)
if 'id' in inst2:
delattr(inst2, 'id')
inst2.create()
with fixtures.DownCellFixture([cell0]):
results = context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_all)
self.assertEqual(2, len(results))
for cell_uuid, result in results.items():
if cell_uuid == cell0.uuid:
self.assertIsInstance(result, db_exc.DBError)
else:
self.assertIsInstance(result, objects.InstanceList)
self.assertEqual(1, len(result))
self.assertEqual(inst2.uuid, result[0].uuid)
def test_fixture_when_explicitly_passing_down_cell_mappings(self):
# The test setup creates two cell mappings (cell0 and cell1) by
# default. We'll create one instance per cell and pass cell0 as
# the down cell. We should thus get db_exc.DBError for cell0 and
# correct InstanceList object from cell1.
ctxt = context.get_admin_context()
cell0 = self.cell_mappings['cell0']
cell1 = self.cell_mappings['cell1']
with context.target_cell(ctxt, cell0) as cctxt:
inst1 = fake_instance.fake_instance_obj(cctxt)
if 'id' in inst1:
delattr(inst1, 'id')
inst1.create()
with context.target_cell(ctxt, cell1) as cctxt:
inst2 = fake_instance.fake_instance_obj(cctxt)
if 'id' in inst2:
delattr(inst2, 'id')
inst2.create()
with fixtures.DownCellFixture([cell0]):
results = context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_all)
self.assertEqual(2, len(results))
for cell_uuid, result in results.items():
if cell_uuid == cell0.uuid:
self.assertIsInstance(result, db_exc.DBError)
else:
self.assertIsInstance(result, objects.InstanceList)
self.assertEqual(1, len(result))
self.assertEqual(inst2.uuid, result[0].uuid)
def test_scatter_gather_cells(self):
self._create_cell_mappings()
# Create an instance in cell0
with context.target_cell(self.context, self.mapping0) as cctxt:
instance = objects.Instance(context=cctxt, uuid=uuids.instance0,
project_id='fake-project')
instance.create()
# Create an instance in first cell
with context.target_cell(self.context, self.mapping1) as cctxt:
instance = objects.Instance(context=cctxt, uuid=uuids.instance1,
project_id='fake-project')
instance.create()
# Create an instance in second cell
with context.target_cell(self.context, self.mapping2) as cctxt:
instance = objects.Instance(context=cctxt, uuid=uuids.instance2,
project_id='fake-project')
instance.create()
filters = {'deleted': False, 'project_id': 'fake-project'}
results = context.scatter_gather_all_cells(
self.context, objects.InstanceList.get_by_filters, filters,
sort_dir='asc')
instances = objects.InstanceList()
for result in results.values():
instances = instances + result
# Should have 3 instances across cells
self.assertEqual(3, len(instances))
# Verify we skip cell0 when specified
results = context.scatter_gather_skip_cell0(
self.context, objects.InstanceList.get_by_filters, filters)
instances = objects.InstanceList()
for result in results.values():
instances = instances + result
# Should have gotten only the instances from the last two cells
self.assertEqual(2, len(instances))
self.assertIn(self.mapping1.uuid, results)
self.assertIn(self.mapping2.uuid, results)
instance_uuids = [inst.uuid for inst in instances]
self.assertIn(uuids.instance1, instance_uuids)
self.assertIn(uuids.instance2, instance_uuids)
# Try passing one cell
results = context.scatter_gather_cells(
self.context, [self.mapping1], 60,
objects.InstanceList.get_by_filters, filters)
instances = objects.InstanceList()
for result in results.values():
instances = instances + result
# Should have gotten only one instance from cell1
self.assertEqual(1, len(instances))
self.assertIn(self.mapping1.uuid, results)
self.assertEqual(uuids.instance1, instances[0].uuid)
def test_scatter_gather_all_cells(self, mock_get_all, mock_scatter):
ctxt = context.get_context()
mapping0 = objects.CellMapping(database_connection='fake://db0',
transport_url='none:///',
uuid=objects.CellMapping.CELL0_UUID)
mapping1 = objects.CellMapping(database_connection='fake://db1',
transport_url='fake://mq1',
uuid=uuids.cell1)
mock_get_all.return_value = objects.CellMappingList(
objects=[mapping0, mapping1])
filters = {'deleted': False}
context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_by_filters, filters, sort_dir='foo')
mock_scatter.assert_called_once_with(
ctxt, mock_get_all.return_value, 60,
objects.InstanceList.get_by_filters, filters, sort_dir='foo')
def test_scatter_gather_all_cells(self, mock_get_all, mock_scatter):
ctxt = context.get_context()
mapping0 = objects.CellMapping(database_connection='fake://db0',
transport_url='none:///',
uuid=objects.CellMapping.CELL0_UUID)
mapping1 = objects.CellMapping(database_connection='fake://db1',
transport_url='fake://mq1',
uuid=uuids.cell1)
mock_get_all.return_value = objects.CellMappingList(
objects=[mapping0, mapping1])
filters = {'deleted': False}
context.scatter_gather_all_cells(
ctxt, objects.InstanceList.get_by_filters, filters, sort_dir='foo')
mock_scatter.assert_called_once_with(
ctxt, mock_get_all.return_value, 60,
objects.InstanceList.get_by_filters, filters, sort_dir='foo')
def get_minimum_version_all_cells(context, binaries, require_all=False):
"""Get the minimum service version, checking all cells.
This attempts to calculate the minimum service version for a set
of binaries across all the cells in the system. If require_all
is False, then any cells that fail to report a version will be
ignored (assuming they won't be candidates for scheduling and thus
excluding them from the minimum version calculation is reasonable).
If require_all is True, then a failing cell will cause this to raise
exception.CellTimeout, as would be appropriate for gating some
data migration until everything is new enough.
Note that services that do not report a positive version are excluded
from this, as it crosses all cells which will naturally not have all
services.
"""
if not all(binary.startswith('nova-') for binary in binaries):
LOG.warning(
'get_minimum_version_all_cells called with '
'likely-incorrect binaries `%s\'', ','.join(binaries))
raise exception.ObjectActionError(
action='get_minimum_version_all_cells',
reason='Invalid binary prefix')
# NOTE(danms): Instead of using Service.get_minimum_version_multi(), we
# replicate the call directly to the underlying DB method here because
# we want to defeat the caching and we need to filter non-present
# services differently from the single-cell method.
results = nova_context.scatter_gather_all_cells(
context, Service._db_service_get_minimum_version, binaries)
min_version = None
for cell_uuid, result in results.items():
if result is nova_context.did_not_respond_sentinel:
LOG.warning(
'Cell %s did not respond when getting minimum '
'service version', cell_uuid)
if require_all:
raise exception.CellTimeout()
elif isinstance(result, Exception):
LOG.warning('Failed to get minimum service version for cell %s',
cell_uuid)
if require_all:
# NOTE(danms): Okay, this isn't necessarily a timeout, but
# it's functionally the same from the caller's perspective
# and we logged the fact that it was actually a failure
# for the forensic investigator during the scatter/gather
# routine.
raise exception.CellTimeout()
else:
# NOTE(danms): Don't consider a zero or None result as the minimum
# since we're crossing cells and will likely not have all the
# services being probed.
relevant_versions = [
version for version in result.values() if version
]
if relevant_versions:
min_version_cell = min(relevant_versions)
min_version = (min(min_version, min_version_cell)
if min_version else min_version_cell)
# NOTE(danms): If we got no matches at all (such as at first startup)
# then report that as zero to be consistent with the other such
# methods.
return min_version or 0
def get_records_sorted(self, ctx, filters, limit, marker, **kwargs):
"""Get a cross-cell list of records matching filters.
This iterates cells in parallel generating a unified and sorted
list of records as efficiently as possible. It takes care to
iterate the list as infrequently as possible. We wrap the results
in RecordWrapper objects so that they are sortable by
heapq.merge(), which requires that the '<' operator just works.
Our sorting requirements are encapsulated into the
RecordSortContext provided to the constructor for this object.
This function is a generator of records from the database like what you
would get from instance_get_all_by_filters_sort() in the DB API.
NOTE: Since we do these in parallel, a nonzero limit will be passed
to each database query, although the limit will be enforced in the
output of this function. Meaning, we will still query $limit from each
database, but only return $limit total results.
"""
if marker:
# A marker identifier was provided from the API. Call this
# the 'global' marker as it determines where we start the
# process across all cells. Look up the record in
# whatever cell it is in and record the values for the
# sort keys so we can find the marker instance in each
# cell (called the 'local' marker).
global_marker_cell, global_marker_record = self.get_marker_record(
ctx, marker)
global_marker_values = [global_marker_record[key]
for key in self.sort_ctx.sort_keys]
def do_query(cctx):
"""Generate RecordWrapper(record) objects from a cell.
We do this inside the thread (created by
scatter_gather_all_cells()) so that we return wrappers and
avoid having to iterate the combined result list in the
caller again. This is run against each cell by the
scatter_gather routine.
"""
# The local marker is an identifier of a record in a cell
# that is found by the special method
# get_marker_by_values(). It should be the next record
# in order according to the sort provided, but after the
# marker instance which may have been in another cell.
local_marker = None
# Since the regular DB query routines take a marker and assume that
# the marked record was the last entry of the previous page, we
# may need to prefix it to our result query if we're not the cell
# that had the actual marker record.
local_marker_prefix = []
marker_id = self.marker_identifier
if marker:
if cctx.cell_uuid == global_marker_cell:
local_marker = marker
else:
local_marker = self.get_marker_by_values(
cctx, global_marker_values)
if local_marker:
if local_marker != marker:
# We did find a marker in our cell, but it wasn't
# the global marker. Thus, we will use it as our
# marker in the main query below, but we also need
# to prefix that result with this marker instance
# since the result below will not return it and it
# has not been returned to the user yet. Note that
# we do _not_ prefix the marker instance if our
# marker was the global one since that has already
# been sent to the user.
local_marker_filters = copy.copy(filters)
if marker_id not in local_marker_filters:
# If an $id filter was provided, it will
# have included our marker already if this
# instance is desired in the output
# set. If it wasn't, we specifically query
# for it. If the other filters would have
# excluded it, then we'll get an empty set
# here and not include it in the output as
# expected.
local_marker_filters[marker_id] = [local_marker]
local_marker_prefix = self.get_by_filters(
cctx, local_marker_filters, limit=1, marker=None,
**kwargs)
else:
# There was a global marker but everything in our
# cell is _before_ that marker, so we return
# nothing. If we didn't have this clause, we'd
# pass marker=None to the query below and return a
# full unpaginated set for our cell.
return
if local_marker_prefix:
# Per above, if we had a matching marker object, that is
# the first result we should generate.
yield RecordWrapper(cctx, self.sort_ctx,
local_marker_prefix[0])
# If a batch size was provided, use that as the limit per
# batch. If not, then ask for the entire $limit in a single
# batch.
batch_size = self.batch_size or limit
# Keep track of how many we have returned in all batches
return_count = 0
# If limit was unlimited then keep querying batches until
# we run out of results. Otherwise, query until the total count
# we have returned exceeds the limit.
while limit is None or return_count < limit:
batch_count = 0
# Do not query a full batch if it would cause our total
# to exceed the limit
if limit:
query_size = min(batch_size, limit - return_count)
else:
query_size = batch_size
# Get one batch
query_result = self.get_by_filters(
cctx, filters,
limit=query_size or None, marker=local_marker,
**kwargs)
# Yield wrapped results from the batch, counting as we go
# (to avoid traversing the list to count). Also, update our
# local_marker each time so that local_marker is the end of
# this batch in order to find the next batch.
for item in query_result:
local_marker = item[self.marker_identifier]
yield RecordWrapper(cctx, self.sort_ctx, item)
batch_count += 1
# No results means we are done for this cell
if not batch_count:
break
return_count += batch_count
LOG.debug(('Listed batch of %(batch)i results from cell '
'out of %(limit)s limit. Returned %(total)i '
'total so far.'),
{'batch': batch_count,
'total': return_count,
'limit': limit or 'no'})
# NOTE(danms): The calls to do_query() will return immediately
# with a generator. There is no point in us checking the
# results for failure or timeout since we have not actually
# run any code in do_query() until the first iteration
# below. The query_wrapper() utility handles inline
# translation of failures and timeouts to sentinels which will
# be generated and consumed just like any normal result below.
if self.cells:
results = context.scatter_gather_cells(ctx, self.cells,
context.CELL_TIMEOUT,
query_wrapper, do_query)
else:
results = context.scatter_gather_all_cells(ctx,
query_wrapper, do_query)
# If a limit was provided, it was passed to the per-cell query
# routines. That means we have NUM_CELLS * limit items across
# results. So, we need to consume from that limit below and
# stop returning results. Call that total_limit since we will
# modify it in the loop below, but do_query() above also looks
# at the original provided limit.
total_limit = limit or 0
# Generate results from heapq so we can return the inner
# instance instead of the wrapper. This is basically free
# as it works as our caller iterates the results.
feeder = heapq.merge(*results.values())
while True:
try:
item = next(feeder)
except StopIteration:
return
if context.is_cell_failure_sentinel(item._db_record):
if not CONF.api.list_records_by_skipping_down_cells:
raise exception.NovaException(
_('Cell %s is not responding but configuration '
'indicates that we should fail.') % item.cell_uuid)
LOG.warning('Cell %s is not responding and hence is '
'being omitted from the results',
item.cell_uuid)
if item._db_record == context.did_not_respond_sentinel:
self._cells_timed_out.add(item.cell_uuid)
elif isinstance(item._db_record, Exception):
self._cells_failed.add(item.cell_uuid)
# We might have received one batch but timed out or failed
# on a later one, so be sure we fix the accounting.
if item.cell_uuid in self._cells_responded:
self._cells_responded.remove(item.cell_uuid)
continue
yield item._db_record
self._cells_responded.add(item.cell_uuid)
total_limit -= 1
if total_limit == 0:
# We'll only hit this if limit was nonzero and we just
# generated our last one
return
def get_records_sorted(self, ctx, filters, limit, marker, **kwargs):
"""Get a cross-cell list of records matching filters.
This iterates cells in parallel generating a unified and sorted
list of records as efficiently as possible. It takes care to
iterate the list as infrequently as possible. We wrap the results
in RecordWrapper objects so that they are sortable by
heapq.merge(), which requires that the '<' operator just works.
Our sorting requirements are encapsulated into the
RecordSortContext provided to the constructor for this object.
This function is a generator of records from the database like what you
would get from instance_get_all_by_filters_sort() in the DB API.
NOTE: Since we do these in parallel, a nonzero limit will be passed
to each database query, although the limit will be enforced in the
output of this function. Meaning, we will still query $limit from each
database, but only return $limit total results.
"""
if marker:
# A marker identifier was provided from the API. Call this
# the 'global' marker as it determines where we start the
# process across all cells. Look up the record in
# whatever cell it is in and record the values for the
# sort keys so we can find the marker instance in each
# cell (called the 'local' marker).
global_marker_record = self.get_marker_record(ctx, marker)
global_marker_values = [global_marker_record[key]
for key in self.sort_ctx.sort_keys]
def do_query(ctx):
"""Generate RecordWrapper(record) objects from a cell.
We do this inside the thread (created by
scatter_gather_all_cells()) so that we return wrappers and
avoid having to iterate the combined result list in the
caller again. This is run against each cell by the
scatter_gather routine.
"""
# The local marker is an identifier of a record in a cell
# that is found by the special method
# get_marker_by_values(). It should be the next record
# in order according to the sort provided, but after the
# marker instance which may have been in another cell.
local_marker = None
# Since the regular DB query routines take a marker and assume that
# the marked record was the last entry of the previous page, we
# may need to prefix it to our result query if we're not the cell
# that had the actual marker record.
local_marker_prefix = []
marker_id = self.marker_identifier
if marker:
# FIXME(danms): If we knew which cell we were in here, we could
# avoid looking up the marker again. But, we don't currently.
local_marker = self.get_marker_by_values(ctx,
global_marker_values)
if local_marker:
if local_marker != marker:
# We did find a marker in our cell, but it wasn't
# the global marker. Thus, we will use it as our
# marker in the main query below, but we also need
# to prefix that result with this marker instance
# since the result below will not return it and it
# has not been returned to the user yet. Note that
# we do _not_ prefix the marker instance if our
# marker was the global one since that has already
# been sent to the user.
local_marker_filters = copy.copy(filters)
if marker_id not in local_marker_filters:
# If an $id filter was provided, it will
# have included our marker already if this
# instance is desired in the output
# set. If it wasn't, we specifically query
# for it. If the other filters would have
# excluded it, then we'll get an empty set
# here and not include it in the output as
# expected.
local_marker_filters[marker_id] = [local_marker]
local_marker_prefix = self.get_by_filters(
ctx, local_marker_filters, limit=1, marker=None,
**kwargs)
else:
# There was a global marker but everything in our
# cell is _before_ that marker, so we return
# nothing. If we didn't have this clause, we'd
# pass marker=None to the query below and return a
# full unpaginated set for our cell.
return []
main_query_result = self.get_by_filters(
ctx, filters,
limit=limit, marker=local_marker,
**kwargs)
return (RecordWrapper(self.sort_ctx, inst) for inst in
itertools.chain(local_marker_prefix, main_query_result))
# NOTE(tssurya): When the below routine provides sentinels to indicate
# a timeout on a cell, we ignore that cell to avoid the crash when
# doing the merge below and continue merging the results from the 'up'
# cells.
# TODO(tssurya): Modify this to return the minimal available info from
# the down cells.
if self.cells:
results = context.scatter_gather_cells(ctx, self.cells, 60,
do_query)
else:
results = context.scatter_gather_all_cells(ctx, do_query)
for cell_uuid in list(results):
if results[cell_uuid] in (context.did_not_respond_sentinel,
context.raised_exception_sentinel):
LOG.warning("Cell %s is not responding and hence skipped "
"from the results.", cell_uuid)
results.pop(cell_uuid)
# If a limit was provided, it was passed to the per-cell query
# routines. That means we have NUM_CELLS * limit items across
# results. So, we need to consume from that limit below and
# stop returning results.
limit = limit or 0
# Generate results from heapq so we can return the inner
# instance instead of the wrapper. This is basically free
# as it works as our caller iterates the results.
for i in heapq.merge(*results.values()):
yield i._db_record
limit -= 1
if limit == 0:
# We'll only hit this if limit was nonzero and we just
# generated our last one
return
def get_instances_sorted(ctx, filters, limit, marker, columns_to_join,
sort_keys, sort_dirs):
"""Get a cross-cell list of instances matching filters.
This iterates cells in parallel generating a unified and sorted
list of instances as efficiently as possible. It takes care to
iterate the list as infrequently as possible. We wrap the results
in InstanceWrapper objects so that they are sortable by
heapq.merge(), which requires that the '<' operator just works. We
encapsulate our sorting requirements into an InstanceSortContext
which we pass to all of the wrappers so they behave the way we
want.
This function is a generator of instances from the database like what you
would get from instance_get_all_by_filters_sort() in the DB API.
NOTE: Since we do these in parallel, a nonzero limit will be passed
to each database query, although the limit will be enforced in the
output of this function. Meaning, we will still query $limit from each
database, but only return $limit total results.
"""
if not sort_keys:
# This is the default from the process_sort_params() method in
# the DB API. It doesn't really matter, as this only comes into
# play if the user didn't ask for a specific ordering, but we
# use the same scheme for consistency.
sort_keys = ['created_at', 'id']
sort_dirs = ['asc', 'asc']
sort_ctx = InstanceSortContext(sort_keys, sort_dirs)
if marker:
# A marker UUID was provided from the API. Call this the 'global'
# marker as it determines where we start the process across
# all cells. Look up the instance in whatever cell it is in and
# record the values for the sort keys so we can find the marker
# instance in each cell (called the 'local' marker).
global_marker_instance = _get_marker_instance(ctx, marker)
global_marker_values = [
global_marker_instance[key] for key in sort_keys
]
def do_query(ctx):
"""Generate InstanceWrapper(Instance) objects from a cell.
We do this inside the thread (created by
scatter_gather_all_cells()) so that we return wrappers and
avoid having to iterate the combined result list in the caller
again. This is run against each cell by the scatter_gather
routine.
"""
# The local marker is a uuid of an instance in a cell that is found
# by the special method instance_get_by_sort_filters(). It should
# be the next instance in order according to the sort provided,
# but after the marker instance which may have been in another cell.
local_marker = None
# Since the regular DB query routines take a marker and assume that
# the marked instance was the last entry of the previous page, we
# may need to prefix it to our result query if we're not the cell
# that had the actual marker instance.
local_marker_prefix = []
if marker:
# FIXME(danms): If we knew which cell we were in here, we could
# avoid looking up the marker again. But, we don't currently.
local_marker = db.instance_get_by_sort_filters(
ctx, sort_keys, sort_dirs, global_marker_values)
if local_marker:
if local_marker != marker:
# We did find a marker in our cell, but it wasn't
# the global marker. Thus, we will use it as our
# marker in the main query below, but we also need
# to prefix that result with this marker instance
# since the result below will not return it and it
# has not been returned to the user yet. Note that
# we do _not_ prefix the marker instance if our
# marker was the global one since that has already
# been sent to the user.
local_marker_filters = copy.copy(filters)
if 'uuid' not in local_marker_filters:
# If a uuid filter was provided, it will
# have included our marker already if this instance
# is desired in the output set. If it wasn't, we
# specifically query for it. If the other filters would
# have excluded it, then we'll get an empty set here
# and not include it in the output as expected.
local_marker_filters['uuid'] = [local_marker]
local_marker_prefix = db.instance_get_all_by_filters_sort(
ctx,
local_marker_filters,
limit=1,
marker=None,
columns_to_join=columns_to_join,
sort_keys=sort_keys,
sort_dirs=sort_dirs)
else:
# There was a global marker but everything in our cell is
# _before_ that marker, so we return nothing. If we didn't
# have this clause, we'd pass marker=None to the query below
# and return a full unpaginated set for our cell.
return []
main_query_result = db.instance_get_all_by_filters_sort(
ctx,
filters,
limit=limit,
marker=local_marker,
columns_to_join=columns_to_join,
sort_keys=sort_keys,
sort_dirs=sort_dirs)
return (InstanceWrapper(sort_ctx, inst) for inst in itertools.chain(
local_marker_prefix, main_query_result))
# FIXME(danms): If we raise or timeout on a cell we need to handle
# that here gracefully. The below routine will provide sentinels
# to indicate that, which will crash the merge below, but we don't
# handle this anywhere yet anyway.
results = context.scatter_gather_all_cells(ctx, do_query)
# If a limit was provided, and passed to the per-cell query routines.
# That means we have NUM_CELLS * limit items across results. So, we
# need to consume from that limit below and stop returning results.
limit = limit or 0
# Generate results from heapq so we can return the inner
# instance instead of the wrapper. This is basically free
# as it works as our caller iterates the results.
for i in heapq.merge(*results.values()):
yield i._db_instance
limit -= 1
if limit == 0:
# We'll only hit this if limit was nonzero and we just generated
# our last one
return
def get_records_sorted(self, ctx, filters, limit, marker, **kwargs):
"""Get a cross-cell list of records matching filters.
This iterates cells in parallel generating a unified and sorted
list of records as efficiently as possible. It takes care to
iterate the list as infrequently as possible. We wrap the results
in RecordWrapper objects so that they are sortable by
heapq.merge(), which requires that the '<' operator just works.
Our sorting requirements are encapsulated into the
RecordSortContext provided to the constructor for this object.
This function is a generator of records from the database like what you
would get from instance_get_all_by_filters_sort() in the DB API.
NOTE: Since we do these in parallel, a nonzero limit will be passed
to each database query, although the limit will be enforced in the
output of this function. Meaning, we will still query $limit from each
database, but only return $limit total results.
"""
if marker:
# A marker identifier was provided from the API. Call this
# the 'global' marker as it determines where we start the
# process across all cells. Look up the record in
# whatever cell it is in and record the values for the
# sort keys so we can find the marker instance in each
# cell (called the 'local' marker).
global_marker_record = self.get_marker_record(ctx, marker)
global_marker_values = [
global_marker_record[key] for key in self.sort_ctx.sort_keys
]
def do_query(ctx):
"""Generate RecordWrapper(record) objects from a cell.
We do this inside the thread (created by
scatter_gather_all_cells()) so that we return wrappers and
avoid having to iterate the combined result list in the
caller again. This is run against each cell by the
scatter_gather routine.
"""
# The local marker is an identifier of a record in a cell
# that is found by the special method
# get_marker_by_values(). It should be the next record
# in order according to the sort provided, but after the
# marker instance which may have been in another cell.
local_marker = None
# Since the regular DB query routines take a marker and assume that
# the marked record was the last entry of the previous page, we
# may need to prefix it to our result query if we're not the cell
# that had the actual marker record.
local_marker_prefix = []
marker_id = self.marker_identifier
if marker:
# FIXME(danms): If we knew which cell we were in here, we could
# avoid looking up the marker again. But, we don't currently.
local_marker = self.get_marker_by_values(
ctx, global_marker_values)
if local_marker:
if local_marker != marker:
# We did find a marker in our cell, but it wasn't
# the global marker. Thus, we will use it as our
# marker in the main query below, but we also need
# to prefix that result with this marker instance
# since the result below will not return it and it
# has not been returned to the user yet. Note that
# we do _not_ prefix the marker instance if our
# marker was the global one since that has already
# been sent to the user.
local_marker_filters = copy.copy(filters)
if marker_id not in local_marker_filters:
# If an $id filter was provided, it will
# have included our marker already if this
# instance is desired in the output
# set. If it wasn't, we specifically query
# for it. If the other filters would have
# excluded it, then we'll get an empty set
# here and not include it in the output as
# expected.
local_marker_filters[marker_id] = [local_marker]
local_marker_prefix = self.get_by_filters(
ctx,
local_marker_filters,
limit=1,
marker=None,
**kwargs)
else:
# There was a global marker but everything in our
# cell is _before_ that marker, so we return
# nothing. If we didn't have this clause, we'd
# pass marker=None to the query below and return a
# full unpaginated set for our cell.
return []
main_query_result = self.get_by_filters(ctx,
filters,
limit=limit,
marker=local_marker,
**kwargs)
return (RecordWrapper(self.sort_ctx,
inst) for inst in itertools.chain(
local_marker_prefix, main_query_result))
# NOTE(tssurya): When the below routine provides sentinels to indicate
# a timeout on a cell, we ignore that cell to avoid the crash when
# doing the merge below and continue merging the results from the 'up'
# cells.
# TODO(tssurya): Modify this to return the minimal available info from
# the down cells.
if self.cells:
results = context.scatter_gather_cells(ctx, self.cells, 60,
do_query)
else:
results = context.scatter_gather_all_cells(ctx, do_query)
for cell_uuid in list(results):
if results[cell_uuid] in (context.did_not_respond_sentinel,
context.raised_exception_sentinel):
LOG.warning(
"Cell %s is not responding and hence skipped "
"from the results.", cell_uuid)
results.pop(cell_uuid)
# If a limit was provided, it was passed to the per-cell query
# routines. That means we have NUM_CELLS * limit items across
# results. So, we need to consume from that limit below and
# stop returning results.
limit = limit or 0
# Generate results from heapq so we can return the inner
# instance instead of the wrapper. This is basically free
# as it works as our caller iterates the results.
for i in heapq.merge(*results.values()):
yield i._db_record
limit -= 1
if limit == 0:
# We'll only hit this if limit was nonzero and we just
# generated our last one
return
