def test_scatter_gather_cells_timeout(self, mock_get_inst, mock_get_result, mock_timeout, mock_log_warning): # This is needed because we're mocking get_by_filters. self.useFixture(nova_fixtures.SpawnIsSynchronousFixture()) ctxt = context.get_context() mapping0 = objects.CellMapping(database_connection='fake://db0', transport_url='none:///', uuid=objects.CellMapping.CELL0_UUID) mapping1 = objects.CellMapping(database_connection='fake://db1', transport_url='fake://mq1', uuid=uuids.cell1) mappings = objects.CellMappingList(objects=[mapping0, mapping1]) # Simulate cell1 not responding. mock_get_result.side_effect = [(mapping0.uuid, mock.sentinel.instances), exception.CellTimeout()] results = context.scatter_gather_cells( ctxt, mappings, 30, objects.InstanceList.get_by_filters) self.assertEqual(2, len(results)) self.assertIn(mock.sentinel.instances, results.values()) self.assertIn(context.did_not_respond_sentinel, results.values()) mock_timeout.assert_called_once_with(30, exception.CellTimeout) self.assertTrue(mock_log_warning.called)
def test_scatter_gather_cells_all_timeout(self, mock_get_inst, mock_get_result, mock_timeout, mock_log_warning): """This is a regression test for bug 1847131. test_scatter_gather_cells_timeout did not catch the issue because it yields a result which sets the cell_uuid variable in scope before the CellTimeout is processed and logged. In this test we only raise the CellTimeout so cell_uuid will not be in scope for the log message. """ # This is needed because we're mocking get_by_filters. self.useFixture(nova_fixtures.SpawnIsSynchronousFixture()) ctxt = context.get_context() mapping0 = objects.CellMapping(database_connection='fake://db0', transport_url='none:///', uuid=objects.CellMapping.CELL0_UUID) mappings = objects.CellMappingList(objects=[mapping0]) # Simulate cell0 not responding. mock_get_result.side_effect = exception.CellTimeout() results = context.scatter_gather_cells( ctxt, mappings, 30, objects.InstanceList.get_by_filters, {}) self.assertEqual(1, len(results)) self.assertIn(context.did_not_respond_sentinel, results.values()) mock_timeout.assert_called_once_with(30, exception.CellTimeout) mock_log_warning.assert_called_once_with( 'Timed out waiting for response from cell', exc_info=True)
def _wait_for_json_responses(self, num_responses=1): """Wait for response(s) to be put into the eventlet queue. Since each queue entry actually contains a list of JSON-ified responses, combine them all into a single list to return. Destroy the eventlet queue when done. """ if not self.resp_queue: # Source is not actually expecting a response return responses = [] wait_time = CONF.cells.call_timeout try: for x in xrange(num_responses): json_responses = self.resp_queue.get(timeout=wait_time) responses.extend(json_responses) except queue.Empty: raise exception.CellTimeout() finally: self._cleanup_response_queue() return responses
def get_minimum_version_all_cells(context, binaries, require_all=False): """Get the minimum service version, checking all cells. This attempts to calculate the minimum service version for a set of binaries across all the cells in the system. If require_all is False, then any cells that fail to report a version will be ignored (assuming they won't be candidates for scheduling and thus excluding them from the minimum version calculation is reasonable). If require_all is True, then a failing cell will cause this to raise exception.CellTimeout, as would be appropriate for gating some data migration until everything is new enough. Note that services that do not report a positive version are excluded from this, as it crosses all cells which will naturally not have all services. """ if not all(binary.startswith('nova-') for binary in binaries): LOG.warning( 'get_minimum_version_all_cells called with ' 'likely-incorrect binaries `%s\'', ','.join(binaries)) raise exception.ObjectActionError( action='get_minimum_version_all_cells', reason='Invalid binary prefix') # NOTE(danms): Instead of using Service.get_minimum_version_multi(), we # replicate the call directly to the underlying DB method here because # we want to defeat the caching and we need to filter non-present # services differently from the single-cell method. results = nova_context.scatter_gather_all_cells( context, Service._db_service_get_minimum_version, binaries) min_version = None for cell_uuid, result in results.items(): if result is nova_context.did_not_respond_sentinel: LOG.warning( 'Cell %s did not respond when getting minimum ' 'service version', cell_uuid) if require_all: raise exception.CellTimeout() elif isinstance(result, Exception): LOG.warning('Failed to get minimum service version for cell %s', cell_uuid) if require_all: # NOTE(danms): Okay, this isn't necessarily a timeout, but # it's functionally the same from the caller's perspective # and we logged the fact that it was actually a failure # for the forensic investigator during the scatter/gather # routine. raise exception.CellTimeout() else: # NOTE(danms): Don't consider a zero or None result as the minimum # since we're crossing cells and will likely not have all the # services being probed. relevant_versions = [ version for version in result.values() if version ] if relevant_versions: min_version_cell = min(relevant_versions) min_version = (min(min_version, min_version_cell) if min_version else min_version_cell) # NOTE(danms): If we got no matches at all (such as at first startup) # then report that as zero to be consistent with the other such # methods. return min_version or 0