def test_get_instances_to_sync(self): fake_context = 'fake_context' call_info = {'get_all': 0, 'shuffle': 0} def random_shuffle(_list): call_info['shuffle'] += 1 @staticmethod def instance_get_all_by_filters(context, filters, sort_key, sort_dir, limit, marker): # Pretend we return a full list the first time otherwise we loop # infinitely if marker is not None: return [] self.assertEqual(fake_context, context) self.assertEqual('deleted', sort_key) self.assertEqual('asc', sort_dir) call_info['got_filters'] = filters call_info['get_all'] += 1 instances = [fake_instance.fake_db_instance() for i in range(3)] return instances self.stub_out('nova.objects.InstanceList.get_by_filters', instance_get_all_by_filters) self.stub_out('random.shuffle', random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(1, call_info['get_all']) self.assertEqual({}, call_info['got_filters']) self.assertEqual(0, call_info['shuffle']) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(2, call_info['get_all']) self.assertEqual({}, call_info['got_filters']) self.assertEqual(1, call_info['shuffle']) instances = cells_utils.get_instances_to_sync(fake_context, updated_since='fake-updated-since') self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(3, call_info['get_all']) self.assertEqual({'changes-since': 'fake-updated-since'}, call_info['got_filters']) self.assertEqual(1, call_info['shuffle']) instances = cells_utils.get_instances_to_sync(fake_context, project_id='fake-project', updated_since='fake-updated-since', shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(4, call_info['get_all']) self.assertEqual({'changes-since': 'fake-updated-since', 'project_id': 'fake-project'}, call_info['got_filters']) self.assertEqual(2, call_info['shuffle'])
def test_get_instances_to_sync(self): fake_context = 'fake_context' call_info = {'get_all': 0, 'shuffle': 0} def random_shuffle(_list): call_info['shuffle'] += 1 @staticmethod def instance_get_all_by_filters(context, filters, sort_key, sort_dir): self.assertEqual(fake_context, context) self.assertEqual('deleted', sort_key) self.assertEqual('asc', sort_dir) call_info['got_filters'] = filters call_info['get_all'] += 1 return ['fake_instance1', 'fake_instance2', 'fake_instance3'] self.stubs.Set(objects.InstanceList, 'get_by_filters', instance_get_all_by_filters) self.stubs.Set(random, 'shuffle', random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(1, call_info['get_all']) self.assertEqual({}, call_info['got_filters']) self.assertEqual(0, call_info['shuffle']) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(2, call_info['get_all']) self.assertEqual({}, call_info['got_filters']) self.assertEqual(1, call_info['shuffle']) instances = cells_utils.get_instances_to_sync( fake_context, updated_since='fake-updated-since') self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(3, call_info['get_all']) self.assertEqual({'changes-since': 'fake-updated-since'}, call_info['got_filters']) self.assertEqual(1, call_info['shuffle']) instances = cells_utils.get_instances_to_sync( fake_context, project_id='fake-project', updated_since='fake-updated-since', shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(4, call_info['get_all']) self.assertEqual( { 'changes-since': 'fake-updated-since', 'project_id': 'fake-project' }, call_info['got_filters']) self.assertEqual(2, call_info['shuffle'])
def test_get_instances_to_sync(self): fake_context = 'fake_context' call_info = {'get_all': 0, 'shuffle': 0} def random_shuffle(_list): call_info['shuffle'] += 1 def instance_get_all_by_filters(context, filters, sort_key, sort_order): self.assertEqual(context, fake_context) self.assertEqual(sort_key, 'deleted') self.assertEqual(sort_order, 'asc') call_info['got_filters'] = filters call_info['get_all'] += 1 return ['fake_instance1', 'fake_instance2', 'fake_instance3'] self.stubs.Set(db, 'instance_get_all_by_filters', instance_get_all_by_filters) self.stubs.Set(random, 'shuffle', random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 1) self.assertEqual(call_info['got_filters'], {}) self.assertEqual(call_info['shuffle'], 0) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 2) self.assertEqual(call_info['got_filters'], {}) self.assertEqual(call_info['shuffle'], 1) instances = cells_utils.get_instances_to_sync( fake_context, updated_since='fake-updated-since') self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 3) self.assertEqual(call_info['got_filters'], {'changes-since': 'fake-updated-since'}) self.assertEqual(call_info['shuffle'], 1) instances = cells_utils.get_instances_to_sync( fake_context, project_id='fake-project', updated_since='fake-updated-since', shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 4) self.assertEqual(call_info['got_filters'], { 'changes-since': 'fake-updated-since', 'project_id': 'fake-project' }) self.assertEqual(call_info['shuffle'], 2)
def test_get_instances_to_sync(self): fake_context = 'fake_context' call_info = {'get_all': 0, 'shuffle': 0} def random_shuffle(_list): call_info['shuffle'] += 1 @staticmethod def instance_get_all_by_filters(context, filters, sort_key, sort_dir): self.assertEqual(context, fake_context) self.assertEqual(sort_key, 'deleted') self.assertEqual(sort_dir, 'asc') call_info['got_filters'] = filters call_info['get_all'] += 1 return ['fake_instance1', 'fake_instance2', 'fake_instance3'] self.stubs.Set(objects.InstanceList, 'get_by_filters', instance_get_all_by_filters) self.stubs.Set(random, 'shuffle', random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 1) self.assertEqual(call_info['got_filters'], {}) self.assertEqual(call_info['shuffle'], 0) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 2) self.assertEqual(call_info['got_filters'], {}) self.assertEqual(call_info['shuffle'], 1) instances = cells_utils.get_instances_to_sync(fake_context, updated_since='fake-updated-since') self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 3) self.assertEqual(call_info['got_filters'], {'changes-since': 'fake-updated-since'}) self.assertEqual(call_info['shuffle'], 1) instances = cells_utils.get_instances_to_sync(fake_context, project_id='fake-project', updated_since='fake-updated-since', shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 4) self.assertEqual(call_info['got_filters'], {'changes-since': 'fake-updated-since', 'project_id': 'fake-project'}) self.assertEqual(call_info['shuffle'], 2)
def test_get_instances_to_sync(self): fake_context = "fake_context" call_info = {"get_all": 0, "shuffle": 0} def random_shuffle(_list): call_info["shuffle"] += 1 def instance_get_all_by_filters(context, filters, sort_key, sort_order): self.assertEqual(context, fake_context) self.assertEqual(sort_key, "deleted") self.assertEqual(sort_order, "asc") call_info["got_filters"] = filters call_info["get_all"] += 1 return ["fake_instance1", "fake_instance2", "fake_instance3"] self.stubs.Set(db, "instance_get_all_by_filters", instance_get_all_by_filters) self.stubs.Set(random, "shuffle", random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 1) self.assertEqual(call_info["got_filters"], {}) self.assertEqual(call_info["shuffle"], 0) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 2) self.assertEqual(call_info["got_filters"], {}) self.assertEqual(call_info["shuffle"], 1) instances = cells_utils.get_instances_to_sync(fake_context, updated_since="fake-updated-since") self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 3) self.assertEqual(call_info["got_filters"], {"changes-since": "fake-updated-since"}) self.assertEqual(call_info["shuffle"], 1) instances = cells_utils.get_instances_to_sync( fake_context, project_id="fake-project", updated_since="fake-updated-since", shuffle=True ) self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 4) self.assertEqual( call_info["got_filters"], {"changes-since": "fake-updated-since", "project_id": "fake-project"} ) self.assertEqual(call_info["shuffle"], 2)
def _test_get_instances_pagination(self, mock_shuffle, mock_get_by_filters, shuffle=False, updated_since=None, project_id=None): fake_context = 'fake_context' instances0 = objects.instance._make_instance_list( fake_context, objects.InstanceList(), [fake_instance.fake_db_instance() for i in range(3)], expected_attrs=None) marker0 = instances0[-1]['uuid'] instances1 = objects.instance._make_instance_list( fake_context, objects.InstanceList(), [fake_instance.fake_db_instance() for i in range(3)], expected_attrs=None) marker1 = instances1[-1]['uuid'] mock_get_by_filters.side_effect = [instances0, instances1, []] instances = cells_utils.get_instances_to_sync(fake_context, updated_since, project_id, shuffle=shuffle) self.assertEqual(len([x for x in instances]), 6) filters = {} if updated_since is not None: filters['changes-since'] = updated_since if project_id is not None: filters['project_id'] = project_id limit = 100 expected_calls = [ mock.call(fake_context, filters, sort_key='deleted', sort_dir='asc', limit=limit, marker=None), mock.call(fake_context, filters, sort_key='deleted', sort_dir='asc', limit=limit, marker=marker0), mock.call(fake_context, filters, sort_key='deleted', sort_dir='asc', limit=limit, marker=marker1) ] mock_get_by_filters.assert_has_calls(expected_calls) self.assertEqual(3, mock_get_by_filters.call_count)
def test_sync_instances(self): # Reset this, as this is a broadcast down. self._setup_attrs(up=False) project_id = 'fake_project_id' updated_since_raw = 'fake_updated_since_raw' updated_since_parsed = 'fake_updated_since_parsed' deleted = 'fake_deleted' instance1 = dict(uuid='fake_uuid1', deleted=False) instance2 = dict(uuid='fake_uuid2', deleted=True) fake_instances = [instance1, instance2] self.mox.StubOutWithMock(self.tgt_msg_runner, 'instance_update_at_top') self.mox.StubOutWithMock(self.tgt_msg_runner, 'instance_destroy_at_top') self.mox.StubOutWithMock(timeutils, 'parse_isotime') self.mox.StubOutWithMock(cells_utils, 'get_instances_to_sync') # Middle cell. timeutils.parse_isotime(updated_since_raw).AndReturn( updated_since_parsed) cells_utils.get_instances_to_sync(self.ctxt, updated_since=updated_since_parsed, project_id=project_id, deleted=deleted).AndReturn([]) # Bottom/Target cell timeutils.parse_isotime(updated_since_raw).AndReturn( updated_since_parsed) cells_utils.get_instances_to_sync(self.ctxt, updated_since=updated_since_parsed, project_id=project_id, deleted=deleted).AndReturn(fake_instances) self.tgt_msg_runner.instance_update_at_top(self.ctxt, instance1) self.tgt_msg_runner.instance_destroy_at_top(self.ctxt, instance2) self.mox.ReplayAll() self.src_msg_runner.sync_instances(self.ctxt, project_id, updated_since_raw, deleted)
def sync_instances(self, message, project_id, updated_since, deleted, **kwargs): projid_str = project_id is None and "<all>" or project_id since_str = updated_since is None and "<all>" or updated_since LOG.info(_("Forcing a sync of instances, project_id=" "%(projid_str)s, updated_since=%(since_str)s"), locals()) if updated_since is not None: updated_since = timeutils.parse_isotime(updated_since) instances = cells_utils.get_instances_to_sync( message.ctxt, updated_since=updated_since, project_id=project_id, deleted=deleted ) for instance in instances: self._sync_instance(message.ctxt, instance)
def sync_instances(self, message, project_id, updated_since, deleted, **kwargs): projid_str = project_id is None and "<all>" or project_id since_str = updated_since is None and "<all>" or updated_since LOG.info( _("Forcing a sync of instances, project_id=" "%(projid_str)s, updated_since=%(since_str)s"), locals()) if updated_since is not None: updated_since = timeutils.parse_isotime(updated_since) instances = cells_utils.get_instances_to_sync( message.ctxt, updated_since=updated_since, project_id=project_id, deleted=deleted) for instance in instances: self._sync_instance(message.ctxt, instance)
def _next_instance(): try: instance = next(self.instances_to_heal) except StopIteration: if info["updated_list"]: return threshold = CONF.cells.instance_updated_at_threshold updated_since = None if threshold > 0: updated_since = timeutils.utcnow() - datetime.timedelta(seconds=threshold) self.instances_to_heal = cells_utils.get_instances_to_sync( ctxt, updated_since=updated_since, shuffle=True, uuids_only=True ) info["updated_list"] = True try: instance = next(self.instances_to_heal) except StopIteration: return return instance
def _next_instance(): try: instance = next(self.instances_to_heal) except StopIteration: if info['updated_list']: return threshold = CONF.cells.instance_updated_at_threshold updated_since = None if threshold > 0: updated_since = timeutils.utcnow() - datetime.timedelta( seconds=threshold) self.instances_to_heal = cells_utils.get_instances_to_sync( ctxt, updated_since=updated_since, shuffle=True, uuids_only=True) info['updated_list'] = True try: instance = next(self.instances_to_heal) except StopIteration: return return instance
def _test_get_instances_pagination(self, mock_shuffle, mock_get_by_filters, shuffle=False, updated_since=None, project_id=None): fake_context = 'fake_context' instances0 = objects.instance._make_instance_list(fake_context, objects.InstanceList(), [fake_instance.fake_db_instance() for i in range(3)], expected_attrs=None) marker0 = instances0[-1]['uuid'] instances1 = objects.instance._make_instance_list(fake_context, objects.InstanceList(), [fake_instance.fake_db_instance() for i in range(3)], expected_attrs=None) marker1 = instances1[-1]['uuid'] mock_get_by_filters.side_effect = [instances0, instances1, []] instances = cells_utils.get_instances_to_sync(fake_context, updated_since, project_id, shuffle=shuffle) self.assertEqual(len([x for x in instances]), 6) filters = {} if updated_since is not None: filters['changes-since'] = updated_since if project_id is not None: filters['project_id'] = project_id limit = 100 expected_calls = [mock.call(fake_context, filters, sort_key='deleted', sort_dir='asc', limit=limit, marker=None), mock.call(fake_context, filters, sort_key='deleted', sort_dir='asc', limit=limit, marker=marker0), mock.call(fake_context, filters, sort_key='deleted', sort_dir='asc', limit=limit, marker=marker1)] mock_get_by_filters.assert_has_calls(expected_calls) self.assertEqual(3, mock_get_by_filters.call_count)