def test_get_sorted_with_deleted_marker(self): marker = self.instances[1]['uuid'] before = list( instance_list.get_instances_sorted(self.context, {}, None, marker, [], None, None)) db.instance_destroy(self.context, marker) after = list( instance_list.get_instances_sorted(self.context, {}, None, marker, [], None, None)) self.assertEqual(before, after)
def test_get_sorted_with_large_limit(self): insts = instance_list.get_instances_sorted(self.context, {}, 5000, None, [], ['uuid'], ['asc']) uuids = [inst['uuid'] for inst in insts] self.assertEqual(sorted(uuids), uuids) self.assertEqual(len(self.instances), len(uuids))
def _test_get_sorted_with_limit_marker(self, sort_by, pages=2, pagesize=2): insts = [] page = 0 while True: if page >= pages: limit = None else: limit = pagesize if insts: marker = insts[-1]['uuid'] else: marker = None batch = list( instance_list.get_instances_sorted(self.context, {}, limit, marker, [], [sort_by], ['asc'])) if not batch: break insts.extend(batch) page += 1 # We should have requested exactly (or one more unlimited) pages self.assertIn(page, (pages, pages + 1)) # Make sure the full set matches what we know to be true found = [x[sort_by] for x in insts] had = [x[sort_by] for x in self.instances] if sort_by == 'launched_at': # We're comparing objects and database entries, so we need to # squash the tzinfo of the object ones so we can compare had = [x.replace(tzinfo=None) for x in had] self.assertEqual(sorted(had), found)
def test_get_sorted_paginated_with_faults(self): """Get pages of one with faults. Do this specifically so we make sure we land on faulted marker instances to ensure we don't omit theirs. """ insts = [] while True: if insts: marker = insts[-1]['uuid'] else: marker = None batch = list( instance_list.get_instances_sorted(self.context, {}, 1, marker, ['fault'], ['hostname'], ['asc'])) if not batch: break insts.extend(batch) self.assertEqual(len(self.instances), len(insts)) # Two of the instances in each cell have faults (0th and 2nd) expected_faults = self.NUMBER_OF_CELLS * 2 expected_no_fault = len(self.instances) - expected_faults faults = [inst['fault'] for inst in insts] self.assertEqual(expected_no_fault, faults.count(None))
def test_get_instances_with_down_cells(self, mock_sg): inst_cell0 = self.insts[uuids.cell0] # storing the uuids of the instances from the up cell uuid_initial = [inst['uuid'] for inst in inst_cell0] def wrap(thing): return multi_cell_list.RecordWrapper(ctx, self.context, thing) ctx = nova_context.RequestContext() instances = [wrap(inst) for inst in inst_cell0] # creating one up cell and two down cells ret_val = {} ret_val[uuids.cell0] = instances ret_val[uuids.cell1] = [wrap(exception.BuildRequestNotFound(uuid='f'))] ret_val[uuids.cell2] = [wrap(nova_context.did_not_respond_sentinel)] mock_sg.return_value = ret_val obj, res = instance_list.get_instances_sorted(self.context, {}, None, None, [], None, None) uuid_final = [inst['uuid'] for inst in res] # return the results from the up cell, ignoring the down cell. self.assertEqual(uuid_initial, uuid_final)
def test_get_sorted_with_large_limit(self): obj, insts = instance_list.get_instances_sorted(self.context, {}, 5000, None, [], ['uuid'], ['asc']) uuids = [inst['uuid'] for inst in insts] self.assertEqual(sorted(uuids), uuids) self.assertEqual(len(self.instances), len(uuids))
def test_get_sorted_paginated_with_faults(self): """Get pages of one with faults. Do this specifically so we make sure we land on faulted marker instances to ensure we don't omit theirs. """ insts = [] while True: if insts: marker = insts[-1]['uuid'] else: marker = None batch = list( instance_list.get_instances_sorted(self.context, {}, 1, marker, ['fault'], ['hostname'], ['asc'])[1]) if not batch: break insts.extend(batch) self.assertEqual(len(self.instances), len(insts)) # Two of the instances in each cell have faults (0th and 2nd) expected_faults = self.NUMBER_OF_CELLS * 2 expected_no_fault = len(self.instances) - expected_faults faults = [inst['fault'] for inst in insts] self.assertEqual(expected_no_fault, faults.count(None))
def test_get_instances_with_down_cells(self, mock_sg): inst_cell0 = self.insts[uuids.cell0] # storing the uuids of the instances from the up cell uuid_initial = [inst['uuid'] for inst in inst_cell0] def wrap(thing): return multi_cell_list.RecordWrapper(ctx, self.context, thing) ctx = nova_context.RequestContext() instances = [wrap(inst) for inst in inst_cell0] # creating one up cell and two down cells ret_val = {} ret_val[uuids.cell0] = instances ret_val[uuids.cell1] = [wrap(nova_context.raised_exception_sentinel)] ret_val[uuids.cell2] = [wrap(nova_context.did_not_respond_sentinel)] mock_sg.return_value = ret_val res = instance_list.get_instances_sorted(self.context, {}, None, None, [], None, None) uuid_final = [inst['uuid'] for inst in res] # return the results from the up cell, ignoring the down cell. self.assertEqual(uuid_initial, uuid_final)
def test_get_sorted_with_limit(self): insts = instance_list.get_instances_sorted(self.context, {}, 5, None, [], ['uuid'], ['asc']) uuids = [inst['uuid'] for inst in insts] had_uuids = [inst.uuid for inst in self.instances] self.assertEqual(sorted(had_uuids)[:5], uuids) self.assertEqual(5, len(uuids))
def test_get_instances_with_cell_down_support(self, mock_sg): self.flags(list_records_by_skipping_down_cells=False, group='api') inst_cell0 = self.insts[uuids.cell0] # storing the uuids of the instances from the up cell uuid_initial = [inst['uuid'] for inst in inst_cell0] def wrap(thing): return multi_cell_list.RecordWrapper(ctx, self.context, thing) ctx = nova_context.RequestContext() instances = [wrap(inst) for inst in inst_cell0] # creating one up cell and two down cells ret_val = {} ret_val[uuids.cell0] = instances ret_val[uuids.cell1] = [wrap(exception.BuildRequestNotFound(uuid='f'))] ret_val[uuids.cell2] = [wrap(nova_context.did_not_respond_sentinel)] mock_sg.return_value = ret_val # From the new microversion (2.68) if cell_down_support is True # then CONF.api.list_records_by_skipping_down_cells will be ignored. # Exception will not be raised even if its False. obj, res = instance_list.get_instances_sorted(self.context, {}, None, None, [], None, None, cell_down_support=True) uuid_final = [inst['uuid'] for inst in res] # return the results from the up cell, ignoring the down cell and # constructing partial results later. self.assertEqual(uuid_initial, uuid_final)
def test_get_sorted_with_deleted_marker(self): marker = self.instances[1]['uuid'] before = list( instance_list.get_instances_sorted(self.context, {}, None, marker, [], None, None)[1]) db.instance_destroy(self.context, marker) after = list( instance_list.get_instances_sorted(self.context, {}, None, marker, [], None, None)[1]) self.assertEqual(before, after)
def test_get_instances_with_cell_down_support(self, mock_sg): self.flags(list_records_by_skipping_down_cells=False, group='api') inst_cell0 = self.insts[uuids.cell0] # storing the uuids of the instances from the up cell uuid_initial = [inst['uuid'] for inst in inst_cell0] def wrap(thing): return multi_cell_list.RecordWrapper(ctx, self.context, thing) ctx = nova_context.RequestContext() instances = [wrap(inst) for inst in inst_cell0] # creating one up cell and two down cells ret_val = {} ret_val[uuids.cell0] = instances ret_val[uuids.cell1] = [wrap(exception.BuildRequestNotFound(uuid='f'))] ret_val[uuids.cell2] = [wrap(nova_context.did_not_respond_sentinel)] mock_sg.return_value = ret_val # From the new microversion (2.68) if cell_down_support is True # then CONF.api.list_records_by_skipping_down_cells will be ignored. # Exception will not be raised even if its False. obj, res = instance_list.get_instances_sorted(self.context, {}, None, None, [], None, None, cell_down_support=True) uuid_final = [inst['uuid'] for inst in res] # return the results from the up cell, ignoring the down cell and # constructing partial results later. self.assertEqual(uuid_initial, uuid_final)
def test_get_sorted_with_limit(self): obj, insts = instance_list.get_instances_sorted(self.context, {}, 5, None, [], ['uuid'], ['asc']) uuids = [inst['uuid'] for inst in insts] had_uuids = [inst.uuid for inst in self.instances] self.assertEqual(sorted(had_uuids)[:5], uuids) self.assertEqual(5, len(uuids))
def test_get_instances_sorted(self, mock_cells, mock_inst): mock_cells.return_value = self.cells insts_by_cell = self.insts.values() mock_inst.side_effect = insts_by_cell obj, insts = instance_list.get_instances_sorted( self.context, {}, None, None, [], ['hostname'], ['asc']) insts_one = [inst['hostname'] for inst in insts] # Reverse the order that we get things from the cells so we can # make sure that the result is still sorted the same way insts_by_cell = list(reversed(list(insts_by_cell))) mock_inst.reset_mock() mock_inst.side_effect = insts_by_cell obj, insts = instance_list.get_instances_sorted( self.context, {}, None, None, [], ['hostname'], ['asc']) insts_two = [inst['hostname'] for inst in insts] self.assertEqual(insts_one, insts_two)
def test_get_sorted_with_purged_instance(self): """Test that we handle a mapped but purged instance.""" im = objects.InstanceMapping(self.context, instance_uuid=uuids.missing, project_id=self.context.project_id, user_id=self.context.user_id, cell=self.cells[0]) im.create() self.assertRaises(exception.MarkerNotFound, list, instance_list.get_instances_sorted( self.context, {}, None, uuids.missing, [], None, None)[1])
def test_get_sorted_with_faults(self): """Make sure we get faults when we ask for them.""" insts = list( instance_list.get_instances_sorted(self.context, {}, None, None, ['fault'], ['hostname'], ['asc'])) # Two of the instances in each cell have faults (0th and 2nd) expected_faults = self.NUMBER_OF_CELLS * 2 expected_no_fault = len(self.instances) - expected_faults faults = [inst['fault'] for inst in insts] self.assertEqual(expected_no_fault, faults.count(None))
def test_get_sorted_with_purged_instance(self): """Test that we handle a mapped but purged instance.""" im = objects.InstanceMapping(self.context, instance_uuid=uuids.missing, project_id=self.context.project_id, user_id=self.context.user_id, cell=self.cells[0]) im.create() self.assertRaises( exception.MarkerNotFound, list, instance_list.get_instances_sorted(self.context, {}, None, uuids.missing, [], None, None))
def test_get_sorted_by_defaults(self): filters = {} limit = None marker = None columns = [] sort_keys = None sort_dirs = None obj, insts = instance_list.get_instances_sorted(self.context, filters, limit, marker, columns, sort_keys, sort_dirs) uuids = set([inst['uuid'] for inst in insts]) expected = set([inst['uuid'] for inst in self.instances]) self.assertEqual(expected, uuids)
def test_get_instances_sorted(self, mock_cells, mock_inst): mock_cells.return_value = self.cells insts_by_cell = self.insts.values() mock_inst.side_effect = insts_by_cell obj, insts = instance_list.get_instances_sorted(self.context, {}, None, None, [], ['hostname'], ['asc']) insts_one = [inst['hostname'] for inst in insts] # Reverse the order that we get things from the cells so we can # make sure that the result is still sorted the same way insts_by_cell = list(reversed(list(insts_by_cell))) mock_inst.reset_mock() mock_inst.side_effect = insts_by_cell obj, insts = instance_list.get_instances_sorted(self.context, {}, None, None, [], ['hostname'], ['asc']) insts_two = [inst['hostname'] for inst in insts] self.assertEqual(insts_one, insts_two)
def test_get_sorted_descending(self): filters = {} limit = None marker = None columns = [] sort_keys = ['uuid'] sort_dirs = ['desc'] insts = instance_list.get_instances_sorted(self.context, filters, limit, marker, columns, sort_keys, sort_dirs) uuids = [inst['uuid'] for inst in insts] self.assertEqual(list(reversed(sorted(uuids))), uuids) self.assertEqual(len(self.instances), len(uuids))
def test_get_sorted_by_defaults(self): filters = {} limit = None marker = None columns = [] sort_keys = None sort_dirs = None insts = instance_list.get_instances_sorted(self.context, filters, limit, marker, columns, sort_keys, sort_dirs) uuids = set([inst['uuid'] for inst in insts]) expected = set([inst['uuid'] for inst in self.instances]) self.assertEqual(expected, uuids)
def test_get_sorted_with_faults(self): """Make sure we get faults when we ask for them.""" insts = list( instance_list.get_instances_sorted(self.context, {}, None, None, ['fault'], ['hostname'], ['asc'])[1]) # Two of the instances in each cell have faults (0th and 2nd) expected_faults = self.NUMBER_OF_CELLS * 2 expected_no_fault = len(self.instances) - expected_faults faults = [inst['fault'] for inst in insts] self.assertEqual(expected_no_fault, faults.count(None))
def test_get_sorted_descending(self): filters = {} limit = None marker = None columns = [] sort_keys = ['uuid'] sort_dirs = ['desc'] obj, insts = instance_list.get_instances_sorted(self.context, filters, limit, marker, columns, sort_keys, sort_dirs) uuids = [inst['uuid'] for inst in insts] self.assertEqual(list(reversed(sorted(uuids))), uuids) self.assertEqual(len(self.instances), len(uuids))
def test_get_sorted_with_filter(self): filters = {'instance_type_id': 1} limit = None marker = None columns = [] sort_keys = ['uuid'] sort_dirs = ['asc'] insts = instance_list.get_instances_sorted(self.context, filters, limit, marker, columns, sort_keys, sort_dirs) uuids = [inst['uuid'] for inst in insts] expected = [inst['uuid'] for inst in self.instances if inst['instance_type_id'] == 1] self.assertEqual(list(sorted(expected)), uuids)
def test_get_sorted_with_filter(self): filters = {'instance_type_id': 1} limit = None marker = None columns = [] sort_keys = ['uuid'] sort_dirs = ['asc'] obj, insts = instance_list.get_instances_sorted(self.context, filters, limit, marker, columns, sort_keys, sort_dirs) uuids = [inst['uuid'] for inst in insts] expected = [inst['uuid'] for inst in self.instances if inst['instance_type_id'] == 1] self.assertEqual(list(sorted(expected)), uuids)
def test_instance_list_minimal_cells(self): """Get a list of instances with a subset of cell mappings.""" last_cell = self.cells[-1] with context.target_cell(self.context, last_cell) as cctxt: last_cell_instances = db.instance_get_all(cctxt) last_cell_uuids = [inst['uuid'] for inst in last_cell_instances] instances = list( instance_list.get_instances_sorted(self.context, {}, None, None, [], ['uuid'], ['asc'], cell_mappings=self.cells[:-1])) found_uuids = [inst['hostname'] for inst in instances] had_uuids = [inst['hostname'] for inst in self.instances if inst['uuid'] not in last_cell_uuids] self.assertEqual(sorted(had_uuids), sorted(found_uuids))
def test_instance_list_minimal_cells(self): """Get a list of instances with a subset of cell mappings.""" last_cell = self.cells[-1] with context.target_cell(self.context, last_cell) as cctxt: last_cell_instances = db.instance_get_all(cctxt) last_cell_uuids = [inst['uuid'] for inst in last_cell_instances] instances = list( instance_list.get_instances_sorted(self.context, {}, None, None, [], ['uuid'], ['asc'], cell_mappings=self.cells[:-1])) found_uuids = [inst['hostname'] for inst in instances] had_uuids = [inst['hostname'] for inst in self.instances if inst['uuid'] not in last_cell_uuids] self.assertEqual(sorted(had_uuids), sorted(found_uuids))
def _test_get_paginated_with_filter(self, filters): found_uuids = [] marker = None while True: # Query for those instances, sorted by a different key in # pages of one until we've consumed them all batch = list( instance_list.get_instances_sorted(self.context, filters, 1, marker, [], ['hostname'], ['asc'])) if not batch: break found_uuids.extend([x['uuid'] for x in batch]) marker = found_uuids[-1] return found_uuids
def _test_get_paginated_with_filter(self, filters): found_uuids = [] marker = None while True: # Query for those instances, sorted by a different key in # pages of one until we've consumed them all batch = list( instance_list.get_instances_sorted(self.context, filters, 1, marker, [], ['hostname'], ['asc'])[1]) if not batch: break found_uuids.extend([x['uuid'] for x in batch]) marker = found_uuids[-1] return found_uuids
def _test_get_sorted_with_limit_marker(self, sort_by, pages=2, pagesize=2): insts = [] page = 0 while True: if page >= pages: limit = None else: limit = pagesize if insts: marker = insts[-1]['uuid'] else: marker = None batch = list( instance_list.get_instances_sorted(self.context, {}, limit, marker, [], [sort_by], ['asc'])) if not batch: break insts.extend(batch) page += 1 if page > len(self.instances) * 2: # Do this sanity check in case we introduce (or find) another # repeating page bug like #1721791. Without this we loop # until timeout, which is less obvious. raise Exception('Infinite paging loop') # We should have requested exactly (or one more unlimited) pages self.assertIn(page, (pages, pages + 1)) # Make sure the full set matches what we know to be true found = [x[sort_by] for x in insts] had = [x[sort_by] for x in self.instances] if sort_by in ('launched_at', 'created_at'): # We're comparing objects and database entries, so we need to # squash the tzinfo of the object ones so we can compare had = [x.replace(tzinfo=None) for x in had] self.assertEqual(len(had), len(found)) self.assertEqual(sorted(had), found)
def test_get_instances_with_down_cells(self, mock_sg): inst_cell0 = self.insts[uuids.cell0] # storing the uuids of the instances from the up cell uuid_initial = [inst['uuid'] for inst in inst_cell0] instances = (multi_cell_list.RecordWrapper(self.context, inst) for inst in inst_cell0) # creating one up cell and two down cells ret_val = {} ret_val[uuids.cell0] = instances ret_val[uuids.cell1] = nova_context.raised_exception_sentinel ret_val[uuids.cell2] = nova_context.did_not_respond_sentinel mock_sg.return_value = ret_val res = instance_list.get_instances_sorted(self.context, {}, None, None, [], None, None) uuid_final = [inst['uuid'] for inst in res] # return the results from the up cell, ignoring the down cell. self.assertEqual(uuid_initial, uuid_final)
def test_get_instances_batched(self, mock_cells, mock_inst): mock_cells.return_value = self.cells def fake_get_insts(ctx, filters, limit, *a, **k): for i in range(0, limit): yield self.insts.pop() mock_inst.side_effect = fake_get_insts obj, insts = instance_list.get_instances_sorted(self.context, {}, 50, None, [], ['hostname'], ['desc'], batch_size=10) # Make sure we returned exactly how many were requested insts = list(insts) self.assertEqual(50, len(insts)) # Since the instances are all uniform, we should have a # predictable number of queries to the database. 5 queries # would get us 50 results, plus one more gets triggered by the # sort to fill the buffer for the first cell feeder that runs # dry. self.assertEqual(6, mock_inst.call_count)
def test_get_instances_batched(self, mock_cells, mock_inst): mock_cells.return_value = self.cells def fake_get_insts(ctx, filters, limit, *a, **k): for i in range(0, limit): yield self.insts.pop() mock_inst.side_effect = fake_get_insts obj, insts = instance_list.get_instances_sorted(self.context, {}, 50, None, [], ['hostname'], ['desc'], batch_size=10) # Make sure we returned exactly how many were requested insts = list(insts) self.assertEqual(50, len(insts)) # Since the instances are all uniform, we should have a # predictable number of queries to the database. 5 queries # would get us 50 results, plus one more gets triggered by the # sort to fill the buffer for the first cell feeder that runs # dry. self.assertEqual(6, mock_inst.call_count)
def _test_get_sorted_with_limit_marker(self, sort_by, pages=2, pagesize=2, sort_dir='asc'): """Get multiple pages by a sort key and validate the results. This requests $pages of $pagesize, followed by a final page with no limit, and a final-final page which should be empty. It validates that we got a consistent set of results no patter where the page boundary is, that we got all the results after the unlimited query, and that the final page comes back empty when we use the last instance as a marker. """ insts = [] page = 0 while True: if page >= pages: # We've requested the specified number of limited (by pagesize) # pages, so request a penultimate page with no limit which # should always finish out the result. limit = None else: # Request a limited-size page for the first $pages pages. limit = pagesize if insts: # If we're not on the first page, use the last instance we # received as the marker marker = insts[-1]['uuid'] else: # No marker for the first page marker = None batch = list( instance_list.get_instances_sorted(self.context, {}, limit, marker, [], [sort_by], [sort_dir])[1]) if not batch: # This should only happen when we've pulled the last empty # page because we used the marker of the last instance. If # we end up with a non-deterministic ordering, we'd loop # forever. break insts.extend(batch) page += 1 if page > len(self.instances) * 2: # Do this sanity check in case we introduce (or find) another # repeating page bug like #1721791. Without this we loop # until timeout, which is less obvious. raise Exception('Infinite paging loop') # We should have requested exactly (or one more unlimited) pages self.assertIn(page, (pages, pages + 1)) # Make sure the full set matches what we know to be true found = [x[sort_by] for x in insts] had = [x[sort_by] for x in self.instances] if sort_by in ('launched_at', 'created_at'): # We're comparing objects and database entries, so we need to # squash the tzinfo of the object ones so we can compare had = [x.replace(tzinfo=None) for x in had] self.assertEqual(len(had), len(found)) if sort_dir == 'asc': self.assertEqual(sorted(had), found) else: self.assertEqual(list(reversed(sorted(had))), found)
def test_get_sorted_with_invalid_marker(self): self.assertRaises(exception.MarkerNotFound, list, instance_list.get_instances_sorted( self.context, {}, None, 'not-a-marker', [], None, None)[1])
def _test_get_sorted_with_limit_marker(self, sort_by, pages=2, pagesize=2, sort_dir='asc'): """Get multiple pages by a sort key and validate the results. This requests $pages of $pagesize, followed by a final page with no limit, and a final-final page which should be empty. It validates that we got a consistent set of results no patter where the page boundary is, that we got all the results after the unlimited query, and that the final page comes back empty when we use the last instance as a marker. """ insts = [] page = 0 while True: if page >= pages: # We've requested the specified number of limited (by pagesize) # pages, so request a penultimate page with no limit which # should always finish out the result. limit = None else: # Request a limited-size page for the first $pages pages. limit = pagesize if insts: # If we're not on the first page, use the last instance we # received as the marker marker = insts[-1]['uuid'] else: # No marker for the first page marker = None batch = list( instance_list.get_instances_sorted(self.context, {}, limit, marker, [], [sort_by], [sort_dir])) if not batch: # This should only happen when we've pulled the last empty # page because we used the marker of the last instance. If # we end up with a non-deterministic ordering, we'd loop # forever. break insts.extend(batch) page += 1 if page > len(self.instances) * 2: # Do this sanity check in case we introduce (or find) another # repeating page bug like #1721791. Without this we loop # until timeout, which is less obvious. raise Exception('Infinite paging loop') # We should have requested exactly (or one more unlimited) pages self.assertIn(page, (pages, pages + 1)) # Make sure the full set matches what we know to be true found = [x[sort_by] for x in insts] had = [x[sort_by] for x in self.instances] if sort_by in ('launched_at', 'created_at'): # We're comparing objects and database entries, so we need to # squash the tzinfo of the object ones so we can compare had = [x.replace(tzinfo=None) for x in had] self.assertEqual(len(had), len(found)) if sort_dir == 'asc': self.assertEqual(sorted(had), found) else: self.assertEqual(list(reversed(sorted(had))), found)
def test_get_sorted_with_invalid_marker(self): self.assertRaises( exception.MarkerNotFound, list, instance_list.get_instances_sorted(self.context, {}, None, 'not-a-marker', [], None, None))