def update_available_resources(self, context): # TODO(hongbin): get available resources from capsule_driver # and aggregates resources resources = self.container_driver.get_available_resources() # We allow 'cpu_used' to be missing from the container driver, # but the DB requires it to be non-null so just initialize it to 0. resources.setdefault('cpu_used', 0) # Check if the compute_node is already registered node = self._get_compute_node(context) if not node: # If not, register it and pass the object to the driver node = objects.ComputeNode(context) node.hostname = self.host self._copy_resources(node, resources) node.create(context) LOG.info('Node created for :%(host)s', {'host': self.host}) else: self._copy_resources(node, resources) node.rp_uuid = self._get_node_rp_uuid(context, node) self._setup_pci_tracker(context, node) self.compute_node = node self._update_available_resource(context) # NOTE(sbiswas7): Consider removing the return statement if not needed return node
def test_get_available_resources(self, mock_cpu_used, mock_info, mock_mem, mock_output): self.driver = DockerDriver() mock_output.return_value = LSCPU_ON conf.CONF.set_override('floating_cpu_set', "0") mock_mem.return_value = (100 * units.Ki, 50 * units.Ki, 50 * units.Ki, 50 * units.Ki) mock_info.return_value = (10, 8, 0, 2, 48, 'x86_64', 'linux', 'CentOS', '3.10.0-123', {'dev.type': 'product'}) mock_cpu_used.return_value = 1.0 node_obj = objects.ComputeNode() self.driver.get_available_resources(node_obj) self.assertEqual(_numa_topo_spec, node_obj.numa_topology.to_list()) self.assertEqual(node_obj.mem_total, 100) self.assertEqual(node_obj.mem_free, 50) self.assertEqual(node_obj.mem_available, 50) self.assertEqual(10, node_obj.total_containers) self.assertEqual(8, node_obj.running_containers) self.assertEqual(0, node_obj.paused_containers) self.assertEqual(2, node_obj.stopped_containers) self.assertEqual(48, node_obj.cpus) self.assertEqual(1.0, node_obj.cpu_used) self.assertEqual('x86_64', node_obj.architecture) self.assertEqual('linux', node_obj.os_type) self.assertEqual('CentOS', node_obj.os) self.assertEqual('3.10.0-123', node_obj.kernel_version) self.assertEqual({'dev.type': 'product'}, node_obj.labels)
def test_label_filter_pass(self): self.filt_cls = label_filter.LabelFilter() container = objects.Container(self.context) container.name = 'test-container' extra_spec = {'label:type': 'test'} host = objects.ComputeNode(self.context) host.labels = {'type': 'test'} self.assertTrue(self.filt_cls.host_passes(host, container, extra_spec))
def test_label_filter_fail(self): self.filt_cls = label_filter.LabelFilter() container = objects.Container(self.context) container.name = 'test-container' extra_spec = {'hints': {'label:type': 'test'}} host = objects.ComputeNode(self.context) host.labels = {'type': 'production'} self.assertFalse(self.filt_cls.host_passes(host, container, extra_spec))
def test_get_one_host(self, mock_get_by_uuid, mock_policy): mock_policy.return_value = True test_host = utils.get_test_compute_node() numat = numa.NUMATopology._from_dict(test_host['numa_topology']) test_host['numa_topology'] = numat test_host_obj = objects.ComputeNode(self.context, **test_host) mock_get_by_uuid.return_value = test_host_obj response = self.get('/v1/hosts/%s' % test_host['uuid']) mock_get_by_uuid.assert_called_once_with(mock.ANY, test_host['uuid']) self.assertEqual(200, response.status_int) self.assertEqual(test_host['uuid'], response.json['uuid'])
def test_create(self): with mock.patch.object(self.dbapi, 'create_compute_node', autospec=True) as mock_create: mock_create.return_value = self.fake_compute_node compute_node_dict = dict(self.fake_compute_node) compute_node_dict['numa_topology'] = objects.NUMATopology\ ._from_dict(compute_node_dict['numa_topology']) compute_node = objects.ComputeNode( self.context, **compute_node_dict) compute_node.create(self.context) mock_create.assert_called_once_with( self.context, self.fake_compute_node) self.assertEqual(self.context, compute_node._context)
def test_get_available_resources(self, mock_cpu_used, mock_info, mock_mem, mock_disk, mock_numa_cpu, mock_numa_mem, mock_output): self.driver = DockerDriver() numa_cpu_info = defaultdict(list) numa_cpu_info['0'] = [0, 8] mock_numa_cpu.return_value = numa_cpu_info mock_numa_mem.return_value = [1024 * 32] mock_output.return_value = LSCPU_ON conf.CONF.set_override('floating_cpu_set', "0") mock_mem.return_value = (100 * units.Ki, 50 * units.Ki, 50 * units.Ki, 50 * units.Ki) mock_info.return_value = { 'total_containers': 10, 'running_containers': 8, 'paused_containers': 0, 'stopped_containers': 2, 'cpus': 48, 'architecture': 'x86_64', 'os_type': 'linux', 'os': 'CentOS', 'kernel_version': '3.10.0-123', 'labels': { 'dev.type': 'product' }, 'runtimes': ['runc'], 'enable_cpu_pinning': False, 'docker_root_dir': '/var/lib/docker' } mock_cpu_used.return_value = 1.0 mock_disk.return_value = 80 node_obj = objects.ComputeNode() self.driver.get_available_resources(node_obj) self.assertEqual(_numa_topo_spec, node_obj.numa_topology.to_list()) self.assertEqual(node_obj.mem_total, 100) self.assertEqual(node_obj.mem_free, 50) self.assertEqual(node_obj.mem_available, 50) self.assertEqual(10, node_obj.total_containers) self.assertEqual(8, node_obj.running_containers) self.assertEqual(0, node_obj.paused_containers) self.assertEqual(2, node_obj.stopped_containers) self.assertEqual(48, node_obj.cpus) self.assertEqual(1.0, node_obj.cpu_used) self.assertEqual('x86_64', node_obj.architecture) self.assertEqual('linux', node_obj.os_type) self.assertEqual('CentOS', node_obj.os) self.assertEqual('3.10.0-123', node_obj.kernel_version) self.assertEqual({'dev.type': 'product'}, node_obj.labels) self.assertEqual(80, node_obj.disk_total) self.assertEqual(['runc'], node_obj.runtimes)
def get_test_compute_node(context, **kwargs): """Return a test compute node object with appropriate attributes. NOTE: The object leaves the attributes marked as changed, such that a create() could be used to commit it to the DB. """ db_compute_node = db_utils.get_test_compute_node(**kwargs) compute_node = objects.ComputeNode(context) for key in db_compute_node: if key == 'numa_topology': numa_obj = NUMATopology._from_dict(db_compute_node[key]) compute_node.numa_topology = numa_obj else: setattr(compute_node, key, db_compute_node[key]) return compute_node
def test_container_update(self, mock_claim, mock_container_update, mock_update, mock_get_node): container1 = obj_utils.get_test_container( self.context, cpu=1, memory=1024) container2 = obj_utils.get_test_container( self.context, cpu2=2, memory=2048) node = objects.ComputeNode(self.context) node.cpu = 10 node.memory = 3072 mock_get_node.return_value = node self._resource_tracker.container_update_claim( self.context, container1, container2) self.assertTrue(mock_claim.called) self.assertTrue(mock_container_update.called) self.assertTrue(mock_update.called)
def update_available_resources(self, context): # Check if the compute_node is already registered node = self._get_compute_node(context) if not node: # If not, register it and pass the object to the driver numa_obj = self.container_driver.get_host_numa_topology() node = objects.ComputeNode(context) node.hostname = self.host node.numa_topology = numa_obj node.create(context) LOG.info('Node created for :%(host)s', {'host': self.host}) self.container_driver.get_available_resources(node) self.compute_node = node self._update_available_resource(context) # NOTE(sbiswas7): Consider removing the return statement if not needed return node
def test_get_one_host(self, mock_get_by_uuid): test_host = utils.get_test_compute_node() numat = numa.NUMATopology._from_dict(test_host['numa_topology']) test_host['numa_topology'] = numat test_host_obj = objects.ComputeNode(self.context, **test_host) mock_get_by_uuid.return_value = test_host_obj extra_environ = {'HTTP_ACCEPT': 'application/json'} headers = {'OpenStack-API-Version': 'container 1.4'} response = self.app.get('/v1/hosts/%s' % test_host['uuid'], extra_environ=extra_environ, headers=headers) mock_get_by_uuid.assert_called_once_with( mock.ANY, test_host['uuid']) self.assertEqual(200, response.status_int) self.assertEqual(test_host['uuid'], response.json['uuid'])
def test_get_all_hosts(self, mock_host_list, mock_policy): mock_policy.return_value = True test_host = utils.get_test_compute_node() numat = numa.NUMATopology._from_dict(test_host['numa_topology']) test_host['numa_topology'] = numat hosts = [objects.ComputeNode(self.context, **test_host)] mock_host_list.return_value = hosts response = self.get('/v1/hosts') mock_host_list.assert_called_once_with(mock.ANY, 1000, None, 'hostname', 'asc', filters=None) self.assertEqual(200, response.status_int) actual_hosts = response.json['hosts'] self.assertEqual(1, len(actual_hosts)) self.assertEqual(test_host['uuid'], actual_hosts[0].get('uuid'))
def test_get_all_hosts_with_pagination_marker(self, mock_host_list, mock_policy): mock_policy.return_value = True host_list = [] for id_ in range(4): test_host = utils.create_test_compute_node( context=self.context, uuid=uuidutils.generate_uuid()) numat = numa.NUMATopology._from_dict(test_host['numa_topology']) test_host['numa_topology'] = numat host = objects.ComputeNode(self.context, **test_host) host_list.append(host) mock_host_list.return_value = host_list[-1:] response = self.get('/v1/hosts?limit=3&marker=%s' % host_list[2].uuid) self.assertEqual(200, response.status_int) actual_hosts = response.json['hosts'] self.assertEqual(1, len(actual_hosts)) self.assertEqual(host_list[-1].uuid, actual_hosts[0].get('uuid'))
def test_get_all_hosts(self, mock_host_list): test_host = utils.get_test_compute_node() numat = numa.NUMATopology._from_dict(test_host['numa_topology']) test_host['numa_topology'] = numat hosts = [objects.ComputeNode(self.context, **test_host)] mock_host_list.return_value = hosts extra_environ = {'HTTP_ACCEPT': 'application/json'} headers = {'OpenStack-API-Version': 'container 1.4'} response = self.app.get('/v1/hosts', extra_environ=extra_environ, headers=headers) mock_host_list.assert_called_once_with(mock.ANY, 1000, None, 'hostname', 'asc', filters=None) self.assertEqual(200, response.status_int) actual_hosts = response.json['hosts'] self.assertEqual(1, len(actual_hosts)) self.assertEqual(test_host['uuid'], actual_hosts[0].get('uuid'))
def test_get_all_hosts_with_pagination_marker(self, mock_host_list): host_list = [] for id_ in range(4): test_host = utils.create_test_compute_node( context=self.context, uuid=uuidutils.generate_uuid()) numat = numa.NUMATopology._from_dict(test_host['numa_topology']) test_host['numa_topology'] = numat host = objects.ComputeNode(self.context, **test_host) host_list.append(host) mock_host_list.return_value = host_list[-1:] extra_environ = {'HTTP_ACCEPT': 'application/json'} headers = {'OpenStack-API-Version': 'container 1.4'} response = self.app.get('/v1/hosts?limit=3&marker=%s' % host_list[2].uuid, extra_environ=extra_environ, headers=headers) self.assertEqual(200, response.status_int) actual_hosts = response.json['hosts'] self.assertEqual(1, len(actual_hosts)) self.assertEqual(host_list[-1].uuid, actual_hosts[0].get('uuid'))
def test_select_destinations(self, mock_random_choice, mock_list_by_binary, mock_compute_list, mock_service_is_up): all_services = [ FakeService('service1', 'host1'), FakeService('service2', 'host2'), FakeService('service3', 'host3'), FakeService('service4', 'host4') ] def _return_services(*args, **kwargs): return all_services self.driver.servicegroup_api.service_is_up = mock.Mock( return_value=True) mock_list_by_binary.side_effect = _return_services numa_topology = { "nodes": [{ "id": 0, "cpuset": [1, 2, 3, 4], "pinned_cpus": [], "mem_total": 1024 * 64, "mem_available": 1024 * 64 }, { "id": 1, "cpuset": [5, 6, 7, 8], "pinned_cpus": [], "mem_total": 1024 * 64, "mem_available": 1024 * 64 }] } numa = objects.numa.NUMATopology._from_dict(numa_topology) test_container = utils.get_test_container() containers = [objects.Container(self.context, **test_container)] node1 = objects.ComputeNode(self.context) node1.rp_uuid = mock.sentinel.node1_rp_uuid node1.updated_at = timeutils.utcnow() node1.cpus = 48 node1.cpu_used = 0.0 node1.mem_total = 1024 * 128 node1.mem_used = 1024 * 4 node1.mem_free = 1024 * 124 node1.mem_available = 1024 * 124 node1.disk_total = 80 node1.disk_used = 20 node1.hostname = 'host1' node1.numa_topology = numa node1.labels = {} node1.pci_device_pools = None node1.disk_quota_supported = True node1.runtimes = ['runc'] node1.enable_cpu_pinning = False node2 = objects.ComputeNode(self.context) node2.rp_uuid = mock.sentinel.node2_rp_uuid node2.updated_at = timeutils.utcnow() node2.cpus = 48 node2.cpu_used = 0.0 node2.mem_total = 1024 * 128 node2.mem_used = 1024 * 4 node2.mem_free = 1024 * 124 node2.mem_available = 1024 * 124 node2.disk_total = 80 node2.disk_used = 20 node2.hostname = 'host2' node2.numa_topology = numa node2.labels = {} node2.pci_device_pools = None node2.disk_quota_supported = True node2.runtimes = ['runc'] node2.enable_cpu_pinning = False node3 = objects.ComputeNode(self.context) node3.rp_uuid = mock.sentinel.node3_rp_uuid node3.updated_at = timeutils.utcnow() node3.cpus = 48 node3.cpu_used = 0.0 node3.mem_total = 1024 * 128 node3.mem_used = 1024 * 4 node3.mem_free = 1024 * 124 node3.mem_available = 1024 * 124 node3.disk_total = 80 node3.disk_used = 20 node3.hostname = 'host3' node3.numa_topology = numa node3.labels = {} node3.pci_device_pools = None node3.disk_quota_supported = True node3.runtimes = ['runc'] node3.enable_cpu_pinning = False node4 = objects.ComputeNode(self.context) node4.rp_uuid = mock.sentinel.node4_rp_uuid node4.updated_at = timeutils.utcnow() node4.cpus = 48 node4.cpu_used = 0.0 node4.mem_total = 1024 * 128 node4.mem_used = 1024 * 4 node4.mem_free = 1024 * 124 node4.mem_available = 1024 * 124 node4.disk_total = 80 node4.disk_used = 20 node4.hostname = 'host4' node4.numa_topology = numa node4.labels = {} node4.pci_device_pools = None node4.disk_quota_supported = True node4.runtimes = ['runc'] node4.enable_cpu_pinning = False nodes = [node1, node2, node3, node4] mock_compute_list.return_value = nodes mock_service_is_up.return_value = True extra_spec = {} mock_alloc_reqs_by_rp_uuid = { node3.rp_uuid: [mock.sentinel.node3_alloc_req] } mock_provider_summaries = {node3.rp_uuid: {}} dests = self.driver.select_destinations( self.context, containers, extra_spec, mock_alloc_reqs_by_rp_uuid, mock_provider_summaries, mock.sentinel.alloc_request_version) self.assertEqual(1, len(dests)) (host, node) = (dests[0]['host'], dests[0]['nodename']) self.assertEqual('host3', host) self.assertIsNone(node) container = containers[0] self.mock_placement_client.claim_resources.assert_called_once_with( mock.ANY, container.uuid, mock.sentinel.node3_alloc_req, container.project_id, container.user_id, allocation_request_version=mock.sentinel.alloc_request_version, consumer_generation=None)
def test_select_destinations(self, mock_random_choice, mock_list_by_binary, mock_compute_list, mock_service_is_up): all_services = [ FakeService('service1', 'host1'), FakeService('service2', 'host2'), FakeService('service3', 'host3'), FakeService('service4', 'host4') ] def _return_services(*args, **kwargs): return all_services self.driver.servicegroup_api.service_is_up = mock.Mock( return_value=True) mock_list_by_binary.side_effect = _return_services test_container = utils.get_test_container() containers = [objects.Container(self.context, **test_container)] node1 = objects.ComputeNode(self.context) node1.cpus = 48 node1.cpu_used = 0.0 node1.mem_total = 1024 * 128 node1.mem_used = 1024 * 4 node1.mem_free = 1024 * 124 node1.disk_total = 80 node1.disk_used = 20 node1.hostname = 'host1' node1.numa_topology = None node1.labels = {} node1.pci_device_pools = None node1.disk_quota_supported = True node2 = objects.ComputeNode(self.context) node2.cpus = 48 node2.cpu_used = 0.0 node2.mem_total = 1024 * 128 node2.mem_used = 1024 * 4 node2.mem_free = 1024 * 124 node2.disk_total = 80 node2.disk_used = 20 node2.hostname = 'host2' node2.numa_topology = None node2.labels = {} node2.pci_device_pools = None node2.disk_quota_supported = True node3 = objects.ComputeNode(self.context) node3.cpus = 48 node3.cpu_used = 0.0 node3.mem_total = 1024 * 128 node3.mem_used = 1024 * 4 node3.mem_free = 1024 * 124 node3.disk_total = 80 node3.disk_used = 20 node3.hostname = 'host3' node3.numa_topology = None node3.labels = {} node3.pci_device_pools = None node3.disk_quota_supported = True node4 = objects.ComputeNode(self.context) node4.cpus = 48 node4.cpu_used = 0.0 node4.mem_total = 1024 * 128 node4.mem_used = 1024 * 4 node4.mem_free = 1024 * 124 node4.disk_total = 80 node4.disk_used = 20 node4.hostname = 'host4' node4.numa_topology = None node4.labels = {} node4.pci_device_pools = None node4.disk_quota_supported = True nodes = [node1, node2, node3, node4] mock_compute_list.return_value = nodes def side_effect(hosts): return hosts[2] mock_random_choice.side_effect = side_effect mock_service_is_up.return_value = True extra_spec = {} dests = self.driver.select_destinations(self.context, containers, extra_spec) self.assertEqual(1, len(dests)) (host, node) = (dests[0]['host'], dests[0]['nodename']) self.assertEqual('host3', host) self.assertIsNone(node)
def test_select_destinations(self, mock_random_choice, mock_list_by_binary, mock_compute_list, mock_service_is_up): all_services = [ FakeService('service1', 'host1'), FakeService('service2', 'host2'), FakeService('service3', 'host3'), FakeService('service4', 'host4') ] def _return_services(*args, **kwargs): return all_services self.driver.servicegroup_api.service_is_up = mock.Mock( return_value=True) mock_list_by_binary.side_effect = _return_services numa_topology = { "nodes": [{ "id": 0, "cpuset": [1, 2, 3, 4], "pinned_cpus": [], "mem_total": 1024 * 64, "mem_available": 1024 * 64 }, { "id": 1, "cpuset": [5, 6, 7, 8], "pinned_cpus": [], "mem_total": 1024 * 64, "mem_available": 1024 * 64 }] } numa = objects.numa.NUMATopology._from_dict(numa_topology) test_container = utils.get_test_container() containers = [objects.Container(self.context, **test_container)] node1 = objects.ComputeNode(self.context) node1.cpus = 48 node1.cpu_used = 0.0 node1.mem_total = 1024 * 128 node1.mem_used = 1024 * 4 node1.mem_free = 1024 * 124 node1.mem_available = 1024 * 124 node1.disk_total = 80 node1.disk_used = 20 node1.hostname = 'host1' node1.numa_topology = numa node1.labels = {} node1.pci_device_pools = None node1.disk_quota_supported = True node1.runtimes = ['runc'] node1.enable_cpu_pinning = False node2 = objects.ComputeNode(self.context) node2.cpus = 48 node2.cpu_used = 0.0 node2.mem_total = 1024 * 128 node2.mem_used = 1024 * 4 node2.mem_free = 1024 * 124 node2.mem_available = 1024 * 124 node2.disk_total = 80 node2.disk_used = 20 node2.hostname = 'host2' node2.numa_topology = numa node2.labels = {} node2.pci_device_pools = None node2.disk_quota_supported = True node2.runtimes = ['runc'] node2.enable_cpu_pinning = False node3 = objects.ComputeNode(self.context) node3.cpus = 48 node3.cpu_used = 0.0 node3.mem_total = 1024 * 128 node3.mem_used = 1024 * 4 node3.mem_free = 1024 * 124 node3.mem_available = 1024 * 124 node3.disk_total = 80 node3.disk_used = 20 node3.hostname = 'host3' node3.numa_topology = numa node3.labels = {} node3.pci_device_pools = None node3.disk_quota_supported = True node3.runtimes = ['runc'] node3.enable_cpu_pinning = False node4 = objects.ComputeNode(self.context) node4.cpus = 48 node4.cpu_used = 0.0 node4.mem_total = 1024 * 128 node4.mem_used = 1024 * 4 node4.mem_free = 1024 * 124 node4.mem_available = 1024 * 124 node4.disk_total = 80 node4.disk_used = 20 node4.hostname = 'host4' node4.numa_topology = numa node4.labels = {} node4.pci_device_pools = None node4.disk_quota_supported = True node4.runtimes = ['runc'] node4.enable_cpu_pinning = False nodes = [node1, node2, node3, node4] mock_compute_list.return_value = nodes def side_effect(hosts): return hosts[2] mock_random_choice.side_effect = side_effect mock_service_is_up.return_value = True extra_spec = {} dests = self.driver.select_destinations(self.context, containers, extra_spec) self.assertEqual(1, len(dests)) (host, node) = (dests[0]['host'], dests[0]['nodename']) self.assertEqual('host3', host) self.assertIsNone(node)