def check_allocate_free(self): cluster = JsonCluster( {"nodes": [ {"ssh_config": {"host": "localhost1"}}, {"ssh_config": {"host": "localhost2"}}, {"ssh_config": {"host": "localhost3"}}]}) assert len(cluster) == 3 assert(cluster.num_available_nodes() == 3) nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=1)) nodes_hostnames = self.cluster_hostnames(nodes) assert len(cluster) == 3 assert(cluster.num_available_nodes() == 2) nodes2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2)) nodes2_hostnames = self.cluster_hostnames(nodes2) assert len(cluster) == 3 assert(cluster.num_available_nodes() == 0) assert(nodes_hostnames.isdisjoint(nodes2_hostnames)) cluster.free(nodes) assert(cluster.num_available_nodes() == 1) cluster.free(nodes2) assert(cluster.num_available_nodes() == 3)
def check_parsing(self): """ Checks that RemoteAccounts are generated correctly from input JSON""" node = JsonCluster( { "nodes": [ {"ssh_config": {"host": "hostname"}}]}).alloc(Service.setup_cluster_spec(num_nodes=1))[0] assert node.account.hostname == "hostname" assert node.account.user is None ssh_config = { "host": "hostname", "user": "******", "hostname": "localhost", "port": 22 } node = JsonCluster({"nodes": [{"hostname": "hostname", "user": "******", "ssh_config": ssh_config}]}).alloc(Service.setup_cluster_spec(num_nodes=1))[0] assert node.account.hostname == "hostname" assert node.account.user == "user" # check ssh configs assert node.account.ssh_config.host == "hostname" assert node.account.ssh_config.user == "user" assert node.account.ssh_config.hostname == "localhost" assert node.account.ssh_config.port == 22
def check_with_changing_cluster_availability(self): """Modify cluster usage in between calls to next() """ scheduler = TestScheduler(self.tc_list, self.cluster) # allocate 60 nodes; only test_id 0 should be available nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=60)) assert self.cluster.num_available_nodes() == 40 t = scheduler.next() assert t.test_id == 0 assert scheduler.peek() is None # return 10 nodes, so 50 are available in the cluster # next test from the scheduler should be test id 1 return_nodes = nodes[:10] keep_nodes = nodes[10:] self.cluster.free(return_nodes) assert self.cluster.num_available_nodes() == 50 t = scheduler.next() assert t.test_id == 1 assert scheduler.peek() is None # return remaining nodes, so cluster is fully available # next test from scheduler should be test id 2 return_nodes = keep_nodes self.cluster.free(return_nodes) assert self.cluster.num_available_nodes() == len(self.cluster) t = scheduler.next() assert t.test_id == 2
def check_free_too_many(self): n = 10 cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)]) nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=n)) with pytest.raises(NodeNotPresentError): nodes.append(MockFiniteSubclusterNode()) cluster.free(nodes)
def check_free_too_many(self): n = 10 cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)]) nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=n)) with pytest.raises(NodeNotPresentError): nodes.append(MockFiniteSubclusterNode()) cluster.free(nodes)
def check_with_changing_cluster_availability(self): """Modify cluster usage in between calls to next() """ scheduler = TestScheduler(self.tc_list, self.cluster) # allocate 60 nodes; only test_id 0 should be available nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=60)) assert self.cluster.num_available_nodes() == 40 t = scheduler.next() assert t.test_id == 0 assert scheduler.peek() is None # return 10 nodes, so 50 are available in the cluster # next test from the scheduler should be test id 1 return_nodes = nodes[: 10] keep_nodes = nodes[10:] self.cluster.free(return_nodes) assert self.cluster.num_available_nodes() == 50 t = scheduler.next() assert t.test_id == 1 assert scheduler.peek() is None # return remaining nodes, so cluster is fully available # next test from scheduler should be test id 2 return_nodes = keep_nodes self.cluster.free(return_nodes) assert self.cluster.num_available_nodes() == len(self.cluster) t = scheduler.next() assert t.test_id == 2
def check_cluster_file_read(self, monkeypatch): """check the behavior of VagrantCluster when cluster_file is specified and the file exists. VagrantCluster should read cluster information from cluster_file. """ self._set_monkeypatch_attr(monkeypatch) # To verify that VagrantCluster reads cluster information from the cluster_file, the # content in the file is intentionally made different from that returned by _vagrant_ssh_config(). nodes_expected = [] node1_expected = { "externally_routable_ip": "127.0.0.3", "ssh_config": { "host": "worker3", "hostname": "127.0.0.3", "user": "******", "port": 2222, "password": "******", "identityfile": "/path/to/identfile3" } } nodes_expected.append(node1_expected) node2_expected = { "externally_routable_ip": "127.0.0.2", "ssh_config": { "host": "worker2", "hostname": "127.0.0.2", "user": "******", "port": 2223, "password": None, "identityfile": "/path/to/indentfile2" } } nodes_expected.append(node2_expected) cluster_json_expected = {} cluster_json_expected["nodes"] = nodes_expected json.dump(cluster_json_expected, open(self.cluster_file, 'w+'), indent=2, separators=(',', ': '), sort_keys=True) # Load the cluster from the json file we just created cluster = VagrantCluster(cluster_file=self.cluster_file, is_type_based=False) assert len(cluster) == 2 assert cluster.num_available_nodes() == 2 node2, node3 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2)) assert node3.account.hostname == "worker2" assert node3.account.user == "vagrant" assert node3.account.ssh_hostname == '127.0.0.2' assert node3.account.ssh_config.to_json() == node2_expected["ssh_config"] assert node2.account.hostname == "worker3" assert node2.account.user == "vagrant" assert node2.account.ssh_hostname == '127.0.0.3' assert node2.account.ssh_config.to_json() == node1_expected["ssh_config"]
def check_allocate_free(self): n = 10 cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)]) assert len(cluster) == n assert cluster.num_available_nodes() == n nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=1)) assert len(nodes) == 1 assert len(cluster) == n assert cluster.num_available_nodes() == n - 1 nodes2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2)) assert len(nodes2) == 2 assert len(cluster) == n assert cluster.num_available_nodes() == n - 3 cluster.free(nodes) assert cluster.num_available_nodes() == n - 2 cluster.free(nodes2) assert cluster.num_available_nodes() == n
def check_allocate_free(self): n = 10 cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)]) assert len(cluster) == n assert cluster.num_available_nodes() == n nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=1)) assert len(nodes) == 1 assert len(cluster) == n assert cluster.num_available_nodes() == n - 1 nodes2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2)) assert len(nodes2) == 2 assert len(cluster) == n assert cluster.num_available_nodes() == n - 3 cluster.free(nodes) assert cluster.num_available_nodes() == n - 2 cluster.free(nodes2) assert cluster.num_available_nodes() == n
def check_parsing(self): """ Checks that RemoteAccounts are generated correctly from input JSON""" node = JsonCluster(cluster_json={ "nodes": [{ "ssh_config": { "host": "hostname" } }] }, is_type_based=False).alloc( Service.setup_cluster_spec(num_nodes=1))[0] assert node.account.hostname == "hostname" assert node.account.user is None ssh_config = { "host": "hostname", "user": "******", "hostname": "localhost", "port": 22 } node = JsonCluster(cluster_json={ "nodes": [{ "hostname": "hostname", "user": "******", "ssh_config": ssh_config }] }, is_type_based=False).alloc( Service.setup_cluster_spec(num_nodes=1))[0] assert node.account.hostname == "hostname" assert node.account.user == "user" # check ssh configs assert node.account.ssh_config.host == "hostname" assert node.account.ssh_config.user == "user" assert node.account.ssh_config.hostname == "localhost" assert node.account.ssh_config.port == 22
def check_allocate_free(self): cluster = JsonCluster(cluster_json={ "nodes": [{ "ssh_config": { "host": "localhost1" } }, { "ssh_config": { "host": "localhost2" } }, { "ssh_config": { "host": "localhost3" } }] }, is_type_based=False) assert len(cluster) == 3 assert (cluster.num_available_nodes() == 3) nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=1)) nodes_hostnames = self.cluster_hostnames(nodes) assert len(cluster) == 3 assert (cluster.num_available_nodes() == 2) nodes2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2)) nodes2_hostnames = self.cluster_hostnames(nodes2) assert len(cluster) == 3 assert (cluster.num_available_nodes() == 0) assert (nodes_hostnames.isdisjoint(nodes2_hostnames)) cluster.free(nodes) assert (cluster.num_available_nodes() == 1) cluster.free(nodes2) assert (cluster.num_available_nodes() == 3)
def check_non_empty_cluster_too_small(self): """Ensure that scheduler does not return tests if the cluster does not have enough available nodes. """ scheduler = TestScheduler(self.tc_list, self.cluster) assert len(scheduler) == len(self.tc_list) assert scheduler.peek() is not None # alloc all cluster nodes so none are available self.cluster.alloc( Service.setup_cluster_spec(num_nodes=len(self.cluster))) assert self.cluster.num_available_nodes() == 0 # peeking should not yield an object assert scheduler.peek() is None
def check_non_empty_cluster_too_small(self): """Ensure that scheduler does not return tests if the cluster does not have enough available nodes. """ scheduler = TestScheduler(self.tc_list, self.cluster) assert len(scheduler) == len(self.tc_list) assert scheduler.peek() is not None # alloc all cluster nodes so none are available self.cluster.alloc(Service.setup_cluster_spec(num_nodes=len(self.cluster))) assert self.cluster.num_available_nodes() == 0 # peeking etc should not yield an object assert scheduler.peek() is None with pytest.raises(RuntimeError): scheduler.next()
def check_one_host_parsing(self, monkeypatch): """check the behavior of VagrantCluster when cluster_file is not specified. VagrantCluster should read cluster information from _vagrant_ssh_config(). """ self._set_monkeypatch_attr(monkeypatch) cluster = VagrantCluster(is_type_based=False) assert len(cluster) == 2 assert cluster.num_available_nodes() == 2 node1, node2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2)) assert node1.account.hostname == "worker1" assert node1.account.user == "vagrant" assert node1.account.ssh_hostname == '127.0.0.1' assert node2.account.hostname == "worker2" assert node2.account.user == "vagrant" assert node2.account.ssh_hostname == '127.0.0.2'
def check_request_free(self): available = self.cluster.num_available_nodes() initial_size = len(self.cluster) # Should be able to allocate arbitrarily many nodes nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=100)) assert (len(nodes) == 100) for i, node in enumerate(nodes): assert node.account.hostname == 'localhost%d' % i assert node.account.ssh_hostname == 'localhost' assert node.account.ssh_config.hostname == 'localhost' assert node.account.ssh_config.port == 22 assert node.account.user is None assert (self.cluster.num_available_nodes() == (available - 100)) assert len(self.cluster) == initial_size # This shouldn't change self.cluster.free(nodes) assert (self.cluster.num_available_nodes() == available)
def check_with_changing_cluster_availability(self): """Modify cluster usage in between calls to next() """ scheduler = TestScheduler(self.tc_list, self.cluster) # start with 100-node cluster (configured in setup_method()) # allocate 60 nodes; only test_id 0 (which needs 10 nodes) should be available nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=60)) assert self.cluster.num_available_nodes() == 40 t = scheduler.peek() assert t == self.tc0 scheduler.remove(t) assert scheduler.peek() is None # return 10 nodes, so 50 are available in the cluster # next test from the scheduler should be test id 1 (which needs 50 nodes) return_nodes = nodes[:10] keep_nodes = nodes[10:] self.cluster.free(return_nodes) assert self.cluster.num_available_nodes() == 50 t = scheduler.peek() assert t == self.tc1 scheduler.remove(t) assert scheduler.peek() is None # return remaining nodes, so cluster is fully available # next test from scheduler should be test id 2 (which needs 100 nodes) return_nodes = keep_nodes self.cluster.free(return_nodes) assert self.cluster.num_available_nodes() == len(self.cluster) t = scheduler.peek() assert t == self.tc2 scheduler.remove(t) # scheduler should become empty now assert len(scheduler) == 0 assert scheduler.peek() is None
def check_alloc_too_many(self): n = 10 cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)]) with pytest.raises(InsufficientResourcesError): cluster.alloc(Service.setup_cluster_spec(num_nodes=(n + 1)))
def check_exhausts_supply(self): cluster = create_json_cluster(self.single_node_cluster_json) with pytest.raises(InsufficientResourcesError): cluster.alloc(Service.setup_cluster_spec(num_nodes=2))
def check_alloc_too_many(self): n = 10 cluster = FiniteSubcluster( [MockFiniteSubclusterNode() for _ in range(n)]) with pytest.raises(InsufficientResourcesError): cluster.alloc(Service.setup_cluster_spec(num_nodes=(n + 1)))
def check_exhausts_supply(self): cluster = JsonCluster(cluster_json=self.single_node_cluster_json, is_type_based=False) with pytest.raises(InsufficientResourcesError): cluster.alloc(Service.setup_cluster_spec(num_nodes=2))
def check_exhausts_supply(self): cluster = JsonCluster(self.single_node_cluster_json) with pytest.raises(InsufficientResourcesError): cluster.alloc(Service.setup_cluster_spec(num_nodes=2))