コード例 #1
0
    def domain_n_preserve_remove_node_test(self):
        site = "site1"
        fake_site, lc = self.make_fake_libcloud_site(site)
        dt_name = self._load_dtrs(site, fake_site)

        n = 3
        dt = _make_domain_def(n, dt_name, site)
        def_id = str(uuid.uuid4())
        self.epum_client.add_domain_definition(def_id, example_definition)
        domain_id = str(uuid.uuid4())
        log.debug("Launching domain %s", domain_id)

        self.epum_client.add_domain(domain_id, def_id, dt, caller=self.user)
        self._wait_states(n, lc, states=[NodeState.RUNNING])

        # wait a little while to make sure query thread detects all VMs
        time.sleep(10)

        nodes = get_valid_nodes(lc)
        lc.destroy_node(random.choice(nodes))

        wait(lambda: len(get_valid_nodes(lc)) == n, timeout=60)

        self._wait_remove_domain(domain_id)
        self._wait_for_all_terminated(lc)
コード例 #2
0
ファイル: test_doctor.py プロジェクト: lelou6666/epu
    def test_initialized_system_boot_with_procs(self):
        # tests the case where just the doctor dies in the middle of system boot
        # but after a doctor has already declared the system initialized. In this
        # case we have processes in the UNSCHEDULED_PENDING state that should be
        # rescheduled once system boot ends.

        self.store.set_system_boot(True)
        self.store.set_initialized()
        self.store.set_pd_state(ProcessDispatcherState.SYSTEM_BOOTING)

        p0 = ProcessRecord.new(None, "proc0", {}, ProcessState.UNSCHEDULED_PENDING)
        self.store.add_process(p0)
        p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.UNSCHEDULED_PENDING)
        self.store.add_process(p1)
        p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.UNSCHEDULED_PENDING)
        self.store.add_process(p2)

        restartable_procs = ["proc0", "proc1", "proc2"]
        self._run_in_thread()

        # now end system boot
        self.store.set_system_boot(False)

        wait(lambda: self.store.get_pd_state() == ProcessDispatcherState.OK)

        # check that pending processes were correctly rescheduled
        self.assertEqual(len(self.store.get_queued_processes()), len(restartable_procs))
        for proc in restartable_procs:
            self.assertEqual(self.store.get_process(None, proc).state,
                             ProcessState.REQUESTED)
コード例 #3
0
    def domain_n_preserve_alter_state_test(self):

        site = uuid.uuid4().hex
        fake_site, lc = self.make_fake_libcloud_site(site)
        dt_name = self._load_dtrs(site, fake_site)

        n = 3
        dt = _make_domain_def(n, dt_name, site)
        def_id = str(uuid.uuid4())
        self.epum_client.add_domain_definition(def_id, example_definition)
        domain_id = str(uuid.uuid4())
        log.debug("Launching domain %s", domain_id)

        self.epum_client.add_domain(domain_id, def_id, dt, caller=self.user)
        wait(lambda: len(get_valid_nodes(lc)) == n, timeout=60)

        nodes = get_valid_nodes(lc)
        lc.set_node_state(nodes[0], NodeState.TERMINATED)

        self._wait_states(n, lc, states=[NodeState.RUNNING, NodeState.PENDING])

        print "terminating"
        self._wait_remove_domain(domain_id)

        # check the node list
        nodes = lc.list_nodes(immediate=True)
        for nd in nodes:
            # verify that any node that is still around is terminated
            self.assertEqual(nd.state, NodeState.TERMINATED)
コード例 #4
0
ファイル: test_doctor.py プロジェクト: lelou6666/epu
    def test_uninitialized_system_boot_without_state(self):
        self.store.set_system_boot(True)
        self._run_in_thread()

        assert self.store.wait_initialized(timeout=10)
        self.assertEqual(self.store.get_pd_state(),
                         ProcessDispatcherState.SYSTEM_BOOTING)
        self.store.set_system_boot(False)
        wait(lambda: self.store.get_pd_state() == ProcessDispatcherState.OK)
コード例 #5
0
ファイル: test_integration.py プロジェクト: lelou6666/epu
    def _wait_for_domain(self, domain_id):
        def waiter():
            try:
                domain = self.epum_client.describe_domain(domain_id)
                return domain is not None
            except NotFoundError:
                return False

        wait(waiter, timeout=30)
コード例 #6
0
    def _wait_for_all_terminated(self, lc):
        def wait_terminated():
            nodes = lc.list_nodes(immediate=True)
            return all(node.state == NodeState.TERMINATED for node in nodes)
        try:
            wait(wait_terminated, timeout=60)
        except wait.TimeOutWaitingFor:
            nodes = [n.id for n in lc.list_nodes(immediate=True) if n.state != NodeState.TERMINATED]

            self.fail("Timed out waiting for all nodes to be terminated. Remaining: %s" % nodes)
コード例 #7
0
 def _wait_remove_many_domains(self, domains, delay=0.1):
     domains = set(domains)
     for domain_id in domains:
         self.epum_client.remove_domain(domain_id)
     # wait for intersection between sets to be empty (no domains left)
     try:
         wait(lambda: len(domains & set(self.epum_client.list_domains())) == 0, timeout=60, wait=delay)
     except wait.TimeOutWaitingFor:
         remaining = domains & set(self.epum_client.list_domains())
         self.fail("Timed out waiting for domains to exit. domains: %s" % list(remaining))
コード例 #8
0
    def _get_contender(self, path, ndx=0):
        """returns name, hostname, pid tuple"""

        assert ndx < self.epum_replica_count
        contenders = []
        election = self.kazoo.Election(path)

        def getem():
            contenders[:] = election.contenders()
            return len(contenders) == self.epum_replica_count
        # retry getting contenders. may take them a while to emerge
        wait(getem, timeout=20)
        name, hostname, pid = contenders[ndx].split(':')
        return name, hostname, int(pid)
コード例 #9
0
ファイル: test_doctor.py プロジェクト: lelou6666/epu
    def test_monitor_thread(self):
        self._run_in_thread()

        assert self.store.wait_initialized(timeout=10)
        self.assertEqual(self.store.get_pd_state(),
                         ProcessDispatcherState.OK)

        self.assertIsNotNone(self.doctor.monitor)
        monitor_thread = self.doctor.monitor_thread
        self.assertIsNotNone(monitor_thread)
        self.assertTrue(monitor_thread.is_alive())

        # now cancel doctor. monitor should stop too
        self.doctor.cancel()
        wait(lambda: not monitor_thread.is_alive())
コード例 #10
0
ファイル: test_doctor.py プロジェクト: lelou6666/epu
    def test_initialized_system_boot_without_procs(self):
        # tests the case where just the doctor dies in the middle of system boot
        # but after a doctor has already declared the system initialized. In this
        # case we have no processes to schedule on system boot completion.

        self.store.set_system_boot(True)
        self.store.set_initialized()
        self.store.set_pd_state(ProcessDispatcherState.SYSTEM_BOOTING)

        self._run_in_thread()

        # now end system boot
        self.store.set_system_boot(False)

        wait(lambda: self.store.get_pd_state() == ProcessDispatcherState.OK)
コード例 #11
0
    def domain_add_check_n_remove_test(self):
        site = uuid.uuid4().hex
        fake_site, lc = self.make_fake_libcloud_site(site)
        dt_name = self._load_dtrs(site, fake_site)

        n = 3
        dt = _make_domain_def(n, dt_name, site)
        def_id = str(uuid.uuid4())
        self.epum_client.add_domain_definition(def_id, example_definition)
        domain_id = str(uuid.uuid4())

        log.debug("Launching domain %s", domain_id)

        self.epum_client.add_domain(domain_id, def_id, dt, caller=self.user)
        wait(lambda: len(get_valid_nodes(lc)) == n, timeout=60)

        self._wait_remove_domain(domain_id)
        self._wait_for_all_terminated(lc)
コード例 #12
0
    def _wait_states(self, n, lc, states=None, timeout=20, delay=0.1):
        if states is None:
            states = [NodeState.RUNNING, NodeState.PENDING]

        print "Waiting for %d nodes in states: %s" % (n, states)

        def wait_running_count():
            nodes = lc.list_nodes(immediate=True)
            running_count = 0
            found_states = defaultdict(int)
            for nd in nodes:
                found_states[nd.state] += 1
                if nd.state in states:
                    running_count += 1
            print "Found %d nodes in states: %s" % (len(nodes),
                " ".join("%s:%s" % pair for pair in found_states.iteritems()))
            return running_count == n

        wait(wait_running_count, timeout=timeout, wait=delay)
コード例 #13
0
    def _add_many_domains_terminate_all(self, kill_func=None, places_to_kill=None, n=1, nodes_per_domain=3):
        test_pc = 1
        def_name = str(uuid.uuid4())
        self.epum_client.add_domain_definition(def_name, example_definition)

        domain = _example_domain(nodes_per_domain)
        domains_started = []
        for i in range(n):
            name = "dom%d" % (i)
            self.epum_client.add_domain(name, def_name, domain)
            domains_started.append(name)

        test_pc = self._kill_cb(test_pc, places_to_kill, kill_func)

        domains = self.epum_client.list_domains()
        domains_started.sort()
        domains.sort()
        self.assertEqual(domains, domains_started)

        self.wait_for_libcloud_nodes(n * nodes_per_domain)
        test_pc = self._kill_cb(test_pc, places_to_kill, kill_func)

        self.wait_for_all_domains()

        state = self.provisioner_client.terminate_all()
        self.assertFalse(state)  # cannot all be terminated this quickly

        test_pc = self._kill_cb(test_pc, places_to_kill, kill_func)

        # wait a little while until hopefully termination is underway
        time.sleep(2)
        test_pc = self._kill_cb(test_pc, places_to_kill, kill_func)

        # this will return true when everything is terminated
        wait(self.provisioner_client.terminate_all, timeout=20)

        self.assertFalse(self.get_valid_libcloud_nodes())

        for name in domains_started:
            self.epum_client.remove_domain(name)
        self.wait_for_domain_set([])
コード例 #14
0
    def wait_for_domain_set(self, expected, timeout=30):
        expected = set(expected)

        wait(lambda: set(self.epum_client.list_domains()) == expected,
            timeout=timeout)
コード例 #15
0
ファイル: test_doctor.py プロジェクト: lelou6666/epu
    def test_uninitialized_system_boot_with_state(self):
        self.store.set_system_boot(True)
        self.core.node_state("node1", domain_id_from_engine("engine1"),
            InstanceState.RUNNING)
        resource_id = "eeagent_1"
        self.core.ee_heartbeat(resource_id, make_beat("node1"))

        p0 = ProcessRecord.new(None, "proc0", {}, ProcessState.RUNNING,
                configuration=nosystemrestart_process_config(),
                assigned=resource_id,
                restart_mode=RestartMode.ALWAYS)
        self.store.add_process(p0)
        p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.RUNNING,
                assigned=resource_id)
        self.store.add_process(p1)
        p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.PENDING,
            assigned=resource_id)
        self.store.add_process(p2)
        p3 = ProcessRecord.new(None, "proc3", {}, ProcessState.TERMINATING,
            assigned=resource_id)
        self.store.add_process(p3)

        # this one shouldn't restart
        p4 = ProcessRecord.new(None, "proc4", {}, ProcessState.RUNNING,
                configuration=nosystemrestart_process_config(),
                assigned=resource_id,
                restart_mode=RestartMode.ABNORMAL)
        self.store.add_process(p4)

        # non-running proceses should also potentially be restarted on boot
        p5 = ProcessRecord.new(None, "proc5", {}, ProcessState.WAITING)
        self.store.add_process(p5)
        self.store.enqueue_process(*p5.key)
        p6 = ProcessRecord.new(None, "proc6", {}, ProcessState.REQUESTED)
        self.store.add_process(p6)

        # not this one, due to RestartMode
        p7 = ProcessRecord.new(None, "proc7", {}, ProcessState.REQUESTED,
            configuration=nosystemrestart_process_config(),
            restart_mode=RestartMode.ALWAYS)
        self.store.add_process(p7)
        self.store.enqueue_process(*p7.key)

        resource = self.store.get_resource(resource_id)
        resource.assigned = [p0.key, p1.key, p2.key, p3.key, p4.key]
        self.store.update_resource(resource)

        restartable_procs = ["proc1", "proc2", "proc5", "proc6"]
        dead_procs = ["proc0", "proc4", "proc7"]

        self._run_in_thread()

        assert self.store.wait_initialized(timeout=10)

        self.assertEqual(len(self.store.get_queued_processes()), 0)
        self.assertEqual(len(self.store.get_node_ids()), 0)
        self.assertEqual(len(self.store.get_resource_ids()), 0)

        for proc in restartable_procs:
            self.assertEqual(self.store.get_process(None, proc).state,
                             ProcessState.UNSCHEDULED_PENDING)
        for proc in dead_procs:
            self.assertEqual(self.store.get_process(None, proc).state,
                             ProcessState.TERMINATED)
        self.assertEqual(self.store.get_process(None, "proc3").state,
                         ProcessState.TERMINATED)

        self.assertEqual(self.store.get_pd_state(),
                         ProcessDispatcherState.SYSTEM_BOOTING)

        # now end system boot
        self.store.set_system_boot(False)

        wait(lambda: self.store.get_pd_state() == ProcessDispatcherState.OK)

        # check that pending processes were correctly rescheduled
        self.assertEqual(len(self.store.get_queued_processes()), len(restartable_procs))
        for proc in restartable_procs:
            self.assertEqual(self.store.get_process(None, proc).state,
                             ProcessState.REQUESTED)
コード例 #16
0
    def domain_sensor_engine_test(self):
        site = uuid.uuid4().hex
        fake_site, lc = self.make_fake_libcloud_site(site)
        dt_name = self._load_dtrs(site, fake_site)

        minimum_n = 1
        maximum_n = 3
        scale_up_threshold = 2.0
        scale_up_n_vms = 1
        scale_down_threshold = 0.5
        scale_down_n_vms = 1
        scale_down_sensor_data = [0, 0, 0]
        scale_up_sensor_data = [3, 3, 5]
        metric = 'load'
        sample_function = 'Average'
        dt = _make_sensor_domain_def(metric, sample_function, minimum_n,
                maximum_n, scale_up_threshold,
                scale_up_n_vms, scale_down_threshold, scale_down_n_vms,
                scale_down_sensor_data, dt_name, site)
        def_id = str(uuid.uuid4())
        self.epum_client.add_domain_definition(def_id, sensor_definition)
        domain_id = str(uuid.uuid4())
        log.debug("Launching domain %s", domain_id)

        self.epum_client.add_domain(domain_id, def_id, dt, caller=self.user)

        # make sure we hit the minimum number of nodes
        wait(lambda: len(get_valid_nodes(lc)) >= minimum_n, timeout=60)

        # Now get it to scale up
        print "reconfiguring with sensor data: %s" % scale_up_sensor_data
        new_config = {'engine_conf': {'sensor_data': scale_up_sensor_data}}
        self.epum_client.reconfigure_domain(domain_id, new_config, caller=self.user)

        # make sure we hit the maximum number of nodes
        wait(lambda: len(get_valid_nodes(lc)) == maximum_n, timeout=60)

        # Now get it to scale down
        print "reconfiguring with sensor data: %s" % scale_down_sensor_data
        new_config = {'engine_conf': {'sensor_data': scale_down_sensor_data}}
        self.epum_client.reconfigure_domain(domain_id, new_config, caller=self.user)

        wait(lambda: len(get_valid_nodes(lc)) == minimum_n, timeout=60)

        # Now test the cooldown
        new_config = {
            'engine_conf': {
                'sensor_data': scale_up_sensor_data,
                'cooldown_period': 100,
            }
        }
        self.epum_client.reconfigure_domain(domain_id, new_config, caller=self.user)

        # Wait 10s for a few decides to happen:
        time.sleep(10)

        # And ensure we're still a minimum scaling
        nodes = get_valid_nodes(lc)
        self.assertEqual(len(nodes), minimum_n)

        # Now set cooldown to 10s (which have already passed)
        new_config = {'engine_conf': {'cooldown_period': 10}}
        self.epum_client.reconfigure_domain(domain_id, new_config, caller=self.user)

        # And watch it scale up
        wait(lambda: len(get_valid_nodes(lc)) == maximum_n, timeout=60)

        self._wait_remove_domain(domain_id)

        self._wait_for_all_terminated(lc)
コード例 #17
0
 def wait_for_libcloud_nodes(self, count, timeout=60):
     wait(lambda: len(self.get_valid_nodes()) == count,
         timeout=timeout)
     return self.get_valid_nodes()
コード例 #18
0
 def wait_for_all_domains(self, timeout=30):
     wait(self.verify_all_domain_instances, timeout=timeout)