def setUp(self):
        if not os.environ.get("INT"):
            raise SkipTest("Skipping Slow integration test")
        self.exchange = "hatestexchange-%s" % str(uuid.uuid4())
        self.sysname = "test-%s" % str(uuid.uuid4())

        parsed_deployment = yaml.load(deployment_one_pd_two_eea)
        self.pd_names = parsed_deployment['process-dispatchers'].keys()
        self.eea_names = []
        for node in parsed_deployment['nodes'].values():
            for eeagent in node['eeagents'].keys():
                self.eea_names.append(eeagent)
        policy_params = {'preserve_n': 0}
        executable = {'exec': 'sleep', 'argv': ['1000']}

        self.setup_harness(exchange=self.exchange, sysname=self.sysname)
        self.addCleanup(self.cleanup_harness)

        self.epuharness.start(deployment_str=deployment_one_pd_two_eea)
        self.block_until_ready(deployment_one_pd_two_eea, self.dashi)

        self.process_definition_id = uuid.uuid4().hex
        for pd_name in self.pd_names:
            pd_client = ProcessDispatcherClient(self.dashi, pd_name)
            pd_client.create_definition(self.process_definition_id, None,
                executable, None, None)

        self.haservice = HighAvailabilityService(policy_parameters=policy_params,
                process_dispatchers=self.pd_names, exchange=self.exchange,
                process_definition_id=self.process_definition_id,
                sysname=self.sysname)
        self.haservice_thread = tevent.spawn(self.haservice.start)

        self.dashi = self.haservice.dashi
        self.haservice_client = HighAvailabilityServiceClient(self.dashi, topic=self.haservice.topic)
    def _get_proc_from_pd(self, upid, pd_name):
        pd_client = ProcessDispatcherClient(self.dashi, pd_name)
        procs = pd_client.describe_processes()
        for proc in procs:
            if upid == proc.get('upid'):
                return proc

        return None
    def _get_all_procs(self):
        all_procs = {}
        for pd_name in self.pd_names:
            pd_client = ProcessDispatcherClient(self.dashi, pd_name)
            print "Querying %s" % pd_name
            procs = pd_client.describe_processes()
            all_procs[pd_name] = procs

        return all_procs
Exemplo n.º 4
0
    def block_until_ready(self, deployment_str, dashi):
        """Blocks until all of the services in a deployment are contacted
        """

        deployment = parse_deployment(yaml_str=deployment_str)

        for provisioner_name in deployment.get('provisioners', {}).iterkeys():
            provisioner = ProvisionerClient(dashi, topic=provisioner_name)
            self._block_on_call(provisioner.describe_nodes)

        for epum_name in deployment.get('epums', {}).iterkeys():
            epum = EPUManagementClient(dashi, epum_name)
            self._block_on_call(epum.list_domains)

        for node in deployment.get('nodes', {}).itervalues():
            for eeagent_name in node.get('eeagents', {}).iterkeys():
                eeagent = EEAgentClient(dashi=dashi,
                                        ee_name=eeagent_name,
                                        handle_heartbeat=False)
                self._block_on_call(eeagent.dump, kwargs={'rpc': True})

        for pd_name in deployment.get('process-dispatchers', {}).iterkeys():
            pd = ProcessDispatcherClient(dashi, pd_name)
            self._block_on_call(pd.describe_processes)

        for dt_name in deployment.get('dt_registries', {}).iterkeys():
            dtrs = DTRSClient(dashi, topic=dt_name)
            self._block_on_call(dtrs.list_sites)
Exemplo n.º 5
0
    def get_clients(self, deployment_str, dashi):
        """returns a dictionary of epu clients, indexed by their topic name
        """

        deployment = parse_deployment(yaml_str=deployment_str)

        clients = {}

        for provisioner_name in deployment.get('provisioners', {}).iterkeys():
            client = ProvisionerClient(dashi, topic=provisioner_name)
            clients[provisioner_name] = client

        for epum_name in deployment.get('epums', {}).iterkeys():
            client = EPUManagementClient(dashi, epum_name)
            clients[epum_name] = client

        for node in deployment.get('nodes', {}).itervalues():
            for eeagent_name in node.get('eeagents', {}).iterkeys():
                client = EEAgentClient(dashi=dashi,
                                       ee_name=eeagent_name,
                                       handle_heartbeat=False)
                clients[eeagent_name] = client

        for pd_name in deployment.get('process-dispatchers', {}).iterkeys():
            client = ProcessDispatcherClient(dashi, pd_name)
            clients[pd_name] = client

        for dt_name in deployment.get('dt_registries', {}).iterkeys():
            client = DTRSClient(dashi, topic=dt_name)
            clients[dt_name] = client

        return clients
    def test_missing_proc(self):
        """test_missing_proc
        Kill a proc, and ensure HA starts a replacement
        """

        n = 2
        self._update_policy_params_and_assert({'preserve_n': n})
        self._assert_n_processes(n)

        upid_to_kill = self.haservice.core.managed_upids[0]
        pd = self._find_procs_pd(upid_to_kill)
        assert pd

        pd_client = ProcessDispatcherClient(self.dashi, pd)
        pd_client.terminate_process(upid_to_kill)
        print self._get_all_procs()
        print self._get_all_procs()
        print self._get_all_procs()

        time.sleep(5)
        self._assert_n_processes(n)
        time.sleep(5)
        self._assert_n_processes(n)
        print self._get_all_procs()
Exemplo n.º 7
0
    def announce_node(self, node_name, engine, process_dispatcher, state=None):
        """Announce a node to each process dispatcher.

        @param node_name: the name of the node to advertise
        @param engine: the execution engine of the node
        @param process_dispatcher: the pd to announce to
        @param state: the state to advertise to the pd
        """
        if not state:
            state = InstanceState.RUNNING

        pd_client = ProcessDispatcherClient(self.dashi, process_dispatcher)
        log.info("Announcing %s of engine %s is '%s' to %s" %
                 (node_name, engine, state, process_dispatcher))
        domain_id = domain_id_from_engine(engine)
        for i in range(1, ADVERTISE_RETRIES):
            try:
                pd_client.node_state(node_name, domain_id, state)
                break
            except timeout:
                wait_time = i * i  # Exponentially increasing wait
                log.warning("PD '%s' not available yet. Waiting %ss" %
                            (process_dispatcher, wait_time))
                time.sleep(2**i)
Exemplo n.º 8
0
    def announce_node(self, node_name, engine, process_dispatcher,
            state=None):
        """Announce a node to each process dispatcher.

        @param node_name: the name of the node to advertise
        @param engine: the execution engine of the node
        @param process_dispatcher: the pd to announce to
        @param state: the state to advertise to the pd
        """
        if not state:
            state = InstanceState.RUNNING

        pd_client = ProcessDispatcherClient(self.dashi, process_dispatcher)
        log.info("Announcing %s of engine %s is '%s' to %s" % (node_name,
            engine, state, process_dispatcher))
        domain_id = domain_id_from_engine(engine)
        for i in range(1, ADVERTISE_RETRIES):
            try:
                pd_client.node_state(node_name, domain_id, state)
                break
            except timeout:
                wait_time = i * i  # Exponentially increasing wait
                log.warning("PD '%s' not available yet. Waiting %ss" % (process_dispatcher, wait_time))
                time.sleep(2 ** i)