def setUp(self): if not os.environ.get("INT"): raise SkipTest("Skipping Slow integration test") self.exchange = "hatestexchange-%s" % str(uuid.uuid4()) self.sysname = "test-%s" % str(uuid.uuid4()) parsed_deployment = yaml.load(deployment_one_pd_two_eea) self.pd_names = parsed_deployment['process-dispatchers'].keys() self.eea_names = [] for node in parsed_deployment['nodes'].values(): for eeagent in node['eeagents'].keys(): self.eea_names.append(eeagent) policy_params = {'preserve_n': 0} executable = {'exec': 'sleep', 'argv': ['1000']} self.setup_harness(exchange=self.exchange, sysname=self.sysname) self.addCleanup(self.cleanup_harness) self.epuharness.start(deployment_str=deployment_one_pd_two_eea) self.block_until_ready(deployment_one_pd_two_eea, self.dashi) self.process_definition_id = uuid.uuid4().hex for pd_name in self.pd_names: pd_client = ProcessDispatcherClient(self.dashi, pd_name) pd_client.create_definition(self.process_definition_id, None, executable, None, None) self.haservice = HighAvailabilityService(policy_parameters=policy_params, process_dispatchers=self.pd_names, exchange=self.exchange, process_definition_id=self.process_definition_id, sysname=self.sysname) self.haservice_thread = tevent.spawn(self.haservice.start) self.dashi = self.haservice.dashi self.haservice_client = HighAvailabilityServiceClient(self.dashi, topic=self.haservice.topic)
def _get_proc_from_pd(self, upid, pd_name): pd_client = ProcessDispatcherClient(self.dashi, pd_name) procs = pd_client.describe_processes() for proc in procs: if upid == proc.get('upid'): return proc return None
def _get_all_procs(self): all_procs = {} for pd_name in self.pd_names: pd_client = ProcessDispatcherClient(self.dashi, pd_name) print "Querying %s" % pd_name procs = pd_client.describe_processes() all_procs[pd_name] = procs return all_procs
def block_until_ready(self, deployment_str, dashi): """Blocks until all of the services in a deployment are contacted """ deployment = parse_deployment(yaml_str=deployment_str) for provisioner_name in deployment.get('provisioners', {}).iterkeys(): provisioner = ProvisionerClient(dashi, topic=provisioner_name) self._block_on_call(provisioner.describe_nodes) for epum_name in deployment.get('epums', {}).iterkeys(): epum = EPUManagementClient(dashi, epum_name) self._block_on_call(epum.list_domains) for node in deployment.get('nodes', {}).itervalues(): for eeagent_name in node.get('eeagents', {}).iterkeys(): eeagent = EEAgentClient(dashi=dashi, ee_name=eeagent_name, handle_heartbeat=False) self._block_on_call(eeagent.dump, kwargs={'rpc': True}) for pd_name in deployment.get('process-dispatchers', {}).iterkeys(): pd = ProcessDispatcherClient(dashi, pd_name) self._block_on_call(pd.describe_processes) for dt_name in deployment.get('dt_registries', {}).iterkeys(): dtrs = DTRSClient(dashi, topic=dt_name) self._block_on_call(dtrs.list_sites)
def get_clients(self, deployment_str, dashi): """returns a dictionary of epu clients, indexed by their topic name """ deployment = parse_deployment(yaml_str=deployment_str) clients = {} for provisioner_name in deployment.get('provisioners', {}).iterkeys(): client = ProvisionerClient(dashi, topic=provisioner_name) clients[provisioner_name] = client for epum_name in deployment.get('epums', {}).iterkeys(): client = EPUManagementClient(dashi, epum_name) clients[epum_name] = client for node in deployment.get('nodes', {}).itervalues(): for eeagent_name in node.get('eeagents', {}).iterkeys(): client = EEAgentClient(dashi=dashi, ee_name=eeagent_name, handle_heartbeat=False) clients[eeagent_name] = client for pd_name in deployment.get('process-dispatchers', {}).iterkeys(): client = ProcessDispatcherClient(dashi, pd_name) clients[pd_name] = client for dt_name in deployment.get('dt_registries', {}).iterkeys(): client = DTRSClient(dashi, topic=dt_name) clients[dt_name] = client return clients
def test_missing_proc(self): """test_missing_proc Kill a proc, and ensure HA starts a replacement """ n = 2 self._update_policy_params_and_assert({'preserve_n': n}) self._assert_n_processes(n) upid_to_kill = self.haservice.core.managed_upids[0] pd = self._find_procs_pd(upid_to_kill) assert pd pd_client = ProcessDispatcherClient(self.dashi, pd) pd_client.terminate_process(upid_to_kill) print self._get_all_procs() print self._get_all_procs() print self._get_all_procs() time.sleep(5) self._assert_n_processes(n) time.sleep(5) self._assert_n_processes(n) print self._get_all_procs()
def announce_node(self, node_name, engine, process_dispatcher, state=None): """Announce a node to each process dispatcher. @param node_name: the name of the node to advertise @param engine: the execution engine of the node @param process_dispatcher: the pd to announce to @param state: the state to advertise to the pd """ if not state: state = InstanceState.RUNNING pd_client = ProcessDispatcherClient(self.dashi, process_dispatcher) log.info("Announcing %s of engine %s is '%s' to %s" % (node_name, engine, state, process_dispatcher)) domain_id = domain_id_from_engine(engine) for i in range(1, ADVERTISE_RETRIES): try: pd_client.node_state(node_name, domain_id, state) break except timeout: wait_time = i * i # Exponentially increasing wait log.warning("PD '%s' not available yet. Waiting %ss" % (process_dispatcher, wait_time)) time.sleep(2**i)
def announce_node(self, node_name, engine, process_dispatcher, state=None): """Announce a node to each process dispatcher. @param node_name: the name of the node to advertise @param engine: the execution engine of the node @param process_dispatcher: the pd to announce to @param state: the state to advertise to the pd """ if not state: state = InstanceState.RUNNING pd_client = ProcessDispatcherClient(self.dashi, process_dispatcher) log.info("Announcing %s of engine %s is '%s' to %s" % (node_name, engine, state, process_dispatcher)) domain_id = domain_id_from_engine(engine) for i in range(1, ADVERTISE_RETRIES): try: pd_client.node_state(node_name, domain_id, state) break except timeout: wait_time = i * i # Exponentially increasing wait log.warning("PD '%s' not available yet. Waiting %ss" % (process_dispatcher, wait_time)) time.sleep(2 ** i)