def setUp(self):
if not os.environ.get("INT"):
raise SkipTest("Skipping Slow integration test")
self.exchange = "hatestexchange-%s" % str(uuid.uuid4())
self.sysname = "test-%s" % str(uuid.uuid4())
parsed_deployment = yaml.load(deployment_one_pd_two_eea)
self.pd_names = parsed_deployment['process-dispatchers'].keys()
self.eea_names = []
for node in parsed_deployment['nodes'].values():
for eeagent in node['eeagents'].keys():
self.eea_names.append(eeagent)
policy_params = {'preserve_n': 0}
executable = {'exec': 'sleep', 'argv': ['1000']}
self.setup_harness(exchange=self.exchange, sysname=self.sysname)
self.addCleanup(self.cleanup_harness)
self.epuharness.start(deployment_str=deployment_one_pd_two_eea)
self.block_until_ready(deployment_one_pd_two_eea, self.dashi)
self.process_definition_id = uuid.uuid4().hex
for pd_name in self.pd_names:
pd_client = ProcessDispatcherClient(self.dashi, pd_name)
pd_client.create_definition(self.process_definition_id, None,
executable, None, None)
self.haservice = HighAvailabilityService(policy_parameters=policy_params,
process_dispatchers=self.pd_names, exchange=self.exchange,
process_definition_id=self.process_definition_id,
sysname=self.sysname)
self.haservice_thread = tevent.spawn(self.haservice.start)
self.dashi = self.haservice.dashi
self.haservice_client = HighAvailabilityServiceClient(self.dashi, topic=self.haservice.topic)
def _get_proc_from_pd(self, upid, pd_name):
pd_client = ProcessDispatcherClient(self.dashi, pd_name)
procs = pd_client.describe_processes()
for proc in procs:
if upid == proc.get('upid'):
return proc
return None
def _get_all_procs(self):
all_procs = {}
for pd_name in self.pd_names:
pd_client = ProcessDispatcherClient(self.dashi, pd_name)
print "Querying %s" % pd_name
procs = pd_client.describe_processes()
all_procs[pd_name] = procs
return all_procs
def block_until_ready(self, deployment_str, dashi):
"""Blocks until all of the services in a deployment are contacted
"""
deployment = parse_deployment(yaml_str=deployment_str)
for provisioner_name in deployment.get('provisioners', {}).iterkeys():
provisioner = ProvisionerClient(dashi, topic=provisioner_name)
self._block_on_call(provisioner.describe_nodes)
for epum_name in deployment.get('epums', {}).iterkeys():
epum = EPUManagementClient(dashi, epum_name)
self._block_on_call(epum.list_domains)
for node in deployment.get('nodes', {}).itervalues():
for eeagent_name in node.get('eeagents', {}).iterkeys():
eeagent = EEAgentClient(dashi=dashi,
ee_name=eeagent_name,
handle_heartbeat=False)
self._block_on_call(eeagent.dump, kwargs={'rpc': True})
for pd_name in deployment.get('process-dispatchers', {}).iterkeys():
pd = ProcessDispatcherClient(dashi, pd_name)
self._block_on_call(pd.describe_processes)
for dt_name in deployment.get('dt_registries', {}).iterkeys():
dtrs = DTRSClient(dashi, topic=dt_name)
self._block_on_call(dtrs.list_sites)
def get_clients(self, deployment_str, dashi):
"""returns a dictionary of epu clients, indexed by their topic name
"""
deployment = parse_deployment(yaml_str=deployment_str)
clients = {}
for provisioner_name in deployment.get('provisioners', {}).iterkeys():
client = ProvisionerClient(dashi, topic=provisioner_name)
clients[provisioner_name] = client
for epum_name in deployment.get('epums', {}).iterkeys():
client = EPUManagementClient(dashi, epum_name)
clients[epum_name] = client
for node in deployment.get('nodes', {}).itervalues():
for eeagent_name in node.get('eeagents', {}).iterkeys():
client = EEAgentClient(dashi=dashi,
ee_name=eeagent_name,
handle_heartbeat=False)
clients[eeagent_name] = client
for pd_name in deployment.get('process-dispatchers', {}).iterkeys():
client = ProcessDispatcherClient(dashi, pd_name)
clients[pd_name] = client
for dt_name in deployment.get('dt_registries', {}).iterkeys():
client = DTRSClient(dashi, topic=dt_name)
clients[dt_name] = client
return clients
def test_missing_proc(self):
"""test_missing_proc
Kill a proc, and ensure HA starts a replacement
"""
n = 2
self._update_policy_params_and_assert({'preserve_n': n})
self._assert_n_processes(n)
upid_to_kill = self.haservice.core.managed_upids[0]
pd = self._find_procs_pd(upid_to_kill)
assert pd
pd_client = ProcessDispatcherClient(self.dashi, pd)
pd_client.terminate_process(upid_to_kill)
print self._get_all_procs()
print self._get_all_procs()
print self._get_all_procs()
time.sleep(5)
self._assert_n_processes(n)
time.sleep(5)
self._assert_n_processes(n)
print self._get_all_procs()
def announce_node(self, node_name, engine, process_dispatcher, state=None):
"""Announce a node to each process dispatcher.
@param node_name: the name of the node to advertise
@param engine: the execution engine of the node
@param process_dispatcher: the pd to announce to
@param state: the state to advertise to the pd
"""
if not state:
state = InstanceState.RUNNING
pd_client = ProcessDispatcherClient(self.dashi, process_dispatcher)
log.info("Announcing %s of engine %s is '%s' to %s" %
(node_name, engine, state, process_dispatcher))
domain_id = domain_id_from_engine(engine)
for i in range(1, ADVERTISE_RETRIES):
try:
pd_client.node_state(node_name, domain_id, state)
break
except timeout:
wait_time = i * i # Exponentially increasing wait
log.warning("PD '%s' not available yet. Waiting %ss" %
(process_dispatcher, wait_time))
time.sleep(2**i)
def announce_node(self, node_name, engine, process_dispatcher,
state=None):
"""Announce a node to each process dispatcher.
@param node_name: the name of the node to advertise
@param engine: the execution engine of the node
@param process_dispatcher: the pd to announce to
@param state: the state to advertise to the pd
"""
if not state:
state = InstanceState.RUNNING
pd_client = ProcessDispatcherClient(self.dashi, process_dispatcher)
log.info("Announcing %s of engine %s is '%s' to %s" % (node_name,
engine, state, process_dispatcher))
domain_id = domain_id_from_engine(engine)
for i in range(1, ADVERTISE_RETRIES):
try:
pd_client.node_state(node_name, domain_id, state)
break
except timeout:
wait_time = i * i # Exponentially increasing wait
log.warning("PD '%s' not available yet. Waiting %ss" % (process_dispatcher, wait_time))
time.sleep(2 ** i)
