def run(self):
log.info("Elected as provisioner leader!")
# TODO need better handling of time, this is susceptible to system clock changes
while self.is_leader:
if self.store.is_disabled():
disabled_agreed = self.store.is_disabled_agreed()
log.debug("terminate_all: disabled_agreed=%s", disabled_agreed)
if not disabled_agreed:
log.info("provisioner termination detected but not all processes agree yet. waiting.")
else:
log.info("provisioner termination beginning")
self.core.terminate_all()
if self.terminator_thread is None:
self.terminator_thread = tevent.spawn(self.run_terminator, _fail_fast=True)
if self.site_query_thread is None:
self.site_query_thread = tevent.spawn(self.run_site_query_thread, _fail_fast=True)
if self.context_query_thread is None:
self.context_query_thread = tevent.spawn(self.run_context_query_thread, _fail_fast=True)
if self.record_reaper_thread is None:
self.record_reaper_thread = tevent.spawn(self.run_record_reaper_thread, _fail_fast=True)
with self.condition:
if self.is_leader:
self.condition.wait(self.query_delay)
def test_fail_fast():
mock_exit = Mock()
def raises_typerror():
raise TypeError("This thread failed!")
tevent.spawn(raises_typerror, _fail_fast=True, _exit=mock_exit)
def test_initialize(self):
self.doctor.is_leader = True
self.doctor.config[self.doctor.CONFIG_MONITOR_HEARTBEATS] = False
def stop_being_leader_in_one_sec():
time.sleep(1)
with self.doctor.condition:
self.doctor.is_leader = False
self.doctor.condition.notify_all()
tevent.spawn(stop_being_leader_in_one_sec)
self.doctor._initialize()
def test_wait_resource(self):
props = {"engine": "engine1"}
r1 = ResourceRecord.new("r1", "n1", 1, properties=props)
self.store.add_resource(r1)
self.wait_resource("r1", lambda r: r.resource_id == "r1")
def makeitso():
r1.slot_count = 2
self.store.update_resource(r1)
tevent.spawn(makeitso)
self.wait_resource("r1", lambda r: r.slot_count == 2)
def inaugurate(self):
"""Callback from the election fired when this leader is elected
This routine is expected to never return as long as we want to
remain the leader.
"""
with self.condition:
if self.is_leader:
raise Exception("already the leader???")
self.is_leader = True
pd_state = self.store.get_pd_state()
log.info("Elected as Process Dispatcher Doctor. State is %s", pd_state)
if pd_state == ProcessDispatcherState.UNINITIALIZED:
self.initialize_pd()
pd_state = self.store.get_pd_state()
if pd_state == ProcessDispatcherState.SYSTEM_BOOTING:
self.watching_system_boot = True
self._watch_system_boot()
with self.condition:
if self.is_leader and self.config.get(self.CONFIG_MONITOR_HEARTBEATS, True):
self.monitor = ExecutionResourceMonitor(self.core, self.store)
self.monitor_thread = tevent.spawn(self.monitor.monitor)
while self.is_leader:
with self.condition:
self.condition.wait()
log.debug("Waiting on monitor thread to exit")
self.monitor_thread.join()
self.monitor = None
self.monitor_thread = None
def test_spawn():
global global_var
global global_args
global global_kwargs
def thread_function(*args, **kwargs):
global global_var
global global_kwargs
global global_args
global_var = True
global_kwargs = kwargs
global_args = args
time.sleep(1)
global_var = False
test_args = ('such', 'good', 'args',)
test_kwargs = {'my': 'really', 'great': 'parameters'}
mythread = tevent.spawn(thread_function, *test_args, **test_kwargs)
assert mythread.__class__ == threading.Thread().__class__
time.sleep(0.5)
assert global_var is True
assert global_args == test_args
assert global_kwargs == test_kwargs
def contend_leader(self, leader):
"""Provide a leader object to participate in an election
"""
assert self._leader is None
self._leader = leader
self._election_thread = tevent.spawn(self._run_election,
self.election, leader, "leader")
def _initialize(self):
pd_state = self.store.get_pd_state()
log.info("Elected as Process Dispatcher Doctor. State is %s", pd_state)
if pd_state == ProcessDispatcherState.UNINITIALIZED:
self.initialize_pd()
pd_state = self.store.get_pd_state()
if pd_state == ProcessDispatcherState.SYSTEM_BOOTING:
self.watching_system_boot = True
self._watch_system_boot()
with self.condition:
if self.is_leader and self.config.get(self.CONFIG_MONITOR_HEARTBEATS, True):
self.monitor = ExecutionResourceMonitor(self.core, self.store)
self.monitor_thread = tevent.spawn(self.monitor.monitor)
while self.is_leader:
with self.condition:
self.condition.wait()
log.debug("Waiting on monitor thread to exit")
if self.monitor_thread is not None:
self.monitor_thread.join()
self.monitor = None
self.monitor_thread = None
def setUp(self):
if not os.environ.get("INT"):
raise SkipTest("Skipping Slow integration test")
self.exchange = "hatestexchange-%s" % str(uuid.uuid4())
self.sysname = "test-%s" % str(uuid.uuid4())
parsed_deployment = yaml.load(deployment_one_pd_two_eea)
self.pd_names = parsed_deployment['process-dispatchers'].keys()
self.eea_names = []
for node in parsed_deployment['nodes'].values():
for eeagent in node['eeagents'].keys():
self.eea_names.append(eeagent)
policy_params = {'preserve_n': 0}
executable = {'exec': 'sleep', 'argv': ['1000']}
self.setup_harness(exchange=self.exchange, sysname=self.sysname)
self.addCleanup(self.cleanup_harness)
self.epuharness.start(deployment_str=deployment_one_pd_two_eea)
self.block_until_ready(deployment_one_pd_two_eea, self.dashi)
self.process_definition_id = uuid.uuid4().hex
for pd_name in self.pd_names:
pd_client = ProcessDispatcherClient(self.dashi, pd_name)
pd_client.create_definition(self.process_definition_id, None,
executable, None, None)
self.haservice = HighAvailabilityService(policy_parameters=policy_params,
process_dispatchers=self.pd_names, exchange=self.exchange,
process_definition_id=self.process_definition_id,
sysname=self.sysname)
self.haservice_thread = tevent.spawn(self.haservice.start)
self.dashi = self.haservice.dashi
self.haservice_client = HighAvailabilityServiceClient(self.dashi, topic=self.haservice.topic)
def test_leader(self):
core = Mock()
store = Mock()
def get_terminating():
time.sleep(0.5)
return []
def side_effect(event, *args, **kwargs):
event.set()
nodes_event = threading.Event()
ctx_event = threading.Event()
store.get_terminating = MagicMock(side_effect=get_terminating)
core._get_nodes_by_id = MagicMock(return_value=[])
core.query_nodes = MagicMock(side_effect=partial(side_effect, nodes_event))
core.query_contexts = MagicMock(side_effect=partial(side_effect, ctx_event))
leader = ProvisionerLeader(store, core)
leader.initialize()
store.contend_leader.assert_called_with(leader)
leader_thread = tevent.spawn(leader.inaugurate)
# the leader should call core.query_nodes() and core.query_contexts()
# wait for that.
assert nodes_event.wait(1)
assert ctx_event.wait(1)
assert core.query_nodes.called
assert core.query_contexts.called
nodes_event.clear()
ctx_event.clear()
core.query_nodes.reset()
core.query_contexts.reset()
# reset and trigger another query cycle (peeking into impl)
leader._force_cycle()
assert nodes_event.wait(1)
assert ctx_event.wait(1)
assert core.query_nodes.called
assert core.query_contexts.called
leader.depose()
self.assertFalse(leader.is_leader)
leader_thread.join(1)
self.assertFalse(leader_thread.is_alive())
def test_terminator_death(self):
# This test will bring down the nose process, so we run it in a child
# process.
child_pid = os.fork()
if child_pid == 0:
core = Mock()
store = Mock()
def dies():
raise TypeError("thread dies!")
core._get_nodes_by_id = MagicMock(return_value=[])
leader = ProvisionerLeader(store, core)
leader.run_terminator = dies
leader.initialize()
store.contend_leader.assert_called_with(leader)
leader_thread = tevent.spawn(leader.inaugurate)
tevent.joinall([leader_thread])
else:
pid, exit = os.wait()
# exit is a 16-bit number, whose low byte is the signal number that
# killed the process, and whose high byte is the exit status (if
# the signal number is zero); the high bit of the low byte is set
# if a core file was produced.
#
# Check that the signal number is zero and the exit code is the
# expected one.
self.assertEqual(exit & 0xff, 0)
self.assertEqual(exit >> 8 & 0xff, os.EX_SOFTWARE)
def _disabled_watch(self, event):
tevent.spawn(self._update_disabled_state)
def _connection_state_listener(self, state):
# called by kazoo when the connection state changes.
# handle in background
tevent.spawn(self._handle_connection_state, state)
def _make_leader(self):
assert self.leader
assert not self.is_leading
self.is_leading = True
self.leader_thread = tevent.spawn(self.leader.inaugurate)
def _spawn_process(self, process):
thread = tevent.spawn(process)
self.threads.append(thread)
def _run_in_thread(self):
self.docthread = tevent.spawn(self.doctor.inaugurate)
time.sleep(0.05)
