def test_double_reader_writer(self):
lock = lock_utils.ReaderWriterLock()
activated = collections.deque()
active = threading_utils.Event()
def double_reader():
with lock.read_lock():
active.set()
while not lock.has_pending_writers:
time.sleep(0.001)
with lock.read_lock():
activated.append(lock.owner)
def happy_writer():
with lock.write_lock():
activated.append(lock.owner)
reader = threading_utils.daemon_thread(double_reader)
reader.start()
self.assertTrue(active.wait(test_utils.WAIT_TIMEOUT))
writer = threading_utils.daemon_thread(happy_writer)
writer.start()
reader.join()
writer.join()
self.assertEqual(2, len(activated))
self.assertEqual(['r', 'w'], list(activated))
def main():
if six.PY3:
# TODO(harlowja): Hack to make eventlet work right, remove when the
# following is fixed: https://github.com/eventlet/eventlet/issues/230
from taskflow.utils import eventlet_utils as _eu # noqa
try:
import eventlet as _eventlet # noqa
except ImportError:
pass
with contextlib.closing(fake_client.FakeClient()) as c:
created = []
for i in compat_range(0, PRODUCERS):
p = threading_utils.daemon_thread(producer, i + 1, c)
created.append(p)
p.start()
consumed = collections.deque()
for i in compat_range(0, WORKERS):
w = threading_utils.daemon_thread(worker, i + 1, c, consumed)
created.append(w)
w.start()
while created:
t = created.pop()
t.join()
# At the end there should be nothing leftover, let's verify that.
board = backends.fetch('verifier', SHARED_CONF.copy(), client=c)
board.connect()
with contextlib.closing(board):
if board.job_count != 0 or len(consumed) != EXPECTED_UNITS:
return 1
return 0
def test_wait_arrival(self):
ev = threading.Event()
jobs = []
def poster(wait_post=0.2):
if not ev.wait(test_utils.WAIT_TIMEOUT):
raise RuntimeError("Waiter did not appear ready"
" in %s seconds" % test_utils.WAIT_TIMEOUT)
time.sleep(wait_post)
self.board.post('test', p_utils.temporary_log_book())
def waiter():
ev.set()
it = self.board.wait()
jobs.extend(it)
with connect_close(self.board):
t1 = threading_utils.daemon_thread(poster)
t1.start()
t2 = threading_utils.daemon_thread(waiter)
t2.start()
for t in (t1, t2):
t.join()
self.assertEqual(1, len(jobs))
def main():
if six.PY3:
# TODO(harlowja): Hack to make eventlet work right, remove when the
# following is fixed: https://github.com/eventlet/eventlet/issues/230
from taskflow.utils import eventlet_utils as _eu # noqa
try:
import eventlet as _eventlet # noqa
except ImportError:
pass
with contextlib.closing(fake_client.FakeClient()) as c:
created = []
for i in compat_range(0, PRODUCERS):
p = threading_utils.daemon_thread(producer, i + 1, c)
created.append(p)
p.start()
consumed = collections.deque()
for i in compat_range(0, WORKERS):
w = threading_utils.daemon_thread(worker, i + 1, c, consumed)
created.append(w)
w.start()
while created:
t = created.pop()
t.join()
# At the end there should be nothing leftover, let's verify that.
board = backends.fetch('verifier', SHARED_CONF.copy(), client=c)
board.connect()
with contextlib.closing(board):
if board.job_count != 0 or len(consumed) != EXPECTED_UNITS:
return 1
return 0
def test_wait_arrival(self):
ev = threading.Event()
jobs = []
def poster(wait_post=0.2):
if not ev.wait(test_utils.WAIT_TIMEOUT):
raise RuntimeError("Waiter did not appear ready"
" in %s seconds" % test_utils.WAIT_TIMEOUT)
time.sleep(wait_post)
self.board.post('test', p_utils.temporary_log_book())
def waiter():
ev.set()
it = self.board.wait()
jobs.extend(it)
with connect_close(self.board):
t1 = threading_utils.daemon_thread(poster)
t1.start()
t2 = threading_utils.daemon_thread(waiter)
t2.start()
for t in (t1, t2):
t.join()
self.assertEqual(1, len(jobs))
def test_double_reader_writer(self):
lock = lock_utils.ReaderWriterLock()
activated = collections.deque()
active = threading_utils.Event()
def double_reader():
with lock.read_lock():
active.set()
while not lock.has_pending_writers:
time.sleep(0.001)
with lock.read_lock():
activated.append(lock.owner)
def happy_writer():
with lock.write_lock():
activated.append(lock.owner)
reader = threading_utils.daemon_thread(double_reader)
reader.start()
self.assertTrue(active.wait(test_utils.WAIT_TIMEOUT))
writer = threading_utils.daemon_thread(happy_writer)
writer.start()
reader.join()
writer.join()
self.assertEqual(2, len(activated))
self.assertEqual(['r', 'w'], list(activated))
def start(self):
"""Starts proxy thread and associated topic notification thread."""
if not _is_alive(self._proxy_thread):
self._proxy_thread = tu.daemon_thread(self._proxy.start)
self._proxy_thread.start()
self._proxy.wait()
if not _is_alive(self._periodic_thread):
self._periodic.reset()
self._periodic_thread = tu.daemon_thread(self._periodic.start)
self._periodic_thread.start()
def start(self):
"""Starts proxy thread and associated topic notification thread."""
if not _is_alive(self._proxy_thread):
self._proxy_thread = tu.daemon_thread(self._proxy.start)
self._proxy_thread.start()
self._proxy.wait()
if not _is_alive(self._periodic_thread):
self._periodic.reset()
self._periodic_thread = tu.daemon_thread(self._periodic.start)
self._periodic_thread.start()
def test_double_acquire_many(self):
activated = collections.deque()
n_lock = lock_utils.MultiLock((threading.RLock(), threading.RLock()))
def critical_section():
start = time.time()
time.sleep(0.05)
end = time.time()
activated.append((start, end))
def run():
with n_lock:
critical_section()
with n_lock:
critical_section()
critical_section()
threads = []
for i in range(0, 20):
t = threading_utils.daemon_thread(run)
threads.append(t)
t.start()
while threads:
t = threads.pop()
t.join()
for (start, end) in activated:
self.assertEqual(1, _find_overlaps(activated, start, end))
def test_acquired_pass(self):
activated = collections.deque()
lock1 = threading.Lock()
lock2 = threading.Lock()
n_lock = lock_utils.MultiLock((lock1, lock2))
def critical_section():
start = time.time()
time.sleep(0.05)
end = time.time()
activated.append((start, end))
def run():
with n_lock:
critical_section()
threads = []
for _i in range(0, 20):
t = threading_utils.daemon_thread(run)
threads.append(t)
t.start()
while threads:
t = threads.pop()
t.join()
for (start, end) in activated:
self.assertEqual(1, _find_overlaps(activated, start, end))
self.assertFalse(lock1.locked())
self.assertFalse(lock2.locked())
def check_worker_state(self, context, startup=True):
LOG.debug('Periodic Timer for worker health check expired')
dead = []
for instance in self.reaper_instances:
if not instance.worker.is_alive() or instance.missed_acks > 5:
LOG.error('Worker for aggregates %s, found dead.',
instance.aggregates)
instance.stop_handling()
instance.worker.join(timeout=0.1)
dead.append(instance)
else:
# Each worker will reset this to 0. If the worker is stuck
# after 3 periodic checks then it is presumed dead and it's
# revived.
instance.missed_acks += 1
for instance in dead:
self.reaper_instances.remove(instance)
LOG.info('Reviving worker for aggregates %s.',
instance.aggregates)
new_instance = reaper.Reaper(instance.aggregates)
new_instance.worker = threading_utils.daemon_thread(
new_instance.job_handler)
self.reaper_instances.append(new_instance)
new_instance.worker.start()
def test_response(self):
barrier = threading_utils.Event()
on_response = mock.MagicMock()
on_response.side_effect = lambda *args, **kwargs: barrier.set()
handlers = {pr.RESPONSE: on_response}
p = proxy.Proxy(TEST_TOPIC,
TEST_EXCHANGE,
handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
resp = pr.Response(pr.RUNNING)
p.publish(resp, TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
self.assertTrue(barrier.is_set())
p.stop()
t.join()
self.assertTrue(on_response.called)
on_response.assert_called_with(resp.to_dict(), mock.ANY)
def test_notify(self):
barrier = threading_utils.Event()
on_notify = mock.MagicMock()
on_notify.side_effect = lambda *args, **kwargs: barrier.set()
handlers = {pr.NOTIFY: on_notify}
p = proxy.Proxy(TEST_TOPIC,
TEST_EXCHANGE,
handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
p.publish(pr.Notify(), TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
p.stop()
t.join()
self.assertTrue(on_notify.called)
on_notify.assert_called_with({}, mock.ANY)
def test_start_stop(self):
events = collections.deque()
def before_start(t):
events.append('bs')
def before_join(t):
events.append('bj')
self.death.set()
def after_start(t):
events.append('as')
def after_join(t):
events.append('aj')
for _i in range(0, self.thread_count):
self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death),
before_join=before_join,
after_join=after_join,
before_start=before_start,
after_start=after_start)
self.assertEqual(self.thread_count, self.bundle.start())
self.assertEqual(self.thread_count, len(self.bundle))
self.assertEqual(self.thread_count, self.bundle.stop())
for event in ['as', 'bs', 'bj', 'aj']:
self.assertEqual(self.thread_count,
len([e for e in events if e == event]))
self.assertEqual(0, self.bundle.stop())
self.assertTrue(self.death.is_set())
def test_double_acquire_many(self):
activated = collections.deque()
acquires = collections.deque()
n_lock = lock_utils.MultiLock((threading.RLock(), threading.RLock()))
def critical_section():
start = now()
time.sleep(NAPPY_TIME)
end = now()
activated.append((start, end))
def run():
with n_lock as gotten:
acquires.append(gotten)
critical_section()
with n_lock as gotten:
acquires.append(gotten)
critical_section()
critical_section()
threads = []
for i in range(0, self.THREAD_COUNT):
t = threading_utils.daemon_thread(run)
threads.append(t)
t.start()
while threads:
t = threads.pop()
t.join()
self.assertTrue(all(acquires))
self.assertEqual(self.THREAD_COUNT * 2, len(acquires))
self.assertEqual(self.THREAD_COUNT * 3, len(activated))
for (start, end) in activated:
self.assertEqual(1, _find_overlaps(activated, start, end))
def test_stop(self):
self.conn_inst_mock.drain_events.side_effect = socket.timeout
# create proxy
pr = self.proxy(reset_master_mock=True)
# check that proxy is not running yes
self.assertFalse(pr.is_running)
# start proxy in separate thread
t = threading_utils.daemon_thread(pr.start)
t.start()
# make sure proxy is started
pr.wait()
# check that proxy is running now
self.assertTrue(pr.is_running)
# stop proxy and wait for thread to finish
pr.stop()
# wait for thread to finish
t.join()
self.assertFalse(pr.is_running)
def test_run_max_dispatches(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
components.board.notifier.register(base.REMOVAL, on_consume)
with close_many(components.client, components.conductor):
t = threading_utils.daemon_thread(
lambda: components.conductor.run(max_dispatches=5))
t.start()
lb, fd = pu.temporary_flow_detail(components.persistence)
engines.save_factory_details(fd,
test_factory, [False], {},
backend=components.persistence)
for _ in range(5):
components.board.post('poke',
lb,
details={'flow_uuid': fd.uuid})
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
components.board.post('poke', lb, details={'flow_uuid': fd.uuid})
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
def test_run(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
components.board.notifier.register(base.REMOVAL, on_consume)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
lb, fd = pu.temporary_flow_detail(components.persistence)
engines.save_factory_details(fd,
test_factory, [False], {},
backend=components.persistence)
components.board.post('poke', lb, details={'flow_uuid': fd.uuid})
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
persistence = components.persistence
with contextlib.closing(persistence.get_connection()) as conn:
lb = conn.get_logbook(lb.uuid)
fd = lb.find(fd.uuid)
self.assertIsNotNone(fd)
self.assertEqual(st.SUCCESS, fd.state)
def test_threaded_access_property(self):
called = collections.deque()
class A(object):
@misc.cachedproperty
def b(self):
called.append(1)
# NOTE(harlowja): wait for a little and give some time for
# another thread to potentially also get in this method to
# also create the same property...
time.sleep(random.random() * 0.5)
return 'b'
a = A()
threads = []
try:
for _i in range(0, 20):
t = threading_utils.daemon_thread(lambda: a.b)
threads.append(t)
for t in threads:
t.start()
finally:
while threads:
t = threads.pop()
t.join()
self.assertEqual(1, len(called))
self.assertEqual('b', a.b)
def test_no_double_writers(self):
lock = lock_utils.ReaderWriterLock()
watch = timing.StopWatch(duration=5)
watch.start()
dups = collections.deque()
active = collections.deque()
def acquire_check(me):
with lock.write_lock():
if len(active) >= 1:
dups.append(me)
dups.extend(active)
active.append(me)
try:
time.sleep(random.random() / 100)
finally:
active.remove(me)
def run():
me = threading.current_thread()
while not watch.expired():
acquire_check(me)
threads = []
for i in range(0, self.THREAD_COUNT):
t = threading_utils.daemon_thread(run)
threads.append(t)
t.start()
while threads:
t = threads.pop()
t.join()
self.assertEqual([], list(dups))
self.assertEqual([], list(active))
def test_response(self):
barrier = threading_utils.Event()
on_response = mock.MagicMock()
on_response.side_effect = lambda *args, **kwargs: barrier.set()
handlers = {pr.RESPONSE: dispatcher.Handler(on_response)}
p = proxy.Proxy(TEST_TOPIC, TEST_EXCHANGE, handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
resp = pr.Response(pr.RUNNING)
p.publish(resp, TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
self.assertTrue(barrier.is_set())
p.stop()
t.join()
self.assertTrue(on_response.called)
on_response.assert_called_with(resp.to_dict(), mock.ANY)
def test_double_acquire_many(self):
activated = collections.deque()
acquires = collections.deque()
n_lock = lock_utils.MultiLock((threading.RLock(), threading.RLock()))
def critical_section():
start = now()
time.sleep(NAPPY_TIME)
end = now()
activated.append((start, end))
def run():
with n_lock as gotten:
acquires.append(gotten)
critical_section()
with n_lock as gotten:
acquires.append(gotten)
critical_section()
critical_section()
threads = []
for i in range(0, self.THREAD_COUNT):
t = threading_utils.daemon_thread(run)
threads.append(t)
t.start()
while threads:
t = threads.pop()
t.join()
self.assertTrue(all(acquires))
self.assertEqual(self.THREAD_COUNT * 2, len(acquires))
self.assertEqual(self.THREAD_COUNT * 3, len(activated))
for (start, end) in activated:
self.assertEqual(1, _find_overlaps(activated, start, end))
def test_no_double_writers(self):
lock = lock_utils.ReaderWriterLock()
watch = timing.StopWatch(duration=5)
watch.start()
dups = collections.deque()
active = collections.deque()
def acquire_check(me):
with lock.write_lock():
if len(active) >= 1:
dups.append(me)
dups.extend(active)
active.append(me)
try:
time.sleep(random.random() / 100)
finally:
active.remove(me)
def run():
me = threading.current_thread()
while not watch.expired():
acquire_check(me)
threads = []
for i in range(0, self.THREAD_COUNT):
t = threading_utils.daemon_thread(run)
threads.append(t)
t.start()
while threads:
t = threads.pop()
t.join()
self.assertEqual([], list(dups))
self.assertEqual([], list(active))
def test_fail_run(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading_utils.Event()
def on_consume(state, details):
consumed_event.set()
components.board.notifier.register(jobboard.REMOVAL, on_consume)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
lb, fd = pu.temporary_flow_detail(components.persistence)
engines.save_factory_details(fd, test_factory,
[True], {},
backend=components.persistence)
components.board.post('poke', lb,
details={'flow_uuid': fd.uuid})
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
self.assertTrue(components.conductor.stop(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
persistence = components.persistence
with contextlib.closing(persistence.get_connection()) as conn:
lb = conn.get_logbook(lb.uuid)
fd = lb.find(fd.uuid)
self.assertIsNotNone(fd)
self.assertEqual(st.REVERTED, fd.state)
def test_run_max_dispatches(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
components.board.notifier.register(base.REMOVAL, on_consume)
with close_many(components.client, components.conductor):
t = threading_utils.daemon_thread(
lambda: components.conductor.run(max_dispatches=5))
t.start()
lb, fd = pu.temporary_flow_detail(components.persistence)
engines.save_factory_details(fd, test_factory,
[False], {},
backend=components.persistence)
for _ in range(5):
components.board.post('poke', lb,
details={'flow_uuid': fd.uuid})
self.assertTrue(consumed_event.wait(
test_utils.WAIT_TIMEOUT))
components.board.post('poke', lb,
details={'flow_uuid': fd.uuid})
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
def test_threaded_access_property(self):
called = collections.deque()
class A(object):
@misc.cachedproperty
def b(self):
called.append(1)
# NOTE(harlowja): wait for a little and give some time for
# another thread to potentially also get in this method to
# also create the same property...
time.sleep(random.random() * 0.5)
return 'b'
a = A()
threads = []
try:
for _i in range(0, 20):
t = threading_utils.daemon_thread(lambda: a.b)
threads.append(t)
for t in threads:
t.start()
finally:
while threads:
t = threads.pop()
t.join()
self.assertEqual(1, len(called))
self.assertEqual('b', a.b)
def test_combined_store(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
flow_store = {'x': True, 'y': False}
job_store = {'z': None}
components.board.notifier.register(base.REMOVAL, on_consume)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
lb, fd = pu.temporary_flow_detail(components.persistence,
meta={'store': flow_store})
engines.save_factory_details(fd, test_store_factory,
[], {},
backend=components.persistence)
components.board.post('poke', lb,
details={'flow_uuid': fd.uuid,
'store': job_store})
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
persistence = components.persistence
with contextlib.closing(persistence.get_connection()) as conn:
lb = conn.get_logbook(lb.uuid)
fd = lb.find(fd.uuid)
self.assertIsNotNone(fd)
self.assertEqual(st.SUCCESS, fd.state)
def test_stop(self):
self.conn_inst_mock.drain_events.side_effect = socket.timeout
# create proxy
pr = self.proxy(reset_master_mock=True)
# check that proxy is not running yes
self.assertFalse(pr.is_running)
# start proxy in separate thread
t = threading_utils.daemon_thread(pr.start)
t.start()
# make sure proxy is started
pr.wait()
# check that proxy is running now
self.assertTrue(pr.is_running)
# stop proxy and wait for thread to finish
pr.stop()
# wait for thread to finish
t.join()
self.assertFalse(pr.is_running)
def test_start_stop(self):
events = collections.deque()
def before_start(t):
events.append('bs')
def before_join(t):
events.append('bj')
self.death.set()
def after_start(t):
events.append('as')
def after_join(t):
events.append('aj')
for _i in range(0, self.thread_count):
self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death),
before_join=before_join,
after_join=after_join,
before_start=before_start,
after_start=after_start)
self.assertEqual(self.thread_count, self.bundle.start())
self.assertEqual(self.thread_count, len(self.bundle))
self.assertEqual(self.thread_count, self.bundle.stop())
for event in ['as', 'bs', 'bj', 'aj']:
self.assertEqual(self.thread_count,
len([e for e in events if e == event]))
self.assertEqual(0, self.bundle.stop())
self.assertTrue(self.death.is_set())
def test_multi_message(self):
message_count = 30
barrier = latch.Latch(message_count)
countdown = lambda data, message: barrier.countdown()
on_notify = mock.MagicMock()
on_notify.side_effect = countdown
on_response = mock.MagicMock()
on_response.side_effect = countdown
on_request = mock.MagicMock()
on_request.side_effect = countdown
handlers = {
pr.NOTIFY: dispatcher.Handler(on_notify),
pr.RESPONSE: dispatcher.Handler(on_response),
pr.REQUEST: dispatcher.Handler(on_request),
}
p = proxy.Proxy(TEST_TOPIC, TEST_EXCHANGE, handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
for i in range(0, message_count):
j = i % 3
if j == 0:
p.publish(pr.Notify(), TEST_TOPIC)
elif j == 1:
p.publish(pr.Response(pr.RUNNING), TEST_TOPIC)
else:
p.publish(pr.Request(test_utils.DummyTask("dummy_%s" % i),
uuidutils.generate_uuid(),
pr.EXECUTE, [], None), TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
self.assertEqual(0, barrier.needed)
p.stop()
t.join()
self.assertTrue(on_notify.called)
self.assertTrue(on_response.called)
self.assertTrue(on_request.called)
self.assertEqual(10, on_notify.call_count)
self.assertEqual(10, on_response.call_count)
self.assertEqual(10, on_request.call_count)
call_count = sum([
on_notify.call_count,
on_response.call_count,
on_request.call_count,
])
self.assertEqual(message_count, call_count)
def start(self):
"""Starts message processing thread."""
if self._helper is not None:
raise RuntimeError("Worker executor must be stopped before"
" it can be started")
self._helper = tu.daemon_thread(self._proxy.start)
self._helper.start()
self._proxy.wait()
def test_bind_invalid(self):
self.assertRaises(ValueError, self.bundle.bind, 1)
for k in ['after_start', 'before_start', 'before_join', 'after_join']:
kwargs = {
k: 1,
}
self.assertRaises(ValueError, self.bundle.bind,
lambda: tu.daemon_thread(_spinner, self.death),
**kwargs)
def test_bundle_length(self):
self.assertEqual(0, len(self.bundle))
for i in range(0, self.thread_count):
self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death))
self.assertEqual(1, self.bundle.start())
self.assertEqual(i + 1, len(self.bundle))
self.death.set()
self.assertEqual(self.thread_count, self.bundle.stop())
self.assertEqual(self.thread_count, len(self.bundle))
def test_alive_thread(self):
death = tu.Event()
t = tu.daemon_thread(_spinner, death)
self.assertFalse(tu.is_alive(t))
t.start()
self.assertTrue(tu.is_alive(t))
death.set()
t.join()
self.assertFalse(tu.is_alive(t))
def test_alive_thread(self):
death = tu.Event()
t = tu.daemon_thread(_spinner, death)
self.assertFalse(tu.is_alive(t))
t.start()
self.assertTrue(tu.is_alive(t))
death.set()
t.join()
self.assertFalse(tu.is_alive(t))
def test_bundle_length(self):
self.assertEqual(0, len(self.bundle))
for i in range(0, self.thread_count):
self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death))
self.assertEqual(1, self.bundle.start())
self.assertEqual(i + 1, len(self.bundle))
self.death.set()
self.assertEqual(self.thread_count, self.bundle.stop())
self.assertEqual(self.thread_count, len(self.bundle))
def __init__(self,
uuid,
exchange,
topics,
transition_timeout=pr.REQUEST_TIMEOUT,
url=None,
transport=None,
transport_options=None,
retry_options=None):
self._uuid = uuid
self._requests_cache = wt.RequestsCache()
self._transition_timeout = transition_timeout
type_handlers = {
pr.RESPONSE:
dispatcher.Handler(self._process_response,
validator=pr.Response.validate),
}
self._proxy = proxy.Proxy(uuid,
exchange,
type_handlers=type_handlers,
on_wait=self._on_wait,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
# NOTE(harlowja): This is the most simplest finder impl. that
# doesn't have external dependencies (outside of what this engine
# already requires); it though does create periodic 'polling' traffic
# to workers to 'learn' of the tasks they can perform (and requires
# pre-existing knowledge of the topics those workers are on to gather
# and update this information).
self._finder = wt.ProxyWorkerFinder(uuid, self._proxy, topics)
self._finder.notifier.register(wt.WorkerFinder.WORKER_ARRIVED,
self._on_worker)
self._helpers = tu.ThreadBundle()
self._helpers.bind(lambda: tu.daemon_thread(self._proxy.start),
after_start=lambda t: self._proxy.wait(),
before_join=lambda t: self._proxy.stop())
p_worker = periodics.PeriodicWorker.create([self._finder])
if p_worker:
self._helpers.bind(lambda: tu.daemon_thread(p_worker.start),
before_join=lambda t: p_worker.stop(),
after_join=lambda t: p_worker.reset(),
before_start=lambda t: p_worker.reset())
def start(self):
if threading_utils.is_alive(self._worker):
raise RuntimeError("Worker thread must be stopped via stop()"
" before starting/restarting")
super(ParallelProcessTaskExecutor, self).start()
self._dispatcher.setup()
self._worker = threading_utils.daemon_thread(
asyncore.loop, map=self._dispatcher.map,
timeout=self._wait_timeout)
self._worker.start()
def test_run_empty(self):
components = self.make_components()
components.conductor.connect()
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
self.assertTrue(
components.conductor.stop(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
t.join()
def test_bind_invalid(self):
self.assertRaises(ValueError, self.bundle.bind, 1)
for k in ['after_start', 'before_start',
'before_join', 'after_join']:
kwargs = {
k: 1,
}
self.assertRaises(ValueError, self.bundle.bind,
lambda: tu.daemon_thread(_spinner, self.death),
**kwargs)
def test_run_empty(self):
components = self.make_components()
components.conductor.connect()
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
t.join()
def __init__(self,
uuid,
exchange,
topics,
transition_timeout=pr.REQUEST_TIMEOUT,
url=None,
transport=None,
transport_options=None,
retry_options=None):
self._uuid = uuid
self._topics = topics
self._requests_cache = wt.RequestsCache()
self._workers = wt.TopicWorkers()
self._transition_timeout = transition_timeout
type_handlers = {
pr.NOTIFY: [
self._process_notify,
functools.partial(pr.Notify.validate, response=True),
],
pr.RESPONSE: [
self._process_response,
pr.Response.validate,
],
}
self._proxy = proxy.Proxy(uuid,
exchange,
type_handlers,
on_wait=self._on_wait,
url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
self._periodic = wt.PeriodicWorker(tt.Timeout(pr.NOTIFY_PERIOD),
[self._notify_topics])
self._helpers = tu.ThreadBundle()
self._helpers.bind(lambda: tu.daemon_thread(self._proxy.start),
after_start=lambda t: self._proxy.wait(),
before_join=lambda t: self._proxy.stop())
self._helpers.bind(lambda: tu.daemon_thread(self._periodic.start),
before_join=lambda t: self._periodic.stop(),
after_join=lambda t: self._periodic.reset(),
before_start=lambda t: self._periodic.reset())
def generate_reviewer(client, saver, name=NAME):
"""Creates a review producer thread with the given name prefix."""
real_name = "%s_reviewer" % name
no_more = threading.Event()
jb = boards.fetch(real_name,
JOBBOARD_CONF,
client=client,
persistence=saver)
def make_save_book(saver, review_id):
# Record what we want to happen (sometime in the future).
book = models.LogBook("book_%s" % review_id)
detail = models.FlowDetail("flow_%s" % review_id,
uuidutils.generate_uuid())
book.add(detail)
# Associate the factory method we want to be called (in the future)
# with the book, so that the conductor will be able to call into
# that factory to retrieve the workflow objects that represent the
# work.
#
# These args and kwargs *can* be used to save any specific parameters
# into the factory when it is being called to create the workflow
# objects (typically used to tell a factory how to create a unique
# workflow that represents this review).
factory_args = ()
factory_kwargs = {}
engines.save_factory_details(detail, create_review_workflow,
factory_args, factory_kwargs)
with contextlib.closing(saver.get_connection()) as conn:
conn.save_logbook(book)
return book
def run():
"""Periodically publishes 'fake' reviews to analyze."""
jb.connect()
review_generator = review_iter()
with contextlib.closing(jb):
while not no_more.is_set():
review = six.next(review_generator)
details = {
'store': {
'review': review,
},
}
job_name = "%s_%s" % (real_name, review['id'])
print("Posting review '%s'" % review['id'])
jb.post(job_name,
book=make_save_book(saver, review['id']),
details=details)
time.sleep(REVIEW_CREATION_DELAY)
# Return the unstarted thread, and a callback that can be used
# shutdown that thread (to avoid running forever).
return (threading_utils.daemon_thread(target=run), no_more.set)
def _setup_workers(self, watched_aggregates):
if len(watched_aggregates) == 0:
LOG.debug('One worker for all infrastructure will be started')
watched_aggregates = [watched_aggregates]
for aggregates in watched_aggregates:
if not isinstance(aggregates, list):
aggregates = [aggregates]
instance = reaper.Reaper(aggregates)
instance.worker = threading_utils.daemon_thread(
instance.job_handler)
self.reaper_instances.append(instance)
def generate_reviewer(client, saver, name=NAME):
"""Creates a review producer thread with the given name prefix."""
real_name = "%s_reviewer" % name
no_more = threading.Event()
jb = boards.fetch(real_name, JOBBOARD_CONF,
client=client, persistence=saver)
def make_save_book(saver, review_id):
# Record what we want to happen (sometime in the future).
book = logbook.LogBook("book_%s" % review_id)
detail = logbook.FlowDetail("flow_%s" % review_id,
uuidutils.generate_uuid())
book.add(detail)
# Associate the factory method we want to be called (in the future)
# with the book, so that the conductor will be able to call into
# that factory to retrieve the workflow objects that represent the
# work.
#
# These args and kwargs *can* be used to save any specific parameters
# into the factory when it is being called to create the workflow
# objects (typically used to tell a factory how to create a unique
# workflow that represents this review).
factory_args = ()
factory_kwargs = {}
engines.save_factory_details(detail, create_review_workflow,
factory_args, factory_kwargs)
with contextlib.closing(saver.get_connection()) as conn:
conn.save_logbook(book)
return book
def run():
"""Periodically publishes 'fake' reviews to analyze."""
jb.connect()
review_generator = review_iter()
with contextlib.closing(jb):
while not no_more.is_set():
review = six.next(review_generator)
details = {
'store': {
'review': review,
},
}
job_name = "%s_%s" % (real_name, review['id'])
print("Posting review '%s'" % review['id'])
jb.post(job_name,
book=make_save_book(saver, review['id']),
details=details)
time.sleep(REVIEW_CREATION_DELAY)
# Return the unstarted thread, and a callback that can be used
# shutdown that thread (to avoid running forever).
return (threading_utils.daemon_thread(target=run), no_more.set)
def main():
with contextlib.closing(fake_client.FakeClient()) as c:
created = []
for i in compat_range(0, PRODUCERS):
p = threading_utils.daemon_thread(producer, i + 1, c)
created.append(p)
p.start()
consumed = collections.deque()
for i in compat_range(0, WORKERS):
w = threading_utils.daemon_thread(worker, i + 1, c, consumed)
created.append(w)
w.start()
while created:
t = created.pop()
t.join()
# At the end there should be nothing leftover, let's verify that.
board = backends.fetch('verifier', SHARED_CONF.copy(), client=c)
board.connect()
with contextlib.closing(board):
if board.job_count != 0 or len(consumed) != EXPECTED_UNITS:
return 1
return 0
def _fetch_server(self, task_classes):
endpoints = []
for cls in task_classes:
endpoints.append(endpoint.Endpoint(cls))
server = worker_server.Server(
TEST_TOPIC, TEST_EXCHANGE,
futures.ThreadPoolExecutor(1), endpoints,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
server_thread = threading_utils.daemon_thread(server.start)
return (server, server_thread)
def main():
with contextlib.closing(fake_client.FakeClient()) as c:
created = []
for i in compat_range(0, PRODUCERS):
p = threading_utils.daemon_thread(producer, i + 1, c)
created.append(p)
p.start()
consumed = collections.deque()
for i in compat_range(0, WORKERS):
w = threading_utils.daemon_thread(worker, i + 1, c, consumed)
created.append(w)
w.start()
while created:
t = created.pop()
t.join()
# At the end there should be nothing leftover, let's verify that.
board = backends.fetch('verifier', SHARED_CONF.copy(), client=c)
board.connect()
with contextlib.closing(board):
if board.job_count != 0 or len(consumed) != EXPECTED_UNITS:
return 1
return 0
def create_fractal():
logging.basicConfig(level=logging.ERROR)
# Setup our transport configuration and merge it into the worker and
# engine configuration so that both of those use it correctly.
shared_conf = dict(BASE_SHARED_CONF)
shared_conf.update({
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
})
if len(sys.argv) >= 2:
output_filename = sys.argv[1]
else:
output_filename = None
worker_conf = dict(WORKER_CONF)
worker_conf.update(shared_conf)
engine_conf = dict(ENGINE_CONF)
engine_conf.update(shared_conf)
workers = []
worker_topics = []
print('Calculating your mandelbrot fractal of size %sx%s.' % IMAGE_SIZE)
try:
# Create a set of workers to simulate actual remote workers.
print('Running %s workers.' % (WORKERS))
for i in compat_range(0, WORKERS):
worker_conf['topic'] = 'calculator_%s' % (i + 1)
worker_topics.append(worker_conf['topic'])
w = worker.Worker(**worker_conf)
runner = threading_utils.daemon_thread(w.run)
runner.start()
w.wait()
workers.append((runner, w.stop))
# Now use those workers to do something.
engine_conf['topics'] = worker_topics
results = calculate(engine_conf)
print('Execution finished.')
finally:
# And cleanup.
print('Stopping workers.')
while workers:
r, stopper = workers.pop()
stopper()
r.join()
print("Writing image...")
write_image(results, output_filename=output_filename)
def create_fractal():
logging.basicConfig(level=logging.ERROR)
# Setup our transport configuration and merge it into the worker and
# engine configuration so that both of those use it correctly.
shared_conf = dict(BASE_SHARED_CONF)
shared_conf.update({
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
})
if len(sys.argv) >= 2:
output_filename = sys.argv[1]
else:
output_filename = None
worker_conf = dict(WORKER_CONF)
worker_conf.update(shared_conf)
engine_conf = dict(ENGINE_CONF)
engine_conf.update(shared_conf)
workers = []
worker_topics = []
print('Calculating your mandelbrot fractal of size %sx%s.' % IMAGE_SIZE)
try:
# Create a set of workers to simulate actual remote workers.
print('Running %s workers.' % (WORKERS))
for i in compat_range(0, WORKERS):
worker_conf['topic'] = 'calculator_%s' % (i + 1)
worker_topics.append(worker_conf['topic'])
w = worker.Worker(**worker_conf)
runner = threading_utils.daemon_thread(w.run)
runner.start()
w.wait()
workers.append((runner, w.stop))
# Now use those workers to do something.
engine_conf['topics'] = worker_topics
results = calculate(engine_conf)
print('Execution finished.')
finally:
# And cleanup.
print('Stopping workers.')
while workers:
r, stopper = workers.pop()
stopper()
r.join()
print("Writing image...")
write_image(results, output_filename=output_filename)
def __init__(self, uuid, exchange, topics,
transition_timeout=pr.REQUEST_TIMEOUT,
url=None, transport=None, transport_options=None,
retry_options=None):
self._uuid = uuid
self._requests_cache = wt.RequestsCache()
self._transition_timeout = transition_timeout
type_handlers = {
pr.RESPONSE: [
self._process_response,
pr.Response.validate,
],
}
self._proxy = proxy.Proxy(uuid, exchange,
type_handlers=type_handlers,
on_wait=self._on_wait, url=url,
transport=transport,
transport_options=transport_options,
retry_options=retry_options)
# NOTE(harlowja): This is the most simplest finder impl. that
# doesn't have external dependencies (outside of what this engine
# already requires); it though does create periodic 'polling' traffic
# to workers to 'learn' of the tasks they can perform (and requires
# pre-existing knowledge of the topics those workers are on to gather
# and update this information).
self._finder = wt.ProxyWorkerFinder(uuid, self._proxy, topics)
self._finder.notifier.register(wt.WorkerFinder.WORKER_ARRIVED,
self._on_worker)
self._helpers = tu.ThreadBundle()
self._helpers.bind(lambda: tu.daemon_thread(self._proxy.start),
after_start=lambda t: self._proxy.wait(),
before_join=lambda t: self._proxy.stop())
p_worker = periodic.PeriodicWorker.create([self._finder])
if p_worker:
self._helpers.bind(lambda: tu.daemon_thread(p_worker.start),
before_join=lambda t: p_worker.stop(),
after_join=lambda t: p_worker.reset(),
before_start=lambda t: p_worker.reset())
def _fetch_server(self, task_classes):
endpoints = []
for cls in task_classes:
endpoints.append(endpoint.Endpoint(cls))
server = worker_server.Server(TEST_TOPIC,
TEST_EXCHANGE,
futures.ThreadPoolExecutor(1),
endpoints,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
server_thread = threading_utils.daemon_thread(server.start)
return (server, server_thread)
def start(self):
if threading_utils.is_alive(self._worker):
raise RuntimeError("Worker thread must be stopped via stop()"
" before starting/restarting")
super(ParallelProcessTaskExecutor, self).start()
# These don't seem restartable; make a new one...
if self._manager.is_shutdown():
self._manager = _ViewableSyncManager()
if not self._manager.is_running():
self._manager.start()
self._dispatcher.reset()
self._queue = self._manager.Queue()
self._worker = threading_utils.daemon_thread(self._dispatcher.run,
self._queue)
self._worker.start()
def test_scanning_periodic(self):
p = PeriodicThingy()
w = periodic.PeriodicWorker.create([p])
self.assertEqual(2, len(w))
t = tu.daemon_thread(target=w.start)
t.start()
time.sleep(0.1)
w.stop()
t.join()
b_calls = [c for c in p.capture if c == 'b']
self.assertGreater(0, len(b_calls))
a_calls = [c for c in p.capture if c == 'a']
self.assertGreater(0, len(a_calls))
def start(self):
if threading_utils.is_alive(self._worker):
raise RuntimeError("Worker thread must be stopped via stop()"
" before starting/restarting")
super(ParallelProcessTaskExecutor, self).start()
# These don't seem restartable; make a new one...
if self._manager.is_shutdown():
self._manager = _ViewableSyncManager()
if not self._manager.is_running():
self._manager.start()
self._dispatcher.reset()
self._queue = self._manager.Queue()
self._worker = threading_utils.daemon_thread(self._dispatcher.run,
self._queue)
self._worker.start()
def generate_conductor(client, saver, name=NAME):
"""Creates a conductor thread with the given name prefix."""
real_name = "%s_conductor" % name
jb = boards.fetch(name, JOBBOARD_CONF,
client=client, persistence=saver)
conductor = conductors.fetch("blocking", real_name, jb,
engine='parallel', wait_timeout=SCAN_DELAY)
def run():
jb.connect()
with contextlib.closing(jb):
conductor.run()
# Return the unstarted thread, and a callback that can be used
# shutdown that thread (to avoid running forever).
return (threading_utils.daemon_thread(target=run), conductor.stop)
