def test_wait_arrival(self):
ev = threading.Event()
jobs = []
def poster(wait_post=0.2):
if not ev.wait(test_utils.WAIT_TIMEOUT):
raise RuntimeError("Waiter did not appear ready"
" in %s seconds" % test_utils.WAIT_TIMEOUT)
time.sleep(wait_post)
self.board.post('test', test_utils.test_factory)
def waiter():
ev.set()
it = self.board.wait(timeout=test_utils.WAIT_TIMEOUT)
jobs.extend(it)
with connect_close(self.board):
t1 = threading_utils.daemon_thread(poster)
t1.start()
t2 = threading_utils.daemon_thread(waiter)
t2.start()
for t in (t1, t2):
t.join()
self.assertEqual(1, len(jobs))
def main():
if six.PY3:
# TODO(harlowja): Hack to make eventlet work right, remove when the
# following is fixed: https://github.com/eventlet/eventlet/issues/230
from zag.utils import eventlet_utils as _eu # noqa
try:
import eventlet as _eventlet # noqa
except ImportError:
pass
with contextlib.closing(fake_client.FakeClient()) as c:
created = []
for i in compat_range(0, PRODUCERS):
p = threading_utils.daemon_thread(producer, i + 1, c)
created.append(p)
p.start()
consumed = collections.deque()
for i in compat_range(0, WORKERS):
w = threading_utils.daemon_thread(worker, i + 1, c, consumed)
created.append(w)
w.start()
while created:
t = created.pop()
t.join()
# At the end there should be nothing leftover, let's verify that.
board = backends.fetch('verifier', SHARED_CONF.copy(), client=c)
board.connect()
with contextlib.closing(board):
if board.job_count != 0 or len(consumed) != EXPECTED_UNITS:
return 1
return 0
def test_class_based_flow_factories(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
store = {'x': True, 'y': False, 'z': None}
components.board.notifier.register(base.REMOVAL, on_consume)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
job = components.board.post('poke', ClassBasedFactory, store=store)
lb = job.book
fd = job.load_flow_detail()
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
persistence = components.persistence
with contextlib.closing(persistence.get_connection()) as conn:
lb = conn.get_logbook(lb.uuid)
fd = lb.find(fd.uuid)
self.assertIsNotNone(fd)
self.assertEqual(st.SUCCESS, fd.state)
def test_job_compilation_errors(self):
components = self.make_components()
components.conductor.connect()
job_trashed_event = threading.Event()
job_abandoned_event = threading.Event()
def on_job_trashed(event, details):
if event == 'job_trashed':
job_trashed_event.set()
def on_job_abandoned(event, details):
if event == 'job_abandoned':
job_abandoned_event.set()
components.conductor.notifier.register("job_trashed", on_job_trashed)
components.conductor.notifier.register("job_abandoned",
on_job_abandoned)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
components.board.post('poke', compiler_failure_factory)
job_abandoned_event.wait(test_utils.WAIT_TIMEOUT)
self.assertTrue(job_abandoned_event.is_set())
job_trashed_event.wait(test_utils.WAIT_TIMEOUT)
self.assertTrue(job_trashed_event.is_set())
components.conductor.stop()
def test_valid_listener_factories(self):
def logging_listener_factory(job, engine):
return timing_listener.DurationListener(engine)
components = self.make_components(
listener_factories=[logging_listener_factory])
components.conductor.connect()
consumed_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
store = {'x': True, 'y': False, 'z': None}
components.board.notifier.register(base.REMOVAL, on_consume)
mock_method = 'zag.listeners.timing.DurationListener._receiver'
with mock.patch(mock_method) as mock_receiver:
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
components.board.post('poke', test_store_factory, store=store)
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
components.conductor.stop()
self.assertTrue(
components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
self.assertGreaterEqual(1, mock_receiver.call_count)
def test_stop(self):
self.conn_inst_mock.drain_events.side_effect = socket.timeout
# create proxy
pr = self.proxy(reset_master_mock=True)
# check that proxy is not running yes
self.assertFalse(pr.is_running)
# start proxy in separate thread
t = threading_utils.daemon_thread(pr.start)
t.start()
# make sure proxy is started
pr.wait()
# check that proxy is running now
self.assertTrue(pr.is_running)
# stop proxy and wait for thread to finish
pr.stop()
# wait for thread to finish
t.join()
self.assertFalse(pr.is_running)
def test_start_stop(self):
events = collections.deque()
def before_start(t):
events.append('bs')
def before_join(t):
events.append('bj')
self.death.set()
def after_start(t):
events.append('as')
def after_join(t):
events.append('aj')
for _i in range(0, self.thread_count):
self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death),
before_join=before_join,
after_join=after_join,
before_start=before_start,
after_start=after_start)
self.assertEqual(self.thread_count, self.bundle.start())
self.assertEqual(self.thread_count, len(self.bundle))
self.assertEqual(self.thread_count, self.bundle.stop())
for event in ['as', 'bs', 'bj', 'aj']:
self.assertEqual(self.thread_count,
len([e for e in events if e == event]))
self.assertEqual(0, self.bundle.stop())
self.assertTrue(self.death.is_set())
def test_notify(self):
barrier = threading.Event()
on_notify = mock.MagicMock()
on_notify.side_effect = lambda *args, **kwargs: barrier.set()
handlers = {pr.NOTIFY: dispatcher.Handler(on_notify)}
p = proxy.Proxy(TEST_TOPIC,
TEST_EXCHANGE,
handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
p.publish(pr.Notify(), TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
p.stop()
t.join()
self.assertTrue(on_notify.called)
on_notify.assert_called_with({}, mock.ANY)
def test_response(self):
barrier = threading.Event()
on_response = mock.MagicMock()
on_response.side_effect = lambda *args, **kwargs: barrier.set()
handlers = {pr.RESPONSE: dispatcher.Handler(on_response)}
p = proxy.Proxy(TEST_TOPIC,
TEST_EXCHANGE,
handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
resp = pr.Response(pr.RUNNING)
p.publish(resp, TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
self.assertTrue(barrier.is_set())
p.stop()
t.join()
self.assertTrue(on_response.called)
on_response.assert_called_with(resp.to_dict(), mock.ANY)
def test_delayed_job(self):
components = self.make_components()
components.conductor.connect()
claimed_event = threading.Event()
def on_claimed(event, details):
if event == 'job_claimed':
claimed_event.set()
flow_store = {'x': True, 'y': False, 'z': None}
components.conductor.notifier.register("job_claimed", on_claimed)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
job = components.board.post_delayed(180,
'poke',
test_store_factory,
store=flow_store)
lb = job.book
fd = job.load_flow_detail()
self.assertFalse(claimed_event.wait(2))
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
persistence = components.persistence
with contextlib.closing(persistence.get_connection()) as conn:
lb = conn.get_logbook(lb.uuid)
fd = lb.find(fd.uuid)
self.assertIsNotNone(fd)
self.assertIsNone(fd.state)
def test_bundle_length(self):
self.assertEqual(0, len(self.bundle))
for i in range(0, self.thread_count):
self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death))
self.assertEqual(1, self.bundle.start())
self.assertEqual(i + 1, len(self.bundle))
self.death.set()
self.assertEqual(self.thread_count, self.bundle.stop())
self.assertEqual(self.thread_count, len(self.bundle))
def test_alive_thread(self):
death = threading.Event()
t = tu.daemon_thread(_spinner, death)
self.assertFalse(tu.is_alive(t))
t.start()
self.assertTrue(tu.is_alive(t))
death.set()
t.join()
self.assertFalse(tu.is_alive(t))
def start(self):
if threading_utils.is_alive(self._worker):
raise RuntimeError("Worker thread must be stopped via stop()"
" before starting/restarting")
super(ParallelProcessTaskExecutor, self).start()
self._dispatcher.setup()
self._worker = threading_utils.daemon_thread(
asyncore.loop, map=self._dispatcher.map,
timeout=self._wait_timeout)
self._worker.start()
def test_bind_invalid(self):
self.assertRaises(ValueError, self.bundle.bind, 1)
for k in ['after_start', 'before_start',
'before_join', 'after_join']:
kwargs = {
k: 1,
}
self.assertRaises(ValueError, self.bundle.bind,
lambda: tu.daemon_thread(_spinner, self.death),
**kwargs)
def test_run_empty(self):
components = self.make_components()
components.conductor.connect()
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
t.join()
def generate_reviewer(client, saver, name=NAME):
"""Creates a review producer thread with the given name prefix."""
real_name = "%s_reviewer" % name
no_more = threading.Event()
jb = boards.fetch(real_name,
JOBBOARD_CONF,
client=client,
persistence=saver)
def make_save_book(saver, review_id):
# Record what we want to happen (sometime in the future).
book = models.LogBook("book_%s" % review_id)
detail = models.FlowDetail("flow_%s" % review_id,
uuidutils.generate_uuid())
book.add(detail)
# Associate the factory method we want to be called (in the future)
# with the book, so that the conductor will be able to call into
# that factory to retrieve the workflow objects that represent the
# work.
#
# These args and kwargs *can* be used to save any specific parameters
# into the factory when it is being called to create the workflow
# objects (typically used to tell a factory how to create a unique
# workflow that represents this review).
factory_args = ()
factory_kwargs = {}
engines.save_factory_details(detail, create_review_workflow,
factory_args, factory_kwargs)
with contextlib.closing(saver.get_connection()) as conn:
conn.save_logbook(book)
return book
def run():
"""Periodically publishes 'fake' reviews to analyze."""
jb.connect()
review_generator = review_iter()
with contextlib.closing(jb):
while not no_more.is_set():
review = six.next(review_generator)
details = {
'store': {
'review': review,
},
}
job_name = "%s_%s" % (real_name, review['id'])
print("Posting review '%s'" % review['id'])
jb.post(job_name,
book=make_save_book(saver, review['id']),
details=details)
time.sleep(REVIEW_CREATION_DELAY)
# Return the unstarted thread, and a callback that can be used
# shutdown that thread (to avoid running forever).
return (threading_utils.daemon_thread(target=run), no_more.set)
def create_fractal():
logging.basicConfig(level=logging.ERROR)
# Setup our transport configuration and merge it into the worker and
# engine configuration so that both of those use it correctly.
shared_conf = dict(BASE_SHARED_CONF)
shared_conf.update({
'transport': 'memory',
'transport_options': {
'polling_interval': 0.1,
},
})
if len(sys.argv) >= 2:
output_filename = sys.argv[1]
else:
output_filename = None
worker_conf = dict(WORKER_CONF)
worker_conf.update(shared_conf)
engine_conf = dict(ENGINE_CONF)
engine_conf.update(shared_conf)
workers = []
worker_topics = []
print('Calculating your mandelbrot fractal of size %sx%s.' % IMAGE_SIZE)
try:
# Create a set of workers to simulate actual remote workers.
print('Running %s workers.' % (WORKERS))
for i in compat_range(0, WORKERS):
worker_conf['topic'] = 'calculator_%s' % (i + 1)
worker_topics.append(worker_conf['topic'])
w = worker.Worker(**worker_conf)
runner = threading_utils.daemon_thread(w.run)
runner.start()
w.wait()
workers.append((runner, w.stop))
# Now use those workers to do something.
engine_conf['topics'] = worker_topics
results = calculate(engine_conf)
print('Execution finished.')
finally:
# And cleanup.
print('Stopping workers.')
while workers:
r, stopper = workers.pop()
stopper()
r.join()
print("Writing image...")
write_image(results, output_filename=output_filename)
def _fetch_server(self, task_classes):
endpoints = []
for cls in task_classes:
endpoints.append(endpoint.Endpoint(cls))
server = worker_server.Server(
TEST_TOPIC,
TEST_EXCHANGE,
futurist.ThreadPoolExecutor(max_workers=1),
endpoints,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
server_thread = threading_utils.daemon_thread(server.start)
return (server, server_thread)
def __init__(self, coordination_url, member_id, watch_groups):
super(ToozWorkerFinder, self).__init__()
self._coordinator = coordination.get_coordinator(
coordination_url, member_id)
self._watch_groups = watch_groups
self._active_groups = {}
self._available_workers = 0
self._helpers = tu.ThreadBundle()
p_worker = periodics.PeriodicWorker.create([self])
if p_worker:
self._helpers.bind(lambda: tu.daemon_thread(p_worker.start),
before_join=lambda t: p_worker.stop(),
after_join=lambda t: p_worker.reset(),
before_start=lambda t: p_worker.reset())
self._activator = misc.Activator([self._coordinator, self._helpers])
def generate_conductor(client, saver, name=NAME):
"""Creates a conductor thread with the given name prefix."""
real_name = "%s_conductor" % name
jb = boards.fetch(name, JOBBOARD_CONF, client=client, persistence=saver)
conductor = conductors.fetch("blocking",
real_name,
jb,
engine='parallel',
wait_timeout=SCAN_DELAY)
def run():
jb.connect()
with contextlib.closing(jb):
conductor.run()
# Return the unstarted thread, and a callback that can be used
# shutdown that thread (to avoid running forever).
return (threading_utils.daemon_thread(target=run), conductor.stop)
def __init__(self, coordinator_url, member_id, join_groups, worker_topic,
worker_endpoints):
self._coordinator = coordination.get_coordinator(
coordinator_url, member_id)
self._join_groups = frozenset(join_groups)
self._helpers = tu.ThreadBundle()
self._member_id = member_id
self._capabilities = {
'topic': worker_topic,
'tasks': [e.name for e in worker_endpoints],
}
p_worker = periodics.PeriodicWorker.create([self])
if p_worker:
self._helpers.bind(lambda: tu.daemon_thread(p_worker.start),
before_join=lambda t: p_worker.stop(),
after_join=lambda t: p_worker.reset(),
before_start=lambda t: p_worker.reset())
self._activator = misc.Activator([self._coordinator, self._helpers])
def test_start_stop_order(self):
start_events = collections.deque()
death_events = collections.deque()
def before_start(i, t):
start_events.append((i, 'bs'))
def before_join(i, t):
death_events.append((i, 'bj'))
self.death.set()
def after_start(i, t):
start_events.append((i, 'as'))
def after_join(i, t):
death_events.append((i, 'aj'))
for i in range(0, self.thread_count):
self.bundle.bind(lambda: tu.daemon_thread(_spinner, self.death),
before_join=functools.partial(before_join, i),
after_join=functools.partial(after_join, i),
before_start=functools.partial(before_start, i),
after_start=functools.partial(after_start, i))
self.assertEqual(self.thread_count, self.bundle.start())
self.assertEqual(self.thread_count, len(self.bundle))
self.assertEqual(self.thread_count, self.bundle.stop())
self.assertEqual(0, self.bundle.stop())
self.assertTrue(self.death.is_set())
expected_start_events = []
for i in range(0, self.thread_count):
expected_start_events.extend([
(i, 'bs'), (i, 'as'),
])
self.assertEqual(expected_start_events, list(start_events))
expected_death_events = []
j = self.thread_count - 1
for _i in range(0, self.thread_count):
expected_death_events.extend([
(j, 'bj'), (j, 'aj'),
])
j -= 1
self.assertEqual(expected_death_events, list(death_events))
def test_fail_run(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
job_consumed_event = threading.Event()
job_abandoned_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
def on_job_consumed(event, details):
if event == 'job_consumed':
job_consumed_event.set()
def on_job_abandoned(event, details):
if event == 'job_abandoned':
job_abandoned_event.set()
components.board.notifier.register(base.REMOVAL, on_consume)
components.conductor.notifier.register("job_consumed", on_job_consumed)
components.conductor.notifier.register("job_abandoned",
on_job_abandoned)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
job = components.board.post('poke', test_blowup_factory)
lb = job.book
fd = job.load_flow_detail()
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
self.assertTrue(job_consumed_event.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(job_abandoned_event.wait(1))
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
persistence = components.persistence
with contextlib.closing(persistence.get_connection()) as conn:
lb = conn.get_logbook(lb.uuid)
fd = lb.find(fd.uuid)
self.assertIsNotNone(fd)
self.assertEqual(st.REVERTED, fd.state)
def test_run_max_dispatches(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
def on_consume(state, details):
consumed_event.set()
components.board.notifier.register(base.REMOVAL, on_consume)
with close_many(components.client, components.conductor):
t = threading_utils.daemon_thread(
lambda: components.conductor.run(max_dispatches=5))
t.start()
for _ in range(5):
components.board.post('poke', test_utils.test_factory)
self.assertTrue(consumed_event.wait(test_utils.WAIT_TIMEOUT))
components.board.post('poke', test_utils.test_factory)
components.conductor.stop()
self.assertTrue(components.conductor.wait(test_utils.WAIT_TIMEOUT))
self.assertFalse(components.conductor.dispatching)
def test_stop_aborts_engine(self):
components = self.make_components()
components.conductor.connect()
consumed_event = threading.Event()
job_consumed_event = threading.Event()
job_abandoned_event = threading.Event()
running_start_event = threading.Event()
def on_running_start(event, details):
running_start_event.set()
def on_consume(state, details):
consumed_event.set()
def on_job_consumed(event, details):
if event == 'job_consumed':
job_consumed_event.set()
def on_job_abandoned(event, details):
if event == 'job_abandoned':
job_abandoned_event.set()
components.board.notifier.register(base.REMOVAL, on_consume)
components.conductor.notifier.register("job_consumed", on_job_consumed)
components.conductor.notifier.register("job_abandoned",
on_job_abandoned)
components.conductor.notifier.register("running_start",
on_running_start)
with close_many(components.conductor, components.client):
t = threading_utils.daemon_thread(components.conductor.run)
t.start()
components.board.post('poke', sleep_factory, store={'duration': 2})
running_start_event.wait(test_utils.WAIT_TIMEOUT)
components.conductor.stop()
job_abandoned_event.wait(test_utils.WAIT_TIMEOUT)
self.assertTrue(job_abandoned_event.is_set())
self.assertFalse(job_consumed_event.is_set())
self.assertFalse(consumed_event.is_set())
def test_multi_message(self):
message_count = 30
barrier = latch.Latch(message_count)
countdown = lambda data, message: barrier.countdown()
on_notify = mock.MagicMock()
on_notify.side_effect = countdown
on_response = mock.MagicMock()
on_response.side_effect = countdown
on_request = mock.MagicMock()
on_request.side_effect = countdown
handlers = {
pr.NOTIFY: dispatcher.Handler(on_notify),
pr.RESPONSE: dispatcher.Handler(on_response),
pr.REQUEST: dispatcher.Handler(on_request),
}
p = proxy.Proxy(TEST_TOPIC,
TEST_EXCHANGE,
handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
for i in range(0, message_count):
j = i % 3
if j == 0:
p.publish(pr.Notify(), TEST_TOPIC)
elif j == 1:
p.publish(pr.Response(pr.RUNNING), TEST_TOPIC)
else:
p.publish(
pr.Request(test_utils.DummyTask("dummy_%s" % i),
uuidutils.generate_uuid(), pr.EXECUTE, [],
None), TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
self.assertEqual(0, barrier.needed)
p.stop()
t.join()
self.assertTrue(on_notify.called)
self.assertTrue(on_response.called)
self.assertTrue(on_request.called)
self.assertEqual(10, on_notify.call_count)
self.assertEqual(10, on_response.call_count)
self.assertEqual(10, on_request.call_count)
call_count = sum([
on_notify.call_count,
on_response.call_count,
on_request.call_count,
])
self.assertEqual(message_count, call_count)
# Create & start our worker that will actually run tasks..
#
# It will join 'coordinator_groups' groups and advertise its capabilities in
# those groups (so that engines may find it).
worker_id = uuidutils.generate_uuid()
advertiser_factory = wt.ToozWorkerAdvertiser.generate_factory({
'coordinator': {
'url': coordinator_url,
'groups': coordinator_groups,
},
})
w = worker.Worker(exchange,
uuidutils.generate_uuid(), [OneTask, TwoTask],
transport=transport,
advertiser_factory=advertiser_factory)
w_runner = tu.daemon_thread(target=w.run, banner_writer=banner_writer)
print("Booting up the worker...")
w_runner.start()
w.wait()
# Now make the engine that will use the previous workers to do some work...
finder_factory = wt.ToozWorkerFinder.generate_factory({
'coordinator': {
'url': coordinator_url,
'groups': coordinator_groups,
},
})
# Create a custom executor for the engine to use.
#
def test_daemon_thread(self):
death = threading.Event()
t = tu.daemon_thread(_spinner, death)
self.assertTrue(t.daemon)
},
})
worker_conf = dict(WORKER_CONF)
worker_conf.update(shared_conf)
engine_options = dict(shared_conf)
workers = []
worker_topics = []
try:
# Create a set of workers to simulate actual remote workers.
print('Running %s workers.' % (worker_count))
for i in range(0, worker_count):
worker_conf['topic'] = 'worker-%s' % (i + 1)
worker_topics.append(worker_conf['topic'])
w = worker.Worker(**worker_conf)
runner = threading_utils.daemon_thread(w.run)
runner.start()
w.wait()
workers.append((runner, w.stop))
# Now use those workers to do something.
print('Executing some work.')
engine_options['topics'] = worker_topics
result = run(engine_options)
print('Execution finished.')
# This is done so that the test examples can work correctly
# even when the keys change order (which will happen in various
# python versions).
print("Result = %s" % json.dumps(result, sort_keys=True))
finally:
# And cleanup.
