def main(): if six.PY3: # TODO(harlowja): Hack to make eventlet work right, remove when the # following is fixed: https://github.com/eventlet/eventlet/issues/230 from zag.utils import eventlet_utils as _eu # noqa try: import eventlet as _eventlet # noqa except ImportError: pass with contextlib.closing(fake_client.FakeClient()) as c: created = [] for i in compat_range(0, PRODUCERS): p = threading_utils.daemon_thread(producer, i + 1, c) created.append(p) p.start() consumed = collections.deque() for i in compat_range(0, WORKERS): w = threading_utils.daemon_thread(worker, i + 1, c, consumed) created.append(w) w.start() while created: t = created.pop() t.join() # At the end there should be nothing leftover, let's verify that. board = backends.fetch('verifier', SHARED_CONF.copy(), client=c) board.connect() with contextlib.closing(board): if board.job_count != 0 or len(consumed) != EXPECTED_UNITS: return 1 return 0
def run_poster(): # This just posts a single job and then ends... print("Starting poster with pid: %s" % ME) my_name = "poster-%s" % ME persist_backend = persistence_backends.fetch(PERSISTENCE_URI) with contextlib.closing(persist_backend): with contextlib.closing(persist_backend.get_connection()) as conn: conn.upgrade() job_backend = job_backends.fetch(my_name, JB_CONF, persistence=persist_backend) job_backend.connect() with contextlib.closing(job_backend): # Create information in the persistence backend about the # unit of work we want to complete and the factory that # can be called to create the tasks that the work unit needs # to be done. lb = models.LogBook("post-from-%s" % my_name) fd = models.FlowDetail("song-from-%s" % my_name, uuidutils.generate_uuid()) lb.add(fd) with contextlib.closing(persist_backend.get_connection()) as conn: conn.save_logbook(lb) engines.save_factory_details(fd, make_bottles, [HOW_MANY_BOTTLES], {}, backend=persist_backend) # Post, and be done with it! jb = job_backend.post("song-from-%s" % my_name, book=lb) print("Posted: %s" % jb) print("Goodbye...")
def test_redis_entry_point(self): conf = { 'board': 'redis', } p = impl_memory.MemoryBackend() with contextlib.closing(backends.fetch('test', conf, persistence=p)) as be: self.assertIsInstance(be, impl_redis.RedisJobBoard)
def test_zk_entry_point(self): p = impl_memory.MemoryBackend() conf = { 'board': 'zookeeper', } with contextlib.closing(backends.fetch('test', conf, persistence=p)) as be: self.assertIsInstance(be, impl_zookeeper.ZookeeperJobBoard)
def setUp(self): super(TestClaimListener, self).setUp() self.client = fake_client.FakeClient() self.addCleanup(self.client.stop) self.board = jobs.fetch('test', 'zookeeper', client=self.client, persistence=impl_memory.MemoryBackend()) self.addCleanup(self.board.close) self.board.connect()
def generate_reviewer(client, saver, name=NAME): """Creates a review producer thread with the given name prefix.""" real_name = "%s_reviewer" % name no_more = threading.Event() jb = boards.fetch(real_name, JOBBOARD_CONF, client=client, persistence=saver) def make_save_book(saver, review_id): # Record what we want to happen (sometime in the future). book = models.LogBook("book_%s" % review_id) detail = models.FlowDetail("flow_%s" % review_id, uuidutils.generate_uuid()) book.add(detail) # Associate the factory method we want to be called (in the future) # with the book, so that the conductor will be able to call into # that factory to retrieve the workflow objects that represent the # work. # # These args and kwargs *can* be used to save any specific parameters # into the factory when it is being called to create the workflow # objects (typically used to tell a factory how to create a unique # workflow that represents this review). factory_args = () factory_kwargs = {} engines.save_factory_details(detail, create_review_workflow, factory_args, factory_kwargs) with contextlib.closing(saver.get_connection()) as conn: conn.save_logbook(book) return book def run(): """Periodically publishes 'fake' reviews to analyze.""" jb.connect() review_generator = review_iter() with contextlib.closing(jb): while not no_more.is_set(): review = six.next(review_generator) details = { 'store': { 'review': review, }, } job_name = "%s_%s" % (real_name, review['id']) print("Posting review '%s'" % review['id']) jb.post(job_name, book=make_save_book(saver, review['id']), details=details) time.sleep(REVIEW_CREATION_DELAY) # Return the unstarted thread, and a callback that can be used # shutdown that thread (to avoid running forever). return (threading_utils.daemon_thread(target=run), no_more.set)
def test_zk_entry_point_existing_client(self): existing_client = fake_client.FakeClient() conf = { 'board': 'zookeeper', } kwargs = { 'client': existing_client, 'persistence': impl_memory.MemoryBackend(), } with contextlib.closing(backends.fetch('test', conf, **kwargs)) as be: self.assertIsInstance(be, impl_zookeeper.ZookeeperJobBoard) self.assertIs(existing_client, be._client)
def run_conductor(only_run_once=False): # This continuously runs consumers until its stopped via ctrl-c or other # kill signal... event_watches = {} # This will be triggered by the conductor doing various activities # with engines, and is quite nice to be able to see the various timing # segments (which is useful for debugging, or watching, or figuring out # where to optimize). def on_conductor_event(cond, event, details): print("Event '%s' has been received..." % event) print("Details = %s" % details) if event.endswith("_start"): w = timeutils.StopWatch() w.start() base_event = event[0:-len("_start")] event_watches[base_event] = w if event.endswith("_end"): base_event = event[0:-len("_end")] try: w = event_watches.pop(base_event) w.stop() print("It took %0.3f seconds for event '%s' to finish" % (w.elapsed(), base_event)) except KeyError: pass if event == 'running_end' and only_run_once: cond.stop() print("Starting conductor with pid: %s" % ME) my_name = "conductor-%s" % ME persist_backend = persistence_backends.fetch(PERSISTENCE_URI) with contextlib.closing(persist_backend): with contextlib.closing(persist_backend.get_connection()) as conn: conn.upgrade() job_backend = job_backends.fetch(my_name, JB_CONF, persistence=persist_backend) job_backend.connect() with contextlib.closing(job_backend): cond = conductor_backends.fetch('blocking', my_name, job_backend, persistence=persist_backend) on_conductor_event = functools.partial(on_conductor_event, cond) cond.notifier.register(cond.notifier.ANY, on_conductor_event) # Run forever, and kill -9 or ctrl-c me... try: cond.run() finally: cond.stop() cond.wait()
def generate_conductor(client, saver, name=NAME): """Creates a conductor thread with the given name prefix.""" real_name = "%s_conductor" % name jb = boards.fetch(name, JOBBOARD_CONF, client=client, persistence=saver) conductor = conductors.fetch("blocking", real_name, jb, engine='parallel', wait_timeout=SCAN_DELAY) def run(): jb.connect() with contextlib.closing(jb): conductor.run() # Return the unstarted thread, and a callback that can be used # shutdown that thread (to avoid running forever). return (threading_utils.daemon_thread(target=run), conductor.stop)