def start(conf): persistence = _get_persistence_backend(conf) if conf.taskflow.db_upgrade: with contextlib.closing(persistence.get_connection()) as conn: LOG.info(_LI('Checking for database schema upgrade')) conn.upgrade() my_name = uuidutils.generate_uuid() LOG.info(_LI('I am %s'), my_name) board = _get_jobboard_backend(conf, persistence=persistence) conductor = conductors.fetch( 'nonblocking', my_name, board, engine='parallel', max_simultaneous_jobs=conf.max_simultaneous_jobs, persistence=persistence) board.connect() LOG.debug('Starting taskflow conductor loop') threading.Thread(target=conductor.run).start() return persistence, board, conductor
def make_components(self, name='testing', wait_timeout=0.1): client = fake_client.FakeClient() persistence = impl_memory.MemoryBackend() board = impl_zookeeper.ZookeeperJobBoard(name, {}, client=client, persistence=persistence) conductor = backends.fetch(self.KIND, name, board, persistence=persistence, wait_timeout=wait_timeout) return ComponentBundle(board, client, persistence, conductor)
def make_components(self): client = fake_client.FakeClient() persistence = impl_memory.MemoryBackend() board = impl_zookeeper.ZookeeperJobBoard('testing', {}, client=client, persistence=persistence) conductor_kwargs = self.conductor_kwargs.copy() conductor_kwargs['persistence'] = persistence conductor = backends.fetch(self.kind, 'testing', board, **conductor_kwargs) return ComponentBundle(board, client, persistence, conductor)
def run(): # This continuously consumes until its stopped via ctrl-c or some other # kill signal... event_watches = {} # This will be triggered by the conductor doing various activities # with engines, and is quite nice to be able to see the various timing # segments (which is useful for debugging, or watching, or figuring out # where to optimize). def on_conductor_event(cond, event, details): print('Event \'%s\' has been received...' % event) print('Details = %s' % details) if event.endswith('_start'): w = timing.StopWatch() w.start() base_event = event[0:-len('_start')] event_watches[base_event] = w if event.endswith('_end'): base_event = event[0:-len('_end')] try: w = event_watches.pop(base_event) w.stop() print('It took %0.3f seconds for event \'%s\' to finish' % (w.elapsed(), base_event)) except KeyError: pass if event == 'running_end': cond.stop() conductor_id = os.getpid() print('Starting GAPIC conductor with pid: %s' % conductor_id) my_name = 'conductor-%s' % conductor_id persist_backend = backend_helper.default_persistence_backend() with contextlib.closing(persist_backend): with contextlib.closing(persist_backend.get_connection()) as conn: conn.upgrade() job_backend = backend_helper.default_jobboard_backend(my_name) job_backend.connect() with contextlib.closing(job_backend): cond = conductor_backends.fetch('blocking', my_name, job_backend, persistence=persist_backend) # on_conductor_event = functools.partial(on_conductor_event, cond) # cond.notifier.register(cond.notifier.ANY, on_conductor_event) # Run forever, and kill -9 or ctrl-c me... try: print('Conductor %s is running' % my_name) cond.run() finally: print('Conductor %s is stopping' % my_name) cond.stop() cond.wait()
def run_conductor(only_run_once=False): # This continuously consumers until its stopped via ctrl-c or other # kill signal... event_watches = {} # This will be triggered by the conductor doing various activities # with engines, and is quite nice to be able to see the various timing # segments (which is useful for debugging, or watching, or figuring out # where to optimize). def on_conductor_event(cond, event, details): print("Event '%s' has been received..." % event) print("Details = %s" % details) if event.endswith("_start"): w = timing.StopWatch() w.start() base_event = event[0:-len("_start")] event_watches[base_event] = w if event.endswith("_end"): base_event = event[0:-len("_end")] try: w = event_watches.pop(base_event) w.stop() print("It took %0.3f seconds for event '%s' to finish" % (w.elapsed(), base_event)) except KeyError: pass if event == 'running_end' and only_run_once: cond.stop() print("Starting conductor with pid: %s" % ME) my_name = "conductor-%s" % ME persist_backend = persistence_backends.fetch(PERSISTENCE_URI) with contextlib.closing(persist_backend): with contextlib.closing(persist_backend.get_connection()) as conn: conn.upgrade() job_backend = job_backends.fetch(my_name, JB_CONF, persistence=persist_backend) job_backend.connect() with contextlib.closing(job_backend): cond = conductor_backends.fetch('blocking', my_name, job_backend, persistence=persist_backend) on_conductor_event = functools.partial(on_conductor_event, cond) cond.notifier.register(cond.notifier.ANY, on_conductor_event) # Run forever, and kill -9 or ctrl-c me... try: cond.run() finally: cond.stop() cond.wait()
def start(self): """Interface to start the ConductorService.""" version_string = version.version_info.version_string() LOG.debug("Starting runner %s on board %s", version_string, self._jobboard_name) persistence = self._persistence jobboard = self._jobboard try: # Create persistence and/or jobboard if they weren't passed in if persistence is None: persistence = tf_client.create_persistence( conf=self._persistence_conf) if jobboard is None: jobboard = tf_client.create_jobboard( board_name=self._jobboard_name, conf=self._jobboard_conf, persistence=persistence, ) self._conductor = conductors.fetch( kind="blocking", name=self._host, jobboard=jobboard, persistence=persistence, engine=self._engine_conf['engine'], wait_timeout=self._wait_timeout) time.sleep(0.5) if enable_cleanup: conductor_notifier = self._conductor.notifier conductor_notifier.register('job_consumed', self.cleanup_job_details) if threading.current_thread().name == 'MainThread': t = threading.Thread(target=self._conductor.run) t.start() signal.pause() else: self._conductor.run() finally: # Close persistence and jobboard if they were created by us if self._persistence is None: persistence.close() if self._jobboard is None: jobboard.close() self._shutdown_event.set()
def generate_conductor(client, saver, name=NAME): """Creates a conductor thread with the given name prefix.""" real_name = "%s_conductor" % name jb = boards.fetch(name, JOBBOARD_CONF, client=client, persistence=saver) conductor = conductors.fetch("blocking", real_name, jb, engine='parallel', wait_timeout=SCAN_DELAY) def run(): jb.connect() with contextlib.closing(jb): conductor.run() # Return the unstarted thread, and a callback that can be used # shutdown that thread (to avoid running forever). return (threading_utils.daemon_thread(target=run), conductor.stop)