def start(conf):
persistence = _get_persistence_backend(conf)
if conf.taskflow.db_upgrade:
with contextlib.closing(persistence.get_connection()) as conn:
LOG.info(_LI('Checking for database schema upgrade'))
conn.upgrade()
my_name = uuidutils.generate_uuid()
LOG.info(_LI('I am %s'), my_name)
board = _get_jobboard_backend(conf, persistence=persistence)
conductor = conductors.fetch(
'nonblocking',
my_name,
board,
engine='parallel',
max_simultaneous_jobs=conf.max_simultaneous_jobs,
persistence=persistence)
board.connect()
LOG.debug('Starting taskflow conductor loop')
threading.Thread(target=conductor.run).start()
return persistence, board, conductor
def make_components(self, name='testing', wait_timeout=0.1):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard(name, {},
client=client,
persistence=persistence)
conductor = backends.fetch(self.KIND, name, board,
persistence=persistence,
wait_timeout=wait_timeout)
return ComponentBundle(board, client, persistence, conductor)
def make_components(self):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard('testing', {},
client=client,
persistence=persistence)
conductor_kwargs = self.conductor_kwargs.copy()
conductor_kwargs['persistence'] = persistence
conductor = backends.fetch(self.kind, 'testing', board,
**conductor_kwargs)
return ComponentBundle(board, client, persistence, conductor)
def make_components(self):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard('testing', {},
client=client,
persistence=persistence)
conductor_kwargs = self.conductor_kwargs.copy()
conductor_kwargs['persistence'] = persistence
conductor = backends.fetch(self.kind, 'testing', board,
**conductor_kwargs)
return ComponentBundle(board, client, persistence, conductor)
def make_components(self, name='testing', wait_timeout=0.1):
client = fake_client.FakeClient()
persistence = impl_memory.MemoryBackend()
board = impl_zookeeper.ZookeeperJobBoard(name, {},
client=client,
persistence=persistence)
conductor = backends.fetch(self.KIND,
name,
board,
persistence=persistence,
wait_timeout=wait_timeout)
return ComponentBundle(board, client, persistence, conductor)
def run():
# This continuously consumes until its stopped via ctrl-c or some other
# kill signal...
event_watches = {}
# This will be triggered by the conductor doing various activities
# with engines, and is quite nice to be able to see the various timing
# segments (which is useful for debugging, or watching, or figuring out
# where to optimize).
def on_conductor_event(cond, event, details):
print('Event \'%s\' has been received...' % event)
print('Details = %s' % details)
if event.endswith('_start'):
w = timing.StopWatch()
w.start()
base_event = event[0:-len('_start')]
event_watches[base_event] = w
if event.endswith('_end'):
base_event = event[0:-len('_end')]
try:
w = event_watches.pop(base_event)
w.stop()
print('It took %0.3f seconds for event \'%s\' to finish' %
(w.elapsed(), base_event))
except KeyError:
pass
if event == 'running_end':
cond.stop()
conductor_id = os.getpid()
print('Starting GAPIC conductor with pid: %s' % conductor_id)
my_name = 'conductor-%s' % conductor_id
persist_backend = backend_helper.default_persistence_backend()
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = backend_helper.default_jobboard_backend(my_name)
job_backend.connect()
with contextlib.closing(job_backend):
cond = conductor_backends.fetch('blocking',
my_name,
job_backend,
persistence=persist_backend)
# on_conductor_event = functools.partial(on_conductor_event, cond)
# cond.notifier.register(cond.notifier.ANY, on_conductor_event)
# Run forever, and kill -9 or ctrl-c me...
try:
print('Conductor %s is running' % my_name)
cond.run()
finally:
print('Conductor %s is stopping' % my_name)
cond.stop()
cond.wait()
def run_conductor(only_run_once=False):
# This continuously consumers until its stopped via ctrl-c or other
# kill signal...
event_watches = {}
# This will be triggered by the conductor doing various activities
# with engines, and is quite nice to be able to see the various timing
# segments (which is useful for debugging, or watching, or figuring out
# where to optimize).
def on_conductor_event(cond, event, details):
print("Event '%s' has been received..." % event)
print("Details = %s" % details)
if event.endswith("_start"):
w = timing.StopWatch()
w.start()
base_event = event[0:-len("_start")]
event_watches[base_event] = w
if event.endswith("_end"):
base_event = event[0:-len("_end")]
try:
w = event_watches.pop(base_event)
w.stop()
print("It took %0.3f seconds for event '%s' to finish" %
(w.elapsed(), base_event))
except KeyError:
pass
if event == 'running_end' and only_run_once:
cond.stop()
print("Starting conductor with pid: %s" % ME)
my_name = "conductor-%s" % ME
persist_backend = persistence_backends.fetch(PERSISTENCE_URI)
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = job_backends.fetch(my_name,
JB_CONF,
persistence=persist_backend)
job_backend.connect()
with contextlib.closing(job_backend):
cond = conductor_backends.fetch('blocking',
my_name,
job_backend,
persistence=persist_backend)
on_conductor_event = functools.partial(on_conductor_event, cond)
cond.notifier.register(cond.notifier.ANY, on_conductor_event)
# Run forever, and kill -9 or ctrl-c me...
try:
cond.run()
finally:
cond.stop()
cond.wait()
def start(self):
"""Interface to start the ConductorService."""
version_string = version.version_info.version_string()
LOG.debug("Starting runner %s on board %s", version_string,
self._jobboard_name)
persistence = self._persistence
jobboard = self._jobboard
try:
# Create persistence and/or jobboard if they weren't passed in
if persistence is None:
persistence = tf_client.create_persistence(
conf=self._persistence_conf)
if jobboard is None:
jobboard = tf_client.create_jobboard(
board_name=self._jobboard_name,
conf=self._jobboard_conf,
persistence=persistence,
)
self._conductor = conductors.fetch(
kind="blocking",
name=self._host,
jobboard=jobboard,
persistence=persistence,
engine=self._engine_conf['engine'],
wait_timeout=self._wait_timeout)
time.sleep(0.5)
if enable_cleanup:
conductor_notifier = self._conductor.notifier
conductor_notifier.register('job_consumed',
self.cleanup_job_details)
if threading.current_thread().name == 'MainThread':
t = threading.Thread(target=self._conductor.run)
t.start()
signal.pause()
else:
self._conductor.run()
finally:
# Close persistence and jobboard if they were created by us
if self._persistence is None:
persistence.close()
if self._jobboard is None:
jobboard.close()
self._shutdown_event.set()
def run_conductor(only_run_once=False):
# This continuously consumers until its stopped via ctrl-c or other
# kill signal...
event_watches = {}
# This will be triggered by the conductor doing various activities
# with engines, and is quite nice to be able to see the various timing
# segments (which is useful for debugging, or watching, or figuring out
# where to optimize).
def on_conductor_event(cond, event, details):
print("Event '%s' has been received..." % event)
print("Details = %s" % details)
if event.endswith("_start"):
w = timing.StopWatch()
w.start()
base_event = event[0:-len("_start")]
event_watches[base_event] = w
if event.endswith("_end"):
base_event = event[0:-len("_end")]
try:
w = event_watches.pop(base_event)
w.stop()
print("It took %0.3f seconds for event '%s' to finish"
% (w.elapsed(), base_event))
except KeyError:
pass
if event == 'running_end' and only_run_once:
cond.stop()
print("Starting conductor with pid: %s" % ME)
my_name = "conductor-%s" % ME
persist_backend = persistence_backends.fetch(PERSISTENCE_URI)
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = job_backends.fetch(my_name, JB_CONF,
persistence=persist_backend)
job_backend.connect()
with contextlib.closing(job_backend):
cond = conductor_backends.fetch('blocking', my_name, job_backend,
persistence=persist_backend)
on_conductor_event = functools.partial(on_conductor_event, cond)
cond.notifier.register(cond.notifier.ANY, on_conductor_event)
# Run forever, and kill -9 or ctrl-c me...
try:
cond.run()
finally:
cond.stop()
cond.wait()
def generate_conductor(client, saver, name=NAME):
"""Creates a conductor thread with the given name prefix."""
real_name = "%s_conductor" % name
jb = boards.fetch(name, JOBBOARD_CONF,
client=client, persistence=saver)
conductor = conductors.fetch("blocking", real_name, jb,
engine='parallel', wait_timeout=SCAN_DELAY)
def run():
jb.connect()
with contextlib.closing(jb):
conductor.run()
# Return the unstarted thread, and a callback that can be used
# shutdown that thread (to avoid running forever).
return (threading_utils.daemon_thread(target=run), conductor.stop)
def generate_conductor(client, saver, name=NAME):
"""Creates a conductor thread with the given name prefix."""
real_name = "%s_conductor" % name
jb = boards.fetch(name, JOBBOARD_CONF, client=client, persistence=saver)
conductor = conductors.fetch("blocking",
real_name,
jb,
engine='parallel',
wait_timeout=SCAN_DELAY)
def run():
jb.connect()
with contextlib.closing(jb):
conductor.run()
# Return the unstarted thread, and a callback that can be used
# shutdown that thread (to avoid running forever).
return (threading_utils.daemon_thread(target=run), conductor.stop)
def start(conf):
persistence = _get_persistence_backend(conf)
if conf.taskflow.db_upgrade:
with contextlib.closing(persistence.get_connection()) as conn:
LOG.info(_LI('Checking for database schema upgrade'))
conn.upgrade()
my_name = uuidutils.generate_uuid()
LOG.info(_LI('I am %s'), my_name)
board = _get_jobboard_backend(conf, persistence=persistence)
conductor = conductors.fetch(
'nonblocking', my_name, board,
engine='parallel',
max_simultaneous_jobs=conf.max_simultaneous_jobs,
persistence=persistence)
board.connect()
LOG.debug('Starting taskflow conductor loop')
threading.Thread(target=conductor.run).start()
return persistence, board, conductor
