def multithreaded_handling(self, request):
backend_conf = {
'board': CONF.reaper.job_backend,
'path': "/var/lib/%s" % CONF.reaper.job_backend,
'host': CONF.reaper.backend_host
}
with backends.backend(self.board_name, backend_conf.copy()) as board:
board.post("ReaperJob", book=None, details=request.to_dict())
def job_board(self, persistence):
jobboard_backend_conf = {
'board': 'redis',
'host': CONF.task_flow.jobboard_backend_hosts[0],
'port': CONF.task_flow.jobboard_backend_port,
'password': CONF.task_flow.jobboard_backend_password,
'namespace': CONF.task_flow.jobboard_backend_namespace,
}
jobboard_backend_conf.update(
CONF.task_flow.jobboard_redis_backend_ssl_options)
return job_backends.backend(CONF.task_flow.jobboard_backend_namespace,
jobboard_backend_conf,
persistence=persistence)
def job_board(self, persistence):
job_backends_hosts = ','.join([
'%s:%s' % (host, CONF.task_flow.jobboard_backend_port)
for host in CONF.task_flow.jobboard_backend_hosts
])
jobboard_backend_conf = {
'board': 'zookeeper',
'hosts': job_backends_hosts,
'path': '/' + CONF.task_flow.jobboard_backend_namespace,
}
jobboard_backend_conf.update(
CONF.task_flow.jobboard_zookeeper_ssl_options)
return job_backends.backend(CONF.task_flow.jobboard_backend_namespace,
jobboard_backend_conf,
persistence=persistence)
def producer(ident, client):
# Create a personal board (using the same client so that it works in
# the same process) and start posting jobs on the board that we want
# some entity to perform.
name = "P-%s" % (ident)
safe_print(name, "started")
with backends.backend(name, SHARED_CONF.copy(), client=client) as board:
for i in compat_range(0, PRODUCER_UNITS):
job_name = "%s-%s" % (name, i)
details = {
'color': random.choice(['red', 'blue']),
}
job = board.post(job_name, book=None, details=details)
safe_print(name, "'%s' [posted]" % (job))
time.sleep(PRODUCER_DELAY)
safe_print(name, "finished", prefix=">>>")
def producer(ident, client):
# Create a personal board (using the same client so that it works in
# the same process) and start posting jobs on the board that we want
# some entity to perform.
name = "P-%s" % (ident)
safe_print(name, "started")
with backends.backend(name, SHARED_CONF.copy(), client=client) as board:
for i in compat_range(0, PRODUCER_UNITS):
job_name = "%s-%s" % (name, i)
details = {
'color': random.choice(['red', 'blue']),
}
job = board.post(job_name, book=None, details=details)
safe_print(name, "'%s' [posted]" % (job))
time.sleep(PRODUCER_DELAY)
safe_print(name, "finished", prefix=">>>")
def worker(ident, client, consumed):
# Create a personal board (using the same client so that it works in
# the same process) and start looking for jobs on the board that we want
# to perform.
name = "W-%s" % (ident)
safe_print(name, "started")
claimed_jobs = 0
consumed_jobs = 0
abandoned_jobs = 0
with backends.backend(name, SHARED_CONF.copy(), client=client) as board:
while len(consumed) != EXPECTED_UNITS:
favorite_color = random.choice(['blue', 'red'])
for job in board.iterjobs(ensure_fresh=True, only_unclaimed=True):
# See if we should even bother with it...
if job.details.get('color') != favorite_color:
continue
safe_print(name, "'%s' [attempting claim]" % (job))
try:
board.claim(job, name)
claimed_jobs += 1
safe_print(name, "'%s' [claimed]" % (job))
except (excp.NotFound, excp.UnclaimableJob):
safe_print(name, "'%s' [claim unsuccessful]" % (job))
else:
try:
dispatch_work(job)
board.consume(job, name)
safe_print(name, "'%s' [consumed]" % (job))
consumed_jobs += 1
consumed.append(job)
except Exception:
board.abandon(job, name)
abandoned_jobs += 1
safe_print(name, "'%s' [abandoned]" % (job))
time.sleep(WORKER_DELAY)
safe_print(name,
"finished (claimed %s jobs, consumed %s jobs,"
" abandoned %s jobs)" %
(claimed_jobs, consumed_jobs, abandoned_jobs),
prefix=">>>")
def worker(ident, client, consumed):
# Create a personal board (using the same client so that it works in
# the same process) and start looking for jobs on the board that we want
# to perform.
name = "W-%s" % (ident)
safe_print(name, "started")
claimed_jobs = 0
consumed_jobs = 0
abandoned_jobs = 0
with backends.backend(name, SHARED_CONF.copy(), client=client) as board:
while len(consumed) != EXPECTED_UNITS:
favorite_color = random.choice(['blue', 'red'])
for job in board.iterjobs(ensure_fresh=True, only_unclaimed=True):
# See if we should even bother with it...
if job.details.get('color') != favorite_color:
continue
safe_print(name, "'%s' [attempting claim]" % (job))
try:
board.claim(job, name)
claimed_jobs += 1
safe_print(name, "'%s' [claimed]" % (job))
except (excp.NotFound, excp.UnclaimableJob):
safe_print(name, "'%s' [claim unsuccessful]" % (job))
else:
try:
dispatch_work(job)
board.consume(job, name)
safe_print(name, "'%s' [consumed]" % (job))
consumed_jobs += 1
consumed.append(job)
except Exception:
board.abandon(job, name)
abandoned_jobs += 1
safe_print(name, "'%s' [abandoned]" % (job))
time.sleep(WORKER_DELAY)
safe_print(name,
"finished (claimed %s jobs, consumed %s jobs,"
" abandoned %s jobs)" % (claimed_jobs, consumed_jobs,
abandoned_jobs), prefix=">>>")
def job_board(self, conf, persistence, **kwargs):
return job_backends.backend(
self.distributed_task_conf.canary_worker_jobboard,
conf.copy(), persistence=persistence)
def job_board(self, conf, persistence, **kwargs):
return job_backends.backend(
self.distributed_task_conf.poppy_service_worker_jobboard,
conf.copy(),
persistence=persistence)
def job_board(self, conf, persistence, **kwargs):
return job_backends.backend(
'poppy_service_jobs',
conf.copy(), persistence=persistence)
