def multithreaded_handling(self, request): backend_conf = { 'board': CONF.reaper.job_backend, 'path': "/var/lib/%s" % CONF.reaper.job_backend, 'host': CONF.reaper.backend_host } with backends.backend(self.board_name, backend_conf.copy()) as board: board.post("ReaperJob", book=None, details=request.to_dict())
def job_board(self, persistence): jobboard_backend_conf = { 'board': 'redis', 'host': CONF.task_flow.jobboard_backend_hosts[0], 'port': CONF.task_flow.jobboard_backend_port, 'password': CONF.task_flow.jobboard_backend_password, 'namespace': CONF.task_flow.jobboard_backend_namespace, } jobboard_backend_conf.update( CONF.task_flow.jobboard_redis_backend_ssl_options) return job_backends.backend(CONF.task_flow.jobboard_backend_namespace, jobboard_backend_conf, persistence=persistence)
def job_board(self, persistence): job_backends_hosts = ','.join([ '%s:%s' % (host, CONF.task_flow.jobboard_backend_port) for host in CONF.task_flow.jobboard_backend_hosts ]) jobboard_backend_conf = { 'board': 'zookeeper', 'hosts': job_backends_hosts, 'path': '/' + CONF.task_flow.jobboard_backend_namespace, } jobboard_backend_conf.update( CONF.task_flow.jobboard_zookeeper_ssl_options) return job_backends.backend(CONF.task_flow.jobboard_backend_namespace, jobboard_backend_conf, persistence=persistence)
def producer(ident, client): # Create a personal board (using the same client so that it works in # the same process) and start posting jobs on the board that we want # some entity to perform. name = "P-%s" % (ident) safe_print(name, "started") with backends.backend(name, SHARED_CONF.copy(), client=client) as board: for i in compat_range(0, PRODUCER_UNITS): job_name = "%s-%s" % (name, i) details = { 'color': random.choice(['red', 'blue']), } job = board.post(job_name, book=None, details=details) safe_print(name, "'%s' [posted]" % (job)) time.sleep(PRODUCER_DELAY) safe_print(name, "finished", prefix=">>>")
def worker(ident, client, consumed): # Create a personal board (using the same client so that it works in # the same process) and start looking for jobs on the board that we want # to perform. name = "W-%s" % (ident) safe_print(name, "started") claimed_jobs = 0 consumed_jobs = 0 abandoned_jobs = 0 with backends.backend(name, SHARED_CONF.copy(), client=client) as board: while len(consumed) != EXPECTED_UNITS: favorite_color = random.choice(['blue', 'red']) for job in board.iterjobs(ensure_fresh=True, only_unclaimed=True): # See if we should even bother with it... if job.details.get('color') != favorite_color: continue safe_print(name, "'%s' [attempting claim]" % (job)) try: board.claim(job, name) claimed_jobs += 1 safe_print(name, "'%s' [claimed]" % (job)) except (excp.NotFound, excp.UnclaimableJob): safe_print(name, "'%s' [claim unsuccessful]" % (job)) else: try: dispatch_work(job) board.consume(job, name) safe_print(name, "'%s' [consumed]" % (job)) consumed_jobs += 1 consumed.append(job) except Exception: board.abandon(job, name) abandoned_jobs += 1 safe_print(name, "'%s' [abandoned]" % (job)) time.sleep(WORKER_DELAY) safe_print(name, "finished (claimed %s jobs, consumed %s jobs," " abandoned %s jobs)" % (claimed_jobs, consumed_jobs, abandoned_jobs), prefix=">>>")
def job_board(self, conf, persistence, **kwargs): return job_backends.backend( self.distributed_task_conf.canary_worker_jobboard, conf.copy(), persistence=persistence)
def job_board(self, conf, persistence, **kwargs): return job_backends.backend( self.distributed_task_conf.poppy_service_worker_jobboard, conf.copy(), persistence=persistence)
def job_board(self, conf, persistence, **kwargs): return job_backends.backend( 'poppy_service_jobs', conf.copy(), persistence=persistence)