def work(self): """children only: watch tubes for all jobs, start working""" beanstalk = connect_beanstalkd() for job in self.jobs.keys(): beanstalk.watch(job) beanstalk.ignore('default') try: while True: job = beanstalk.reserve() job_name = job.stats()['tube'] if job_name in self.jobs: logger.debug("Calling %s with arg: %s" % (job_name, job.body)) try: self.jobs[job_name](job.body) except Exception, e: tp, value, tb = sys.exc_info() logger.error('Error while calling "%s" with arg "%s": ' '%s' % ( job_name, job.body, e, )) logger.debug("%s:%s" % (tp.__name__, value)) logger.debug("\n".join(traceback.format_tb(tb))) job.bury() else: job.delete() else: job.release() except KeyboardInterrupt: sys.exit(0)
def work(self): """children only: watch tubes for all jobs, start working""" beanstalk = connect_beanstalkd() for job in self.jobs.keys(): beanstalk.watch(job) beanstalk.ignore('default') try: while True: job = beanstalk.reserve() job_name = job.stats()['tube'] if job_name in self.jobs: logger.debug("Calling %s" % job_name) try: self.jobs[job_name](job.body) except Exception, e: tp, value, tb = sys.exc_info() logger.error('Error while calling "%s": ' '%s' % ( job_name, e, ) ) logger.debug("%s:%s" % (tp.__name__, value)) logger.debug("\n".join(traceback.format_tb(tb))) job.bury() else: job.delete() else: job.release() except KeyboardInterrupt: sys.exit(0)
def work(self): """children only: watch tubes for all jobs, start working""" beanstalk = connect_beanstalkd() for job in self.jobs.keys(): beanstalk.watch(job) beanstalk.ignore('default') logger = logging.getLogger('beanstalk_worker.work') try: while True: try : job = beanstalk.reserve() if self.__checkShutdown(job) : continue else : job_uuid = self.__get_uuid__(job) #Jobs may be requeued, in the event that we detect beanstalk went down or is an #an inconsistent state. Therefore, we'll just ignore this job if it already exists in the system and #has finished successfully if not self.__hasJobBeenProcessed(job_uuid) : # Create the JobRecord here, since the JobRecord gets created in # BaseAsyncTasks.run() # only if job queuing has failed. job_name = job.stats()['tube'] JobRecord.updateJobStatus(job_uuid, JobStatus.RESERVED) if job_name in self.jobs: logger.debug("Calling %s with arg: %s" % (job_name, job.body)) try: self.jobs[job_name](job.body) except Exception, e: logger.error('Error while calling "%s" with arg "%s": ' '%s' % ( job_name, job.body, e, ) ) job.bury() JobRecord.updateJobStatus(job_uuid, JobStatus.BURIED) else: job.delete() JobRecord.updateJobStatus(job_uuid, JobStatus.FINISHED_SUCCESSFULLY) else: job.release() else : job.delete() except Exception, unknownErr : logger.exception("An unexpected beanstalk error occured! %s" % str(unknownErr))
def handle(self, *args, **options): bc = connect_beanstalkd() buried_jobs = bc.stats()["current-jobs-buried"] for t in bc.tubes(): bc.use(t) while buried_jobs > 0: try: j = bc.peek_buried() j.delete() buried_jobs -= 1 self.stdout.write("Cleaning buried jobs: %s..." % buried_jobs, ending="\r") self.stdout.flush() except AttributeError: break
def work(self): """children only: watch tubes for all jobs, start working""" try: while True: try: # Reattempt Beanstalk connection if connection attempt fails or is dropped beanstalk = connect_beanstalkd() for job in self.jobs.keys(): beanstalk.watch(job) beanstalk.ignore('default') # Connected to Beanstalk queue, continually process jobs until an error occurs self.process_jobs(beanstalk) except (BeanstalkError, SocketError) as e: logger.info("Beanstalk connection error: " + str(e)) time.sleep(2.0) logger.info("retrying Beanstalk connection...") except KeyboardInterrupt: sys.exit(0)
def work(self): """children only: watch tubes for all jobs, start working""" try: while True: try: # Reattempt Beanstalk connection if connection attempt fails or is dropped beanstalk = connect_beanstalkd(server=self.beanstalk_server, port=self.beanstalk_port) for job in self.jobs.keys(): beanstalk.watch(job) beanstalk.ignore("default") # Connected to Beanstalk queue, continually process jobs until an error occurs # Each worker will have their own connection db.connections["default"].close() self.process_jobs(beanstalk) except (BeanstalkError, SocketError) as e: msg = "Beanstalk connection error: " + str(e) logger.info(msg) client = Client(dsn=settings.RAVEN_CONFIG["dsn"]) client.captureMessage(msg, stack=True, level=logging.ERROR) time.sleep(2.0) logger.info("retrying Beanstalk connection...") except KeyboardInterrupt: raise except Exception as e: msg = "Beanstalk error: " + str(e) client = Client(dsn=settings.RAVEN_CONFIG["dsn"]) client.captureMessage(msg, stack=True, level=logging.ERROR) logger.info(msg) time.sleep(2.0) logger.info("retrying Beanstalk connection...") except KeyboardInterrupt: sys.exit(0)