def work(self):
"""children only: watch tubes for all jobs, start working"""
beanstalk = connect_beanstalkd()
for job in self.jobs.keys():
beanstalk.watch(job)
beanstalk.ignore('default')
try:
while True:
job = beanstalk.reserve()
job_name = job.stats()['tube']
if job_name in self.jobs:
logger.debug("Calling %s with arg: %s" %
(job_name, job.body))
try:
self.jobs[job_name](job.body)
except Exception, e:
tp, value, tb = sys.exc_info()
logger.error('Error while calling "%s" with arg "%s": '
'%s' % (
job_name,
job.body,
e,
))
logger.debug("%s:%s" % (tp.__name__, value))
logger.debug("\n".join(traceback.format_tb(tb)))
job.bury()
else:
job.delete()
else:
job.release()
except KeyboardInterrupt:
sys.exit(0)
def work(self):
"""children only: watch tubes for all jobs, start working"""
beanstalk = connect_beanstalkd()
for job in self.jobs.keys():
beanstalk.watch(job)
beanstalk.ignore('default')
try:
while True:
job = beanstalk.reserve()
job_name = job.stats()['tube']
if job_name in self.jobs:
logger.debug("Calling %s" % job_name)
try:
self.jobs[job_name](job.body)
except Exception, e:
tp, value, tb = sys.exc_info()
logger.error('Error while calling "%s": '
'%s' % (
job_name,
e,
)
)
logger.debug("%s:%s" % (tp.__name__, value))
logger.debug("\n".join(traceback.format_tb(tb)))
job.bury()
else:
job.delete()
else:
job.release()
except KeyboardInterrupt:
sys.exit(0)
def work(self):
"""children only: watch tubes for all jobs, start working"""
beanstalk = connect_beanstalkd()
for job in self.jobs.keys():
beanstalk.watch(job)
beanstalk.ignore('default')
logger = logging.getLogger('beanstalk_worker.work')
try:
while True:
try :
job = beanstalk.reserve()
if self.__checkShutdown(job) :
continue
else :
job_uuid = self.__get_uuid__(job)
#Jobs may be requeued, in the event that we detect beanstalk went down or is an
#an inconsistent state. Therefore, we'll just ignore this job if it already exists in the system and
#has finished successfully
if not self.__hasJobBeenProcessed(job_uuid) :
# Create the JobRecord here, since the JobRecord gets created in
# BaseAsyncTasks.run()
# only if job queuing has failed.
job_name = job.stats()['tube']
JobRecord.updateJobStatus(job_uuid, JobStatus.RESERVED)
if job_name in self.jobs:
logger.debug("Calling %s with arg: %s" % (job_name, job.body))
try:
self.jobs[job_name](job.body)
except Exception, e:
logger.error('Error while calling "%s" with arg "%s": '
'%s' % (
job_name,
job.body,
e,
)
)
job.bury()
JobRecord.updateJobStatus(job_uuid, JobStatus.BURIED)
else:
job.delete()
JobRecord.updateJobStatus(job_uuid, JobStatus.FINISHED_SUCCESSFULLY)
else:
job.release()
else :
job.delete()
except Exception, unknownErr :
logger.exception("An unexpected beanstalk error occured! %s" % str(unknownErr))
def handle(self, *args, **options):
bc = connect_beanstalkd()
buried_jobs = bc.stats()["current-jobs-buried"]
for t in bc.tubes():
bc.use(t)
while buried_jobs > 0:
try:
j = bc.peek_buried()
j.delete()
buried_jobs -= 1
self.stdout.write("Cleaning buried jobs: %s..." % buried_jobs, ending="\r")
self.stdout.flush()
except AttributeError:
break
def work(self):
"""children only: watch tubes for all jobs, start working"""
try:
while True:
try:
# Reattempt Beanstalk connection if connection attempt fails or is dropped
beanstalk = connect_beanstalkd()
for job in self.jobs.keys():
beanstalk.watch(job)
beanstalk.ignore('default')
# Connected to Beanstalk queue, continually process jobs until an error occurs
self.process_jobs(beanstalk)
except (BeanstalkError, SocketError) as e:
logger.info("Beanstalk connection error: " + str(e))
time.sleep(2.0)
logger.info("retrying Beanstalk connection...")
except KeyboardInterrupt:
sys.exit(0)
def work(self):
"""children only: watch tubes for all jobs, start working"""
try:
while True:
try:
# Reattempt Beanstalk connection if connection attempt fails or is dropped
beanstalk = connect_beanstalkd()
for job in self.jobs.keys():
beanstalk.watch(job)
beanstalk.ignore('default')
# Connected to Beanstalk queue, continually process jobs until an error occurs
self.process_jobs(beanstalk)
except (BeanstalkError, SocketError) as e:
logger.info("Beanstalk connection error: " + str(e))
time.sleep(2.0)
logger.info("retrying Beanstalk connection...")
except KeyboardInterrupt:
sys.exit(0)
def work(self):
"""children only: watch tubes for all jobs, start working"""
try:
while True:
try:
# Reattempt Beanstalk connection if connection attempt fails or is dropped
beanstalk = connect_beanstalkd(server=self.beanstalk_server, port=self.beanstalk_port)
for job in self.jobs.keys():
beanstalk.watch(job)
beanstalk.ignore("default")
# Connected to Beanstalk queue, continually process jobs until an error occurs
# Each worker will have their own connection
db.connections["default"].close()
self.process_jobs(beanstalk)
except (BeanstalkError, SocketError) as e:
msg = "Beanstalk connection error: " + str(e)
logger.info(msg)
client = Client(dsn=settings.RAVEN_CONFIG["dsn"])
client.captureMessage(msg, stack=True, level=logging.ERROR)
time.sleep(2.0)
logger.info("retrying Beanstalk connection...")
except KeyboardInterrupt:
raise
except Exception as e:
msg = "Beanstalk error: " + str(e)
client = Client(dsn=settings.RAVEN_CONFIG["dsn"])
client.captureMessage(msg, stack=True, level=logging.ERROR)
logger.info(msg)
time.sleep(2.0)
logger.info("retrying Beanstalk connection...")
except KeyboardInterrupt:
sys.exit(0)
