def process(self, job):
if self.nofork:
self.dispatch(job)
else:
from multiprocessing import Process
p = Process(target=self.dispatch, args=(job,))
p.start()
setprocname("dreque: Forked %d at %d" % (p.pid, time.time()))
p.join()
if p.exitcode != 0:
raise Exception("Job failed")
def process(self, job):
if self.dispatcher == "fork":
child = Process(target=self.dispatch_child, args=(job, ))
child.start()
self.child = child
setprocname("dreque: Forked %d at %d" % (child.pid, time.time()))
while True:
try:
child.join()
except OSError, exc:
if 'Interrupted system call' not in exc:
raise
continue
break
self.child = None
if child.exitcode != 0:
raise Exception("Job failed with exitcode %d" % child.exitcode)
def work(self, interval=5):
self.register_worker()
self.register_signal_handlers()
setprocname("dreque: Starting")
self._shutdown = None
try:
while not self._shutdown:
worked = self.work_once()
if interval == 0:
break
if not worked:
setprocname("dreque: Waiting for %s" % ",".join(self.queues))
time.sleep(interval)
finally:
self.unregister_worker()
def process(self, job):
if self.dispatcher == "fork":
child = Process(target=self.dispatch_child, args=(job,))
child.start()
self.child = child
setprocname("dreque: Forked %d at %d" % (child.pid, time.time()))
while True:
try:
child.join()
except OSError, exc:
if 'Interrupted system call' not in exc:
raise
continue
break
self.child = None
if child.exitcode != 0:
raise Exception("Job failed with exitcode %d" % child.exitcode)
def work(self, interval=5):
self.register_worker()
self.register_signal_handlers()
setprocname("dreque: Starting")
self._shutdown = None
try:
while not self._shutdown:
worked = self.work_once()
if interval == 0:
break
if not worked:
setprocname("dreque: Waiting for %s" %
",".join(self.queues))
time.sleep(interval)
finally:
self.unregister_worker()
def work(self, interval=5):
self.register_worker()
setprocname("dreque: Starting")
try:
while True:
job = self.dequeue(self.queues)
if not job:
if interval == 0:
break
setprocname("dreque: Waiting for %s" % ",".join(self.queues))
time.sleep(interval)
continue
try:
self.working_on(job)
self.process(job)
except Exception, exc:
import traceback
self.log.info("Job failed (%s): %s\n%s" % (job, str(exc), traceback.format_exc()))
# Requeue
queue = job.pop("queue")
if 'fails' not in job:
job['fails'] = 1
else:
job['fails'] += 1
if job['fails'] < 10:
self.push_delayed(queue, job, 2**job['fails'])
self.failed()
else:
self.done_working()
if interval == 0:
break
finally:
self.unregister_worker()
def dispatch(self, job):
setprocname("dreque: Processing %s since %d" %
(job['queue'], time.time()))
func = self.lookup_function(job['func'])
kwargs = dict((str(k), v) for k, v in job['kwargs'].items())
func(*job['args'], **kwargs)
def dispatch(self, job):
setprocname("dreque: Processing %s since %d" % (job['queue'], time.time()))
func = self.lookup_function(job['func'])
kwargs = dict((str(k), v) for k, v in job['kwargs'].items())
func(*job['args'], **kwargs)