Example #1
0
    def process(self, job):
        if self.nofork:
            self.dispatch(job)
        else:
            from multiprocessing import Process

            p = Process(target=self.dispatch, args=(job,))
            p.start()
            setprocname("dreque: Forked %d at %d" % (p.pid, time.time()))
            p.join()

            if p.exitcode != 0:
                raise Exception("Job failed")
Example #2
0
    def process(self, job):
        if self.dispatcher == "fork":
            child = Process(target=self.dispatch_child, args=(job, ))
            child.start()
            self.child = child
            setprocname("dreque: Forked %d at %d" % (child.pid, time.time()))
            while True:
                try:
                    child.join()
                except OSError, exc:
                    if 'Interrupted system call' not in exc:
                        raise
                    continue
                break
            self.child = None

            if child.exitcode != 0:
                raise Exception("Job failed with exitcode %d" % child.exitcode)
Example #3
0
    def work(self, interval=5):
        self.register_worker()
        self.register_signal_handlers()

        setprocname("dreque: Starting")

        self._shutdown = None
        try:
            while not self._shutdown:
                worked = self.work_once()
                if interval == 0:
                    break

                if not worked:
                    setprocname("dreque: Waiting for %s" % ",".join(self.queues))
                    time.sleep(interval)
        finally:
            self.unregister_worker()
Example #4
0
    def process(self, job):
        if self.dispatcher == "fork":
            child = Process(target=self.dispatch_child, args=(job,))
            child.start()
            self.child = child
            setprocname("dreque: Forked %d at %d" % (child.pid, time.time()))
            while True:
                try:
                    child.join()
                except OSError, exc:
                    if 'Interrupted system call' not in exc:
                        raise
                    continue
                break
            self.child = None

            if child.exitcode != 0:
                raise Exception("Job failed with exitcode %d" % child.exitcode)
Example #5
0
    def work(self, interval=5):
        self.register_worker()
        self.register_signal_handlers()

        setprocname("dreque: Starting")

        self._shutdown = None
        try:
            while not self._shutdown:
                worked = self.work_once()
                if interval == 0:
                    break

                if not worked:
                    setprocname("dreque: Waiting for %s" %
                                ",".join(self.queues))
                    time.sleep(interval)
        finally:
            self.unregister_worker()
Example #6
0
    def work(self, interval=5):
        self.register_worker()

        setprocname("dreque: Starting")

        try:
            while True:
                job = self.dequeue(self.queues)
                if not job:
                    if interval == 0:
                        break
                    setprocname("dreque: Waiting for %s" % ",".join(self.queues))
                    time.sleep(interval)
                    continue

                try:
                    self.working_on(job)
                    self.process(job)
                except Exception, exc:
                    import traceback
                    self.log.info("Job failed (%s): %s\n%s" % (job, str(exc), traceback.format_exc()))
                    # Requeue
                    queue = job.pop("queue")
                    if 'fails' not in job:
                        job['fails'] = 1
                    else:
                        job['fails'] += 1
                    if job['fails'] < 10:
                        self.push_delayed(queue, job, 2**job['fails'])
                    self.failed()
                else:
                    self.done_working()

                if interval == 0:
                    break
        finally:
            self.unregister_worker()
Example #7
0
 def dispatch(self, job):
     setprocname("dreque: Processing %s since %d" %
                 (job['queue'], time.time()))
     func = self.lookup_function(job['func'])
     kwargs = dict((str(k), v) for k, v in job['kwargs'].items())
     func(*job['args'], **kwargs)
Example #8
0
 def dispatch(self, job):
     setprocname("dreque: Processing %s since %d" % (job['queue'], time.time()))
     func = self.lookup_function(job['func'])
     kwargs = dict((str(k), v) for k, v in job['kwargs'].items())
     func(*job['args'], **kwargs)