예제 #1
0
파일: asynpool.py 프로젝트: Nextdoor/celery
    def flush(self):
        if self._state == TERMINATE:
            return
        # cancel all tasks that have not been accepted so that NACK is sent.
        for job in values(self._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()

        self.maintain_pool()

        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                owned_by = {}
                for job in values(self._cache):
                    writer = _get_job_writer(job)
                    if writer is not None:
                        owned_by[writer] = job

                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if gen.__name__ == "_write_job" and gen_not_started(gen):
                            # has not started writing the job so can
                            # discard the task, but we must also remove
                            # it from the Pool._cache.
                            try:
                                job = owned_by[gen]
                            except KeyError:
                                pass
                            else:
                                # removes from Pool._cache
                                job.discard()
                            self._active_writers.discard(gen)
                        else:
                            try:
                                job = owned_by[gen]
                            except KeyError:
                                pass
                            else:
                                job_proc = job._write_to
                                if job_proc._is_alive():
                                    self._flush_writer(job_proc, gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
            self._active_writes.clear()
            self._busy_workers.clear()
예제 #2
0
파일: processes.py 프로젝트: jarieb/celery
    def flush(self):
        # cancel all tasks that have not been accepted so that NACK is sent.
        for job in values(self._pool._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()
        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._pool._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if gen.__name__ == "_write_job" and gen.gi_frame and gen.gi_frame.f_lasti != -1:
                            # has not started writing the job so can
                            # safely discard
                            self._active_writers.discard(gen)
                        else:
                            try:
                                next(gen)
                            except StopIteration:
                                self._active_writers.discard(gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
예제 #3
0
    def flush(self):
        if self._state == TERMINATE:
            return
        # cancel all tasks that have not been accepted so that NACK is sent.
        for job in values(self._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()

        self.maintain_pool()

        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                owned_by = {}
                for job in values(self._cache):
                    writer = _get_job_writer(job)
                    if writer is not None:
                        owned_by[writer] = job

                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if (gen.__name__ == '_write_job' and
                                gen_not_started(gen)):
                            # has not started writing the job so can
                            # discard the task, but we must also remove
                            # it from the Pool._cache.
                            try:
                                job = owned_by[gen]
                            except KeyError:
                                pass
                            else:
                                # removes from Pool._cache
                                job.discard()
                            self._active_writers.discard(gen)
                        else:
                            try:
                                job = owned_by[gen]
                            except KeyError:
                                pass
                            else:
                                if job._write_to.exitcode is None:
                                    self._flush_writer(gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
            self._active_writes.clear()
            self._busy_workers.clear()
예제 #4
0
    def flush(self):
        if self._state == TERMINATE:
            return
        # cancel all tasks that have not been accepted so that NACK is sent.
        for job in values(self._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()
        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if (gen.__name__ == '_write_job' and
                                gen_not_started(gen)):
                            # has not started writing the job so can
                            # discard the task, but we must also remove
                            # it from the Pool._cache.
                            job_to_discard = None
                            for job in values(self._cache):
                                if job._writer() is gen:  # _writer is saferef
                                    # removes from Pool._cache
                                    job_to_discard = job
                                    break
                            if job_to_discard:
                                job_to_discard.discard()
                            self._active_writers.discard(gen)
                        else:
                            try:
                                next(gen)
                            except StopIteration:
                                self._active_writers.discard(gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
예제 #5
0
    def flush(self):
        # cancel all tasks that have not been accepted so that NACK is sent.
        for job in values(self._pool._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()
        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._pool._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if (gen.__name__ == '_write_job' and
                                gen_not_started(gen)):
                            # has not started writing the job so can
                            # discard the task, but we must also remove
                            # it from the Pool._cache.
                            job_to_discard = None
                            for job in values(self._pool._cache):
                                if job._writer() is gen:  # _writer is saferef
                                    # removes from Pool._cache
                                    job_to_discard = job
                                    break
                            if job_to_discard:
                                job_to_discard.discard()
                            self._active_writers.discard(gen)
                        else:
                            try:
                                next(gen)
                            except StopIteration:
                                self._active_writers.discard(gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
예제 #6
0
 def flush(self):
     if self.outbound_buffer:
         self.outbound_buffer.clear()
     try:
         if self._pool._state == RUN:
             # flush outgoing buffers
             intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
             while self._active_writers:
                 writers = list(self._active_writers)
                 for gen in writers:
                     if gen.gi_frame.f_lasti != -1:  # generator started?
                         try:
                             next(gen)
                         except StopIteration:
                             self._active_writers.discard(gen)
                 # workers may have exited in the meantime.
                 self.maintain_pool()
                 sleep(next(intervals))  # don't busyloop
     finally:
         self.outbound_buffer.clear()
         self._active_writers.clear()
예제 #7
0
    def flush(self):
        # cancel all tasks that have not been accepted to that NACK is sent.
        for job in values(self._pool._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()
        try:
            # but we must continue to write the payloads we already started
            # otherwise we will not recover the message boundaries.
            # the messages will be NACK'ed later.
            if self._pool._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if (gen.__name__ == '_write_job'
                                and gen.gi_frame.f_lasti != -1):
                            # has not started writing the job so can
                            # safely discard
                            self._active_writers.discard(gen)
                        else:
                            try:
                                next(gen)
                            except StopIteration:
                                self._active_writers.discard(gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
예제 #8
0
파일: canvas.py 프로젝트: aliscott/celery
 def skew(self, start=1.0, stop=None, step=1.0):
     _next_skew = fxrange(start, stop, step, repeatlast=True).next
     for task in self.tasks:
         task.set(countdown=_next_skew())
     return self
예제 #9
0
 def skew(self, start=1.0, stop=None, step=1.0):
     _next_skew = fxrange(start, stop, step, repeatlast=True).next
     for task in self.tasks:
         task.set(countdown=_next_skew())
     return self
예제 #10
0
파일: canvas.py 프로젝트: AnSavvides/celery
 def skew(self, start=1.0, stop=None, step=1.0):
     it = fxrange(start, stop, step, repeatlast=True)
     for task in self.tasks:
         task.set(countdown=next(it))
     return self
예제 #11
0
 def test_fxrange_no_repeatlast(self):
     self.assertEqual(list(utils.fxrange(1.0, 3.0, 1.0)), [1.0, 2.0, 3.0])
예제 #12
0
 def test_fxrange_no_repeatlast(self):
     self.assertEqual(list(utils.fxrange(1.0, 3.0, 1.0)),
                      [1.0, 2.0, 3.0])
예제 #13
0
 def skew(self, start=1.0, stop=None, step=1.0):
     it = fxrange(start, stop, step, repeatlast=True)
     for task in self.tasks:
         task.set(countdown=next(it))
     return self