def register_worker(self, worker_id=None, extra_metadata={}): self._reclaim() # set up this worker worker_id = worker_id or uuid.uuid1() worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(), worker_id) worker_recorder.record_initial_metadata(extra_metadata) return worker_id
def run_once(self, worker_id): """ Run the queue for one step. """ envelopes = [] worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(), worker_id) # The Big Pipeline pipeline = self.redis.pipeline() while len(envelopes) < self._batch_size: envelope = self.pop(worker_id, pipeline=pipeline) if not envelope: break envelope['pop_time'] = time.time() response_time = envelope['pop_time'] - float(envelope['first_ts']) self._event_registrar.on_pop(item=envelope['item'], item_key=self.item_key(envelope['item']), response_time=response_time, pipeline=pipeline) envelopes.append(envelope) if not envelopes: pipeline.execute() return None # clear expired envelopes envelopes_to_process = list(envelopes) for envelope in envelopes: if envelope['ttl'] and (envelope['first_ts'] + envelope['ttl'] < time.time()): envelopes_to_process.remove(envelope) self._event_registrar.on_expire(item=envelope['item'], item_key=self.item_key(envelope['item']), pipeline=pipeline, pretty_printed_item=self.pretty_printer(envelope['item'])) worker_recorder.record_expire(pipeline=pipeline) def handle_error(envelope): self._event_registrar.on_error(item=envelope['item'], item_key=self.item_key(envelope['item']), pipeline=pipeline, pretty_printed_item=self.pretty_printer(envelope['item'])) worker_recorder.record_error(pipeline=pipeline) self.error_queue.queue_error(envelope) try: self.process_items([envelope['item'] for envelope in envelopes_to_process]) except errors.ExpiredError: for envelope in envelopes: self._event_registrar.on_expire(item=envelope['item'], item_key=self.item_key(envelope['item']), pipeline=pipeline, pretty_printed_item=self.pretty_printer(envelope['item'])) worker_recorder.record_expire(pipeline=pipeline) except tuple(self._retry_error_classes): for envelope in envelopes_to_process: if envelope['attempts'] >= self._max_attempts - 1: handle_error(envelope) else: self._event_registrar.on_retry(item=envelope['item'], item_key=self.item_key(envelope['item']), pipeline=pipeline, pretty_printed_item=self.pretty_printer(envelope['item'])) worker_recorder.record_retry(pipeline=pipeline) # When we retry, first_ts stsys the same self.push(envelope['item'], pipeline=pipeline, envelope=envelope, attempts=envelope['attempts'] + 1) except Exception: for envelope in envelopes_to_process: handle_error(envelope) else: for envelope in envelopes_to_process: self._event_registrar.on_success(item=envelope['item'], item_key=self.item_key(envelope['item']), pipeline=pipeline, pretty_printed_item=self.pretty_printer(envelope['item'])) worker_recorder.record_success(pipeline=pipeline) finally: for envelope in envelopes: self.complete(envelope, worker_id, pipeline=pipeline) complete_time = time.time() turnaround_time = complete_time - float(envelope['first_ts']) processing_time = complete_time - envelope['pop_time'] self._event_registrar.on_complete(item=envelope['item'], item_key=self.item_key(envelope['item']), turnaround_time=turnaround_time, processing_time=processing_time, pipeline=pipeline) pipeline.execute() return envelopes
def run_once(self): """ Run the queue for one step. """ worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(), self.worker_id) # The Big Pipeline pipeline = self.redis.pipeline() envelope = self.pop(pipeline=pipeline) if not envelope: self._event_registrar.on_empty(pipeline=pipeline) pipeline.execute() return None item = envelope['item'] pop_time = time.time() response_time = pop_time - float(envelope['ts']) item_error_classes = self.error_classes_for_envelope(envelope) self._event_registrar.on_pop(item=item, item_key=self.item_key(item), response_time=response_time, pipeline=pipeline) def handle_error(): self._event_registrar.on_error( item=item, item_key=self.item_key(item), pipeline=pipeline, exc_info=sys.exc_info(), pretty_printed_item=self.pretty_printer(item)) worker_recorder.record_error(pipeline=pipeline) self.error_queue.queue_error(envelope, pipeline=pipeline) try: if envelope['ttl'] and (envelope['first_ts'] + envelope['ttl'] < time.time()): raise errors.ExpiredError self.process_item(envelope['item']) except errors.ExpiredError: self._event_registrar.on_expire( item=item, item_key=self.item_key(item), pipeline=pipeline, pretty_printed_item=self.pretty_printer(item)) worker_recorder.record_expire(pipeline=pipeline) except tuple(item_error_classes): # If we've tried this item three times already, cut our losses and # treat it like other errors. max_attempts = envelope.get('max_attempts', self._max_attempts) if envelope['attempts'] >= max_attempts - 1: handle_error() else: self._event_registrar.on_retry( item=item, item_key=self.item_key(item), pipeline=pipeline, pretty_printed_item=self.pretty_printer(item)) worker_recorder.record_retry(pipeline=pipeline) # When we retry, first_ts stays the same try: self.push(envelope['item'], pipeline=pipeline, envelope=envelope, max_attempts=max_attempts, attempts=envelope['attempts'] + 1, error_classes=item_error_classes) except Exception as e: print e except Exception: handle_error() else: self._event_registrar.on_success( item=item, item_key=self.item_key(item), pipeline=pipeline, pretty_printed_item=self.pretty_printer(item)) worker_recorder.record_success(pipeline=pipeline) finally: self.complete(envelope, pipeline=pipeline) complete_time = time.time() turnaround_time = complete_time - float(envelope['ts']) processing_time = complete_time - pop_time self._event_registrar.on_complete(item=item, item_key=self.item_key(item), turnaround_time=turnaround_time, processing_time=processing_time, pipeline=pipeline) pipeline.execute() return envelope
def register_worker(self): worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(), self.worker_id) worker_recorder.record_initial_metadata()