def register_worker(self, worker_id=None, extra_metadata={}):
self._reclaim()
# set up this worker
worker_id = worker_id or uuid.uuid1()
worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(),
worker_id)
worker_recorder.record_initial_metadata(extra_metadata)
return worker_id
def run_once(self, worker_id):
"""
Run the queue for one step.
"""
envelopes = []
worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(),
worker_id)
# The Big Pipeline
pipeline = self.redis.pipeline()
while len(envelopes) < self._batch_size:
envelope = self.pop(worker_id, pipeline=pipeline)
if not envelope:
break
envelope['pop_time'] = time.time()
response_time = envelope['pop_time'] - float(envelope['first_ts'])
self._event_registrar.on_pop(item=envelope['item'],
item_key=self.item_key(envelope['item']),
response_time=response_time,
pipeline=pipeline)
envelopes.append(envelope)
if not envelopes:
pipeline.execute()
return None
# clear expired envelopes
envelopes_to_process = list(envelopes)
for envelope in envelopes:
if envelope['ttl'] and (envelope['first_ts'] + envelope['ttl'] < time.time()):
envelopes_to_process.remove(envelope)
self._event_registrar.on_expire(item=envelope['item'],
item_key=self.item_key(envelope['item']),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(envelope['item']))
worker_recorder.record_expire(pipeline=pipeline)
def handle_error(envelope):
self._event_registrar.on_error(item=envelope['item'],
item_key=self.item_key(envelope['item']),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(envelope['item']))
worker_recorder.record_error(pipeline=pipeline)
self.error_queue.queue_error(envelope)
try:
self.process_items([envelope['item'] for envelope in envelopes_to_process])
except errors.ExpiredError:
for envelope in envelopes:
self._event_registrar.on_expire(item=envelope['item'],
item_key=self.item_key(envelope['item']),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(envelope['item']))
worker_recorder.record_expire(pipeline=pipeline)
except tuple(self._retry_error_classes):
for envelope in envelopes_to_process:
if envelope['attempts'] >= self._max_attempts - 1:
handle_error(envelope)
else:
self._event_registrar.on_retry(item=envelope['item'],
item_key=self.item_key(envelope['item']),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(envelope['item']))
worker_recorder.record_retry(pipeline=pipeline)
# When we retry, first_ts stsys the same
self.push(envelope['item'], pipeline=pipeline, envelope=envelope,
attempts=envelope['attempts'] + 1)
except Exception:
for envelope in envelopes_to_process:
handle_error(envelope)
else:
for envelope in envelopes_to_process:
self._event_registrar.on_success(item=envelope['item'],
item_key=self.item_key(envelope['item']),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(envelope['item']))
worker_recorder.record_success(pipeline=pipeline)
finally:
for envelope in envelopes:
self.complete(envelope, worker_id, pipeline=pipeline)
complete_time = time.time()
turnaround_time = complete_time - float(envelope['first_ts'])
processing_time = complete_time - envelope['pop_time']
self._event_registrar.on_complete(item=envelope['item'],
item_key=self.item_key(envelope['item']),
turnaround_time=turnaround_time,
processing_time=processing_time,
pipeline=pipeline)
pipeline.execute()
return envelopes
def run_once(self):
"""
Run the queue for one step.
"""
worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(),
self.worker_id)
# The Big Pipeline
pipeline = self.redis.pipeline()
envelope = self.pop(pipeline=pipeline)
if not envelope:
self._event_registrar.on_empty(pipeline=pipeline)
pipeline.execute()
return None
item = envelope['item']
pop_time = time.time()
response_time = pop_time - float(envelope['ts'])
item_error_classes = self.error_classes_for_envelope(envelope)
self._event_registrar.on_pop(item=item, item_key=self.item_key(item),
response_time=response_time,
pipeline=pipeline)
def handle_error():
self._event_registrar.on_error(
item=item,
item_key=self.item_key(item),
pipeline=pipeline, exc_info=sys.exc_info(),
pretty_printed_item=self.pretty_printer(item))
worker_recorder.record_error(pipeline=pipeline)
self.error_queue.queue_error(envelope, pipeline=pipeline)
try:
if envelope['ttl'] and (envelope['first_ts'] + envelope['ttl'] < time.time()):
raise errors.ExpiredError
self.process_item(envelope['item'])
except errors.ExpiredError:
self._event_registrar.on_expire(
item=item,
item_key=self.item_key(item),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(item))
worker_recorder.record_expire(pipeline=pipeline)
except tuple(item_error_classes):
# If we've tried this item three times already, cut our losses and
# treat it like other errors.
max_attempts = envelope.get('max_attempts', self._max_attempts)
if envelope['attempts'] >= max_attempts - 1:
handle_error()
else:
self._event_registrar.on_retry(
item=item,
item_key=self.item_key(item),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(item))
worker_recorder.record_retry(pipeline=pipeline)
# When we retry, first_ts stays the same
try:
self.push(envelope['item'], pipeline=pipeline, envelope=envelope,
max_attempts=max_attempts,
attempts=envelope['attempts'] + 1,
error_classes=item_error_classes)
except Exception as e:
print e
except Exception:
handle_error()
else:
self._event_registrar.on_success(
item=item,
item_key=self.item_key(item),
pipeline=pipeline,
pretty_printed_item=self.pretty_printer(item))
worker_recorder.record_success(pipeline=pipeline)
finally:
self.complete(envelope, pipeline=pipeline)
complete_time = time.time()
turnaround_time = complete_time - float(envelope['ts'])
processing_time = complete_time - pop_time
self._event_registrar.on_complete(item=item, item_key=self.item_key(item),
turnaround_time=turnaround_time,
processing_time=processing_time,
pipeline=pipeline)
pipeline.execute()
return envelope
def register_worker(self):
worker_recorder = WorkerMetadataRecorder(self.redis, self._queue_key(),
self.worker_id)
worker_recorder.record_initial_metadata()
