def test_bulk_reset(self):
for i in range(0, 5):
error = create_error(_change(id=i), attempts=const.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS)
error.save()
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 0)
PillowError.bulk_reset_attempts(datetime.utcnow())
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 5)
def test_bulk_reset_cutoff(self):
for i in range(0, 3):
error = create_error({'id': i}, attempts=1)
if i >= 1:
error.total_attempts = PillowError.multi_attempts_cutoff() + 1
error.save()
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 0)
PillowError.bulk_reset_attempts(datetime.utcnow())
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 2)
def test_bulk_reset_cutoff(self):
for i in range(0, 3):
error = create_error(_change(id=i), attempts=1)
if i >= 1:
error.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
error.save()
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 0)
PillowError.bulk_reset_attempts(datetime.utcnow())
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 2)
def test_get_errors_to_process(self):
# Only re-process errors with
# current_attempt < const.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS
date = datetime.utcnow()
for i in range(0, 5):
error = create_error(_change(id=i), attempts=i + 1)
error.date_next_attempt = date.replace(day=i + 1)
error.save()
errors = PillowError.get_errors_to_process(date.replace(day=1), ).all()
self.assertEqual(len(errors), 1)
errors = PillowError.get_errors_to_process(date.replace(day=5), ).all()
self.assertEqual(len(errors), 3)
def test_bulk_reset_cutoff(self):
for i in range(0, 3):
error = create_error({'id': i}, attempts=1)
if i >= 1:
error.total_attempts = PillowError.multi_attempts_cutoff() + 1
error.save()
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 0)
PillowError.bulk_reset_attempts(datetime.utcnow())
errors = PillowError.get_errors_to_process(datetime.utcnow()).all()
self.assertEqual(len(errors), 2)
def test_get_errors_to_process_queued_update(self):
date = datetime.utcnow()
error = create_error({'id': 1}, attempts=0)
error.date_next_attempt = date
error.save()
errors = PillowError.get_errors_to_process(date, ).all()
self.assertEqual(len(errors), 1)
# check that calling update on the return value has the desired effect
errors.update(queued=True)
errors = PillowError.get_errors_to_process(date, ).all()
self.assertEqual(len(errors), 0)
def test_get_errors_to_process_max_limit(self):
date = datetime.utcnow()
def make_error(id, current_attempt, total_attempts):
error = create_error(_change(id=id))
error.date_next_attempt = date
error.current_attempt = current_attempt
error.total_attempts = total_attempts
error.save()
# current_attempts <= limit, total_attempts <= limit
make_error('to-process1',
const.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS,
const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF)
# current_attempts = 0, total_attempts > limit
make_error('to-process2', 0,
const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1)
# current_attempts > limit, total_attempts <= limit
make_error('not-processed1',
const.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS + 1,
const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF)
# current_attempts <= limit, total_attempts > limit
make_error('not-processed2',
const.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS,
const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1)
errors = PillowError.get_errors_to_process(date).all()
self.assertEqual(len(errors), 2)
docs_to_process = {e.doc_id for e in errors}
self.assertEqual({'to-process1', 'to-process2'}, docs_to_process)
def test_get_errors_to_process(self):
# Only re-process errors with
# current_attempt < const.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS
date = datetime.utcnow()
for i in range(0, 5):
error = create_error(_change(id=i), attempts=i+1)
error.date_next_attempt = date.replace(day=i+1)
error.save()
errors = PillowError.get_errors_to_process(
date.replace(day=1),
).all()
self.assertEqual(len(errors), 1)
errors = PillowError.get_errors_to_process(
date.replace(day=5),
).all()
self.assertEqual(len(errors), 3)
def test_get_errors_to_process_queued_update(self):
date = datetime.utcnow()
error = create_error({'id': 1}, attempts=0)
error.date_next_attempt = date
error.save()
errors = PillowError.get_errors_to_process(
date,
).all()
self.assertEqual(len(errors), 1)
# check that calling update on the return value has the desired effect
errors.update(queued=True)
errors = PillowError.get_errors_to_process(
date,
).all()
self.assertEqual(len(errors), 0)
def test_get_errors_to_process_queued(self):
date = datetime.utcnow()
error = create_error({'id': 1}, attempts=0)
error.date_next_attempt = date
error.save()
queued_error = create_error({'id': 2}, attempts=0)
queued_error.date_next_attempt = date
queued_error.queued = True
queued_error.save()
errors = PillowError.get_errors_to_process(date, ).all()
self.assertEqual(len(errors), 1)
self.assertEqual(error.id, errors[0]['id'])
def test_get_errors_to_process_queued(self):
date = datetime.utcnow()
error = create_error({'id': 1}, attempts=0)
error.date_next_attempt = date
error.save()
queued_error = create_error({'id': 2}, attempts=0)
queued_error.date_next_attempt = date
queued_error.queued = True
queued_error.save()
errors = PillowError.get_errors_to_process(
date,
).all()
self.assertEqual(len(errors), 1)
self.assertEqual(error.id, errors[0]['id'])
def test_get_errors_to_process_max_limit(self):
# see settings.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF
date = datetime.utcnow()
def make_error(id, current_attempt, total_attempts):
error = create_error({'id': id})
error.date_next_attempt = date
error.current_attempt = current_attempt
error.total_attempts = total_attempts
error.save()
# current_attempts <= limit, total_attempts <= limit
make_error(
'to-process1',
settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS,
settings.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF
)
# current_attempts = 0, total_attempts > limit
make_error(
'to-process2',
0,
settings.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
)
# current_attempts > limit, total_attempts <= limit
make_error(
'not-processed1',
settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS + 1,
settings.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF
)
# current_attempts <= limit, total_attempts > limit
make_error(
'not-processed2',
settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS,
settings.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
)
errors = PillowError.get_errors_to_process(date, fetch_full=True).all()
self.assertEqual(len(errors), 2)
docs_to_process = {e.doc_id for e in errors}
self.assertEqual({'to-process1', 'to-process2'}, docs_to_process)
def _get_items(utcnow):
errors = PillowError.get_errors_to_process(utcnow=utcnow, limit=1000)
error_pks = [error['id'] for error in errors]
PillowError.objects.filter(pk__in=error_pks).update(queued=True)
return [dict(id=e['id'], key=e['date_next_attempt']) for e in errors]
def _get_items(self, utcnow):
errors = PillowError.get_errors_to_process(
utcnow=utcnow,
)
return (dict(id=e['id'], key=e['date_next_attempt']) for e in errors)
def _get_items(self, utcnow):
errors = PillowError.get_errors_to_process(utcnow=utcnow, )
return (dict(id=e['id'], key=e['date_next_attempt']) for e in errors)
def _get_items(utcnow):
errors = PillowError.get_errors_to_process(utcnow=utcnow, limit=1000)
error_pks = [error['id'] for error in errors]
PillowError.objects.filter(pk__in=error_pks).update(queued=True)
return [dict(id=e['id'], key=e['date_next_attempt']) for e in errors]
def _get_items(utcnow):
errors = PillowError.get_errors_to_process(utcnow=utcnow, limit=1000)
return [
QueueItem(id=e.id, key=e.date_next_attempt, object=e)
for e in errors
]
def _get_items(self, utcnow):
errors = PillowError.get_errors_to_process(utcnow=utcnow,
limit=BATCH_SIZE)
return list(errors)
def _get_items(utcnow):
errors = PillowError.get_errors_to_process(utcnow=utcnow, limit=1000)
return [dict(id=e['id'], key=e['date_next_attempt']) for e in errors]
