def test_passing_args_and_kwargs(self):
[DeferIterationTestModel.objects.create() for i in range(25)]
defer_iteration_with_finalize(
DeferIterationTestModel.objects.all(),
callback,
finalize,
touch=False, # kwarg to not touch the objects at all
_shards=_SHARD_COUNT)
self.process_task_queues()
self.assertEqual(
0,
DeferIterationTestModel.objects.filter(touched=True).count())
def test_instances_hit(self):
[DeferIterationTestModel.objects.create() for i in range(25)]
defer_iteration_with_finalize(DeferIterationTestModel.objects.all(),
callback,
finalize,
_shards=_SHARD_COUNT)
self.process_task_queues()
self.assertEqual(
25,
DeferIterationTestModel.objects.filter(touched=True).count())
self.assertEqual(
25,
DeferIterationTestModel.objects.filter(finalized=True).count())
def test_shard_continue_on_error(self):
[DeferIterationTestModel.objects.create(pk=i + 1) for i in range(25)]
global sporadic_error_counter
sporadic_error_counter = 0
defer_iteration_with_finalize(DeferIterationTestModel.objects.all(),
sporadic_error,
finalize,
_shards=_SHARD_COUNT)
self.process_task_queues()
self.assertEqual(
25,
DeferIterationTestModel.objects.filter(touched=True).count())
self.assertEqual(
25,
DeferIterationTestModel.objects.filter(finalized=True).count())
def test_shard_iterated_in_order(self):
DeferStringKeyModel.objects.create(name="D", other="A")
DeferStringKeyModel.objects.create(name="B", other="B")
DeferStringKeyModel.objects.create(name="A", other="C")
DeferStringKeyModel.objects.create(name="C", other="D")
DeferStringKeyModel.objects.create(name="E", other="E")
defer_iteration_with_finalize(DeferStringKeyModel.objects.all(),
update_timestamp,
noop,
_shards=2)
self.process_task_queues()
instances = DeferStringKeyModel.objects.order_by("time_hit")
last_id = "\0"
for instance in instances:
self.assertTrue(instance.pk > last_id)
last_id = instance.pk
def unindex_document(document):
"""
Deletes a document from its index
"""
try:
record = document._record or DocumentRecord.objects.get(pk=document.id)
except DocumentRecord.DoesNotExist:
return 0
# Find all the things to delete
qs = TokenFieldIndex.objects.filter(record_id=document.id,
revision=document.revision).all()
defer_iteration_with_finalize(qs,
_destroy_record,
_finalize,
_shards=1,
_queue=_SEARCH_QUEUE)
record.delete()
return 1
def test_excluded_missed(self):
[
DeferIterationTestModel.objects.create(ignored=(i < 5))
for i in range(25)
]
defer_iteration_with_finalize(
DeferIterationTestModel.objects.filter(ignored=False),
callback,
finalize,
_shards=_SHARD_COUNT)
self.process_task_queues()
self.assertEqual(
5,
DeferIterationTestModel.objects.filter(ignored=True).count())
self.assertEqual(
20,
DeferIterationTestModel.objects.filter(touched=True).count())
self.assertEqual(
25,
DeferIterationTestModel.objects.filter(finalized=True).count())
def reindex_document(document):
"""
Deletes old tokens, bumps the revision
then indexes the document
"""
try:
record = document._record or DocumentRecord.objects.get(pk=document.id)
except DocumentRecord.DoesNotExist:
return
qs = TokenFieldIndex.objects.filter(record_id=document.id,
revision=document.revision)
defer_iteration_with_finalize(qs,
_destroy_record,
_finalize,
_shards=1,
_queue=_SEARCH_QUEUE)
# Generate a brand new revision ID for this document
record.revision = uuid.uuid4()
record.save()
index_document(document.index_name, document)