def start(self, *args, **kwargs):
if 'map' not in self.__class__.__dict__:
raise TypeError(
'No static map method defined on class {cls}'.format(
self.__class__))
if 'finish' in self.__class__.__dict__:
finish = self.finish
else:
finish = None
# We have to pass dotted paths to functions here because staticmethods don't have
# any concept of self, or the class they are defined in.
return map_queryset(
self.model.objects.using(self.db).all(),
".".join([qualname(self.__class__), "run_map"]),
finalize_func=".".join([qualname(self.__class__), "finish"])
if finish else None,
_shards=self.shard_count,
_output_writer=self.output_writer_spec,
_output_writer_kwargs=None,
_job_name=self.job_name,
_queue_name=kwargs.pop('queue_name', self.queue_name),
*args,
**kwargs)
def test_no_start_pipeline(self):
counter = Counter.objects.create()
pipeline = map_queryset(TestModel.objects.all(),
count,
counter_id=counter.pk,
start_pipeline=False)
self.assertIsNone(pipeline._pipeline_key)
def test_no_start_pipeline(self):
counter = Counter.objects.create()
pipeline = map_queryset(
TestModel.objects.all(),
count,
counter_id=counter.pk,
start_pipeline=False
)
self.assertIsNone(pipeline._pipeline_key)
def test_filtering(self):
counter = Counter.objects.create()
pipeline = map_queryset(TestModel.objects.filter(is_true=True),
count,
finalize_func=delete,
counter_id=counter.pk)
counter = Counter.objects.create()
self.process_task_queues()
pipeline = get_pipeline_by_id(pipeline.pipeline_id)
self.assertTrue(pipeline.has_finalized)
counter.refresh_from_db()
self.assertEqual(0, counter.count)
def test_filtering(self):
counter = Counter.objects.create()
pipeline = map_queryset(
TestModel.objects.filter(is_true=True),
count,
finalize_func=delete,
counter_id=counter.pk
)
counter = Counter.objects.create()
self.process_task_queues()
pipeline = get_pipeline_by_id(pipeline.pipeline_id)
self.assertTrue(pipeline.has_finalized)
counter.refresh_from_db()
self.assertEqual(0, counter.count)
def test_filters_apply(self):
counter = Counter.objects.create()
pipeline = map_queryset(TestModel.objects.filter(pk__gt=2),
count,
finalize_func=delete,
counter_id=counter.pk)
self.process_task_queues()
pipeline = get_pipeline_by_id(pipeline.pipeline_id)
self.assertTrue(pipeline.has_finalized)
counter.refresh_from_db()
self.assertEqual(3, counter.count)
self.assertFalse(TestModel.objects.count())
def test_filters_apply(self):
counter = Counter.objects.create()
pipeline = map_queryset(
TestModel.objects.filter(pk__gt=2),
count,
finalize_func=delete,
counter_id=counter.pk
)
self.process_task_queues()
pipeline = get_pipeline_by_id(pipeline.pipeline_id)
self.assertTrue(pipeline.has_finalized)
counter.refresh_from_db()
self.assertEqual(3, counter.count)
self.assertFalse(TestModel.objects.count())
def test_slicing(self):
counter = Counter.objects.create()
pipeline = map_queryset(
TestModel.objects.all(),
slow_count,
finalize_func=delete,
counter_id=counter.pk,
# mapreduce default slice duration is 15 seconds
# slow down processing enough to split into two slices
sleep_duration=4,
_shards=1)
self.process_task_queues()
pipeline = get_pipeline_by_id(pipeline.pipeline_id)
self.assertTrue(pipeline.has_finalized)
counter.refresh_from_db()
self.assertEqual(5, counter.count)
self.assertFalse(TestModel.objects.count())
def test_slicing(self):
counter = Counter.objects.create()
pipeline = map_queryset(
TestModel.objects.all(),
slow_count,
finalize_func=delete,
counter_id=counter.pk,
# mapreduce default slice duration is 15 seconds
# slow down processing enough to split into two slices
sleep_duration=4,
_shards=1
)
self.process_task_queues()
pipeline = get_pipeline_by_id(pipeline.pipeline_id)
self.assertTrue(pipeline.has_finalized)
counter.refresh_from_db()
self.assertEqual(5, counter.count)
self.assertFalse(TestModel.objects.count())
def start(self, *args, **kwargs):
if 'map' not in self.__class__.__dict__:
raise TypeError('No static map method defined on class {cls}'.format(self.__class__))
if 'finish' in self.__class__.__dict__:
finish = self.finish
else:
finish = None
# We have to pass dotted paths to functions here because staticmethods don't have
# any concept of self, or the class they are defined in.
return map_queryset(
self.model.objects.using(self.db).all(),
".".join([qualname(self.__class__), "run_map"]),
finalize_func=".".join([qualname(self.__class__), "finish"]) if finish else None,
_shards=self.shard_count,
_output_writer=self.output_writer_spec,
_output_writer_kwargs=None,
_job_name=self.job_name,
_queue_name=kwargs.pop('queue_name', self.queue_name),
*args,
**kwargs
)
