def _shuffled_task_kwargs(self, options):
"""
Iterate over all task keyword arguments in random order.
Randomizing them will help even out the load on the task workers,
though it will not entirely prevent the possibility of spikes. It will
also make the overall time to completion more predictable.
"""
all_args = []
estimate_first_attempted = options['estimate_first_attempted']
for course_key in self._get_course_keys(options):
# This is a tuple to reduce memory consumption.
# The dictionaries with their extra overhead will be created
# and consumed one at a time.
for task_arg_tuple in tasks._course_task_args(
course_key, **options):
all_args.append(task_arg_tuple)
all_args.sort(key=lambda x: hashlib.md5('{!r}'.format(x).encode(
'utf-8')).digest())
for args in all_args:
yield {
'course_key': args[0],
'offset': args[1],
'batch_size': args[2],
'estimate_first_attempted': estimate_first_attempted,
}
def test_course_task_args(self, test_batch_size):
offset_expected = 0
for course_key, offset, batch_size in _course_task_args(
batch_size=test_batch_size, course_key=self.course.id, from_settings=False
):
self.assertEqual(course_key, six.text_type(self.course.id))
self.assertEqual(batch_size, test_batch_size)
self.assertEqual(offset, offset_expected)
offset_expected += test_batch_size
def test_course_task_args(self, test_batch_size):
offset_expected = 0
for course_key, offset, batch_size in _course_task_args(
batch_size=test_batch_size, course_key=self.course.id, from_settings=False
):
self.assertEqual(course_key, six.text_type(self.course.id))
self.assertEqual(batch_size, test_batch_size)
self.assertEqual(offset, offset_expected)
offset_expected += test_batch_size
def test_course_task_args(self, test_batch_size):
offset_expected = 0
for course_key, offset, batch_size in _course_task_args(
batch_size=test_batch_size, course_key=self.course.id, from_settings=False
):
assert course_key == str(self.course.id)
assert batch_size == test_batch_size
assert offset == offset_expected
offset_expected += test_batch_size
