def parallelize(self, task_func, task_arg_gen):
"""
Given a callable and a task arg generator, apply the callable to
the arguments in parallel. To save memory the tasks are spawned in
blocks with maximum size defined by the method .concurrent_tasks().
It is possible to pass a function side_effect(ret) which takes the
return value of the callable and does something with it, such as
saving or printing it. The order is not preserved.
:param task_func: a `celery` task callable
:param task_args: an iterable over positional arguments
NB: if the environment variable OQ_NO_DISTRIBUTE is set the
tasks are run sequentially in the current process.
"""
taskname = task_func.__name__
logs.LOG.debug('building arglist')
arglist = list(task_arg_gen)
total = len(arglist)
logs.LOG.progress('spawning %d tasks of kind %s', total, taskname)
ntasks = 0
for argblock in general.block_splitter(
arglist, self.concurrent_tasks()):
tasks.parallelize(task_func, argblock, lambda _: None)
ntasks += len(argblock)
percent = math.ceil(float(ntasks) / total * 100)
logs.LOG.progress('> %s %3d%% complete', taskname, percent)
def test_failing_subtask(self):
try:
tasks.parallelize(failing_task, [(42, )], None)
except NotImplementedError as exc:
self.assertEqual(42, exc.args[0])
else:
raise Exception("Exception not raised.")
def test_failing_subtask(self):
try:
tasks.parallelize(failing_task, [(42, )], None)
except NotImplementedError as exc:
self.assertEqual(42, exc.args[0])
else:
raise Exception("Exception not raised.")
def test_failing_subtask(self):
try:
tasks.parallelize(failing_task, [(42, )], None)
except RuntimeError as exc:
self.assertIn('NotImplementedError: 42', str(exc))
else:
raise Exception("Exception not raised.")
def parallelize(self, task_func, task_arg_gen, task_completed):
"""
Given a callable and a task arg generator, build an argument list and
apply the callable to the arguments in parallel. The order is not
preserved.
Every time a task completes the method .task_completed() is called
which by default simply display the progress percentage.
:param task_func: a `celery` task callable
:param task_args: an iterable over positional arguments
NB: if the environment variable OQ_NO_DISTRIBUTE is set the
tasks are run sequentially in the current process.
"""
arglist = self.initialize_percent(task_func, task_arg_gen)
tasks.parallelize(task_func, arglist, task_completed)
def parallelize(self, task_func, task_arg_gen):
"""
Given a callable and a task arg generator, build an argument list and
apply the callable to the arguments in parallel. The order is not preserved.
Every time a task completes the method .log_percent() is called
and a progress message is displayed if the percentage has changed.
:param task_func: a `celery` task callable
:param task_args: an iterable over positional arguments
NB: if the environment variable OQ_NO_DISTRIBUTE is set the
tasks are run sequentially in the current process.
"""
self.taskname = task_func.__name__
arglist = list(task_arg_gen)
self.num_tasks = len(arglist)
self.tasksdone = 0
self.percent = 0.0
logs.LOG.progress(
'spawning %d tasks of kind %s', self.num_tasks, self.taskname)
tasks.parallelize(task_func, arglist, self.log_percent)
def test_parallelize(self):
lst = []
res = tasks.parallelize(just_say_hello, [(i, ) for i in range(5)],
lst.append)
self.assertEqual(res, None)
self.assertEqual(lst, ['hello'] * 5)
def test_parallelize(self):
lst = []
res = tasks.parallelize(just_say_hello, [(i, ) for i in range(5)],
lst.append)
self.assertEqual(res, None)
self.assertEqual(lst, ['hello'] * 5)