def group_one(task, seq, *args, **kwargs):
"""
Create a group of tasks, each task handle one element of seq
"""
tasks = []
if "attr_name" in kwargs:
attr_name = kwargs['attr_name']
del kwargs["attr_name"]
else:
attr_name = None
for elem in seq:
if attr_name:
kwargs[attr_name] = elem
tasks.append(
task.subtask(args=list(args),
kwargs=dict(kwargs),
immutable=True))
else:
tasks.append(
task.subtask(args=[elem] + list(args),
kwargs=dict(kwargs),
immutable=True))
return group(tasks)
def process_models_with_chunks(task, iterable, count_per_chunk,
delta_countdown=None):
def model_chunker(iterable, count_per_chunk):
results_ids = []
for obj in iterable.only('id'):
if len(results_ids) == count_per_chunk:
yield results_ids
results_ids = []
results_ids.append(obj.id)
if results_ids:
yield results_ids
count = 0
tasks = []
for model_ids in model_chunker(iterable, count_per_chunk):
options = None
if delta_countdown:
options = {
'countdown': delta_countdown * count,
}
t = task.subtask((model_ids, iterable.model), options=options)
tasks.append(t)
count += 1
return group(*tasks).apply_async()
def group_chunks(task, seq, n, *args, **kwargs):
"""
Creates a group of tasks, each subtask has <n> elements to handle
"""
tasks = []
for i in xrange(0, len(seq), n):
tasks.append(
task.subtask(args=[seq[i:i + n]] + list(args),
kwargs=kwargs,
immutable=True))
return group(tasks)
def group_chunks(task, seq, n, *args, **kwargs):
"""
Creates a group of tasks, each subtask has <n> elements to handle
"""
tasks = []
for i in xrange(0, len(seq), n):
tasks.append(
task.subtask(args=[seq[i:i + n]] + list(args), kwargs=kwargs,
immutable=True)
)
return group(tasks)
def group_one(task, seq, *args, **kwargs):
"""
Create a group of tasks, each task handle one element of seq
"""
tasks = []
if "attr_name" in kwargs:
attr_name = kwargs['attr_name']
del kwargs["attr_name"]
else:
attr_name = None
for elem in seq:
if attr_name:
kwargs[attr_name] = elem
tasks.append(task.subtask(args=list(args), kwargs=dict(kwargs),
immutable=True))
else:
tasks.append(task.subtask(args=[elem] + list(args),
kwargs=dict(kwargs), immutable=True))
return group(tasks)