def test_get_indices(self):
# Task execution for running 6 items with concurrency=3.
task_ex = models.TaskExecution(spec={'action': 'myaction'},
runtime_context={
'with_items_context': {
'capacity': 3,
'count': 6
}
},
action_executions=[],
workflow_executions=[])
# Set 3 items: 2 success and 1 error unaccepted.
task_ex.action_executions += [
self.get_action_ex(True, states.SUCCESS, 0),
self.get_action_ex(True, states.SUCCESS, 1),
self.get_action_ex(False, states.ERROR, 2)
]
# Then call get_indices and expect [2, 3, 4].
indices = with_items.get_indices_for_loop(task_ex)
# TODO(rakhmerov): Restore concurrency support.
# With disabled 'concurrency' support we expect indices 2,3,4,5
# because overall count is 6 and two indices were already processed.
self.assertListEqual([2, 3, 4, 5], indices)
def test_get_indices(self):
# Task execution for running 6 items with concurrency=3.
task_ex = models.TaskExecution(
spec={
'action': 'myaction'
},
runtime_context={
'with_items_context': {
'capacity': 3,
'count': 6
}
},
action_executions=[],
workflow_executions=[]
)
# Set 3 items: 2 success and 1 error unaccepted.
task_ex.action_executions += [
self.get_action_ex(True, states.SUCCESS, 0),
self.get_action_ex(True, states.SUCCESS, 1),
self.get_action_ex(False, states.ERROR, 2)
]
# Then call get_indices and expect [2, 3, 4].
indices = with_items.get_indices_for_loop(task_ex)
self.assertListEqual([2, 3, 4], indices)
def _get_with_items_input(wf_spec, task_ex, task_spec, ctx):
"""Calculate input array for separating each action input.
Example:
DSL:
with_items:
- itemX in <% $.arrayI %>
- itemY in <% $.arrayJ %>
Assume arrayI = [1, 2], arrayJ = ['a', 'b'].
with_items_input = {
"itemX": [1, 2],
"itemY": ['a', 'b']
}
Then we get separated input:
inputs_per_item = [
{'itemX': 1, 'itemY': 'a'},
{'itemX': 2, 'itemY': 'b'}
]
:return: the list of tuples containing indexes
and the corresponding input dict.
"""
with_items_inputs = expr.evaluate_recursively(
task_spec.get_with_items(), ctx
)
with_items.validate_input(with_items_inputs)
inputs_per_item = []
for key, value in with_items_inputs.items():
for index, item in enumerate(value):
iter_context = {key: item}
if index >= len(inputs_per_item):
inputs_per_item.append(iter_context)
else:
inputs_per_item[index].update(iter_context)
action_inputs = []
for item_input in inputs_per_item:
new_ctx = utils.merge_dicts(item_input, ctx)
action_inputs.append(_get_workflow_or_action_input(
wf_spec, task_ex, task_spec, new_ctx
))
with_items.prepare_runtime_context(task_ex, task_spec, action_inputs)
indices = with_items.get_indices_for_loop(task_ex)
with_items.decrease_capacity(task_ex, len(indices))
if indices:
current_inputs = operator.itemgetter(*indices)(action_inputs)
return zip(
indices,
current_inputs if isinstance(current_inputs, tuple)
else [current_inputs]
)
return []
def _get_with_items_input(self):
"""Calculate input array for separating each action input.
Example:
DSL:
with_items:
- itemX in <% $.arrayI %>
- itemY in <% $.arrayJ %>
Assume arrayI = [1, 2], arrayJ = ['a', 'b'].
with_items_input = {
"itemX": [1, 2],
"itemY": ['a', 'b']
}
Then we get separated input:
inputs_per_item = [
{'itemX': 1, 'itemY': 'a'},
{'itemX': 2, 'itemY': 'b'}
]
:return: the list of tuples containing indexes
and the corresponding input dict.
"""
with_items_inputs = expr.evaluate_recursively(
self.task_spec.get_with_items(), self.ctx)
with_items.validate_input(with_items_inputs)
inputs_per_item = []
for key, value in with_items_inputs.items():
for index, item in enumerate(value):
iter_context = {key: item}
if index >= len(inputs_per_item):
inputs_per_item.append(iter_context)
else:
inputs_per_item[index].update(iter_context)
action_inputs = []
for item_input in inputs_per_item:
new_ctx = utils.merge_dicts(item_input, self.ctx)
action_inputs.append(self._get_action_input(new_ctx))
with_items.prepare_runtime_context(self.task_ex, self.task_spec,
action_inputs)
indices = with_items.get_indices_for_loop(self.task_ex)
with_items.decrease_capacity(self.task_ex, len(indices))
if indices:
current_inputs = operator.itemgetter(*indices)(action_inputs)
return zip(
indices, current_inputs
if isinstance(current_inputs, tuple) else [current_inputs])
return []
