def _schedule_actions(self):
with_items_values = self._get_with_items_values()
if with_items.is_new(self.task_ex):
with_items.validate_values(with_items_values)
action_count = len(six.next(iter(with_items_values.values())))
with_items.prepare_runtime_context(self.task_ex, self.task_spec,
action_count)
input_dicts = self._get_input_dicts(with_items_values)
if not input_dicts:
self.complete(states.SUCCESS)
return
for i, input_dict in input_dicts:
target = self._get_target(input_dict)
action = self._build_action()
action.validate_input(input_dict)
action.schedule(input_dict,
target,
index=i,
safe_rerun=self.task_spec.get_safe_rerun())
with_items.decrease_capacity(self.task_ex, 1)
def _run_existing_task(task_ex, task_spec, wf_spec):
input_dicts = _get_input_dictionaries(wf_spec, task_ex, task_spec,
task_ex.in_context)
# TODO(rakhmerov): May be it shouldn't be here. Need to think.
if task_spec.get_with_items():
with_items.prepare_runtime_context(task_ex, task_spec, input_dicts)
# In some cases we can have no input, e.g. in case of 'with-items'.
if input_dicts:
for index, input_d in enumerate(input_dicts):
_run_action_or_workflow(task_ex, task_spec, input_d, index)
else:
_schedule_noop_action(task_ex, task_spec)
def _run_existing_task(task_ex, task_spec, wf_spec):
input_dicts = _get_input_dictionaries(
wf_spec,
task_ex,
task_spec,
task_ex.in_context
)
# TODO(rakhmerov): May be it shouldn't be here. Need to think.
if task_spec.get_with_items():
with_items.prepare_runtime_context(task_ex, task_spec, input_dicts)
# In some cases we can have no input, e.g. in case of 'with-items'.
if input_dicts:
for index, input_d in enumerate(input_dicts):
_run_action_or_workflow(task_ex, task_spec, input_d, index)
else:
_schedule_noop_action(task_ex, task_spec)
def _get_with_items_input(wf_spec, task_ex, task_spec, ctx):
"""Calculate input array for separating each action input.
Example:
DSL:
with_items:
- itemX in <% $.arrayI %>
- itemY in <% $.arrayJ %>
Assume arrayI = [1, 2], arrayJ = ['a', 'b'].
with_items_input = {
"itemX": [1, 2],
"itemY": ['a', 'b']
}
Then we get separated input:
inputs_per_item = [
{'itemX': 1, 'itemY': 'a'},
{'itemX': 2, 'itemY': 'b'}
]
:return: the list of tuples containing indexes
and the corresponding input dict.
"""
with_items_inputs = expr.evaluate_recursively(
task_spec.get_with_items(), ctx
)
with_items.validate_input(with_items_inputs)
inputs_per_item = []
for key, value in with_items_inputs.items():
for index, item in enumerate(value):
iter_context = {key: item}
if index >= len(inputs_per_item):
inputs_per_item.append(iter_context)
else:
inputs_per_item[index].update(iter_context)
action_inputs = []
for item_input in inputs_per_item:
new_ctx = utils.merge_dicts(item_input, ctx)
action_inputs.append(_get_workflow_or_action_input(
wf_spec, task_ex, task_spec, new_ctx
))
with_items.prepare_runtime_context(task_ex, task_spec, action_inputs)
indices = with_items.get_indices_for_loop(task_ex)
with_items.decrease_capacity(task_ex, len(indices))
if indices:
current_inputs = operator.itemgetter(*indices)(action_inputs)
return zip(
indices,
current_inputs if isinstance(current_inputs, tuple)
else [current_inputs]
)
return []
def _get_with_items_input(self):
"""Calculate input array for separating each action input.
Example:
DSL:
with_items:
- itemX in <% $.arrayI %>
- itemY in <% $.arrayJ %>
Assume arrayI = [1, 2], arrayJ = ['a', 'b'].
with_items_input = {
"itemX": [1, 2],
"itemY": ['a', 'b']
}
Then we get separated input:
inputs_per_item = [
{'itemX': 1, 'itemY': 'a'},
{'itemX': 2, 'itemY': 'b'}
]
:return: the list of tuples containing indexes
and the corresponding input dict.
"""
with_items_inputs = expr.evaluate_recursively(
self.task_spec.get_with_items(), self.ctx)
with_items.validate_input(with_items_inputs)
inputs_per_item = []
for key, value in with_items_inputs.items():
for index, item in enumerate(value):
iter_context = {key: item}
if index >= len(inputs_per_item):
inputs_per_item.append(iter_context)
else:
inputs_per_item[index].update(iter_context)
action_inputs = []
for item_input in inputs_per_item:
new_ctx = utils.merge_dicts(item_input, self.ctx)
action_inputs.append(self._get_action_input(new_ctx))
with_items.prepare_runtime_context(self.task_ex, self.task_spec,
action_inputs)
indices = with_items.get_indices_for_loop(self.task_ex)
with_items.decrease_capacity(self.task_ex, len(indices))
if indices:
current_inputs = operator.itemgetter(*indices)(action_inputs)
return zip(
indices, current_inputs
if isinstance(current_inputs, tuple) else [current_inputs])
return []
