def _schedule_actions(self): with_items_values = self._get_with_items_values() if with_items.is_new(self.task_ex): with_items.validate_values(with_items_values) action_count = len(six.next(iter(with_items_values.values()))) with_items.prepare_runtime_context(self.task_ex, self.task_spec, action_count) input_dicts = self._get_input_dicts(with_items_values) if not input_dicts: self.complete(states.SUCCESS) return for i, input_dict in input_dicts: target = self._get_target(input_dict) action = self._build_action() action.validate_input(input_dict) action.schedule(input_dict, target, index=i, safe_rerun=self.task_spec.get_safe_rerun()) with_items.decrease_capacity(self.task_ex, 1)
def _run_existing_task(task_ex, task_spec, wf_spec): input_dicts = _get_input_dictionaries(wf_spec, task_ex, task_spec, task_ex.in_context) # TODO(rakhmerov): May be it shouldn't be here. Need to think. if task_spec.get_with_items(): with_items.prepare_runtime_context(task_ex, task_spec, input_dicts) # In some cases we can have no input, e.g. in case of 'with-items'. if input_dicts: for index, input_d in enumerate(input_dicts): _run_action_or_workflow(task_ex, task_spec, input_d, index) else: _schedule_noop_action(task_ex, task_spec)
def _run_existing_task(task_ex, task_spec, wf_spec): input_dicts = _get_input_dictionaries( wf_spec, task_ex, task_spec, task_ex.in_context ) # TODO(rakhmerov): May be it shouldn't be here. Need to think. if task_spec.get_with_items(): with_items.prepare_runtime_context(task_ex, task_spec, input_dicts) # In some cases we can have no input, e.g. in case of 'with-items'. if input_dicts: for index, input_d in enumerate(input_dicts): _run_action_or_workflow(task_ex, task_spec, input_d, index) else: _schedule_noop_action(task_ex, task_spec)
def _get_with_items_input(wf_spec, task_ex, task_spec, ctx): """Calculate input array for separating each action input. Example: DSL: with_items: - itemX in <% $.arrayI %> - itemY in <% $.arrayJ %> Assume arrayI = [1, 2], arrayJ = ['a', 'b']. with_items_input = { "itemX": [1, 2], "itemY": ['a', 'b'] } Then we get separated input: inputs_per_item = [ {'itemX': 1, 'itemY': 'a'}, {'itemX': 2, 'itemY': 'b'} ] :return: the list of tuples containing indexes and the corresponding input dict. """ with_items_inputs = expr.evaluate_recursively( task_spec.get_with_items(), ctx ) with_items.validate_input(with_items_inputs) inputs_per_item = [] for key, value in with_items_inputs.items(): for index, item in enumerate(value): iter_context = {key: item} if index >= len(inputs_per_item): inputs_per_item.append(iter_context) else: inputs_per_item[index].update(iter_context) action_inputs = [] for item_input in inputs_per_item: new_ctx = utils.merge_dicts(item_input, ctx) action_inputs.append(_get_workflow_or_action_input( wf_spec, task_ex, task_spec, new_ctx )) with_items.prepare_runtime_context(task_ex, task_spec, action_inputs) indices = with_items.get_indices_for_loop(task_ex) with_items.decrease_capacity(task_ex, len(indices)) if indices: current_inputs = operator.itemgetter(*indices)(action_inputs) return zip( indices, current_inputs if isinstance(current_inputs, tuple) else [current_inputs] ) return []
def _get_with_items_input(self): """Calculate input array for separating each action input. Example: DSL: with_items: - itemX in <% $.arrayI %> - itemY in <% $.arrayJ %> Assume arrayI = [1, 2], arrayJ = ['a', 'b']. with_items_input = { "itemX": [1, 2], "itemY": ['a', 'b'] } Then we get separated input: inputs_per_item = [ {'itemX': 1, 'itemY': 'a'}, {'itemX': 2, 'itemY': 'b'} ] :return: the list of tuples containing indexes and the corresponding input dict. """ with_items_inputs = expr.evaluate_recursively( self.task_spec.get_with_items(), self.ctx) with_items.validate_input(with_items_inputs) inputs_per_item = [] for key, value in with_items_inputs.items(): for index, item in enumerate(value): iter_context = {key: item} if index >= len(inputs_per_item): inputs_per_item.append(iter_context) else: inputs_per_item[index].update(iter_context) action_inputs = [] for item_input in inputs_per_item: new_ctx = utils.merge_dicts(item_input, self.ctx) action_inputs.append(self._get_action_input(new_ctx)) with_items.prepare_runtime_context(self.task_ex, self.task_spec, action_inputs) indices = with_items.get_indices_for_loop(self.task_ex) with_items.decrease_capacity(self.task_ex, len(indices)) if indices: current_inputs = operator.itemgetter(*indices)(action_inputs) return zip( indices, current_inputs if isinstance(current_inputs, tuple) else [current_inputs]) return []