def test_get_indices(self): # Task execution for running 6 items with concurrency=3. task_ex = models.TaskExecution(spec={'action': 'myaction'}, runtime_context={ 'with_items_context': { 'capacity': 3, 'count': 6 } }, action_executions=[], workflow_executions=[]) # Set 3 items: 2 success and 1 error unaccepted. task_ex.action_executions += [ self.get_action_ex(True, states.SUCCESS, 0), self.get_action_ex(True, states.SUCCESS, 1), self.get_action_ex(False, states.ERROR, 2) ] # Then call get_indices and expect [2, 3, 4]. indices = with_items.get_indices_for_loop(task_ex) # TODO(rakhmerov): Restore concurrency support. # With disabled 'concurrency' support we expect indices 2,3,4,5 # because overall count is 6 and two indices were already processed. self.assertListEqual([2, 3, 4, 5], indices)
def test_get_indices(self): # Task execution for running 6 items with concurrency=3. task_ex = models.TaskExecution( spec={ 'action': 'myaction' }, runtime_context={ 'with_items_context': { 'capacity': 3, 'count': 6 } }, action_executions=[], workflow_executions=[] ) # Set 3 items: 2 success and 1 error unaccepted. task_ex.action_executions += [ self.get_action_ex(True, states.SUCCESS, 0), self.get_action_ex(True, states.SUCCESS, 1), self.get_action_ex(False, states.ERROR, 2) ] # Then call get_indices and expect [2, 3, 4]. indices = with_items.get_indices_for_loop(task_ex) self.assertListEqual([2, 3, 4], indices)
def _get_with_items_input(wf_spec, task_ex, task_spec, ctx): """Calculate input array for separating each action input. Example: DSL: with_items: - itemX in <% $.arrayI %> - itemY in <% $.arrayJ %> Assume arrayI = [1, 2], arrayJ = ['a', 'b']. with_items_input = { "itemX": [1, 2], "itemY": ['a', 'b'] } Then we get separated input: inputs_per_item = [ {'itemX': 1, 'itemY': 'a'}, {'itemX': 2, 'itemY': 'b'} ] :return: the list of tuples containing indexes and the corresponding input dict. """ with_items_inputs = expr.evaluate_recursively( task_spec.get_with_items(), ctx ) with_items.validate_input(with_items_inputs) inputs_per_item = [] for key, value in with_items_inputs.items(): for index, item in enumerate(value): iter_context = {key: item} if index >= len(inputs_per_item): inputs_per_item.append(iter_context) else: inputs_per_item[index].update(iter_context) action_inputs = [] for item_input in inputs_per_item: new_ctx = utils.merge_dicts(item_input, ctx) action_inputs.append(_get_workflow_or_action_input( wf_spec, task_ex, task_spec, new_ctx )) with_items.prepare_runtime_context(task_ex, task_spec, action_inputs) indices = with_items.get_indices_for_loop(task_ex) with_items.decrease_capacity(task_ex, len(indices)) if indices: current_inputs = operator.itemgetter(*indices)(action_inputs) return zip( indices, current_inputs if isinstance(current_inputs, tuple) else [current_inputs] ) return []
def _get_with_items_input(self): """Calculate input array for separating each action input. Example: DSL: with_items: - itemX in <% $.arrayI %> - itemY in <% $.arrayJ %> Assume arrayI = [1, 2], arrayJ = ['a', 'b']. with_items_input = { "itemX": [1, 2], "itemY": ['a', 'b'] } Then we get separated input: inputs_per_item = [ {'itemX': 1, 'itemY': 'a'}, {'itemX': 2, 'itemY': 'b'} ] :return: the list of tuples containing indexes and the corresponding input dict. """ with_items_inputs = expr.evaluate_recursively( self.task_spec.get_with_items(), self.ctx) with_items.validate_input(with_items_inputs) inputs_per_item = [] for key, value in with_items_inputs.items(): for index, item in enumerate(value): iter_context = {key: item} if index >= len(inputs_per_item): inputs_per_item.append(iter_context) else: inputs_per_item[index].update(iter_context) action_inputs = [] for item_input in inputs_per_item: new_ctx = utils.merge_dicts(item_input, self.ctx) action_inputs.append(self._get_action_input(new_ctx)) with_items.prepare_runtime_context(self.task_ex, self.task_spec, action_inputs) indices = with_items.get_indices_for_loop(self.task_ex) with_items.decrease_capacity(self.task_ex, len(indices)) if indices: current_inputs = operator.itemgetter(*indices)(action_inputs) return zip( indices, current_inputs if isinstance(current_inputs, tuple) else [current_inputs]) return []