def setup_retry_in_task_state(self, task_state_entry, in_ctx_idxs):
# Setup the retry in the task state.
task_id = task_state_entry['id']
task_retry_spec = self.graph.get_task_retry_spec(task_id)
task_state_entry['retry'] = copy.deepcopy(task_retry_spec)
task_state_entry['retry']['tally'] = 0
# Get task context for evaluating the expression in delay and count.
in_ctx = self.get_task_context(in_ctx_idxs)
# Evaluate the retry delay value.
if ('delay' in task_state_entry['retry'] and
isinstance(task_state_entry['retry']['delay'], six.string_types)):
delay_value = expr_base.evaluate(task_state_entry['retry']['delay'], in_ctx)
if not isinstance(delay_value, int):
raise ValueError('The retry delay for task "%s" is not an integer.' % task_id)
task_state_entry['retry']['delay'] = delay_value
# Evaluate the retry count value.
if ('count' in task_state_entry['retry'] and
isinstance(task_state_entry['retry']['count'], six.string_types)):
count_value = expr_base.evaluate(task_state_entry['retry']['count'], in_ctx)
if not isinstance(count_value, int):
raise ValueError('The retry count for task "%s" is not an integer.' % task_id)
task_state_entry['retry']['count'] = count_value
def render(self, in_ctx):
action_specs = []
if not self.has_items():
action_spec = {
"action": expr_base.evaluate(self.action, in_ctx),
"input": expr_base.evaluate(getattr(self, "input", {}),
in_ctx),
}
action_specs.append(action_spec)
else:
items_spec = self.get_items_spec()
if " in " not in items_spec.items:
items_expr = items_spec.items.strip()
else:
start_idx = items_spec.items.index(" in ") + 4
items_expr = items_spec.items[start_idx:].strip()
items = expr_base.evaluate(items_expr, in_ctx)
if not isinstance(items, list):
raise TypeError('The value of "%s" is not type of list.' %
items_expr)
item_keys = (
None if " in " not in items_spec.items else
items_spec.items[:items_spec.items.index(" in ")].replace(
" ", "").split(","))
for idx, item in enumerate(items):
if item_keys and (isinstance(item, tuple)
or isinstance(item, list)):
item = dict(zip(item_keys, list(item)))
elif item_keys and len(item_keys) == 1:
item = {item_keys[0]: item}
item_ctx_value = ctx_util.set_current_item(in_ctx, item)
action_spec = {
"action":
expr_base.evaluate(self.action, item_ctx_value),
"input":
expr_base.evaluate(getattr(self, "input", {}),
item_ctx_value),
"item_id":
idx,
}
action_specs.append(action_spec)
return self, action_specs
def render(self, in_ctx):
action_specs = []
if not self.has_items():
action_spec = {
'action': expr_base.evaluate(self.action, in_ctx),
'input': expr_base.evaluate(getattr(self, 'input', {}), in_ctx)
}
action_specs.append(action_spec)
else:
items_spec = self.get_items_spec()
items_expr = (items_spec.items.strip()
if ' in ' not in items_spec.items else
items_spec.items[items_spec.items.index(' in ') +
4:].strip())
items = expr_base.evaluate(items_expr, in_ctx)
if not isinstance(items, list):
raise TypeError('The value of "%s" is not type of list.' %
items_expr)
item_keys = (
None if ' in ' not in items_spec.items else
items_spec.items[:items_spec.items.index(' in ')].replace(
' ', '').split(','))
for idx, item in enumerate(items):
if item_keys and (isinstance(item, tuple)
or isinstance(item, list)):
item = dict(zip(item_keys, list(item)))
elif item_keys and len(item_keys) == 1:
item = {item_keys[0]: item}
item_ctx_value = ctx_util.set_current_item(in_ctx, item)
action_spec = {
'action':
expr_base.evaluate(self.action, item_ctx_value),
'input':
expr_base.evaluate(getattr(self, 'input', {}),
item_ctx_value),
'item_id':
idx
}
action_specs.append(action_spec)
return self, action_specs
def render(self, in_ctx):
action_specs = []
if self.has_items():
raise NotImplementedError('Task with items is not implemented.')
action_spec = {
'action': expr.evaluate(self.action, in_ctx),
'input': expr.evaluate(getattr(self, 'input', {}), in_ctx)
}
action_specs.append(action_spec)
return self, action_specs
def render(self, in_ctx):
action_specs = []
if self.has_items():
raise NotImplementedError("Task with items is not implemented.")
action_spec = {
"action": expr_base.evaluate(self.action, in_ctx),
"input": expr_base.evaluate(getattr(self, "input", {}), in_ctx),
}
action_specs.append(action_spec)
return self, action_specs
def finalize_context(self, next_task_name, task_transition_meta, in_ctx):
criteria = task_transition_meta[3].get("criteria") or []
expected_criteria_pattern = r"<\% task_status\(\w+\) in \['succeeded'\] \%>"
new_ctx = {}
errors = []
if not re.match(expected_criteria_pattern, criteria[0]):
return in_ctx, new_ctx, errors
task_publish_spec = getattr(self, "publish") or {}
try:
new_ctx = {
var_name: expr_base.evaluate(var_expr, in_ctx)
for var_name, var_expr in six.iteritems(task_publish_spec)
}
except exc.ExpressionEvaluationException as e:
errors.append(str(e))
out_ctx = dict_util.merge_dicts(in_ctx, new_ctx, overwrite=True)
for key in list(out_ctx.keys()):
if key.startswith("__"):
out_ctx.pop(key)
return out_ctx, new_ctx, errors
def finalize_context(self, next_task_name, task_transition_meta, in_ctx):
rolling_ctx = copy.deepcopy(in_ctx)
new_ctx = {}
errors = []
task_transition_specs = getattr(self, 'next') or []
task_transition_spec = task_transition_specs[task_transition_meta[3]
['ref']]
next_task_names = getattr(task_transition_spec, 'do') or []
if next_task_name in next_task_names:
for task_publish_spec in (getattr(task_transition_spec, 'publish')
or {}):
var_name = list(task_publish_spec.items())[0][0]
default_var_value = list(task_publish_spec.items())[0][1]
try:
rendered_var_value = expr_base.evaluate(
default_var_value, rolling_ctx)
rolling_ctx[var_name] = rendered_var_value
new_ctx[var_name] = rendered_var_value
except exc.ExpressionEvaluationException as e:
errors.append(e)
out_ctx = dict_util.merge_dicts(in_ctx, new_ctx, overwrite=True)
for key in list(out_ctx.keys()):
if key.startswith('__'):
out_ctx.pop(key)
return out_ctx, new_ctx, errors
def test_multi_block_eval(self):
expr = ("{% for i in ctx().x %}{{ i }}{% endfor %}"
"{% for i in ctx().y %}{{ i }}{% endfor %}")
data = {"x": ["a", "b", "c"], "y": ["d", "e", "f"]}
self.assertEqual("abcdef", expr_base.evaluate(expr, data))
def test_eval_list(self):
expr = ['{{ ctx().foo }}', '{{ ctx().marco }}', 'foo{{ ctx().foo }}']
data = {'foo': 'bar', 'marco': 'polo'}
self.assertListEqual(['bar', 'polo', 'foobar'],
expressions.evaluate(expr, data))
def test_eval_list(self):
expr = ['<% ctx().foo %>', '<% ctx().marco %>', 'foo<% ctx().foo %>']
data = {'foo': 'bar', 'marco': 'polo'}
self.assertListEqual(['bar', 'polo', 'foobar'],
expr_base.evaluate(expr, data))
def test_multi_block_eval(self):
expr = ('{% for i in ctx().x %}{{ i }}{% endfor %}'
'{% for i in ctx().y %}{{ i }}{% endfor %}')
data = {'x': ['a', 'b', 'c'], 'y': ['d', 'e', 'f']}
self.assertEqual('abcdef', expressions.evaluate(expr, data))
def test_eval_list(self):
expr = ["{{ ctx().foo }}", "{{ ctx().marco }}", "foo{{ ctx().foo }}"]
data = {"foo": "bar", "marco": "polo"}
self.assertListEqual(["bar", "polo", "foobar"],
expr_base.evaluate(expr, data))
def auto_complete(yaql_expression, yaml_string, legacy=False):
"""
Returns complement suggestions to the given YAQL expression
:param str yaql_expression: the YAQL expression
:param str | dict yaml_string:
:return: dictionary with two keys:
"yaql_valid"- is the YAQL expressio valid?
"suggestions"- list of suggestions
:rtype: dict
"""
yaql_exp_valid = True
res = []
try:
if yaql_expression[-1] != "]":
if yaql_expression.count("[") > yaql_expression.count("]"):
# we are in a middle of $....[ ...
first_dlr_index = yaql_expression.rfind("$")
expression_prefix = yaql_expression[:first_dlr_index - 1]
sub_value = yaql_expression[first_dlr_index + 2:]
else:
# only dots
last_dot_index = yaql_expression.rindex(".")
sub_value = yaql_expression[last_dot_index + 1:]
expression_prefix = yaql_expression[:last_dot_index]
partial_value = evaluate(expression_prefix, yaml_string, legacy)
res = []
if partial_value:
if isinstance(partial_value, list) or isinstance(
partial_value, types.GeneratorType):
for index, item in enumerate(list(partial_value)):
if isinstance(item, list):
res.extend(range(len(item)))
elif isinstance(item, dict):
res.extend(_get_matched_values(item, sub_value))
elif isinstance(partial_value, dict):
res = _get_matched_values(partial_value, sub_value)
except (YamlException, YaqlException):
pass
try:
evaluate(yaql_expression, yaml_string)
except YaqlException:
yaql_exp_valid = False
return {"yaql_valid": yaql_exp_valid, "suggestions": list(set(res))}
def test_eval_dict_of_list(self):
expr = {"nested": ["<% ctx().foo %>", "<% ctx().marco %>"]}
data = {"foo": "bar", "marco": "polo"}
expected = {"nested": ["bar", "polo"]}
self.assertDictEqual(expected, expr_base.evaluate(expr, data))
def test_eval_list_of_list(self):
expr = [['{{ ctx().foo }}', '{{ ctx().marco }}']]
data = {'foo': 'bar', 'marco': 'polo'}
expected = [['bar', 'polo']]
self.assertListEqual(expected, expressions.evaluate(expr, data))
def test_eval_dict_of_list(self):
expr = {'nested': ['{{ ctx().foo }}', '{{ ctx().marco }}']}
data = {'foo': 'bar', 'marco': 'polo'}
expected = {'nested': ['bar', 'polo']}
self.assertDictEqual(expected, expressions.evaluate(expr, data))
def test_eval_list_of_dict(self):
expr = [{"foo": "<% ctx().bar %>", "<% ctx().marco %>": "polo"}]
data = {"bar": "bar", "marco": "marco"}
expected = [{"foo": "bar", "marco": "polo"}]
self.assertListEqual(expected, expr_base.evaluate(expr, data))
def test_eval_list_of_list(self):
expr = [["<% ctx().foo %>", "<% ctx().marco %>"]]
data = {"foo": "bar", "marco": "polo"}
expected = [["bar", "polo"]]
self.assertListEqual(expected, expr_base.evaluate(expr, data))
def test_eval_dict_of_dict(self):
expr = {"nested": {"foo": "<% ctx().bar %>", "<% ctx().marco %>": "polo"}}
data = {"bar": "bar", "marco": "marco"}
expected = {"nested": {"foo": "bar", "marco": "polo"}}
self.assertDictEqual(expected, expr_base.evaluate(expr, data))
def test_eval_list_of_dict(self):
expr = [{'foo': '{{ ctx().bar }}', '{{ ctx().marco }}': 'polo'}]
data = {'bar': 'bar', 'marco': 'marco'}
expected = [{'foo': 'bar', 'marco': 'polo'}]
self.assertListEqual(expected, expressions.evaluate(expr, data))
def test_eval_dict_of_list(self):
expr = {'nested': ['<% ctx().foo %>', '<% ctx().marco %>']}
data = {'foo': 'bar', 'marco': 'polo'}
expected = {'nested': ['bar', 'polo']}
self.assertDictEqual(expected, expr_base.evaluate(expr, data))
def test_eval_list_of_list(self):
expr = [['<% ctx().foo %>', '<% ctx().marco %>']]
data = {'foo': 'bar', 'marco': 'polo'}
expected = [['bar', 'polo']]
self.assertListEqual(expected, expr_base.evaluate(expr, data))
def test_eval_list_of_dict(self):
expr = [{'foo': '<% ctx().bar %>', '<% ctx().marco %>': 'polo'}]
data = {'bar': 'bar', 'marco': 'marco'}
expected = [{'foo': 'bar', 'marco': 'polo'}]
self.assertListEqual(expected, expr_base.evaluate(expr, data))
def get_task(self, task_id, route):
try:
task_ctx = self.get_task_initial_context(task_id, route)
except ValueError:
task_ctx = self.get_workflow_initial_context()
state_ctx = {'__state': self.workflow_state.serialize()}
current_task = {'id': task_id, 'route': route}
task_ctx = ctx_util.set_current_task(task_ctx, current_task)
task_ctx = dict_util.merge_dicts(task_ctx, state_ctx, True)
task_spec = self.spec.tasks.get_task(task_id).copy()
task_spec, action_specs = task_spec.render(task_ctx)
task = {
'id': task_id,
'route': route,
'ctx': task_ctx,
'spec': task_spec,
'actions': action_specs
}
# If there is a task delay specified, evaluate the delay value.
if getattr(task_spec, 'delay', None):
task_delay = task_spec.delay
if isinstance(task_delay, six.string_types):
task_delay = expr_base.evaluate(task_delay, task_ctx)
if not isinstance(task_delay, int):
raise TypeError(
'The value of task delay is not type of integer.')
task['delay'] = task_delay
# Add items and related meta data to the task details.
if task_spec.has_items():
items_spec = getattr(task_spec, 'with')
concurrency = getattr(items_spec, 'concurrency', None)
task['items_count'] = len(action_specs)
task['concurrency'] = expr_base.evaluate(concurrency, task_ctx)
return task
def test_eval_recursive(self):
expr = '{{ ctx().fee }}'
data = {
'fee': '{{ ctx().fi }}',
'fi': '{{ ctx().fo }}',
'fo': '{{ ctx().fum }}',
'fum': 'fee-fi-fo-fum'
}
self.assertEqual('fee-fi-fo-fum', expressions.evaluate(expr, data))
def test_eval_recursive(self):
expr = '<% ctx().fee %>'
data = {
'fee': '<% ctx().fi %>',
'fi': '<% ctx().fo %>',
'fo': '<% ctx().fum %>',
'fum': 'fee-fi-fo-fum'
}
self.assertEqual('fee-fi-fo-fum', expr_base.evaluate(expr, data))
def render_vars(self, in_ctx):
vars_specs = getattr(self, 'vars') or {}
rendered_vars = {}
errors = []
try:
rendered_vars = expr_base.evaluate(vars_specs, in_ctx)
except exc.ExpressionEvaluationException as e:
errors.append(str(e))
return rendered_vars, errors
def test_eval_recursive(self):
expr = "{{ ctx().fee }}"
data = {
"fee": "{{ ctx().fi }}",
"fi": "{{ ctx().fo }}",
"fo": "{{ ctx().fum }}",
"fum": "fee-fi-fo-fum",
}
self.assertEqual("fee-fi-fo-fum", expr_base.evaluate(expr, data))
def test_eval_recursive(self):
expr = "<% ctx().fee %>"
data = {
"fee": "<% ctx().fi %>",
"fi": "<% ctx().fo %>",
"fo": "<% ctx().fum %>",
"fum": "fee-fi-fo-fum",
}
self.assertEqual("fee-fi-fo-fum", expr_base.evaluate(expr, data))
def get_task(self, task_id):
task_node = self.graph.get_task(task_id)
task_name = task_node['name']
try:
task_ctx = self.get_task_initial_context(task_id)['value']
except ValueError:
task_ctx = self.get_workflow_initial_context()
current_task = {'id': task_id, 'name': task_name}
task_ctx = ctx.set_current_task(task_ctx, current_task)
task_spec = self.spec.tasks.get_task(task_name).copy()
task_spec, action_specs = task_spec.render(task_ctx)
task = {
'id': task_id,
'name': task_name,
'ctx': task_ctx,
'spec': task_spec,
'actions': action_specs
}
# If there is a task delay specified, evaluate the delay value.
if getattr(task_spec, 'delay', None):
task_delay = task_spec.delay
if isinstance(task_delay, six.string_types):
task_delay = expr.evaluate(task_delay, task_ctx)
if not isinstance(task_delay, int):
raise TypeError('The value of task delay is not type of integer.')
task['delay'] = task_delay
# Add items and related meta data to the task details.
if task_spec.has_items():
items_spec = getattr(task_spec, 'with')
task['items_count'] = len(action_specs)
task['concurrency'] = expr.evaluate(getattr(items_spec, 'concurrency', None), task_ctx)
return task