def setUp(self):
"""Create some parameters and templates for use in tests."""
params_map = {'a': [-3, -1], 'i': [0, 1], 'j': [0, 1, 2], 'k': [0, 1]}
# k has template is deliberately bad
templates = {
'a': '_a%(a)d', 'i': '_i%(i)d', 'j': '_j%(j)d', 'k': '_k%(z)d'}
self.name_expander = NameExpander((params_map, templates))
self.graph_expander = GraphExpander((params_map, templates))
def setUp(self):
"""Create some parameters and templates for use in tests."""
params_map = {'a': [-3, -1], 'i': [0, 1], 'j': [0, 1, 2], 'k': [0, 1]}
# k has template is deliberately bad
templates = {
'a': '_a%(a)d', 'i': '_i%(i)d', 'j': '_j%(j)d', 'k': '_k%(z)d'}
self.name_expander = NameExpander((params_map, templates))
self.graph_expander = GraphExpander((params_map, templates))
def test_parameter_graph_mixing_offset_and_conditional(self):
"""Test for bug reported in issue #2608 on GitHub."""
for test_case in self._param_expand_params():
params_map, templates, expanded_str, expanded_values = \
test_case
graph_expander = GraphExpander((params_map, templates))
# Ignore white spaces.
expanded = [expanded.replace(' ', '') for expanded in
graph_expander.expand(expanded_str)]
self.assertEqual(
len(expanded_values),
len(expanded),
f"Invalid length for expected {expanded_values} and "
f"{expanded}")
# When testing, we don't really care for white spaces,as they
# are removed in the GraphParser anyway. That's why we have
# ''.replace(' ', '').
for expected in expanded_values:
self.assertTrue(
expected.replace(' ', '') in expanded,
f"Expected value {expected.replace(' ', '')} "
f"not in {expanded}")
def parse_graph(self, graph_string):
"""Parse the graph string for a single graph section.
(Assumes any general line-continuation markers have been processed).
1. Strip comments, whitespace, and blank lines.
(all whitespace is removed up front so we don't have to consider
it in regexes and strip it from matched elements)
2. Join incomplete lines starting or ending with '=>'.
3. Replicate and expand any parameterized lines.
4. Split and process by pairs "left-expression => right-node":
i. Replace families with members (any or all semantics).
ii. Record parsed dependency information for each right-side node.
"""
# Strip comments, whitespace, and blank lines.
non_blank_lines = []
bad_lines = []
for line in graph_string.split('\n'):
modified_line = self.__class__.REC_COMMENT.sub('', line)
# Ignore empty lines
if not modified_line or modified_line.isspace():
continue
# Catch simple bad lines that would be accepted once
# spaces are removed, e.g. 'foo bar => baz'
if self.REC_GRAPH_BAD_SPACES_LINE.search(modified_line):
bad_lines.append(line)
continue
# Apparently this is the fastest way to strip all whitespace!:
modified_line = "".join(modified_line.split())
non_blank_lines.append(modified_line)
# Check if there were problem lines and abort
if bad_lines:
self._report_invalid_lines(bad_lines)
# Join incomplete lines (beginning or ending with an arrow).
full_lines = []
part_lines = []
for i, _ in enumerate(non_blank_lines):
this_line = non_blank_lines[i]
if i == 0:
# First line can't start with an arrow.
if this_line.startswith(ARROW):
raise GraphParseError("leading arrow: %s" % this_line)
try:
next_line = non_blank_lines[i + 1]
except IndexError:
next_line = ''
if this_line.endswith(ARROW):
# Last line can't end with an arrow.
raise GraphParseError("trailing arrow: %s" % this_line)
part_lines.append(this_line)
if (this_line.endswith(ARROW) or next_line.startswith(ARROW)):
continue
full_line = ''.join(part_lines)
# Record inter-suite dependence and remove the marker notation.
# ("foo<SUITE::TASK:fail> => bar" becomes:fail "foo => bar").
repl = Replacement('\\1')
full_line = self.__class__.REC_SUITE_STATE.sub(repl, full_line)
for item in repl.match_groups:
l_task, r_all, r_suite, r_task, r_status = item
if r_status:
r_status = r_status[1:]
else:
r_status = self.__class__.TRIG_SUCCEED[1:]
self.suite_state_polling_tasks[l_task] = (r_suite, r_task,
r_status, r_all)
full_lines.append(full_line)
part_lines = []
# Check for double-char conditional operators (a common mistake),
# and bad node syntax (order of qualifiers).
bad_lines = []
for line in full_lines:
if self.__class__.OP_AND_ERR in line:
raise GraphParseError("the graph AND operator is '%s': %s" %
(self.__class__.OP_AND, line))
if self.__class__.OP_OR_ERR in line:
raise GraphParseError("the graph OR operator is '%s': %s" %
(self.__class__.OP_OR, line))
# Check node syntax. First drop all non-node characters.
node_str = line
for s in ['=>', '|', '&', '(', ')', '!']:
node_str = node_str.replace(s, ' ')
# Drop all valid @triggers, longest first to avoid sub-strings.
nodes = self.__class__.REC_ACTION.findall(node_str)
nodes.sort(key=len, reverse=True)
for node in nodes:
node_str = node_str.replace(node, '')
# Then drop all valid nodes, longest first to avoid sub-strings.
bad_lines = [
node_str for node in node_str.split()
if self.__class__.REC_NODE_FULL.sub('', node, 1)
]
if bad_lines:
self._report_invalid_lines(bad_lines)
# Expand parameterized lines (or detect undefined parameters).
line_set = set()
graph_expander = GraphExpander(self.parameters)
for line in full_lines:
if not self.__class__.REC_PARAMS.search(line):
line_set.add(line)
continue
for l in graph_expander.expand(line):
line_set.add(l)
# Process chains of dependencies as pairs: left => right.
# Parameterization can duplicate some dependencies, so use a set.
pairs = set()
for line in line_set:
# "foo => bar => baz" becomes [foo, bar, baz]
chain = line.split(ARROW)
# Auto-trigger lone nodes and initial nodes in a chain.
for name, offset, _ in self.__class__.REC_NODES.findall(chain[0]):
if not offset and not name.startswith('@'):
pairs.add((None, name))
for i in range(0, len(chain) - 1):
pairs.add((chain[i], chain[i + 1]))
for pair in pairs:
self._proc_dep_pair(pair[0], pair[1])
class TestParamExpand(unittest.TestCase):
"""Unit tests for the parameter expansion module."""
def setUp(self):
"""Create some parameters and templates for use in tests."""
params_map = {'a': [-3, -1], 'i': [0, 1], 'j': [0, 1, 2], 'k': [0, 1]}
# k has template is deliberately bad
templates = {
'a': '_a%(a)d',
'i': '_i%(i)d',
'j': '_j%(j)d',
'k': '_k%(z)d'
}
self.name_expander = NameExpander((params_map, templates))
self.graph_expander = GraphExpander((params_map, templates))
def test_name_one_param(self):
"""Test name expansion and returned value for a single parameter."""
self.assertEqual(self.name_expander.expand('foo<j>'), [('foo_j0', {
'j': 0
}), ('foo_j1', {
'j': 1
}), ('foo_j2', {
'j': 2
})])
def test_name_two_params(self):
"""Test name expansion and returned values for two parameters."""
self.assertEqual(self.name_expander.expand('foo<i,j>'),
[('foo_i0_j0', {
'i': 0,
'j': 0
}), ('foo_i0_j1', {
'i': 0,
'j': 1
}), ('foo_i0_j2', {
'i': 0,
'j': 2
}), ('foo_i1_j0', {
'i': 1,
'j': 0
}), ('foo_i1_j1', {
'i': 1,
'j': 1
}), ('foo_i1_j2', {
'i': 1,
'j': 2
})])
def test_name_two_names(self):
"""Test name expansion for two names."""
self.assertEqual(self.name_expander.expand('foo<i>, bar<j>'),
[('foo_i0', {
'i': 0
}), ('foo_i1', {
'i': 1
}), ('bar_j0', {
'j': 0
}), ('bar_j1', {
'j': 1
}), ('bar_j2', {
'j': 2
})])
def test_name_specific_val_1(self):
"""Test singling out a specific value, in name expansion."""
self.assertEqual(self.name_expander.expand('foo<i=0>'), [('foo_i0', {
'i': 0
})])
def test_name_specific_val_2(self):
"""Test specific value in the first parameter of a pair."""
self.assertEqual(self.name_expander.expand('foo<i=0,j>'),
[('foo_i0_j0', {
'i': 0,
'j': 0
}), ('foo_i0_j1', {
'i': 0,
'j': 1
}), ('foo_i0_j2', {
'i': 0,
'j': 2
})])
def test_name_specific_val_3(self):
"""Test specific value in the second parameter of a pair."""
self.assertEqual(self.name_expander.expand('foo<i,j=1>'),
[('foo_i0_j1', {
'i': 0,
'j': 1
}), ('foo_i1_j1', {
'i': 1,
'j': 1
})])
def test_name_fail_bare_value(self):
"""Test foo<0,j> fails."""
# It should be foo<i=0,j>.
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<0,j>')
def test_name_fail_undefined_param(self):
"""Test that an undefined parameter gets failed."""
# m is not defined.
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<m,j>')
def test_name_fail_param_value_too_high(self):
"""Test that an out-of-range parameter gets failed."""
# i stops at 3.
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<i=4,j>')
def test_name_multiple(self):
"""Test expansion of two names, with one and two parameters."""
self.assertEqual(self.name_expander.expand('foo<i>, bar<i,j>'),
[('foo_i0', {
'i': 0
}), ('foo_i1', {
'i': 1
}), ('bar_i0_j0', {
'i': 0,
'j': 0
}), ('bar_i0_j1', {
'i': 0,
'j': 1
}), ('bar_i0_j2', {
'i': 0,
'j': 2
}), ('bar_i1_j0', {
'i': 1,
'j': 0
}), ('bar_i1_j1', {
'i': 1,
'j': 1
}), ('bar_i1_j2', {
'i': 1,
'j': 2
})])
def test_graph_expand_1(self):
"""Test graph expansion with two parameters each side of an arrow."""
self.assertEqual(
self.graph_expander.expand("bar<i,j>=>baz<i,j>"),
set([
"bar_i0_j1=>baz_i0_j1", "bar_i1_j2=>baz_i1_j2",
"bar_i0_j2=>baz_i0_j2", "bar_i1_j1=>baz_i1_j1",
"bar_i1_j0=>baz_i1_j0", "bar_i0_j0=>baz_i0_j0"
]))
def test_graph_expand_2(self):
"""Test graph expansion to 'branch and merge' a workflow."""
self.assertEqual(
self.graph_expander.expand("pre=>bar<i>=>baz<i,j>=>post"),
set([
"pre=>bar_i0=>baz_i0_j1=>post", "pre=>bar_i1=>baz_i1_j2=>post",
"pre=>bar_i0=>baz_i0_j2=>post", "pre=>bar_i1=>baz_i1_j1=>post",
"pre=>bar_i1=>baz_i1_j0=>post", "pre=>bar_i0=>baz_i0_j0=>post"
]))
def test_graph_expand_3(self):
"""Test graph expansion -ve integers."""
self.assertEqual(self.graph_expander.expand("bar<a>"),
set(["bar_a-1", "bar_a-3"]))
def test_graph_expand_offset_1(self):
"""Test graph expansion with a -ve offset."""
self.assertEqual(
self.graph_expander.expand("bar<i-1,j>=>baz<i,j>"),
set([
"baz_i0_j0", "baz_i0_j1", "baz_i0_j2", "bar_i0_j0=>baz_i1_j0",
"bar_i0_j1=>baz_i1_j1", "bar_i0_j2=>baz_i1_j2"
]))
def test_graph_expand_offset_2(self):
"""Test graph expansion with a +ve offset."""
self.assertEqual(self.graph_expander.expand("baz<i>=>baz<i+1>"),
set(["baz_i0=>baz_i1"]))
def test_graph_expand_specific(self):
"""Test graph expansion with a specific value."""
self.assertEqual(
self.graph_expander.expand("bar<i=1,j>=>baz<i,j>"),
set([
"bar_i1_j0=>baz_i0_j0", "bar_i1_j1=>baz_i0_j1",
"bar_i1_j2=>baz_i0_j2", "bar_i1_j0=>baz_i1_j0",
"bar_i1_j1=>baz_i1_j1", "bar_i1_j2=>baz_i1_j2"
]))
def test_graph_fail_bare_value(self):
"""Test that a bare parameter value fails in the graph."""
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<0,j>=>bar<i,j>')
def test_graph_fail_undefined_param(self):
"""Test that an undefined parameter value fails in the graph."""
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<m,j>=>bar<i,j>')
def test_graph_fail_param_value_too_high(self):
"""Test that an out-of-range parameter value fails in the graph."""
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<i=4,j><i,j>')
def test_template_fail_missing_param(self):
"""Test a template string specifying a non-existent parameter."""
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<k>')
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<k>')
class TestParamExpand(unittest.TestCase):
"""Unit tests for the parameter expansion module."""
def setUp(self):
"""Create some parameters and templates for use in tests."""
params_map = {'a': [-3, -1], 'i': [0, 1], 'j': [0, 1, 2], 'k': [0, 1]}
# k has template is deliberately bad
templates = {
'a': '_a%(a)d',
'i': '_i%(i)d',
'j': '_j%(j)d',
'k': '_k%(z)d'
}
self.name_expander = NameExpander((params_map, templates))
self.graph_expander = GraphExpander((params_map, templates))
def test_name_one_param(self):
"""Test name expansion and returned value for a single parameter."""
self.assertEqual(self.name_expander.expand('foo<j>'), [('foo_j0', {
'j': 0
}), ('foo_j1', {
'j': 1
}), ('foo_j2', {
'j': 2
})])
def test_name_two_params(self):
"""Test name expansion and returned values for two parameters."""
self.assertEqual(self.name_expander.expand('foo<i,j>'),
[('foo_i0_j0', {
'i': 0,
'j': 0
}), ('foo_i0_j1', {
'i': 0,
'j': 1
}), ('foo_i0_j2', {
'i': 0,
'j': 2
}), ('foo_i1_j0', {
'i': 1,
'j': 0
}), ('foo_i1_j1', {
'i': 1,
'j': 1
}), ('foo_i1_j2', {
'i': 1,
'j': 2
})])
def test_name_two_names(self):
"""Test name expansion for two names."""
self.assertEqual(self.name_expander.expand('foo<i>, bar<j>'),
[('foo_i0', {
'i': 0
}), ('foo_i1', {
'i': 1
}), ('bar_j0', {
'j': 0
}), ('bar_j1', {
'j': 1
}), ('bar_j2', {
'j': 2
})])
def test_name_specific_val_1(self):
"""Test singling out a specific value, in name expansion."""
self.assertEqual(self.name_expander.expand('foo<i=0>'), [('foo_i0', {
'i': 0
})])
def test_name_specific_val_2(self):
"""Test specific value in the first parameter of a pair."""
self.assertEqual(self.name_expander.expand('foo<i=0,j>'),
[('foo_i0_j0', {
'i': 0,
'j': 0
}), ('foo_i0_j1', {
'i': 0,
'j': 1
}), ('foo_i0_j2', {
'i': 0,
'j': 2
})])
def test_name_specific_val_3(self):
"""Test specific value in the second parameter of a pair."""
self.assertEqual(self.name_expander.expand('foo<i,j=1>'),
[('foo_i0_j1', {
'i': 0,
'j': 1
}), ('foo_i1_j1', {
'i': 1,
'j': 1
})])
def test_name_fail_bare_value(self):
"""Test foo<0,j> fails."""
# It should be foo<i=0,j>.
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<0,j>')
def test_name_fail_undefined_param(self):
"""Test that an undefined parameter gets failed."""
# m is not defined.
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<m,j>')
def test_name_fail_param_value_too_high(self):
"""Test that an out-of-range parameter gets failed."""
# i stops at 3.
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<i=4,j>')
def test_name_multiple(self):
"""Test expansion of two names, with one and two parameters."""
self.assertEqual(self.name_expander.expand('foo<i>, bar<i,j>'),
[('foo_i0', {
'i': 0
}), ('foo_i1', {
'i': 1
}), ('bar_i0_j0', {
'i': 0,
'j': 0
}), ('bar_i0_j1', {
'i': 0,
'j': 1
}), ('bar_i0_j2', {
'i': 0,
'j': 2
}), ('bar_i1_j0', {
'i': 1,
'j': 0
}), ('bar_i1_j1', {
'i': 1,
'j': 1
}), ('bar_i1_j2', {
'i': 1,
'j': 2
})])
def test_graph_expand_1(self):
"""Test graph expansion with two parameters each side of an arrow."""
self.assertEqual(
self.graph_expander.expand("bar<i,j>=>baz<i,j>"),
set([
"bar_i0_j1=>baz_i0_j1", "bar_i1_j2=>baz_i1_j2",
"bar_i0_j2=>baz_i0_j2", "bar_i1_j1=>baz_i1_j1",
"bar_i1_j0=>baz_i1_j0", "bar_i0_j0=>baz_i0_j0"
]))
def test_graph_expand_2(self):
"""Test graph expansion to 'branch and merge' a workflow."""
self.assertEqual(
self.graph_expander.expand("pre=>bar<i>=>baz<i,j>=>post"),
set([
"pre=>bar_i0=>baz_i0_j1=>post", "pre=>bar_i1=>baz_i1_j2=>post",
"pre=>bar_i0=>baz_i0_j2=>post", "pre=>bar_i1=>baz_i1_j1=>post",
"pre=>bar_i1=>baz_i1_j0=>post", "pre=>bar_i0=>baz_i0_j0=>post"
]))
def test_graph_expand_3(self):
"""Test graph expansion -ve integers."""
self.assertEqual(self.graph_expander.expand("bar<a>"),
set(["bar_a-1", "bar_a-3"]))
def test_graph_expand_offset_1(self):
"""Test graph expansion with a -ve offset."""
self.assertEqual(
self.graph_expander.expand("bar<i-1,j>=>baz<i,j>"),
set([
"bar_i-32768_j0=>baz_i0_j0", "bar_i-32768_j1=>baz_i0_j1",
"bar_i-32768_j2=>baz_i0_j2", "bar_i0_j0=>baz_i1_j0",
"bar_i0_j1=>baz_i1_j1", "bar_i0_j2=>baz_i1_j2"
]))
def test_graph_expand_offset_2(self):
"""Test graph expansion with a +ve offset."""
self.assertEqual(self.graph_expander.expand("baz<i>=>baz<i+1>"),
set(["baz_i0=>baz_i1", "baz_i1=>baz_i-32768"]))
def test_graph_expand_specific(self):
"""Test graph expansion with a specific value."""
self.assertEqual(
self.graph_expander.expand("bar<i=1,j>=>baz<i,j>"),
set([
"bar_i1_j0=>baz_i0_j0", "bar_i1_j1=>baz_i0_j1",
"bar_i1_j2=>baz_i0_j2", "bar_i1_j0=>baz_i1_j0",
"bar_i1_j1=>baz_i1_j1", "bar_i1_j2=>baz_i1_j2"
]))
def test_graph_fail_bare_value(self):
"""Test that a bare parameter value fails in the graph."""
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<0,j>=>bar<i,j>')
def test_graph_fail_undefined_param(self):
"""Test that an undefined parameter value fails in the graph."""
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<m,j>=>bar<i,j>')
def test_graph_fail_param_value_too_high(self):
"""Test that an out-of-range parameter value fails in the graph."""
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<i=4,j><i,j>')
def test_template_fail_missing_param(self):
"""Test a template string specifying a non-existent parameter."""
self.assertRaises(ParamExpandError, self.name_expander.expand,
'foo<k>')
self.assertRaises(ParamExpandError, self.graph_expander.expand,
'foo<k>')
def _param_expand_params(self):
"""Test data for test_parameter_graph_mixing_offset_and_conditional.
params_map, templates, expanded_str, expanded_values
params_map : map of parameters used in the graph expression
templates : parameters template
expanded_str : graph string, using params/template
expanded_values: values expected to exist after params expanded
"""
return (
# original case from #2608
({
'm': ["cat", "dog"]
}, {
'm': '_%(m)s'
}, "foo<m-1> & baz => foo<m>",
['foo_-32768 & baz => foo_cat', 'foo_cat & baz => foo_dog']),
# cases from comments from #2608
# see cylc/cylc-flow/pull/3452#issuecomment-670782800
(
# single element, so bar<m-1> does not exist
{
'm': ["cat"]
},
{
'm': '_%(m)s'
},
"foo & bar<m-1> & baz => qux",
["foo & bar_-32768 & baz => qux"]),
# cases from comments from #2608
# see cylc/cylc-flow/pull/3452#issuecomment-670776749
({
'm': ["1", "2"]
}, {
'm': '_%(m)s'
}, "foo<m-1> => bar<m> => baz",
["foo_-32768 => bar_1 => baz", "foo_1 => bar_2 => baz"]),
# cases from comments from #2608
# see cylc/cylc-flow/pull/3452#discussion_r430967867
({
'm': ["cat", "dog"]
}, {
'm': '_%(m)s'
}, "baz & foo<m-1> & pub => foo<m>", [
"baz & foo_-32768 & pub => foo_cat",
"baz & foo_cat & pub => foo_dog"
]),
({
'm': ["cat", "dog"]
}, {
'm': '_%(m)s'
}, "bar & foo<m-1> & pub<m-1> & qux => foo<m>", [
"bar & foo_-32768 & pub_-32768 & qux => foo_cat",
"bar & foo_cat & pub_cat & qux => foo_dog"
]),
# GraphParser strips spaces!
({
'm': ["cat"]
}, {
'm': '_%(m)s'
}, "foo&bar<m-1>&baz=>qux", ["foo&bar_-32768&baz=>qux"]),
({
'm': ["cat", "dog"]
}, {
'm': '_%(m)s'
}, "foo&bar<m-1>&baz=>qux",
["foo&bar_-32768&baz=>qux", "foo&bar_cat&baz=>qux"]),
# must support & and | in graph expressions
({
'm': ["cat", "dog"]
}, {
'm': '_%(m)s'
}, "foo|bar<m-1>|baz=>qux",
["foo|bar_-32768|baz=>qux", "foo|bar_cat|baz=>qux"]),
({
'm': ["cat", "dog"]
}, {
'm': '_%(m)s'
}, "foo&bar<m-1>|baz=>qux",
["foo&bar_-32768|baz=>qux", "foo&bar_cat|baz=>qux"]),
({
'm': ["cat", "dog"]
}, {
'm': '_%(m)s'
}, "foo&bar<m-1>|baz=>qux",
["foo&bar_-32768|baz=>qux", "foo&bar_cat|baz=>qux"]),
({
'm': ["cat"]
}, {
'm': '_%(m)s'
}, "foo => bar<m-1> => baz", ["foo=>bar_-32768=>baz"]))
def test_parameter_graph_mixing_offset_and_conditional(self):
"""Test for bug reported in issue #2608 on GitHub."""
for test_case in self._param_expand_params():
params_map, templates, expanded_str, expanded_values = \
test_case
graph_expander = GraphExpander((params_map, templates))
# Ignore white spaces.
expanded = [
expanded.replace(' ', '')
for expanded in graph_expander.expand(expanded_str)
]
self.assertEqual(
len(expanded_values), len(expanded),
f"Invalid length for expected {expanded_values} and "
f"{expanded}")
# When testing, we don't really care for white spaces,as they
# are removed in the GraphParser anyway. That's why we have
# ''.replace(' ', '').
for expected in expanded_values:
self.assertTrue(
expected.replace(' ', '') in expanded,
f"Expected value {expected.replace(' ', '')} "
f"not in {expanded}")
def parse_graph(self, graph_string: str) -> None:
"""Parse the graph string for a single graph section.
(Assumes any general line-continuation markers have been processed).
1. Strip comments, whitespace, and blank lines.
(all whitespace is removed up front so we don't have to consider
it in regexes and strip it from matched elements)
2. Join incomplete lines starting or ending with '=>'.
3. Replicate and expand any parameterized lines.
4. Split and process by pairs "left-expression => right-node":
i. Replace families with members (any or all semantics).
ii. Record parsed dependency information for each right-side node.
"""
# Strip comments, whitespace, and blank lines.
non_blank_lines = []
bad_lines = []
for line in graph_string.split('\n'):
modified_line = self.__class__.REC_COMMENT.sub('', line)
# Ignore empty lines
if not modified_line or modified_line.isspace():
continue
# Catch simple bad lines that would be accepted once
# spaces are removed, e.g. 'foo bar => baz'
if self.REC_GRAPH_BAD_SPACES_LINE.search(modified_line):
bad_lines.append(line)
continue
# Apparently this is the fastest way to strip all whitespace!:
modified_line = "".join(modified_line.split())
non_blank_lines.append(modified_line)
# Check if there were problem lines and abort
if bad_lines:
self._report_invalid_lines(bad_lines)
# Join incomplete lines (beginning or ending with an arrow).
full_lines = []
part_lines = []
for i, _ in enumerate(non_blank_lines):
this_line = non_blank_lines[i]
for seq in self.CONTINUATION_STRS:
if i == 0 and this_line.startswith(seq):
# First line can't start with an arrow.
raise GraphParseError(f"Leading {seq}: {this_line}")
try:
next_line = non_blank_lines[i + 1]
except IndexError:
next_line = ''
for seq in self.CONTINUATION_STRS:
if this_line.endswith(seq):
# Last line can't end with an arrow, & or |.
raise GraphParseError(f"Dangling {seq}:"
f"{this_line}")
part_lines.append(this_line)
# Check that a continuation sequence doesn't end this line and
# begin the next:
if (this_line.endswith(self.CONTINUATION_STRS)
and next_line.startswith(self.CONTINUATION_STRS)):
raise GraphParseError(
'Consecutive lines end and start with continuation '
'characters:\n'
f'{this_line}\n'
f'{next_line}')
# Check that line ends with a valid continuation sequence:
if (any(
this_line.endswith(seq) or next_line.startswith(seq)
for seq in self.CONTINUATION_STRS) and not (any(
this_line.endswith(seq) or next_line.startswith(seq)
for seq in self.BAD_STRS))):
continue
full_line = ''.join(part_lines)
# Record inter-workflow dependence and remove the marker notation.
# ("foo<WORKFLOW::TASK:fail> => bar" becomes:fail "foo => bar").
repl = Replacement('\\1')
full_line = self.__class__.REC_WORKFLOW_STATE.sub(repl, full_line)
for item in repl.match_groups:
l_task, r_all, r_workflow, r_task, r_status = item
if r_status:
r_status = r_status.strip(self.__class__.QUALIFIER)
r_status = TaskTrigger.standardise_name(r_status)
else:
r_status = TASK_OUTPUT_SUCCEEDED
self.workflow_state_polling_tasks[l_task] = (r_workflow,
r_task, r_status,
r_all)
full_lines.append(full_line)
part_lines = []
# Check for double-char conditional operators (a common mistake),
# and bad node syntax (order of qualifiers).
bad_lines = []
for line in full_lines:
if self.__class__.OP_AND_ERR in line:
raise GraphParseError("The graph AND operator is "
f"'{self.__class__.OP_AND}': {line}")
if self.__class__.OP_OR_ERR in line:
raise GraphParseError("The graph OR operator is "
f"'{self.__class__.OP_OR}': {line}")
# Check node syntax. First drop all non-node characters.
node_str = line
for spec in [
self.__class__.ARROW,
self.__class__.OP_OR,
self.__class__.OP_AND,
self.__class__.SUICIDE,
'(',
')',
]:
node_str = node_str.replace(spec, ' ')
# Drop all valid @xtriggers, longest first to avoid sub-strings.
nodes = self.__class__.REC_XTRIG.findall(node_str)
nodes.sort(key=len, reverse=True)
for node in nodes:
node_str = node_str.replace(node, '')
# Then drop all valid nodes, longest first to avoid sub-strings.
bad_lines = [
node_str for node in node_str.split()
if self.__class__.REC_NODE_FULL.sub('', node, 1)
]
if bad_lines:
self._report_invalid_lines(bad_lines)
# Expand parameterized lines (or detect undefined parameters).
line_set = set()
graph_expander = GraphExpander(self.parameters)
for line in full_lines:
if not self.__class__.REC_PARAMS.search(line):
line_set.add(line)
continue
for line_ in graph_expander.expand(line):
line_set.add(line_)
# Process chains of dependencies as pairs: left => right.
# Parameterization can duplicate some dependencies, so use a set.
pairs: Set[Tuple[Optional[str], str]] = set()
for line in line_set:
chain = []
# "foo => bar => baz" becomes [foo, bar, baz]
# "foo => bar_-32768 => baz" becomes [foo]
# "foo_-32768 => bar" becomes []
for node in line.split(self.__class__.ARROW):
# This can happen, e.g. "foo => => bar" produces
# "foo, '', bar", so we add so that later it raises
# an error
if node == '':
chain.append(node)
continue
node = self.REC_NODE_OUT_OF_RANGE.sub('', node)
if node == '':
# For "foo => bar<err> => baz", stop at "bar<err>"
break
else:
chain.append(node)
if not chain:
continue
for item in self.__class__.REC_NODES.findall(chain[0]):
# Auto-trigger lone nodes and initial nodes in a chain.
if not item[0].startswith(self.__class__.XTRIG):
pairs.add((None, ''.join(item)))
for i in range(0, len(chain) - 1):
pairs.add((chain[i], chain[i + 1]))
for pair in pairs:
self._proc_dep_pair(pair)
def parse_graph(self, graph_string):
"""Parse the graph string for a single graph section.
(Assumes any general line-continuation markers have been processed).
1. Strip comments, whitespace, and blank lines.
(all whitespace is removed up front so we don't have to consider
it in regexes and strip it from matched elements)
2. Join incomplete lines starting or ending with '=>'.
3. Replicate and expand any parameterized lines.
4. Split and process by pairs "left-expression => right-node":
i. Replace families with members (any or all semantics).
ii. Record parsed dependency information for each right-side node.
"""
# Strip comments, whitespace, and blank lines.
non_blank_lines = []
bad_lines = []
for line in graph_string.split('\n'):
modified_line = self.__class__.REC_COMMENT.sub('', line)
# Ignore empty lines
if not modified_line or modified_line.isspace():
continue
# Catch simple bad lines that would be accepted once
# spaces are removed, e.g. 'foo bar => baz'
if self.REC_GRAPH_BAD_SPACES_LINE.search(modified_line):
bad_lines.append(line)
continue
# Apparently this is the fastest way to strip all whitespace!:
modified_line = "".join(modified_line.split())
non_blank_lines.append(modified_line)
# Check if there were problem lines and abort
if bad_lines:
self._report_invalid_lines(bad_lines)
# Join incomplete lines (beginning or ending with an arrow).
full_lines = []
part_lines = []
for i, _ in enumerate(non_blank_lines):
this_line = non_blank_lines[i]
if i == 0:
# First line can't start with an arrow.
if this_line.startswith(ARROW):
raise GraphParseError(
"leading arrow: %s" % this_line)
try:
next_line = non_blank_lines[i + 1]
except IndexError:
next_line = ''
if this_line.endswith(ARROW):
# Last line can't end with an arrow.
raise GraphParseError(
"trailing arrow: %s" % this_line)
part_lines.append(this_line)
if (this_line.endswith(ARROW) or next_line.startswith(ARROW)):
continue
full_line = ''.join(part_lines)
# Record inter-suite dependence and remove the marker notation.
# ("foo<SUITE::TASK:fail> => bar" becomes:fail "foo => bar").
repl = Replacement('\\1')
full_line = self.__class__.REC_SUITE_STATE.sub(repl, full_line)
for item in repl.match_groups:
l_task, r_all, r_suite, r_task, r_status = item
if r_status:
r_status = r_status[1:]
else:
r_status = self.__class__.TRIG_SUCCEED[1:]
self.suite_state_polling_tasks[l_task] = (
r_suite, r_task, r_status, r_all)
full_lines.append(full_line)
part_lines = []
# Check for double-char conditional operators (a common mistake),
# and bad node syntax (order of qualifiers).
bad_lines = []
for line in full_lines:
if self.__class__.OP_AND_ERR in line:
raise GraphParseError(
"the graph AND operator is '%s': %s" % (
self.__class__.OP_AND, line))
if self.__class__.OP_OR_ERR in line:
raise GraphParseError(
"the graph OR operator is '%s': %s" % (
self.__class__.OP_OR, line))
# Check node syntax. First drop all non-node characters.
node_str = line
for s in ['=>', '|', '&', '(', ')', '!']:
node_str = node_str.replace(s, ' ')
# Drop all valid @triggers, longest first to avoid sub-strings.
nodes = self.__class__.REC_ACTION.findall(node_str)
nodes.sort(key=len, reverse=True)
for node in nodes:
node_str = node_str.replace(node, '')
# Then drop all valid nodes, longest first to avoid sub-strings.
bad_lines = [node_str for node in node_str.split()
if self.__class__.REC_NODE_FULL.sub('', node, 1)]
if bad_lines:
self._report_invalid_lines(bad_lines)
# Expand parameterized lines (or detect undefined parameters).
line_set = set()
graph_expander = GraphExpander(self.parameters)
for line in full_lines:
if not self.__class__.REC_PARAMS.search(line):
line_set.add(line)
continue
for l in graph_expander.expand(line):
line_set.add(l)
# Process chains of dependencies as pairs: left => right.
# Parameterization can duplicate some dependencies, so use a set.
pairs = set()
for line in line_set:
# "foo => bar => baz" becomes [foo, bar, baz]
chain = line.split(ARROW)
# Auto-trigger lone nodes and initial nodes in a chain.
for name, offset, _ in self.__class__.REC_NODES.findall(chain[0]):
if not offset and not name.startswith('@'):
pairs.add((None, name))
for i in range(0, len(chain) - 1):
pairs.add((chain[i], chain[i + 1]))
for pair in pairs:
self._proc_dep_pair(pair[0], pair[1])
class TestParamExpand(unittest.TestCase):
"""Unit tests for the parameter expansion module."""
def setUp(self):
"""Create some parameters and templates for use in tests."""
params_map = {'a': [-3, -1], 'i': [0, 1], 'j': [0, 1, 2], 'k': [0, 1]}
# k has template is deliberately bad
templates = {
'a': '_a%(a)d', 'i': '_i%(i)d', 'j': '_j%(j)d', 'k': '_k%(z)d'}
self.name_expander = NameExpander((params_map, templates))
self.graph_expander = GraphExpander((params_map, templates))
def test_name_one_param(self):
"""Test name expansion and returned value for a single parameter."""
self.assertEqual(
self.name_expander.expand('foo<j>'),
[('foo_j0', {'j': 0}),
('foo_j1', {'j': 1}),
('foo_j2', {'j': 2})]
)
def test_name_two_params(self):
"""Test name expansion and returned values for two parameters."""
self.assertEqual(
self.name_expander.expand('foo<i,j>'),
[('foo_i0_j0', {'i': 0, 'j': 0}),
('foo_i0_j1', {'i': 0, 'j': 1}),
('foo_i0_j2', {'i': 0, 'j': 2}),
('foo_i1_j0', {'i': 1, 'j': 0}),
('foo_i1_j1', {'i': 1, 'j': 1}),
('foo_i1_j2', {'i': 1, 'j': 2})]
)
def test_name_two_names(self):
"""Test name expansion for two names."""
self.assertEqual(
self.name_expander.expand('foo<i>, bar<j>'),
[('foo_i0', {'i': 0}),
('foo_i1', {'i': 1}),
('bar_j0', {'j': 0}),
('bar_j1', {'j': 1}),
('bar_j2', {'j': 2})]
)
def test_name_specific_val_1(self):
"""Test singling out a specific value, in name expansion."""
self.assertEqual(
self.name_expander.expand('foo<i=0>'),
[('foo_i0', {'i': 0})]
)
def test_name_specific_val_2(self):
"""Test specific value in the first parameter of a pair."""
self.assertEqual(
self.name_expander.expand('foo<i=0,j>'),
[('foo_i0_j0', {'i': 0, 'j': 0}),
('foo_i0_j1', {'i': 0, 'j': 1}),
('foo_i0_j2', {'i': 0, 'j': 2})]
)
def test_name_specific_val_3(self):
"""Test specific value in the second parameter of a pair."""
self.assertEqual(
self.name_expander.expand('foo<i,j=1>'),
[('foo_i0_j1', {'i': 0, 'j': 1}),
('foo_i1_j1', {'i': 1, 'j': 1})]
)
def test_name_fail_bare_value(self):
"""Test foo<0,j> fails."""
# It should be foo<i=0,j>.
self.assertRaises(ParamExpandError,
self.name_expander.expand, 'foo<0,j>')
def test_name_fail_undefined_param(self):
"""Test that an undefined parameter gets failed."""
# m is not defined.
self.assertRaises(ParamExpandError,
self.name_expander.expand, 'foo<m,j>')
def test_name_fail_param_value_too_high(self):
"""Test that an out-of-range parameter gets failed."""
# i stops at 3.
self.assertRaises(ParamExpandError,
self.name_expander.expand, 'foo<i=4,j>')
def test_name_multiple(self):
"""Test expansion of two names, with one and two parameters."""
self.assertEqual(
self.name_expander.expand('foo<i>, bar<i,j>'),
[('foo_i0', {'i': 0}),
('foo_i1', {'i': 1}),
('bar_i0_j0', {'i': 0, 'j': 0}),
('bar_i0_j1', {'i': 0, 'j': 1}),
('bar_i0_j2', {'i': 0, 'j': 2}),
('bar_i1_j0', {'i': 1, 'j': 0}),
('bar_i1_j1', {'i': 1, 'j': 1}),
('bar_i1_j2', {'i': 1, 'j': 2})]
)
def test_graph_expand_1(self):
"""Test graph expansion with two parameters each side of an arrow."""
self.assertEqual(
self.graph_expander.expand("bar<i,j>=>baz<i,j>"),
set(["bar_i0_j1=>baz_i0_j1",
"bar_i1_j2=>baz_i1_j2",
"bar_i0_j2=>baz_i0_j2",
"bar_i1_j1=>baz_i1_j1",
"bar_i1_j0=>baz_i1_j0",
"bar_i0_j0=>baz_i0_j0"])
)
def test_graph_expand_2(self):
"""Test graph expansion to 'branch and merge' a workflow."""
self.assertEqual(
self.graph_expander.expand("pre=>bar<i>=>baz<i,j>=>post"),
set(["pre=>bar_i0=>baz_i0_j1=>post",
"pre=>bar_i1=>baz_i1_j2=>post",
"pre=>bar_i0=>baz_i0_j2=>post",
"pre=>bar_i1=>baz_i1_j1=>post",
"pre=>bar_i1=>baz_i1_j0=>post",
"pre=>bar_i0=>baz_i0_j0=>post"])
)
def test_graph_expand_3(self):
"""Test graph expansion -ve integers."""
self.assertEqual(
self.graph_expander.expand("bar<a>"),
set(["bar_a-1", "bar_a-3"]))
def test_graph_expand_offset_1(self):
"""Test graph expansion with a -ve offset."""
self.assertEqual(
self.graph_expander.expand("bar<i-1,j>=>baz<i,j>"),
set(["baz_i0_j0",
"baz_i0_j1",
"baz_i0_j2",
"bar_i0_j0=>baz_i1_j0",
"bar_i0_j1=>baz_i1_j1",
"bar_i0_j2=>baz_i1_j2"])
)
def test_graph_expand_offset_2(self):
"""Test graph expansion with a +ve offset."""
self.assertEqual(
self.graph_expander.expand("baz<i>=>baz<i+1>"),
set(["baz_i0=>baz_i1"])
)
def test_graph_expand_specific(self):
"""Test graph expansion with a specific value."""
self.assertEqual(
self.graph_expander.expand("bar<i=1,j>=>baz<i,j>"),
set(["bar_i1_j0=>baz_i0_j0",
"bar_i1_j1=>baz_i0_j1",
"bar_i1_j2=>baz_i0_j2",
"bar_i1_j0=>baz_i1_j0",
"bar_i1_j1=>baz_i1_j1",
"bar_i1_j2=>baz_i1_j2"])
)
def test_graph_fail_bare_value(self):
"""Test that a bare parameter value fails in the graph."""
self.assertRaises(ParamExpandError,
self.graph_expander.expand, 'foo<0,j>=>bar<i,j>')
def test_graph_fail_undefined_param(self):
"""Test that an undefined parameter value fails in the graph."""
self.assertRaises(ParamExpandError,
self.graph_expander.expand, 'foo<m,j>=>bar<i,j>')
def test_graph_fail_param_value_too_high(self):
"""Test that an out-of-range parameter value fails in the graph."""
self.assertRaises(ParamExpandError,
self.graph_expander.expand, 'foo<i=4,j><i,j>')
def test_template_fail_missing_param(self):
"""Test a template string specifying a non-existent parameter."""
self.assertRaises(
ParamExpandError, self.name_expander.expand, 'foo<k>')
self.assertRaises(
ParamExpandError, self.graph_expander.expand, 'foo<k>')