def setUp(self):
"""Create some parameters and templates for use in tests."""
params_map = {'a': [-3, -1], 'i': [0, 1], 'j': [0, 1, 2], 'k': [0, 1]}
# k has template is deliberately bad
templates = {
'a': '_a%(a)d', 'i': '_i%(i)d', 'j': '_j%(j)d', 'k': '_k%(z)d'}
self.name_expander = NameExpander((params_map, templates))
self.graph_expander = GraphExpander((params_map, templates))
def test_parameter_graph_mixing_offset_and_conditional(self):
"""Test for bug reported in issue #2608 on GitHub."""
for test_case in self._param_expand_params():
params_map, templates, expanded_str, expanded_values = \
test_case
graph_expander = GraphExpander((params_map, templates))
# Ignore white spaces.
expanded = [expanded.replace(' ', '') for expanded in
graph_expander.expand(expanded_str)]
self.assertEqual(
len(expanded_values),
len(expanded),
f"Invalid length for expected {expanded_values} and "
f"{expanded}")
# When testing, we don't really care for white spaces,as they
# are removed in the GraphParser anyway. That's why we have
# ''.replace(' ', '').
for expected in expanded_values:
self.assertTrue(
expected.replace(' ', '') in expanded,
f"Expected value {expected.replace(' ', '')} "
f"not in {expanded}")
def parse_graph(self, graph_string):
"""Parse the graph string for a single graph section.
(Assumes any general line-continuation markers have been processed).
1. Strip comments, whitespace, and blank lines.
(all whitespace is removed up front so we don't have to consider
it in regexes and strip it from matched elements)
2. Join incomplete lines starting or ending with '=>'.
3. Replicate and expand any parameterized lines.
4. Split and process by pairs "left-expression => right-node":
i. Replace families with members (any or all semantics).
ii. Record parsed dependency information for each right-side node.
"""
# Strip comments, whitespace, and blank lines.
non_blank_lines = []
bad_lines = []
for line in graph_string.split('\n'):
modified_line = self.__class__.REC_COMMENT.sub('', line)
# Ignore empty lines
if not modified_line or modified_line.isspace():
continue
# Catch simple bad lines that would be accepted once
# spaces are removed, e.g. 'foo bar => baz'
if self.REC_GRAPH_BAD_SPACES_LINE.search(modified_line):
bad_lines.append(line)
continue
# Apparently this is the fastest way to strip all whitespace!:
modified_line = "".join(modified_line.split())
non_blank_lines.append(modified_line)
# Check if there were problem lines and abort
if bad_lines:
self._report_invalid_lines(bad_lines)
# Join incomplete lines (beginning or ending with an arrow).
full_lines = []
part_lines = []
for i, _ in enumerate(non_blank_lines):
this_line = non_blank_lines[i]
if i == 0:
# First line can't start with an arrow.
if this_line.startswith(ARROW):
raise GraphParseError("leading arrow: %s" % this_line)
try:
next_line = non_blank_lines[i + 1]
except IndexError:
next_line = ''
if this_line.endswith(ARROW):
# Last line can't end with an arrow.
raise GraphParseError("trailing arrow: %s" % this_line)
part_lines.append(this_line)
if (this_line.endswith(ARROW) or next_line.startswith(ARROW)):
continue
full_line = ''.join(part_lines)
# Record inter-suite dependence and remove the marker notation.
# ("foo<SUITE::TASK:fail> => bar" becomes:fail "foo => bar").
repl = Replacement('\\1')
full_line = self.__class__.REC_SUITE_STATE.sub(repl, full_line)
for item in repl.match_groups:
l_task, r_all, r_suite, r_task, r_status = item
if r_status:
r_status = r_status[1:]
else:
r_status = self.__class__.TRIG_SUCCEED[1:]
self.suite_state_polling_tasks[l_task] = (r_suite, r_task,
r_status, r_all)
full_lines.append(full_line)
part_lines = []
# Check for double-char conditional operators (a common mistake),
# and bad node syntax (order of qualifiers).
bad_lines = []
for line in full_lines:
if self.__class__.OP_AND_ERR in line:
raise GraphParseError("the graph AND operator is '%s': %s" %
(self.__class__.OP_AND, line))
if self.__class__.OP_OR_ERR in line:
raise GraphParseError("the graph OR operator is '%s': %s" %
(self.__class__.OP_OR, line))
# Check node syntax. First drop all non-node characters.
node_str = line
for s in ['=>', '|', '&', '(', ')', '!']:
node_str = node_str.replace(s, ' ')
# Drop all valid @triggers, longest first to avoid sub-strings.
nodes = self.__class__.REC_ACTION.findall(node_str)
nodes.sort(key=len, reverse=True)
for node in nodes:
node_str = node_str.replace(node, '')
# Then drop all valid nodes, longest first to avoid sub-strings.
bad_lines = [
node_str for node in node_str.split()
if self.__class__.REC_NODE_FULL.sub('', node, 1)
]
if bad_lines:
self._report_invalid_lines(bad_lines)
# Expand parameterized lines (or detect undefined parameters).
line_set = set()
graph_expander = GraphExpander(self.parameters)
for line in full_lines:
if not self.__class__.REC_PARAMS.search(line):
line_set.add(line)
continue
for l in graph_expander.expand(line):
line_set.add(l)
# Process chains of dependencies as pairs: left => right.
# Parameterization can duplicate some dependencies, so use a set.
pairs = set()
for line in line_set:
# "foo => bar => baz" becomes [foo, bar, baz]
chain = line.split(ARROW)
# Auto-trigger lone nodes and initial nodes in a chain.
for name, offset, _ in self.__class__.REC_NODES.findall(chain[0]):
if not offset and not name.startswith('@'):
pairs.add((None, name))
for i in range(0, len(chain) - 1):
pairs.add((chain[i], chain[i + 1]))
for pair in pairs:
self._proc_dep_pair(pair[0], pair[1])
def parse_graph(self, graph_string: str) -> None:
"""Parse the graph string for a single graph section.
(Assumes any general line-continuation markers have been processed).
1. Strip comments, whitespace, and blank lines.
(all whitespace is removed up front so we don't have to consider
it in regexes and strip it from matched elements)
2. Join incomplete lines starting or ending with '=>'.
3. Replicate and expand any parameterized lines.
4. Split and process by pairs "left-expression => right-node":
i. Replace families with members (any or all semantics).
ii. Record parsed dependency information for each right-side node.
"""
# Strip comments, whitespace, and blank lines.
non_blank_lines = []
bad_lines = []
for line in graph_string.split('\n'):
modified_line = self.__class__.REC_COMMENT.sub('', line)
# Ignore empty lines
if not modified_line or modified_line.isspace():
continue
# Catch simple bad lines that would be accepted once
# spaces are removed, e.g. 'foo bar => baz'
if self.REC_GRAPH_BAD_SPACES_LINE.search(modified_line):
bad_lines.append(line)
continue
# Apparently this is the fastest way to strip all whitespace!:
modified_line = "".join(modified_line.split())
non_blank_lines.append(modified_line)
# Check if there were problem lines and abort
if bad_lines:
self._report_invalid_lines(bad_lines)
# Join incomplete lines (beginning or ending with an arrow).
full_lines = []
part_lines = []
for i, _ in enumerate(non_blank_lines):
this_line = non_blank_lines[i]
for seq in self.CONTINUATION_STRS:
if i == 0 and this_line.startswith(seq):
# First line can't start with an arrow.
raise GraphParseError(f"Leading {seq}: {this_line}")
try:
next_line = non_blank_lines[i + 1]
except IndexError:
next_line = ''
for seq in self.CONTINUATION_STRS:
if this_line.endswith(seq):
# Last line can't end with an arrow, & or |.
raise GraphParseError(f"Dangling {seq}:"
f"{this_line}")
part_lines.append(this_line)
# Check that a continuation sequence doesn't end this line and
# begin the next:
if (this_line.endswith(self.CONTINUATION_STRS)
and next_line.startswith(self.CONTINUATION_STRS)):
raise GraphParseError(
'Consecutive lines end and start with continuation '
'characters:\n'
f'{this_line}\n'
f'{next_line}')
# Check that line ends with a valid continuation sequence:
if (any(
this_line.endswith(seq) or next_line.startswith(seq)
for seq in self.CONTINUATION_STRS) and not (any(
this_line.endswith(seq) or next_line.startswith(seq)
for seq in self.BAD_STRS))):
continue
full_line = ''.join(part_lines)
# Record inter-workflow dependence and remove the marker notation.
# ("foo<WORKFLOW::TASK:fail> => bar" becomes:fail "foo => bar").
repl = Replacement('\\1')
full_line = self.__class__.REC_WORKFLOW_STATE.sub(repl, full_line)
for item in repl.match_groups:
l_task, r_all, r_workflow, r_task, r_status = item
if r_status:
r_status = r_status.strip(self.__class__.QUALIFIER)
r_status = TaskTrigger.standardise_name(r_status)
else:
r_status = TASK_OUTPUT_SUCCEEDED
self.workflow_state_polling_tasks[l_task] = (r_workflow,
r_task, r_status,
r_all)
full_lines.append(full_line)
part_lines = []
# Check for double-char conditional operators (a common mistake),
# and bad node syntax (order of qualifiers).
bad_lines = []
for line in full_lines:
if self.__class__.OP_AND_ERR in line:
raise GraphParseError("The graph AND operator is "
f"'{self.__class__.OP_AND}': {line}")
if self.__class__.OP_OR_ERR in line:
raise GraphParseError("The graph OR operator is "
f"'{self.__class__.OP_OR}': {line}")
# Check node syntax. First drop all non-node characters.
node_str = line
for spec in [
self.__class__.ARROW,
self.__class__.OP_OR,
self.__class__.OP_AND,
self.__class__.SUICIDE,
'(',
')',
]:
node_str = node_str.replace(spec, ' ')
# Drop all valid @xtriggers, longest first to avoid sub-strings.
nodes = self.__class__.REC_XTRIG.findall(node_str)
nodes.sort(key=len, reverse=True)
for node in nodes:
node_str = node_str.replace(node, '')
# Then drop all valid nodes, longest first to avoid sub-strings.
bad_lines = [
node_str for node in node_str.split()
if self.__class__.REC_NODE_FULL.sub('', node, 1)
]
if bad_lines:
self._report_invalid_lines(bad_lines)
# Expand parameterized lines (or detect undefined parameters).
line_set = set()
graph_expander = GraphExpander(self.parameters)
for line in full_lines:
if not self.__class__.REC_PARAMS.search(line):
line_set.add(line)
continue
for line_ in graph_expander.expand(line):
line_set.add(line_)
# Process chains of dependencies as pairs: left => right.
# Parameterization can duplicate some dependencies, so use a set.
pairs: Set[Tuple[Optional[str], str]] = set()
for line in line_set:
chain = []
# "foo => bar => baz" becomes [foo, bar, baz]
# "foo => bar_-32768 => baz" becomes [foo]
# "foo_-32768 => bar" becomes []
for node in line.split(self.__class__.ARROW):
# This can happen, e.g. "foo => => bar" produces
# "foo, '', bar", so we add so that later it raises
# an error
if node == '':
chain.append(node)
continue
node = self.REC_NODE_OUT_OF_RANGE.sub('', node)
if node == '':
# For "foo => bar<err> => baz", stop at "bar<err>"
break
else:
chain.append(node)
if not chain:
continue
for item in self.__class__.REC_NODES.findall(chain[0]):
# Auto-trigger lone nodes and initial nodes in a chain.
if not item[0].startswith(self.__class__.XTRIG):
pairs.add((None, ''.join(item)))
for i in range(0, len(chain) - 1):
pairs.add((chain[i], chain[i + 1]))
for pair in pairs:
self._proc_dep_pair(pair)