def test_file_parse_error():
error = FileParseError(reason="No reason")
assert str(error) == "No reason"
error = FileParseError("", index=2)
assert str(error) == (" (line 3)\n" "(line numbers match 'cylc view -p')")
error = FileParseError("", line="test")
assert str(error) == ":\n test"
error = FileParseError("", lines=["a", "b"])
assert str(error) == ("\nContext lines:\n" "a\nb\t<--")
def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
raise FileParseError(
'line %d: already encountered %s',
index, itemstr(parents, key, val))
if key in cfig:
oldval = cfig[key]
# this item already exists
if (
parents[0:2] == ['scheduling', 'graph'] or
parents[0:2] == ['scheduling', 'dependencies'] # back compat <=7.X
):
# append the new graph string to the existing one
if not isinstance(cfig, list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
cfig[key] = val
LOG.debug(
'%s: already exists in configuration:\nold: %s\nnew: %s',
key, repr(oldval), repr(cfig[key])) # repr preserves \n
else:
cfig[key] = val
def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
raise FileParseError('line %d: already encountered %s', index,
itemstr(parents, key, val))
if key in cfig:
oldval = cfig[key]
# this item already exists
if (parents[0:2] == ['scheduling', 'graph'] or
# BACK COMPAT: [scheduling][dependencies]
# url:
# https://github.com/cylc/cylc-flow/pull/3191
# from:
# Cylc<=7
# to:
# Cylc8
# remove at:
# Cylc9
parents[0:2] == ['scheduling', 'dependencies']):
# append the new graph string to the existing one
if not isinstance(cfig, list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
cfig[key] = val
LOG.debug('%s: already exists in configuration:\nold: %s\nnew: %s',
key, repr(oldval), repr(cfig[key])) # repr preserves \n
else:
cfig[key] = val
def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
raise FileParseError('line %d: already encountered %s', index,
itemstr(parents, key, val))
if key in cfig:
# this item already exists
if (key == 'graph' and
(parents == ['scheduling', 'dependencies'] or len(parents) == 3
and parents[-3:-1] == ['scheduling', 'dependencies'])):
# append the new graph string to the existing one
LOG.debug('Merging graph strings under %s', itemstr(parents))
if not isinstance(cfig[key], list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
# otherwise override the existing item
LOG.debug('overriding %s old value: %s new value: %s',
itemstr(parents, key), cfig[key], val)
cfig[key] = val
else:
cfig[key] = val
def multiline(flines, value, index, maxline):
"""Consume lines for multiline strings."""
o_index = index
quot = value[:3]
newvalue = value[3:]
# could be a triple-quoted single line:
single_line = _TRIPLE_QUOTE[quot][0]
multi_line = _TRIPLE_QUOTE[quot][1]
mat = single_line.match(value)
if mat:
return value, index
elif newvalue.find(quot) != -1:
# TODO - this should be handled by validation?:
# e.g. non-comment follows single-line triple-quoted string
raise FileParseError('Invalid line', o_index, flines[index])
while index < maxline:
index += 1
newvalue += '\n'
line = flines[index]
if line.find(quot) == -1:
newvalue += line
else:
# end of multiline, process it
break
else:
raise FileParseError(
'Multiline string not closed', o_index, flines[o_index])
mat = multi_line.match(line)
if not mat:
# e.g. end multi-line string followed by a non-comment
raise FileParseError('Invalid line', o_index, line)
# value, comment = mat.groups()
return quot + newvalue + line, index
def _concatenate(lines):
"""concatenate continuation lines"""
index = 0
clines = []
maxline = len(lines)
while index < maxline:
line = lines[index]
# Raise an error if line has a whitespace after the line break
if re.match(_BAD_CONTINUATION_TRAILING_WHITESPACE, line):
msg = ("Syntax error line {0}: Whitespace after the line "
"continuation character (\\).")
raise FileParseError(msg.format(index + 1))
while line.endswith('\\'):
if index == maxline - 1:
# continuation char on the last line
# must be an error - safe to strip it
line = line[:-1]
else:
index += 1
line = line[:-1] + lines[index]
clines.append(line)
index += 1
return clines
def parse(fpath, output_fname=None, template_vars=None):
"""Parse file items line-by-line into a corresponding nested dict."""
# read and process the file (jinja2, include-files, line continuation)
flines = read_and_proc(fpath, template_vars)
if output_fname:
with open(output_fname, 'w') as handle:
handle.write('\n'.join(flines) + '\n')
LOG.debug('Processed configuration dumped: %s', output_fname)
nesting_level = 0
config = OrderedDictWithDefaults()
parents = []
maxline = len(flines) - 1
index = -1
while index < maxline:
index += 1
line = flines[index]
if re.match(_LINECOMMENT, line):
# skip full-line comments
continue
if re.match(_BLANKLINE, line):
# skip blank lines
continue
m = re.match(_HEADING, line)
if m:
# matched a section heading
s_open, sect_name, s_close = m.groups()[1:-1]
nb = len(s_open)
if nb != len(s_close):
raise FileParseError('bracket mismatch', index, line)
elif nb == nesting_level:
# sibling section
parents = parents[:-1] + [sect_name]
elif nb == nesting_level + 1:
# child section
parents = parents + [sect_name]
elif nb < nesting_level:
# back up one or more levels
ndif = nesting_level - nb
parents = parents[:-ndif - 1] + [sect_name]
else:
raise FileParseError(
'Error line ' + str(index + 1) + ': ' + line)
nesting_level = nb
addsect(config, sect_name, parents[:-1])
else:
m = re.match(_KEY_VALUE, line)
if m:
# matched a key=value item
key, _, val = m.groups()[1:]
if val.startswith('"""') or val.startswith("'''"):
# triple quoted - may be a multiline value
val, index = multiline(flines, val, index, maxline)
addict(config, key, val, parents, index)
else:
# no match
raise FileParseError(
'Invalid line ' + str(index + 1) + ': ' + line)
return config
def inline(lines,
dir_,
filename,
for_grep=False,
for_edit=False,
viewcfg=None,
level=None):
"""Recursive inlining of parsec include-files"""
global flist
if level is None:
# avoid being affected by multiple *different* calls to this function
flist = [filename]
else:
flist.append(filename)
single = False
mark = False
label = False
if viewcfg:
mark = viewcfg['mark']
single = viewcfg['single']
label = viewcfg['label']
else:
viewcfg = {}
global done
global modtimes
outf = []
initial_line_index = 0
if level is None:
level = ''
if for_edit:
m = re.match('^(#![jJ]inja2)', lines[0])
if m:
outf.append(m.groups()[0])
initial_line_index = 1
outf.append(
"""# !WARNING! CYLC EDIT INLINED (DO NOT MODIFY THIS LINE).
# !WARNING! This is an inlined parsec config file; include-files are split
# !WARNING! out again on exiting the edit session. If you are editing
# !WARNING! this file manually then a previous inlined session may have
# !WARNING! crashed; exit now and use 'cylc edit -i' to recover (this
# !WARNING! will split the file up again on exiting).""")
else:
if mark:
level += '!'
elif for_edit:
level += ' > '
if for_edit:
msg = ' (DO NOT MODIFY THIS LINE!)'
else:
msg = ''
for line in lines[initial_line_index:]:
m = include_re.match(line)
if m:
q1, match, q2 = m.groups()
if q1 and (q1 != q2):
raise FileParseError(
"mismatched quotes",
line=line,
fpath=filename,
)
inc = os.path.join(dir_, match)
if inc not in done:
if single or for_edit:
done.append(inc)
if for_edit:
backup(inc)
# store original modtime
modtimes[inc] = os.stat(inc).st_mtime
if os.path.isfile(inc):
if for_grep or single or label or for_edit:
outf.append('#++++ START INLINED INCLUDE FILE ' +
match + msg)
h = open(inc, 'r')
finc = [line.rstrip('\n') for line in h]
h.close()
# recursive inclusion
outf.extend(
inline(finc, dir_, inc, for_grep, for_edit, viewcfg,
level))
if for_grep or single or label or for_edit:
outf.append('#++++ END INLINED INCLUDE FILE ' + match +
msg)
else:
flist.append(inc)
raise IncludeFileNotFoundError(flist)
else:
if not for_edit:
outf.append(level + line)
else:
outf.append(line)
else:
# no match
if not for_edit:
outf.append(level + line)
else:
outf.append(line)
return outf
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
extra_vars = process_plugins(Path(fpath).parent)
if not template_vars:
template_vars = {}
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
template_vars['CYLC_VERSION'] = __version__
# Push template_vars into extra_vars so that duplicates come from
# template_vars.
if extra_vars['templating_detected'] is not None:
will_be_overwritten = (template_vars.keys()
& extra_vars['template_variables'].keys())
for key in will_be_overwritten:
LOG.warning(
f'Overriding {key}: {extra_vars["template_variables"][key]} ->'
f' {template_vars[key]}')
extra_vars['template_variables'].update(template_vars)
template_vars = extra_vars['template_variables']
template_vars['CYLC_TEMPLATE_VARS'] = template_vars
# process with EmPy
if do_empy:
if (extra_vars['templating_detected'] == 'empy'
and not re.match(r'^#![Ee]m[Pp]y\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!empy')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if (extra_vars['templating_detected'] == 'jinja2'
and not re.match(r'^#![jJ]inja2\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!jinja2')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def parse(fpath, output_fname=None, template_vars=None, opts=None):
"""Parse file items line-by-line into a corresponding nested dict."""
# read and process the file (jinja2, include-files, line continuation)
flines = read_and_proc(fpath, template_vars, opts=opts)
if output_fname:
with open(output_fname, 'w') as handle:
handle.write('\n'.join(flines) + '\n')
LOG.debug('Processed configuration dumped: %s', output_fname)
nesting_level = 0
config = OrderedDictWithDefaults()
parents = []
maxline = len(flines) - 1
index = -1
while index < maxline:
index += 1
line = flines[index]
if re.match(_LINECOMMENT, line):
# skip full-line comments
continue
if re.match(_BLANKLINE, line):
# skip blank lines
continue
m = re.match(_HEADING, line)
if m:
# matched a section heading
s_open, sect_name, s_close = m.groups()[1:-1]
nb = len(s_open)
if nb != len(s_close):
raise FileParseError('bracket mismatch', index, line)
elif nb == nesting_level:
# sibling section
parents = parents[:-1] + [sect_name]
elif nb == nesting_level + 1:
# child section
parents = parents + [sect_name]
elif nb < nesting_level: # noqa: SIM106
# back up one or more levels
ndif = nesting_level - nb
parents = parents[:-ndif - 1] + [sect_name]
else:
raise FileParseError('Error line', index=index, line=line)
nesting_level = nb
addsect(config, sect_name, parents[:-1])
else:
m = re.match(_KEY_VALUE, line)
if m:
# matched a key=value item
key, _, val = m.groups()[1:]
if val.startswith('"""') or val.startswith("'''"):
# triple quoted - may be a multiline value
val, index = multiline(flines, val, index, maxline)
addict(config, key, val, parents, index)
else:
# no match
help_lines = None
if 'val' in locals() and _UNCLOSED_MULTILINE.search(val):
# this might be an unclosed multiline string
# provide a helpful error message
key_name = ''.join(
[f'[{parent}]' for parent in parents]
) + key
help_lines = [f'Did you forget to close {key_name}?']
raise FileParseError(
'Invalid line',
index=index,
line=line,
help_lines=help_lines
)
return config
def inline(lines, dir_, filename, for_grep=False, viewcfg=None, level=None):
"""Recursive inlining of parsec include-files"""
global flist
if level is None:
# avoid being affected by multiple *different* calls to this function
flist = [filename]
else:
flist.append(filename)
single = False
mark = False
label = False
if viewcfg:
mark = viewcfg['mark']
single = viewcfg['single']
label = viewcfg['label']
else:
viewcfg = {}
global done
outf = []
initial_line_index = 0
if level is None:
level = ''
elif mark:
level += '!'
msg = ''
for line in lines[initial_line_index:]:
m = include_re.match(line)
if m:
q1, match, q2 = m.groups()
if q1 and (q1 != q2):
raise FileParseError(
"mismatched quotes",
line=line,
fpath=filename,
)
inc = os.path.join(dir_, match)
if inc not in done:
if single:
done.append(inc)
if not os.path.isfile(inc):
flist.append(inc)
raise IncludeFileNotFoundError(flist)
if for_grep or single or label:
outf.append('#++++ START INLINED INCLUDE FILE ' + match +
msg)
with open(inc, 'r') as handle:
finc = [line.rstrip('\n') for line in handle]
# recursive inclusion
outf.extend(inline(finc, dir_, inc, for_grep, viewcfg, level))
if for_grep or single or label:
outf.append('#++++ END INLINED INCLUDE FILE ' + match +
msg)
else:
outf.append(level + line)
else:
# no match
outf.append(level + line)
return outf
