def checkspec(self, spec, parents=[]):
"check that the file spec is a nested dict of validators"
for key, value in spec.items():
pars = parents + [key]
if isinstance(value, dict):
self.checkspec(value, pars)
else:
if not isinstance(value, validator):
raise ParsecError("Illegal file spec item: %s" %
itemstr(pars, repr(value)))
def checkspec(spec_root, parents=None):
"""Check that the file spec is a nested dict of specifications"""
stack = [[spec_root, []]]
while stack:
spec, parents = stack.pop()
for key, value in spec.items():
pars = parents + [key]
if isinstance(value, dict):
stack.append([value, pars])
elif not isinstance(value, list):
raise ParsecError("Illegal file spec item: %s" %
itemstr(pars, repr(value)))
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
except IncludeFileNotFoundError, x:
raise FileParseError(str(x))
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from parsec.empysupport import EmPyError, empyprocess
except ImportError:
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
try:
flines = empyprocess(flines, fdir, template_vars)
except EmPyError as exc:
lines = flines[max(exc.lineno - 4, 0):exc.lineno]
msg = traceback.format_exc()
raise FileParseError(msg, lines=lines, error_name="EmPyError")
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
flines = jinja2process(flines, fdir, template_vars)
except (StandardError, TemplateError, UndefinedError) as exc:
def inline(lines,
dir_,
filename,
for_grep=False,
for_edit=False,
viewcfg=None,
level=None):
"""Recursive inlining of parsec include-files"""
global flist
if level is None:
# avoid being affected by multiple *different* calls to this function
flist = [filename]
else:
flist.append(filename)
single = False
mark = False
label = False
if viewcfg:
mark = viewcfg['mark']
single = viewcfg['single']
label = viewcfg['label']
else:
viewcfg = {}
global done
global modtimes
outf = []
initial_line_index = 0
if level is None:
level = ''
if for_edit:
m = re.match('^(#![jJ]inja2)', lines[0])
if m:
outf.append(m.groups()[0])
initial_line_index = 1
outf.append(
"""# !WARNING! CYLC EDIT INLINED (DO NOT MODIFY THIS LINE).
# !WARNING! This is an inlined parsec config file; include-files are split
# !WARNING! out again on exiting the edit session. If you are editing
# !WARNING! this file manually then a previous inlined session may have
# !WARNING! crashed; exit now and use 'cylc edit -i' to recover (this
# !WARNING! will split the file up again on exiting).""")
else:
if mark:
level += '!'
elif for_edit:
level += ' > '
if for_edit:
msg = ' (DO NOT MODIFY THIS LINE!)'
else:
msg = ''
for line in lines[initial_line_index:]:
m = include_re.match(line)
if m:
q1, match, q2 = m.groups()
if q1 and (q1 != q2):
raise ParsecError("ERROR, mismatched quotes: " + line)
inc = os.path.join(dir_, match)
if inc not in done:
if single or for_edit:
done.append(inc)
if for_edit:
backup(inc)
# store original modtime
modtimes[inc] = os.stat(inc).st_mtime
if os.path.isfile(inc):
if for_grep or single or label or for_edit:
outf.append('#++++ START INLINED INCLUDE FILE ' +
match + msg)
h = open(inc, 'rb')
finc = [line.rstrip('\n') for line in h]
h.close()
# recursive inclusion
outf.extend(
inline(finc, dir_, inc, for_grep, for_edit, viewcfg,
level))
if for_grep or single or label or for_edit:
outf.append('#++++ END INLINED INCLUDE FILE ' + match +
msg)
else:
flist.append(inc)
raise IncludeFileNotFoundError(flist)
else:
if not for_edit:
outf.append(level + line)
else:
outf.append(line)
else:
# no match
if not for_edit:
outf.append(level + line)
else:
outf.append(line)
return outf
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
try:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
except IncludeFileNotFoundError as exc:
raise FileParseError(str(exc))
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from parsec.empysupport import EmPyError, empyprocess
except ImportError:
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
try:
flines = empyprocess(flines, fdir, template_vars)
except EmPyError as exc:
lines = flines[max(exc.lineno - 4, 0):exc.lineno]
msg = traceback.format_exc()
raise FileParseError(msg, lines=lines, error_name="EmPyError")
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
flines = jinja2process(flines, fdir, template_vars)
except Exception as exc:
# Extract diagnostic info from the end of the Jinja2 traceback.
exc_lines = traceback.format_exc().splitlines()
suffix = []
for line in reversed(exc_lines):
suffix.append(line)
if re.match(r"\s*File", line):
break
msg = '\n'.join(reversed(suffix))
lines = None
lineno = None
if hasattr(exc, 'lineno'):
lineno = exc.lineno
else:
match = re.search(r'File "<template>", line (\d+)', msg)
if match:
lineno = int(match.groups()[0])
if (lineno and getattr(exc, 'filename', None) is None):
# Jinja2 omits the line if it isn't from an external file.
line_index = lineno - 1
if getattr(exc, 'source', None) is None:
# Jinja2Support strips the shebang line.
lines = flines[1:]
elif isinstance(exc.source, str):
lines = exc.source.splitlines()
if lines:
min_line_index = max(line_index - 3, 0)
lines = lines[min_line_index:line_index + 1]
raise FileParseError(msg,
lines=lines,
error_name="Jinja2Error")
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]