def test_parsec_error_msg(self):
parsec_error = ParsecError()
self.assertEqual('', str(parsec_error))
parsec_error = ParsecError('foo')
self.assertEqual('foo', str(parsec_error))
parsec_error = ParsecError('foo', 'bar', 'baz')
self.assertEqual('foo bar baz', str(parsec_error))
def process_plugins(fpath):
# Load Rose Vars, if a ``rose-suite.conf`` file is present.
extra_vars = {
'env': {},
'template_variables': {},
'templating_detected': None
}
for entry_point in pkg_resources.iter_entry_points('cylc.pre_configure'):
try:
plugin_result = entry_point.resolve()(srcdir=fpath)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError('cylc.pre_configure', entry_point.name,
exc) from None
for section in ['env', 'template_variables']:
if section in plugin_result and plugin_result[section] is not None:
# Raise error if multiple plugins try to update the same keys.
section_update = plugin_result.get(section, {})
keys_collision = (extra_vars[section].keys()
& section_update.keys())
if keys_collision:
raise ParsecError(
f"{entry_point.name} is trying to alter "
f"[{section}]{', '.join(sorted(keys_collision))}.")
extra_vars[section].update(section_update)
if ('templating_detected' in plugin_result
and plugin_result['templating_detected'] is not None
and extra_vars['templating_detected'] is not None
and extra_vars['templating_detected'] !=
plugin_result['templating_detected']):
# Don't allow subsequent plugins with different templating_detected
raise ParsecError("Can't merge templating languages "
f"{extra_vars['templating_detected']} and "
f"{plugin_result['templating_detected']}")
elif ('templating_detected' in plugin_result
and plugin_result['templating_detected'] is not None):
extra_vars['templating_detected'] = plugin_result[
'templating_detected']
return extra_vars
def checkspec(spec_root, parents=None):
"""Check that the file spec is a nested dict of specifications"""
stack = [[spec_root, []]]
while stack:
spec, parents = stack.pop()
for key, value in spec.items():
pars = parents + [key]
if isinstance(value, dict):
stack.append([value, pars])
elif not isinstance(value, list):
raise ParsecError("Illegal file spec item: %s" %
itemstr(pars, repr(value)))
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def test_parsec_error_str(self):
msg = 'Turbulence!'
parsec_error = ParsecError(msg)
self.assertEqual(msg, str(parsec_error))
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
extra_vars = process_plugins(Path(fpath).parent)
if not template_vars:
template_vars = {}
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
template_vars['CYLC_VERSION'] = __version__
# Push template_vars into extra_vars so that duplicates come from
# template_vars.
if extra_vars['templating_detected'] is not None:
will_be_overwritten = (template_vars.keys()
& extra_vars['template_variables'].keys())
for key in will_be_overwritten:
LOG.warning(
f'Overriding {key}: {extra_vars["template_variables"][key]} ->'
f' {template_vars[key]}')
extra_vars['template_variables'].update(template_vars)
template_vars = extra_vars['template_variables']
template_vars['CYLC_TEMPLATE_VARS'] = template_vars
# process with EmPy
if do_empy:
if (extra_vars['templating_detected'] == 'empy'
and not re.match(r'^#![Ee]m[Pp]y\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!empy')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if (extra_vars['templating_detected'] == 'jinja2'
and not re.match(r'^#![jJ]inja2\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!jinja2')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def read_and_proc(fpath, template_vars=None, viewcfg=None, opts=None):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
workflow_lib_python = os.path.join(fdir, "lib", "python")
if (
os.path.isdir(workflow_lib_python)
and workflow_lib_python not in sys.path
):
sys.path.append(workflow_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
extra_vars = process_plugins(Path(fpath).parent, opts)
if not template_vars:
template_vars = {}
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, viewcfg=viewcfg)
template_vars['CYLC_VERSION'] = __version__
template_vars = merge_template_vars(template_vars, extra_vars)
template_vars['CYLC_TEMPLATE_VARS'] = template_vars
# Fail if templating_detected ≠ hashbang
process_with = hashbang_and_plugin_templating_clash(
extra_vars['templating_detected'], flines
)
# process with EmPy
if do_empy:
if (
extra_vars['templating_detected'] == 'empy' and
not process_with and
process_with != 'empy'
):
flines.insert(0, '#!empy')
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except ImportError:
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(
flines, fdir, template_vars
)
# process with Jinja2
if do_jinja2:
if (
extra_vars['templating_detected'] == 'jinja2' and
not process_with and
process_with != 'jinja2'
):
flines.insert(0, '#!jinja2')
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except ImportError:
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(
flines, fdir, template_vars
)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def process_plugins(fpath, opts):
"""Run a Cylc pre-configuration plugin.
Plugins should return a dictionary containing:
'env': A dictionary of environment variables.
'template_variables': A dictionary of template variables.
'templating_detected': Where the plugin identifies a templating
language this is specified here. Expected values are ``jinja2``
or ``empy``.
args:
fpath: Directory where the plugin will look for a config.
opts: Command line options to be passed to the plugin.
Returns: Dictionary in the form:
extra_vars = {
'env': {},
'template_variables': {},
'templating_detected': None
}
"""
# Set out blank dictionary for return:
extra_vars = {
'env': {},
'template_variables': {},
'templating_detected': None
}
# Run entry point pre_configure items, trying to merge values with each.:
for entry_point in iter_entry_points(
'cylc.pre_configure'
):
try:
# If you want it to work on sourcedirs you need to get the options
# to here.
plugin_result = entry_point.resolve()(
srcdir=fpath, opts=opts
)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError(
'cylc.pre_configure',
entry_point.name,
exc
) from None
for section in ['env', 'template_variables']:
if section in plugin_result and plugin_result[section] is not None:
# Raise error if multiple plugins try to update the same keys.
section_update = plugin_result.get(section, {})
keys_collision = (
extra_vars[section].keys() & section_update.keys()
)
if keys_collision:
raise ParsecError(
f"{entry_point.name} is trying to alter "
f"[{section}]{', '.join(sorted(keys_collision))}."
)
extra_vars[section].update(section_update)
if (
'templating_detected' in plugin_result and
plugin_result['templating_detected'] is not None and
extra_vars['templating_detected'] is not None and
extra_vars['templating_detected'] !=
plugin_result['templating_detected']
):
# Don't allow subsequent plugins with different templating_detected
raise ParsecError(
"Can't merge templating languages "
f"{extra_vars['templating_detected']} and "
f"{plugin_result['templating_detected']}"
)
elif(
'templating_detected' in plugin_result and
plugin_result['templating_detected'] is not None
):
extra_vars['templating_detected'] = plugin_result[
'templating_detected'
]
return extra_vars
def inline(lines,
dir_,
filename,
for_grep=False,
for_edit=False,
viewcfg=None,
level=None):
"""Recursive inlining of parsec include-files"""
global flist
if level is None:
# avoid being affected by multiple *different* calls to this function
flist = [filename]
else:
flist.append(filename)
single = False
mark = False
label = False
if viewcfg:
mark = viewcfg['mark']
single = viewcfg['single']
label = viewcfg['label']
else:
viewcfg = {}
global done
global modtimes
outf = []
initial_line_index = 0
if level is None:
level = ''
if for_edit:
m = re.match('^(#![jJ]inja2)', lines[0])
if m:
outf.append(m.groups()[0])
initial_line_index = 1
outf.append(
"""# !WARNING! CYLC EDIT INLINED (DO NOT MODIFY THIS LINE).
# !WARNING! This is an inlined parsec config file; include-files are split
# !WARNING! out again on exiting the edit session. If you are editing
# !WARNING! this file manually then a previous inlined session may have
# !WARNING! crashed; exit now and use 'cylc edit -i' to recover (this
# !WARNING! will split the file up again on exiting).""")
else:
if mark:
level += '!'
elif for_edit:
level += ' > '
if for_edit:
msg = ' (DO NOT MODIFY THIS LINE!)'
else:
msg = ''
for line in lines[initial_line_index:]:
m = include_re.match(line)
if m:
q1, match, q2 = m.groups()
if q1 and (q1 != q2):
raise ParsecError("mismatched quotes: " + line)
inc = os.path.join(dir_, match)
if inc not in done:
if single or for_edit:
done.append(inc)
if for_edit:
backup(inc)
# store original modtime
modtimes[inc] = os.stat(inc).st_mtime
if os.path.isfile(inc):
if for_grep or single or label or for_edit:
outf.append('#++++ START INLINED INCLUDE FILE ' +
match + msg)
h = open(inc, 'r')
finc = [line.rstrip('\n') for line in h]
h.close()
# recursive inclusion
outf.extend(
inline(finc, dir_, inc, for_grep, for_edit, viewcfg,
level))
if for_grep or single or label or for_edit:
outf.append('#++++ END INLINED INCLUDE FILE ' + match +
msg)
else:
flist.append(inc)
raise IncludeFileNotFoundError(flist)
else:
if not for_edit:
outf.append(level + line)
else:
outf.append(line)
else:
# no match
if not for_edit:
outf.append(level + line)
else:
outf.append(line)
return outf
def raise_ParsecError(*a, **k):
raise ParsecError("Mock error")