def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def read_and_proc(fpath, template_vars=None, viewcfg=None, opts=None):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
workflow_lib_python = os.path.join(fdir, "lib", "python")
if (
os.path.isdir(workflow_lib_python)
and workflow_lib_python not in sys.path
):
sys.path.append(workflow_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
extra_vars = process_plugins(Path(fpath).parent, opts)
if not template_vars:
template_vars = {}
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, viewcfg=viewcfg)
template_vars['CYLC_VERSION'] = __version__
template_vars = merge_template_vars(template_vars, extra_vars)
template_vars['CYLC_TEMPLATE_VARS'] = template_vars
# Fail if templating_detected ≠ hashbang
process_with = hashbang_and_plugin_templating_clash(
extra_vars['templating_detected'], flines
)
# process with EmPy
if do_empy:
if (
extra_vars['templating_detected'] == 'empy' and
not process_with and
process_with != 'empy'
):
flines.insert(0, '#!empy')
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except ImportError:
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(
flines, fdir, template_vars
)
# process with Jinja2
if do_jinja2:
if (
extra_vars['templating_detected'] == 'jinja2' and
not process_with and
process_with != 'jinja2'
):
flines.insert(0, '#!jinja2')
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except ImportError:
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(
flines, fdir, template_vars
)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
extra_vars = process_plugins(Path(fpath).parent)
if not template_vars:
template_vars = {}
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
template_vars['CYLC_VERSION'] = __version__
# Push template_vars into extra_vars so that duplicates come from
# template_vars.
if extra_vars['templating_detected'] is not None:
will_be_overwritten = (template_vars.keys()
& extra_vars['template_variables'].keys())
for key in will_be_overwritten:
LOG.warning(
f'Overriding {key}: {extra_vars["template_variables"][key]} ->'
f' {template_vars[key]}')
extra_vars['template_variables'].update(template_vars)
template_vars = extra_vars['template_variables']
template_vars['CYLC_TEMPLATE_VARS'] = template_vars
# process with EmPy
if do_empy:
if (extra_vars['templating_detected'] == 'empy'
and not re.match(r'^#![Ee]m[Pp]y\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!empy')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if (extra_vars['templating_detected'] == 'jinja2'
and not re.match(r'^#![jJ]inja2\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!jinja2')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]