def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
raise FileParseError('ERROR line %d: already encountered %s', index,
itemstr(parents, key, val))
if key in cfig:
# this item already exists
if (key == 'graph' and
(parents == ['scheduling', 'dependencies'] or len(parents) == 3
and parents[-3:-1] == ['scheduling', 'dependencies'])):
# append the new graph string to the existing one
LOG.debug('Merging graph strings under %s', itemstr(parents))
if not isinstance(cfig[key], list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
# otherwise override the existing item
LOG.debug('overriding %s old value: %s new value: %s',
itemstr(parents, key), cfig[key], val)
cfig[key] = val
else:
cfig[key] = val
def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
raise FileParseError(
'line %d: already encountered %s',
index, itemstr(parents, key, val))
if key in cfig:
# this item already exists
if (key == 'graph' and (
parents == ['scheduling', 'dependencies'] or
len(parents) == 3 and
parents[-3:-1] == ['scheduling', 'dependencies'])):
# append the new graph string to the existing one
LOG.debug('Merging graph strings under %s', itemstr(parents))
if not isinstance(cfig[key], list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
# otherwise override the existing item
LOG.debug(
'overriding %s old value: %s new value: %s',
itemstr(parents, key), cfig[key], val)
cfig[key] = val
else:
cfig[key] = val
def jinja2process(flines, dir_, template_vars=None):
"""Pass configure file through Jinja2 processor."""
# Set up Jinja2 environment.
env = jinja2environment(dir_)
# Load file lines into a template, excluding '#!jinja2' so that
# '#!cylc-x.y.z' rises to the top. Callers should handle jinja2
# TemplateSyntaxerror and TemplateError.
if template_vars:
LOG.debug(
'Setting Jinja2 template variables:\n%s', '\n'.join(
['+ %s=%s' % item for item in sorted(template_vars.items())]))
# Jinja2 render method requires a dictionary as argument (not None):
if not template_vars:
template_vars = {}
# CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
# AND TYPEERROR (e.g. for not using "|int" filter on number inputs.
# Convert unicode to plain str, ToDo - still needed for parsec?)
suiterc = []
template = env.from_string('\n'.join(flines[1:]))
for line in str(template.render(template_vars)).splitlines():
# Jinja2 leaves blank lines where source lines contain
# only Jinja2 code; this matters if line continuation
# markers are involved, so we remove blank lines here.
if not line.strip():
continue
# restoring newlines here is only necessary for display by
# the cylc view command:
# ##suiterc.append(line + '\n')
suiterc.append(line)
return suiterc
def addsect(cfig, sname, parents):
"""Add a new section to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if sname in cfig:
# this doesn't warrant a warning unless contained items are repeated
LOG.debug('Section already encountered: %s',
itemstr(parents + [sname]))
else:
cfig[sname] = OrderedDictWithDefaults()
def addsect(cfig, sname, parents):
"""Add a new section to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if sname in cfig:
# this doesn't warrant a warning unless contained items are repeated
LOG.debug(
'Section already encountered: %s', itemstr(parents + [sname]))
else:
cfig[sname] = OrderedDictWithDefaults()
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
try:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
except IncludeFileNotFoundError, x:
raise FileParseError(str(x))
def upgrade(self):
warnings = OrderedDict()
for vn, upgs in self.upgrades.items():
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings.setdefault(vn, [])
warnings[vn].append(msg)
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if warnings:
level = WARNING
if self.descr == self.SITE_CONFIG:
# Site level configuration, user cannot easily fix.
# Only log at debug level.
level = DEBUG
else:
# User level configuration, user should be able to fix.
# Log at warning level.
level = WARNING
LOG.log(
level,
"deprecated items were automatically upgraded in '%s':",
self.descr)
for vn, msgs in warnings.items():
for msg in msgs:
LOG.log(level, ' * (%s) %s', vn, msg)
def upgrade(self):
warnings = OrderedDict()
for vn, upgs in self.upgrades.items():
for u in upgs:
try:
exp = self.expand(u)
except (KeyError, UpgradeError):
continue
for upg in exp:
try:
old = self.get_item(upg['old'])
except KeyError:
# OK: deprecated item not found
pass
else:
msg = self.show_keys(upg['old'])
if upg['new']:
msg += ' -> ' + self.show_keys(upg['new'])
else:
upg['new'] = upg['old']
msg += " - " + upg['cvt'].describe()
if not upg['silent']:
warnings.setdefault(vn, [])
warnings[vn].append(msg)
self.del_item(upg['old'])
if upg['cvt'].describe() != "DELETED (OBSOLETE)":
self.put_item(upg['new'], upg['cvt'].convert(old))
if warnings:
level = WARNING
if self.descr == self.SITE_CONFIG:
# Site level configuration, user cannot easily fix.
# Only log at debug level.
level = DEBUG
else:
# User level configuration, user should be able to fix.
# Log at warning level.
level = WARNING
LOG.log(
level,
"deprecated items were automatically upgraded in '%s':",
self.descr)
for vn, msgs in warnings.items():
for msg in msgs:
LOG.log(level, ' * (%s) %s', vn, msg)
def parse(fpath, output_fname=None, template_vars=None):
"Parse file items line-by-line into a corresponding nested dict."
# read and process the file (jinja2, include-files, line continuation)
flines = read_and_proc(fpath, template_vars)
if output_fname:
with open(output_fname, 'wb') as handle:
handle.write('\n'.join(flines) + '\n')
LOG.debug('Processed configuration dumped: %s', output_fname)
nesting_level = 0
config = OrderedDictWithDefaults()
parents = []
maxline = len(flines) - 1
index = -1
while index < maxline:
index += 1
line = flines[index]
if re.match(_LINECOMMENT, line):
# skip full-line comments
continue
if re.match(_BLANKLINE, line):
# skip blank lines
continue
m = re.match(_HEADING, line)
if m:
# matched a section heading
s_open, sect_name, s_close = m.groups()[1:-1]
nb = len(s_open)
if nb != len(s_close):
raise FileParseError('bracket mismatch', index, line)
elif nb == nesting_level:
# sibling section
parents = parents[:-1] + [sect_name]
elif nb == nesting_level + 1:
# child section
parents = parents + [sect_name]
elif nb < nesting_level:
# back up one or more levels
ndif = nesting_level - nb
parents = parents[:-ndif - 1] + [sect_name]
else:
raise FileParseError('Error line ' + str(index + 1) + ': ' +
line)
nesting_level = nb
addsect(config, sect_name, parents[:-1])
else:
m = re.match(_KEY_VALUE, line)
if m:
# matched a key=value item
key, _, val = m.groups()[1:]
if val.startswith('"""') or val.startswith("'''"):
# triple quoted - may be a multiline value
val, index = multiline(flines, val, index, maxline)
addict(config, key, val, parents, index)
else:
# no match
raise FileParseError('Invalid line ' + str(index + 1) + ': ' +
line)
return config
# inline any cylc include-files
if do_inline:
try:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
except IncludeFileNotFoundError, x:
raise FileParseError(str(x))
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from parsec.empysupport import EmPyError, empyprocess
except ImportError:
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
try:
flines = empyprocess(flines, fdir, template_vars)
except EmPyError as exc:
lines = flines[max(exc.lineno - 4, 0):exc.lineno]
msg = traceback.format_exc()
raise FileParseError(msg, lines=lines, error_name="EmPyError")
# process with Jinja2
if do_jinja2:
def jinja2process(flines, dir_, template_vars=None):
"""Pass configure file through Jinja2 processor."""
# Load file lines into a template, excluding '#!jinja2' so that
# '#!cylc-x.y.z' rises to the top. Callers should handle jinja2
# TemplateSyntaxerror and TemplateError.
if template_vars:
LOG.debug(
'Setting Jinja2 template variables:\n%s',
'\n'.join(
['+ %s=%s' % item for item in sorted(template_vars.items())]))
# Jinja2 render method requires a dictionary as argument (not None):
if not template_vars:
template_vars = {}
# CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
# AND TYPEERROR (e.g. for not using "|int" filter on number inputs.
# Convert unicode to plain str, ToDo - still needed for parsec?)
try:
env = jinja2environment(dir_)
template = env.from_string('\n'.join(flines[1:]))
lines = str(template.render(template_vars)).splitlines()
except TemplateSyntaxError as exc:
filename = None
# extract source lines
if exc.lineno and exc.source and not exc.filename:
# error in suite.rc or cylc include file
lines = exc.source.splitlines()
elif exc.lineno and exc.filename:
# error in jinja2 include file
filename = os.path.relpath(exc.filename, dir_)
with open(exc.filename, 'r') as include_file:
include_file.seek(max(exc.lineno - CONTEXT_LINES, 0), 0)
lines = []
for _ in range(CONTEXT_LINES):
lines.append(include_file.readline().splitlines()[0])
if lines:
# extract context lines from source lines
lines = lines[max(exc.lineno - CONTEXT_LINES, 0):exc.lineno]
raise Jinja2Error(exc, lines=lines, filename=filename)
except Exception as exc:
lineno = get_error_location()
lines = None
if lineno:
lineno += 1 # shebang line ignored by jinja2
lines = flines[max(lineno - CONTEXT_LINES, 0):lineno]
raise Jinja2Error(exc, lines=lines)
suiterc = []
for line in lines:
# Jinja2 leaves blank lines where source lines contain
# only Jinja2 code; this matters if line continuation
# markers are involved, so we remove blank lines here.
if not line.strip():
continue
# restoring newlines here is only necessary for display by
# the cylc view command:
# ##suiterc.append(line + '\n')
suiterc.append(line)
return suiterc
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
try:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
except IncludeFileNotFoundError as exc:
raise FileParseError(str(exc))
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from parsec.empysupport import EmPyError, empyprocess
except ImportError:
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
try:
flines = empyprocess(flines, fdir, template_vars)
except EmPyError as exc:
lines = flines[max(exc.lineno - 4, 0):exc.lineno]
msg = traceback.format_exc()
raise FileParseError(msg, lines=lines, error_name="EmPyError")
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
flines = jinja2process(flines, fdir, template_vars)
except Exception as exc:
# Extract diagnostic info from the end of the Jinja2 traceback.
exc_lines = traceback.format_exc().splitlines()
suffix = []
for line in reversed(exc_lines):
suffix.append(line)
if re.match(r"\s*File", line):
break
msg = '\n'.join(reversed(suffix))
lines = None
lineno = None
if hasattr(exc, 'lineno'):
lineno = exc.lineno
else:
match = re.search(r'File "<template>", line (\d+)', msg)
if match:
lineno = int(match.groups()[0])
if (lineno and getattr(exc, 'filename', None) is None):
# Jinja2 omits the line if it isn't from an external file.
line_index = lineno - 1
if getattr(exc, 'source', None) is None:
# Jinja2Support strips the shebang line.
lines = flines[1:]
elif isinstance(exc.source, str):
lines = exc.source.splitlines()
if lines:
min_line_index = max(line_index - 3, 0)
lines = lines[min_line_index:line_index + 1]
raise FileParseError(msg,
lines=lines,
error_name="Jinja2Error")
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def parse(fpath, output_fname=None, template_vars=None):
"""Parse file items line-by-line into a corresponding nested dict."""
# read and process the file (jinja2, include-files, line continuation)
flines = read_and_proc(fpath, template_vars)
if output_fname:
with open(output_fname, 'w') as handle:
handle.write('\n'.join(flines) + '\n')
LOG.debug('Processed configuration dumped: %s', output_fname)
nesting_level = 0
config = OrderedDictWithDefaults()
parents = []
maxline = len(flines) - 1
index = -1
while index < maxline:
index += 1
line = flines[index]
if re.match(_LINECOMMENT, line):
# skip full-line comments
continue
if re.match(_BLANKLINE, line):
# skip blank lines
continue
m = re.match(_HEADING, line)
if m:
# matched a section heading
s_open, sect_name, s_close = m.groups()[1:-1]
nb = len(s_open)
if nb != len(s_close):
raise FileParseError('bracket mismatch', index, line)
elif nb == nesting_level:
# sibling section
parents = parents[:-1] + [sect_name]
elif nb == nesting_level + 1:
# child section
parents = parents + [sect_name]
elif nb < nesting_level:
# back up one or more levels
ndif = nesting_level - nb
parents = parents[:-ndif - 1] + [sect_name]
else:
raise FileParseError(
'Error line ' + str(index + 1) + ': ' + line)
nesting_level = nb
addsect(config, sect_name, parents[:-1])
else:
m = re.match(_KEY_VALUE, line)
if m:
# matched a key=value item
key, _, val = m.groups()[1:]
if val.startswith('"""') or val.startswith("'''"):
# triple quoted - may be a multiline value
val, index = multiline(flines, val, index, maxline)
addict(config, key, val, parents, index)
else:
# no match
raise FileParseError(
'Invalid line ' + str(index + 1) + ': ' + line)
return config
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]