def test_inline_error_mismatched_quotes(self):
"""The inline function throws an error when you have the
%include statement with a value without the correct balance
for quotes, e.g. %include "abc.txt
"""
with self.assertRaises(ParsecError):
inline(lines=["%include 'abc.txt"],
dir_=None,
filename=None,
level=None)
def test_inline(self):
with tempfile.NamedTemporaryFile() as tf:
filename = tf.name
file_lines = ["#!jinja2", "[section]", "value 1"]
file_lines_with_include = file_lines + [
"%include '{0}'".format(filename)
]
# same as before as there was no include lines
r = inline(lines=file_lines,
dir_=os.path.dirname(tf.name),
filename=filename)
self.assertEqual(file_lines, r)
# here the include line is removed, so the value returned
# is still file_lines, not file_lines_with_include
r = inline(lines=file_lines_with_include,
dir_=os.path.dirname(tf.name),
filename=filename)
self.assertEqual(file_lines, r)
# the for_grep adds some marks helpful for when grep'ing the file
r = inline(lines=file_lines_with_include,
dir_=os.path.dirname(tf.name),
filename=filename,
for_grep=True)
expected = file_lines + [
"#++++ START INLINED INCLUDE FILE {0}".format(filename),
"#++++ END INLINED INCLUDE FILE {0}".format(filename)
]
self.assertEqual(expected, r)
# for_edit would call the backup function, which triggers file
# system operations. So we avoid testing that option here.
# test that whatever is in the included file appears in the output
tf.write("[section2]".encode())
tf.flush()
r = inline(lines=file_lines_with_include,
dir_=os.path.dirname(tf.name),
filename=filename)
expected = file_lines + ["[section2]"]
self.assertEqual(expected, r)
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def main(parser, options, reg, *patterns):
workflow, flow_file = parse_workflow_arg(options, reg)
# cylc search WORKFLOW PATTERN
pattern = '|'.join(patterns)
workflowdir = os.path.dirname(flow_file)
if os.path.isfile(flow_file):
h = open(flow_file, 'r')
lines = h.readlines()
h.close()
lines = inline(lines, workflowdir, flow_file, for_grep=True)
else:
parser.error(f"File not found: {flow_file}")
sections = deque(['(top)'])
line_count = 1
inc_file = None
in_include_file = False
prev_section_key = None
prev_file = None
for line in lines:
m = re.match(r'^#\+\+\+\+ START INLINED INCLUDE FILE ([\w/\.\-]+)',
line)
if m:
inc_file = m.groups()[0]
in_include_file = True
inc_line_count = 0
continue
if not in_include_file:
line_count += 1
else:
inc_line_count += 1
m = re.match(r'^#\+\+\+\+ END INLINED INCLUDE FILE ' + inc_file,
line)
if m:
in_include_file = False
inc_file = None
continue
m = re.match(r'\s*(\[+\s*.+\s*\]+)', line)
if m:
# new section heading detected
heading = m.groups()[0]
level = section_level(heading)
# unwind to the current section level
while len(sections) > level - 1:
sections.pop()
sections.append(heading)
continue
if re.search(pattern, line):
# Found a pattern match.
# Print the file name
if in_include_file:
curr_file = os.path.join(workflowdir, inc_file)
line_no = inc_line_count
else:
curr_file = flow_file
line_no = line_count
if curr_file != prev_file:
prev_file = curr_file
print("\nFILE:", curr_file)
# Print the nested section headings
section_key = '->'.join(sections)
if section_key != prev_section_key:
prev_section_key = section_key
print(' SECTION:', section_key)
# Print the pattern match, with line number
print(' (' + str(line_no) + '):', line.rstrip('\n'))
if not options.search_bin:
sys.exit(0)
# search files in workflow bin directory
bin_ = os.path.join(workflowdir, 'bin')
if not os.path.isdir(bin_):
print("\nWorkflow " + workflow + " has no bin directory",
file=sys.stderr)
sys.exit(0)
for name in os.listdir(bin_):
if name.startswith('.'):
# skip hidden dot-files
# (e.g. vim editor temporary files)
continue
new_file = True
try:
h = open(os.path.join(bin_, name), 'r')
except IOError as exc:
# e.g. there's a sub-directory under bin; ignore it.
print('Unable to open file ' + os.path.join(bin_, name),
file=sys.stderr)
print(exc, file=sys.stderr)
continue
contents = h.readlines()
h.close()
count = 0
for line in contents:
line = line.rstrip('\n')
count += 1
if re.search(pattern, line):
if new_file:
print('\nFILE:', os.path.join(bin_, name))
new_file = False
print(' (' + str(count) + '): ' + line)
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
extra_vars = process_plugins(Path(fpath).parent)
if not template_vars:
template_vars = {}
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(flines,
fdir,
fpath,
False,
viewcfg=viewcfg,
for_edit=asedit)
template_vars['CYLC_VERSION'] = __version__
# Push template_vars into extra_vars so that duplicates come from
# template_vars.
if extra_vars['templating_detected'] is not None:
will_be_overwritten = (template_vars.keys()
& extra_vars['template_variables'].keys())
for key in will_be_overwritten:
LOG.warning(
f'Overriding {key}: {extra_vars["template_variables"][key]} ->'
f' {template_vars[key]}')
extra_vars['template_variables'].update(template_vars)
template_vars = extra_vars['template_variables']
template_vars['CYLC_TEMPLATE_VARS'] = template_vars
# process with EmPy
if do_empy:
if (extra_vars['templating_detected'] == 'empy'
and not re.match(r'^#![Ee]m[Pp]y\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!empy')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if (extra_vars['templating_detected'] == 'jinja2'
and not re.match(r'^#![jJ]inja2\s*', flines[0])):
if not re.match(r'^#!', flines[0]):
flines.insert(0, '#!jinja2')
else:
raise FileParseError(
"Plugins set templating engine = "
f"{extra_vars['templating_detected']}"
f" which does not match {flines[0]} set in flow.cylc.")
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def read_and_proc(fpath, template_vars=None, viewcfg=None, opts=None):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
workflow_lib_python = os.path.join(fdir, "lib", "python")
if (
os.path.isdir(workflow_lib_python)
and workflow_lib_python not in sys.path
):
sys.path.append(workflow_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
extra_vars = process_plugins(Path(fpath).parent, opts)
if not template_vars:
template_vars = {}
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, viewcfg=viewcfg)
template_vars['CYLC_VERSION'] = __version__
template_vars = merge_template_vars(template_vars, extra_vars)
template_vars['CYLC_TEMPLATE_VARS'] = template_vars
# Fail if templating_detected ≠ hashbang
process_with = hashbang_and_plugin_templating_clash(
extra_vars['templating_detected'], flines
)
# process with EmPy
if do_empy:
if (
extra_vars['templating_detected'] == 'empy' and
not process_with and
process_with != 'empy'
):
flines.insert(0, '#!empy')
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except ImportError:
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(
flines, fdir, template_vars
)
# process with Jinja2
if do_jinja2:
if (
extra_vars['templating_detected'] == 'jinja2' and
not process_with and
process_with != 'jinja2'
):
flines.insert(0, '#!jinja2')
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except ImportError:
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(
flines, fdir, template_vars
)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]
def main(parser, options, *args):
flow_file = parse_suite_arg(options, args[0])[1]
if options.geditor:
editor = glbl_cfg().get(['editors', 'gui'])
else:
editor = glbl_cfg().get(['editors', 'terminal'])
suitedir = os.path.dirname(flow_file)
if options.cleanup:
# remove backup files left by inlined editing sessions
cleanup(suitedir)
sys.exit(0)
if not options.inline:
# plain old editing.
# move to suite def dir
os.chdir(suitedir)
# edit the flow.cylc file
if not os.path.isfile(flow_file):
raise UserInputError(f'file not found: {flow_file}')
# in case editor has options, e.g. 'emacs -nw':
command_list = re.split(' ', editor)
command_list.append(flow_file)
command = ' '.join(command_list)
# THIS BLOCKS UNTIL THE COMMAND COMPLETES
retcode = call(command_list)
if retcode != 0:
# the command returned non-zero exist status
raise CylcError(f'{command} failed: {retcode}')
# !!!EDITING FINISHED!!!
sys.exit(0)
# read the flow.cylc file
if os.path.isfile(flow_file):
# back up the original
backup(flow_file)
# record original modtime
modtimes[flow_file] = os.stat(flow_file).st_mtime
# read the file
h = open(flow_file, 'r')
lines0 = h.readlines()
h.close()
if lines0[0].startswith('# !WARNING! CYLC EDIT INLINED'):
print('WARNING: RECOVERING A PREVIOUSLY INLINED FILE')
recovery = True
lines = lines0
else:
recovery = False
lines = inline(lines0, suitedir, flow_file, for_edit=True)
else:
parser.error(f"File not found: {flow_file}")
lines = [i.rstrip() for i in lines]
# overwrite the (now backed up) original with the inlined file:
h = open(flow_file, 'wb')
for line in lines:
h.write((line + '\n').encode())
h.close()
print('PRE-EDIT BACKUPS:')
for file in backups:
src = re.sub(suitedir + '/', '', file)
dst = re.sub(suitedir + '/', '', backups[file])
print(' + ' + src + ' ---> ' + dst)
# in case editor has options, e.g. 'emacs -nw':
command_list = re.split(' ', editor)
command_list.append(flow_file)
command = ' '.join(command_list)
# THIS BLOCKS UNTIL THE COMMAND COMPLETES
retcode = call(command_list)
if retcode != 0:
# the command returned non-zero exist status
raise CylcError(f'{command} failed: {retcode}')
print('EDITING DONE')
# Now back up the inlined file in case of absolute disaster, so as the
# user or his editor corrupting the inlined-include-file marker lines.
inlined_flow_file_backup = (
suitedir + '/flow.cylc.INLINED.EDIT.' +
get_current_time_string(override_use_utc=True, use_basic_format=True))
copy(flow_file, inlined_flow_file_backup)
# read in the edited inlined file
h = open(flow_file, 'r')
lines = h.readlines()
h.close()
# split it back into separate files
split_file(suitedir, lines, flow_file, recovery)
print(f' + edited: {flow_file}')
print(f' + backup: {inlined_flow_file_backup}')
print('INCLUDE-FILES WRITTEN:')
for file in newfiles:
f = re.sub(suitedir + '/', '', file)
if re.search(r'\.EDIT\.NEW\.', f):
print(' + ' + f + ' (!!! WARNING: original changed on disk !!!)')
else:
print(' + ' + f)
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
"""
Read a cylc parsec config file (at fpath), inline any include files,
process with Jinja2, and concatenate continuation lines.
Jinja2 processing must be done before concatenation - it could be
used to generate continuation lines.
"""
fdir = os.path.dirname(fpath)
# Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
suite_lib_python = os.path.join(fdir, "lib", "python")
if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
sys.path.append(suite_lib_python)
LOG.debug('Reading file %s', fpath)
# read the file into a list, stripping newlines
with open(fpath) as f:
flines = [line.rstrip('\n') for line in f]
do_inline = True
do_empy = True
do_jinja2 = True
do_contin = True
if viewcfg:
if not viewcfg['empy']:
do_empy = False
if not viewcfg['jinja2']:
do_jinja2 = False
if not viewcfg['contin']:
do_contin = False
if not viewcfg['inline']:
do_inline = False
# inline any cylc include-files
if do_inline:
flines = inline(
flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)
# process with EmPy
if do_empy:
if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
LOG.debug('Processing with EmPy')
try:
from cylc.flow.parsec.empysupport import empyprocess
except (ImportError, ModuleNotFoundError):
raise ParsecError('EmPy Python package must be installed '
'to process file: ' + fpath)
flines = empyprocess(flines, fdir, template_vars)
# process with Jinja2
if do_jinja2:
if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
LOG.debug('Processing with Jinja2')
try:
from cylc.flow.parsec.jinja2support import jinja2process
except (ImportError, ModuleNotFoundError):
raise ParsecError('Jinja2 Python package must be installed '
'to process file: ' + fpath)
flines = jinja2process(flines, fdir, template_vars)
# concatenate continuation lines
if do_contin:
flines = _concatenate(flines)
# return rstripped lines
return [fl.rstrip() for fl in flines]