Пример #1
0
 def test_inline_error_mismatched_quotes(self):
     """The inline function throws an error when you have the
     %include statement with a value without the correct balance
     for quotes, e.g. %include "abc.txt
     """
     with self.assertRaises(ParsecError):
         inline(lines=["%include 'abc.txt"],
                dir_=None,
                filename=None,
                level=None)
Пример #2
0
    def test_inline(self):
        with tempfile.NamedTemporaryFile() as tf:
            filename = tf.name
            file_lines = ["#!jinja2", "[section]", "value 1"]
            file_lines_with_include = file_lines + [
                "%include '{0}'".format(filename)
            ]

            # same as before as there was no include lines
            r = inline(lines=file_lines,
                       dir_=os.path.dirname(tf.name),
                       filename=filename)
            self.assertEqual(file_lines, r)

            # here the include line is removed, so the value returned
            # is still file_lines, not file_lines_with_include
            r = inline(lines=file_lines_with_include,
                       dir_=os.path.dirname(tf.name),
                       filename=filename)
            self.assertEqual(file_lines, r)

            # the for_grep adds some marks helpful for when grep'ing the file
            r = inline(lines=file_lines_with_include,
                       dir_=os.path.dirname(tf.name),
                       filename=filename,
                       for_grep=True)
            expected = file_lines + [
                "#++++ START INLINED INCLUDE FILE {0}".format(filename),
                "#++++ END INLINED INCLUDE FILE {0}".format(filename)
            ]
            self.assertEqual(expected, r)

            # for_edit would call the backup function, which triggers file
            # system operations. So we avoid testing that option here.

            # test that whatever is in the included file appears in the output
            tf.write("[section2]".encode())
            tf.flush()
            r = inline(lines=file_lines_with_include,
                       dir_=os.path.dirname(tf.name),
                       filename=filename)
            expected = file_lines + ["[section2]"]
            self.assertEqual(expected, r)
Пример #3
0
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
    """
    Read a cylc parsec config file (at fpath), inline any include files,
    process with Jinja2, and concatenate continuation lines.
    Jinja2 processing must be done before concatenation - it could be
    used to generate continuation lines.
    """
    fdir = os.path.dirname(fpath)

    # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
    suite_lib_python = os.path.join(fdir, "lib", "python")
    if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
        sys.path.append(suite_lib_python)

    LOG.debug('Reading file %s', fpath)

    # read the file into a list, stripping newlines
    with open(fpath) as f:
        flines = [line.rstrip('\n') for line in f]

    do_inline = True
    do_empy = True
    do_jinja2 = True
    do_contin = True
    if viewcfg:
        if not viewcfg['empy']:
            do_empy = False
        if not viewcfg['jinja2']:
            do_jinja2 = False
        if not viewcfg['contin']:
            do_contin = False
        if not viewcfg['inline']:
            do_inline = False

    # inline any cylc include-files
    if do_inline:
        flines = inline(
            flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)

    # process with EmPy
    if do_empy:
        if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
            LOG.debug('Processing with EmPy')
            try:
                from cylc.flow.parsec.empysupport import empyprocess
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('EmPy Python package must be installed '
                                  'to process file: ' + fpath)
            flines = empyprocess(flines, fdir, template_vars)

    # process with Jinja2
    if do_jinja2:
        if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
            LOG.debug('Processing with Jinja2')
            try:
                from cylc.flow.parsec.jinja2support import jinja2process
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('Jinja2 Python package must be installed '
                                  'to process file: ' + fpath)
            flines = jinja2process(flines, fdir, template_vars)

    # concatenate continuation lines
    if do_contin:
        flines = _concatenate(flines)

    # return rstripped lines
    return [fl.rstrip() for fl in flines]
Пример #4
0
def main(parser, options, reg, *patterns):
    workflow, flow_file = parse_workflow_arg(options, reg)

    # cylc search WORKFLOW PATTERN
    pattern = '|'.join(patterns)

    workflowdir = os.path.dirname(flow_file)

    if os.path.isfile(flow_file):
        h = open(flow_file, 'r')
        lines = h.readlines()
        h.close()
        lines = inline(lines, workflowdir, flow_file, for_grep=True)
    else:
        parser.error(f"File not found: {flow_file}")

    sections = deque(['(top)'])

    line_count = 1
    inc_file = None
    in_include_file = False
    prev_section_key = None
    prev_file = None

    for line in lines:

        m = re.match(r'^#\+\+\+\+ START INLINED INCLUDE FILE ([\w/\.\-]+)',
                     line)
        if m:
            inc_file = m.groups()[0]
            in_include_file = True
            inc_line_count = 0
            continue

        if not in_include_file:
            line_count += 1
        else:
            inc_line_count += 1
            m = re.match(r'^#\+\+\+\+ END INLINED INCLUDE FILE ' + inc_file,
                         line)
            if m:
                in_include_file = False
                inc_file = None
                continue

        m = re.match(r'\s*(\[+\s*.+\s*\]+)', line)
        if m:
            # new section heading detected
            heading = m.groups()[0]
            level = section_level(heading)
            # unwind to the current section level
            while len(sections) > level - 1:
                sections.pop()
            sections.append(heading)
            continue

        if re.search(pattern, line):
            # Found a pattern match.

            # Print the file name
            if in_include_file:
                curr_file = os.path.join(workflowdir, inc_file)
                line_no = inc_line_count
            else:
                curr_file = flow_file
                line_no = line_count

            if curr_file != prev_file:
                prev_file = curr_file
                print("\nFILE:", curr_file)

            # Print the nested section headings
            section_key = '->'.join(sections)
            if section_key != prev_section_key:
                prev_section_key = section_key
                print('   SECTION:', section_key)

            # Print the pattern match, with line number
            print('      (' + str(line_no) + '):', line.rstrip('\n'))

    if not options.search_bin:
        sys.exit(0)

    # search files in workflow bin directory
    bin_ = os.path.join(workflowdir, 'bin')
    if not os.path.isdir(bin_):
        print("\nWorkflow " + workflow + " has no bin directory",
              file=sys.stderr)
        sys.exit(0)

    for name in os.listdir(bin_):
        if name.startswith('.'):
            # skip hidden dot-files
            # (e.g. vim editor temporary files)
            continue
        new_file = True
        try:
            h = open(os.path.join(bin_, name), 'r')
        except IOError as exc:
            # e.g. there's a sub-directory under bin; ignore it.
            print('Unable to open file ' + os.path.join(bin_, name),
                  file=sys.stderr)
            print(exc, file=sys.stderr)
            continue
        contents = h.readlines()
        h.close()

        count = 0
        for line in contents:
            line = line.rstrip('\n')
            count += 1
            if re.search(pattern, line):
                if new_file:
                    print('\nFILE:', os.path.join(bin_, name))
                    new_file = False
                print('   (' + str(count) + '): ' + line)
Пример #5
0
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
    """
    Read a cylc parsec config file (at fpath), inline any include files,
    process with Jinja2, and concatenate continuation lines.
    Jinja2 processing must be done before concatenation - it could be
    used to generate continuation lines.
    """
    fdir = os.path.dirname(fpath)

    # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
    suite_lib_python = os.path.join(fdir, "lib", "python")
    if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
        sys.path.append(suite_lib_python)

    LOG.debug('Reading file %s', fpath)

    # read the file into a list, stripping newlines
    with open(fpath) as f:
        flines = [line.rstrip('\n') for line in f]

    do_inline = True
    do_empy = True
    do_jinja2 = True
    do_contin = True

    extra_vars = process_plugins(Path(fpath).parent)

    if not template_vars:
        template_vars = {}

    if viewcfg:
        if not viewcfg['empy']:
            do_empy = False
        if not viewcfg['jinja2']:
            do_jinja2 = False
        if not viewcfg['contin']:
            do_contin = False
        if not viewcfg['inline']:
            do_inline = False

    # inline any cylc include-files
    if do_inline:
        flines = inline(flines,
                        fdir,
                        fpath,
                        False,
                        viewcfg=viewcfg,
                        for_edit=asedit)

    template_vars['CYLC_VERSION'] = __version__

    # Push template_vars into extra_vars so that duplicates come from
    # template_vars.
    if extra_vars['templating_detected'] is not None:
        will_be_overwritten = (template_vars.keys()
                               & extra_vars['template_variables'].keys())
        for key in will_be_overwritten:
            LOG.warning(
                f'Overriding {key}: {extra_vars["template_variables"][key]} ->'
                f' {template_vars[key]}')
        extra_vars['template_variables'].update(template_vars)
        template_vars = extra_vars['template_variables']

    template_vars['CYLC_TEMPLATE_VARS'] = template_vars

    # process with EmPy
    if do_empy:
        if (extra_vars['templating_detected'] == 'empy'
                and not re.match(r'^#![Ee]m[Pp]y\s*', flines[0])):
            if not re.match(r'^#!', flines[0]):
                flines.insert(0, '#!empy')
            else:
                raise FileParseError(
                    "Plugins set templating engine = "
                    f"{extra_vars['templating_detected']}"
                    f" which does not match {flines[0]} set in flow.cylc.")
        if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
            LOG.debug('Processing with EmPy')
            try:
                from cylc.flow.parsec.empysupport import empyprocess
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('EmPy Python package must be installed '
                                  'to process file: ' + fpath)
            flines = empyprocess(flines, fdir, template_vars)

    # process with Jinja2
    if do_jinja2:
        if (extra_vars['templating_detected'] == 'jinja2'
                and not re.match(r'^#![jJ]inja2\s*', flines[0])):
            if not re.match(r'^#!', flines[0]):
                flines.insert(0, '#!jinja2')
            else:
                raise FileParseError(
                    "Plugins set templating engine = "
                    f"{extra_vars['templating_detected']}"
                    f" which does not match {flines[0]} set in flow.cylc.")
        if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
            LOG.debug('Processing with Jinja2')
            try:
                from cylc.flow.parsec.jinja2support import jinja2process
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('Jinja2 Python package must be installed '
                                  'to process file: ' + fpath)
            flines = jinja2process(flines, fdir, template_vars)

    # concatenate continuation lines
    if do_contin:
        flines = _concatenate(flines)

    # return rstripped lines
    return [fl.rstrip() for fl in flines]
Пример #6
0
def read_and_proc(fpath, template_vars=None, viewcfg=None, opts=None):
    """
    Read a cylc parsec config file (at fpath), inline any include files,
    process with Jinja2, and concatenate continuation lines.
    Jinja2 processing must be done before concatenation - it could be
    used to generate continuation lines.
    """

    fdir = os.path.dirname(fpath)

    # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
    workflow_lib_python = os.path.join(fdir, "lib", "python")
    if (
        os.path.isdir(workflow_lib_python)
        and workflow_lib_python not in sys.path
    ):
        sys.path.append(workflow_lib_python)

    LOG.debug('Reading file %s', fpath)

    # read the file into a list, stripping newlines
    with open(fpath) as f:
        flines = [line.rstrip('\n') for line in f]

    do_inline = True
    do_empy = True
    do_jinja2 = True
    do_contin = True

    extra_vars = process_plugins(Path(fpath).parent, opts)

    if not template_vars:
        template_vars = {}

    if viewcfg:
        if not viewcfg['empy']:
            do_empy = False
        if not viewcfg['jinja2']:
            do_jinja2 = False
        if not viewcfg['contin']:
            do_contin = False
        if not viewcfg['inline']:
            do_inline = False

    # inline any cylc include-files
    if do_inline:
        flines = inline(
            flines, fdir, fpath, viewcfg=viewcfg)

    template_vars['CYLC_VERSION'] = __version__

    template_vars = merge_template_vars(template_vars, extra_vars)

    template_vars['CYLC_TEMPLATE_VARS'] = template_vars

    # Fail if templating_detected ≠ hashbang
    process_with = hashbang_and_plugin_templating_clash(
        extra_vars['templating_detected'], flines
    )
    # process with EmPy
    if do_empy:
        if (
            extra_vars['templating_detected'] == 'empy' and
            not process_with and
            process_with != 'empy'
        ):
            flines.insert(0, '#!empy')

        if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
            LOG.debug('Processing with EmPy')
            try:
                from cylc.flow.parsec.empysupport import empyprocess
            except ImportError:
                raise ParsecError('EmPy Python package must be installed '
                                  'to process file: ' + fpath)
            flines = empyprocess(
                flines, fdir, template_vars
            )

    # process with Jinja2
    if do_jinja2:
        if (
            extra_vars['templating_detected'] == 'jinja2' and
            not process_with and
            process_with != 'jinja2'
        ):
            flines.insert(0, '#!jinja2')

        if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
            LOG.debug('Processing with Jinja2')
            try:
                from cylc.flow.parsec.jinja2support import jinja2process
            except ImportError:
                raise ParsecError('Jinja2 Python package must be installed '
                                  'to process file: ' + fpath)
            flines = jinja2process(
                flines, fdir, template_vars
            )

    # concatenate continuation lines
    if do_contin:
        flines = _concatenate(flines)

    # return rstripped lines
    return [fl.rstrip() for fl in flines]
Пример #7
0
def main(parser, options, *args):
    flow_file = parse_suite_arg(options, args[0])[1]

    if options.geditor:
        editor = glbl_cfg().get(['editors', 'gui'])
    else:
        editor = glbl_cfg().get(['editors', 'terminal'])

    suitedir = os.path.dirname(flow_file)

    if options.cleanup:
        # remove backup files left by inlined editing sessions
        cleanup(suitedir)
        sys.exit(0)

    if not options.inline:
        # plain old editing.
        # move to suite def dir
        os.chdir(suitedir)

        # edit the flow.cylc file
        if not os.path.isfile(flow_file):
            raise UserInputError(f'file not found: {flow_file}')

        # in case editor has options, e.g. 'emacs -nw':
        command_list = re.split(' ', editor)
        command_list.append(flow_file)
        command = ' '.join(command_list)
        # THIS BLOCKS UNTIL THE COMMAND COMPLETES
        retcode = call(command_list)
        if retcode != 0:
            # the command returned non-zero exist status
            raise CylcError(f'{command} failed: {retcode}')

        # !!!EDITING FINISHED!!!
        sys.exit(0)

    # read the flow.cylc file
    if os.path.isfile(flow_file):
        # back up the original
        backup(flow_file)
        # record original modtime
        modtimes[flow_file] = os.stat(flow_file).st_mtime
        # read the file
        h = open(flow_file, 'r')
        lines0 = h.readlines()
        h.close()
        if lines0[0].startswith('# !WARNING! CYLC EDIT INLINED'):
            print('WARNING: RECOVERING A PREVIOUSLY INLINED FILE')
            recovery = True
            lines = lines0
        else:
            recovery = False
            lines = inline(lines0, suitedir, flow_file, for_edit=True)
    else:
        parser.error(f"File not found: {flow_file}")

    lines = [i.rstrip() for i in lines]

    # overwrite the (now backed up) original with the inlined file:
    h = open(flow_file, 'wb')
    for line in lines:
        h.write((line + '\n').encode())
    h.close()

    print('PRE-EDIT BACKUPS:')
    for file in backups:
        src = re.sub(suitedir + '/', '', file)
        dst = re.sub(suitedir + '/', '', backups[file])
        print(' + ' + src + ' ---> ' + dst)

    # in case editor has options, e.g. 'emacs -nw':
    command_list = re.split(' ', editor)
    command_list.append(flow_file)
    command = ' '.join(command_list)
    # THIS BLOCKS UNTIL THE COMMAND COMPLETES
    retcode = call(command_list)
    if retcode != 0:
        # the command returned non-zero exist status
        raise CylcError(f'{command} failed: {retcode}')
    print('EDITING DONE')

    # Now back up the inlined file in case of absolute disaster, so as the
    # user or his editor corrupting the inlined-include-file marker lines.
    inlined_flow_file_backup = (
        suitedir + '/flow.cylc.INLINED.EDIT.' +
        get_current_time_string(override_use_utc=True, use_basic_format=True))
    copy(flow_file, inlined_flow_file_backup)

    # read in the edited inlined file
    h = open(flow_file, 'r')
    lines = h.readlines()
    h.close()

    # split it back into separate files
    split_file(suitedir, lines, flow_file, recovery)

    print(f' + edited: {flow_file}')
    print(f' + backup: {inlined_flow_file_backup}')
    print('INCLUDE-FILES WRITTEN:')
    for file in newfiles:
        f = re.sub(suitedir + '/', '', file)
        if re.search(r'\.EDIT\.NEW\.', f):
            print(' + ' + f + ' (!!! WARNING: original changed on disk !!!)')
        else:
            print(' + ' + f)
Пример #8
0
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
    """
    Read a cylc parsec config file (at fpath), inline any include files,
    process with Jinja2, and concatenate continuation lines.
    Jinja2 processing must be done before concatenation - it could be
    used to generate continuation lines.
    """
    fdir = os.path.dirname(fpath)

    # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
    suite_lib_python = os.path.join(fdir, "lib", "python")
    if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
        sys.path.append(suite_lib_python)

    LOG.debug('Reading file %s', fpath)

    # read the file into a list, stripping newlines
    with open(fpath) as f:
        flines = [line.rstrip('\n') for line in f]

    do_inline = True
    do_empy = True
    do_jinja2 = True
    do_contin = True
    if viewcfg:
        if not viewcfg['empy']:
            do_empy = False
        if not viewcfg['jinja2']:
            do_jinja2 = False
        if not viewcfg['contin']:
            do_contin = False
        if not viewcfg['inline']:
            do_inline = False

    # inline any cylc include-files
    if do_inline:
        flines = inline(
            flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)

    # process with EmPy
    if do_empy:
        if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
            LOG.debug('Processing with EmPy')
            try:
                from cylc.flow.parsec.empysupport import empyprocess
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('EmPy Python package must be installed '
                                  'to process file: ' + fpath)
            flines = empyprocess(flines, fdir, template_vars)

    # process with Jinja2
    if do_jinja2:
        if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
            LOG.debug('Processing with Jinja2')
            try:
                from cylc.flow.parsec.jinja2support import jinja2process
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('Jinja2 Python package must be installed '
                                  'to process file: ' + fpath)
            flines = jinja2process(flines, fdir, template_vars)

    # concatenate continuation lines
    if do_contin:
        flines = _concatenate(flines)

    # return rstripped lines
    return [fl.rstrip() for fl in flines]