Beispiel #1
0
 def test_parsec_error_msg(self):
     parsec_error = ParsecError()
     self.assertEqual('', str(parsec_error))
     parsec_error = ParsecError('foo')
     self.assertEqual('foo', str(parsec_error))
     parsec_error = ParsecError('foo', 'bar', 'baz')
     self.assertEqual('foo bar baz', str(parsec_error))
Beispiel #2
0
def process_plugins(fpath):
    # Load Rose Vars, if a ``rose-suite.conf`` file is present.
    extra_vars = {
        'env': {},
        'template_variables': {},
        'templating_detected': None
    }
    for entry_point in pkg_resources.iter_entry_points('cylc.pre_configure'):
        try:
            plugin_result = entry_point.resolve()(srcdir=fpath)
        except Exception as exc:
            # NOTE: except Exception (purposefully vague)
            # this is to separate plugin from core Cylc errors
            raise PluginError('cylc.pre_configure', entry_point.name,
                              exc) from None
        for section in ['env', 'template_variables']:
            if section in plugin_result and plugin_result[section] is not None:
                # Raise error if multiple plugins try to update the same keys.
                section_update = plugin_result.get(section, {})
                keys_collision = (extra_vars[section].keys()
                                  & section_update.keys())
                if keys_collision:
                    raise ParsecError(
                        f"{entry_point.name} is trying to alter "
                        f"[{section}]{', '.join(sorted(keys_collision))}.")
                extra_vars[section].update(section_update)

        if ('templating_detected' in plugin_result
                and plugin_result['templating_detected'] is not None
                and extra_vars['templating_detected'] is not None
                and extra_vars['templating_detected'] !=
                plugin_result['templating_detected']):
            # Don't allow subsequent plugins with different templating_detected
            raise ParsecError("Can't merge templating languages "
                              f"{extra_vars['templating_detected']} and "
                              f"{plugin_result['templating_detected']}")
        elif ('templating_detected' in plugin_result
              and plugin_result['templating_detected'] is not None):
            extra_vars['templating_detected'] = plugin_result[
                'templating_detected']

    return extra_vars
Beispiel #3
0
 def checkspec(spec_root, parents=None):
     """Check that the file spec is a nested dict of specifications"""
     stack = [[spec_root, []]]
     while stack:
         spec, parents = stack.pop()
         for key, value in spec.items():
             pars = parents + [key]
             if isinstance(value, dict):
                 stack.append([value, pars])
             elif not isinstance(value, list):
                 raise ParsecError("Illegal file spec item: %s" %
                                   itemstr(pars, repr(value)))
Beispiel #4
0
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
    """
    Read a cylc parsec config file (at fpath), inline any include files,
    process with Jinja2, and concatenate continuation lines.
    Jinja2 processing must be done before concatenation - it could be
    used to generate continuation lines.
    """
    fdir = os.path.dirname(fpath)

    # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
    suite_lib_python = os.path.join(fdir, "lib", "python")
    if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
        sys.path.append(suite_lib_python)

    LOG.debug('Reading file %s', fpath)

    # read the file into a list, stripping newlines
    with open(fpath) as f:
        flines = [line.rstrip('\n') for line in f]

    do_inline = True
    do_empy = True
    do_jinja2 = True
    do_contin = True
    if viewcfg:
        if not viewcfg['empy']:
            do_empy = False
        if not viewcfg['jinja2']:
            do_jinja2 = False
        if not viewcfg['contin']:
            do_contin = False
        if not viewcfg['inline']:
            do_inline = False

    # inline any cylc include-files
    if do_inline:
        flines = inline(
            flines, fdir, fpath, False, viewcfg=viewcfg, for_edit=asedit)

    # process with EmPy
    if do_empy:
        if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
            LOG.debug('Processing with EmPy')
            try:
                from cylc.flow.parsec.empysupport import empyprocess
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('EmPy Python package must be installed '
                                  'to process file: ' + fpath)
            flines = empyprocess(flines, fdir, template_vars)

    # process with Jinja2
    if do_jinja2:
        if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
            LOG.debug('Processing with Jinja2')
            try:
                from cylc.flow.parsec.jinja2support import jinja2process
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('Jinja2 Python package must be installed '
                                  'to process file: ' + fpath)
            flines = jinja2process(flines, fdir, template_vars)

    # concatenate continuation lines
    if do_contin:
        flines = _concatenate(flines)

    # return rstripped lines
    return [fl.rstrip() for fl in flines]
Beispiel #5
0
 def test_parsec_error_str(self):
     msg = 'Turbulence!'
     parsec_error = ParsecError(msg)
     self.assertEqual(msg, str(parsec_error))
Beispiel #6
0
def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
    """
    Read a cylc parsec config file (at fpath), inline any include files,
    process with Jinja2, and concatenate continuation lines.
    Jinja2 processing must be done before concatenation - it could be
    used to generate continuation lines.
    """
    fdir = os.path.dirname(fpath)

    # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
    suite_lib_python = os.path.join(fdir, "lib", "python")
    if os.path.isdir(suite_lib_python) and suite_lib_python not in sys.path:
        sys.path.append(suite_lib_python)

    LOG.debug('Reading file %s', fpath)

    # read the file into a list, stripping newlines
    with open(fpath) as f:
        flines = [line.rstrip('\n') for line in f]

    do_inline = True
    do_empy = True
    do_jinja2 = True
    do_contin = True

    extra_vars = process_plugins(Path(fpath).parent)

    if not template_vars:
        template_vars = {}

    if viewcfg:
        if not viewcfg['empy']:
            do_empy = False
        if not viewcfg['jinja2']:
            do_jinja2 = False
        if not viewcfg['contin']:
            do_contin = False
        if not viewcfg['inline']:
            do_inline = False

    # inline any cylc include-files
    if do_inline:
        flines = inline(flines,
                        fdir,
                        fpath,
                        False,
                        viewcfg=viewcfg,
                        for_edit=asedit)

    template_vars['CYLC_VERSION'] = __version__

    # Push template_vars into extra_vars so that duplicates come from
    # template_vars.
    if extra_vars['templating_detected'] is not None:
        will_be_overwritten = (template_vars.keys()
                               & extra_vars['template_variables'].keys())
        for key in will_be_overwritten:
            LOG.warning(
                f'Overriding {key}: {extra_vars["template_variables"][key]} ->'
                f' {template_vars[key]}')
        extra_vars['template_variables'].update(template_vars)
        template_vars = extra_vars['template_variables']

    template_vars['CYLC_TEMPLATE_VARS'] = template_vars

    # process with EmPy
    if do_empy:
        if (extra_vars['templating_detected'] == 'empy'
                and not re.match(r'^#![Ee]m[Pp]y\s*', flines[0])):
            if not re.match(r'^#!', flines[0]):
                flines.insert(0, '#!empy')
            else:
                raise FileParseError(
                    "Plugins set templating engine = "
                    f"{extra_vars['templating_detected']}"
                    f" which does not match {flines[0]} set in flow.cylc.")
        if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
            LOG.debug('Processing with EmPy')
            try:
                from cylc.flow.parsec.empysupport import empyprocess
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('EmPy Python package must be installed '
                                  'to process file: ' + fpath)
            flines = empyprocess(flines, fdir, template_vars)

    # process with Jinja2
    if do_jinja2:
        if (extra_vars['templating_detected'] == 'jinja2'
                and not re.match(r'^#![jJ]inja2\s*', flines[0])):
            if not re.match(r'^#!', flines[0]):
                flines.insert(0, '#!jinja2')
            else:
                raise FileParseError(
                    "Plugins set templating engine = "
                    f"{extra_vars['templating_detected']}"
                    f" which does not match {flines[0]} set in flow.cylc.")
        if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
            LOG.debug('Processing with Jinja2')
            try:
                from cylc.flow.parsec.jinja2support import jinja2process
            except (ImportError, ModuleNotFoundError):
                raise ParsecError('Jinja2 Python package must be installed '
                                  'to process file: ' + fpath)
            flines = jinja2process(flines, fdir, template_vars)

    # concatenate continuation lines
    if do_contin:
        flines = _concatenate(flines)

    # return rstripped lines
    return [fl.rstrip() for fl in flines]
Beispiel #7
0
def read_and_proc(fpath, template_vars=None, viewcfg=None, opts=None):
    """
    Read a cylc parsec config file (at fpath), inline any include files,
    process with Jinja2, and concatenate continuation lines.
    Jinja2 processing must be done before concatenation - it could be
    used to generate continuation lines.
    """

    fdir = os.path.dirname(fpath)

    # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
    workflow_lib_python = os.path.join(fdir, "lib", "python")
    if (
        os.path.isdir(workflow_lib_python)
        and workflow_lib_python not in sys.path
    ):
        sys.path.append(workflow_lib_python)

    LOG.debug('Reading file %s', fpath)

    # read the file into a list, stripping newlines
    with open(fpath) as f:
        flines = [line.rstrip('\n') for line in f]

    do_inline = True
    do_empy = True
    do_jinja2 = True
    do_contin = True

    extra_vars = process_plugins(Path(fpath).parent, opts)

    if not template_vars:
        template_vars = {}

    if viewcfg:
        if not viewcfg['empy']:
            do_empy = False
        if not viewcfg['jinja2']:
            do_jinja2 = False
        if not viewcfg['contin']:
            do_contin = False
        if not viewcfg['inline']:
            do_inline = False

    # inline any cylc include-files
    if do_inline:
        flines = inline(
            flines, fdir, fpath, viewcfg=viewcfg)

    template_vars['CYLC_VERSION'] = __version__

    template_vars = merge_template_vars(template_vars, extra_vars)

    template_vars['CYLC_TEMPLATE_VARS'] = template_vars

    # Fail if templating_detected ≠ hashbang
    process_with = hashbang_and_plugin_templating_clash(
        extra_vars['templating_detected'], flines
    )
    # process with EmPy
    if do_empy:
        if (
            extra_vars['templating_detected'] == 'empy' and
            not process_with and
            process_with != 'empy'
        ):
            flines.insert(0, '#!empy')

        if flines and re.match(r'^#![Ee]m[Pp]y\s*', flines[0]):
            LOG.debug('Processing with EmPy')
            try:
                from cylc.flow.parsec.empysupport import empyprocess
            except ImportError:
                raise ParsecError('EmPy Python package must be installed '
                                  'to process file: ' + fpath)
            flines = empyprocess(
                flines, fdir, template_vars
            )

    # process with Jinja2
    if do_jinja2:
        if (
            extra_vars['templating_detected'] == 'jinja2' and
            not process_with and
            process_with != 'jinja2'
        ):
            flines.insert(0, '#!jinja2')

        if flines and re.match(r'^#![jJ]inja2\s*', flines[0]):
            LOG.debug('Processing with Jinja2')
            try:
                from cylc.flow.parsec.jinja2support import jinja2process
            except ImportError:
                raise ParsecError('Jinja2 Python package must be installed '
                                  'to process file: ' + fpath)
            flines = jinja2process(
                flines, fdir, template_vars
            )

    # concatenate continuation lines
    if do_contin:
        flines = _concatenate(flines)

    # return rstripped lines
    return [fl.rstrip() for fl in flines]
Beispiel #8
0
def process_plugins(fpath, opts):
    """Run a Cylc pre-configuration plugin.

    Plugins should return a dictionary containing:
        'env': A dictionary of environment variables.
        'template_variables': A dictionary of template variables.
        'templating_detected': Where the plugin identifies a templating
            language this is specified here. Expected values are ``jinja2``
            or ``empy``.

    args:
        fpath: Directory where the plugin will look for a config.
        opts: Command line options to be passed to the plugin.

    Returns: Dictionary in the form:
        extra_vars = {
            'env': {},
            'template_variables': {},
            'templating_detected': None
        }
    """
    # Set out blank dictionary for return:
    extra_vars = {
        'env': {},
        'template_variables': {},
        'templating_detected': None
    }

    # Run entry point pre_configure items, trying to merge values with each.:
    for entry_point in iter_entry_points(
        'cylc.pre_configure'
    ):
        try:
            # If you want it to work on sourcedirs you need to get the options
            # to here.
            plugin_result = entry_point.resolve()(
                srcdir=fpath, opts=opts
            )
        except Exception as exc:
            # NOTE: except Exception (purposefully vague)
            # this is to separate plugin from core Cylc errors
            raise PluginError(
                'cylc.pre_configure',
                entry_point.name,
                exc
            ) from None
        for section in ['env', 'template_variables']:
            if section in plugin_result and plugin_result[section] is not None:
                # Raise error if multiple plugins try to update the same keys.
                section_update = plugin_result.get(section, {})
                keys_collision = (
                    extra_vars[section].keys() & section_update.keys()
                )
                if keys_collision:
                    raise ParsecError(
                        f"{entry_point.name} is trying to alter "
                        f"[{section}]{', '.join(sorted(keys_collision))}."
                    )
                extra_vars[section].update(section_update)

        if (
            'templating_detected' in plugin_result and
            plugin_result['templating_detected'] is not None and
            extra_vars['templating_detected'] is not None and
            extra_vars['templating_detected'] !=
                plugin_result['templating_detected']
        ):
            # Don't allow subsequent plugins with different templating_detected
            raise ParsecError(
                "Can't merge templating languages "
                f"{extra_vars['templating_detected']} and "
                f"{plugin_result['templating_detected']}"
            )
        elif(
            'templating_detected' in plugin_result and
            plugin_result['templating_detected'] is not None
        ):
            extra_vars['templating_detected'] = plugin_result[
                'templating_detected'
            ]

    return extra_vars
Beispiel #9
0
def inline(lines,
           dir_,
           filename,
           for_grep=False,
           for_edit=False,
           viewcfg=None,
           level=None):
    """Recursive inlining of parsec include-files"""

    global flist
    if level is None:
        # avoid being affected by multiple *different* calls to this function
        flist = [filename]
    else:
        flist.append(filename)
    single = False
    mark = False
    label = False
    if viewcfg:
        mark = viewcfg['mark']
        single = viewcfg['single']
        label = viewcfg['label']
    else:
        viewcfg = {}

    global done
    global modtimes

    outf = []
    initial_line_index = 0

    if level is None:
        level = ''
        if for_edit:
            m = re.match('^(#![jJ]inja2)', lines[0])
            if m:
                outf.append(m.groups()[0])
                initial_line_index = 1
            outf.append(
                """# !WARNING! CYLC EDIT INLINED (DO NOT MODIFY THIS LINE).
# !WARNING! This is an inlined parsec config file; include-files are split
# !WARNING! out again on exiting the edit session.  If you are editing
# !WARNING! this file manually then a previous inlined session may have
# !WARNING! crashed; exit now and use 'cylc edit -i' to recover (this
# !WARNING! will split the file up again on exiting).""")

    else:
        if mark:
            level += '!'
        elif for_edit:
            level += ' > '

    if for_edit:
        msg = ' (DO NOT MODIFY THIS LINE!)'
    else:
        msg = ''

    for line in lines[initial_line_index:]:
        m = include_re.match(line)
        if m:
            q1, match, q2 = m.groups()
            if q1 and (q1 != q2):
                raise ParsecError("mismatched quotes: " + line)
            inc = os.path.join(dir_, match)
            if inc not in done:
                if single or for_edit:
                    done.append(inc)
                if for_edit:
                    backup(inc)
                    # store original modtime
                    modtimes[inc] = os.stat(inc).st_mtime
                if os.path.isfile(inc):
                    if for_grep or single or label or for_edit:
                        outf.append('#++++ START INLINED INCLUDE FILE ' +
                                    match + msg)
                    h = open(inc, 'r')
                    finc = [line.rstrip('\n') for line in h]
                    h.close()
                    # recursive inclusion
                    outf.extend(
                        inline(finc, dir_, inc, for_grep, for_edit, viewcfg,
                               level))
                    if for_grep or single or label or for_edit:
                        outf.append('#++++ END INLINED INCLUDE FILE ' + match +
                                    msg)
                else:
                    flist.append(inc)
                    raise IncludeFileNotFoundError(flist)
            else:
                if not for_edit:
                    outf.append(level + line)
                else:
                    outf.append(line)
        else:
            # no match
            if not for_edit:
                outf.append(level + line)
            else:
                outf.append(line)
    return outf
Beispiel #10
0
 def raise_ParsecError(*a, **k):
     raise ParsecError("Mock error")