def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
        """
        Generate prepend-path statements for the given list of paths

        @param key: environment variable to prepend paths to
        @param paths: list of paths to prepend
        @param allow_abs: allow providing of absolute paths
        @param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
        """
        if isinstance(paths, basestring):
            self.log.debug("Wrapping %s into a list before using it to prepend path %s", paths, key)
            paths = [paths]

        abspaths = []
        for path in paths:
            if os.path.isabs(path):
                if allow_abs:
                    abspaths.append(quote_str(path))
                else:
                    raise EasyBuildError(
                        "Absolute path %s passed to prepend_paths which only expects relative paths.", path
                    )
            else:
                # use pathJoin for (non-empty) relative paths
                if path:
                    if expand_relpaths:
                        abspaths.append(self.PATH_JOIN_TEMPLATE % path)
                    else:
                        abspaths.append(quote_str(path))
                else:
                    abspaths.append("root")

        statements = [self.PREPEND_PATH_TEMPLATE % (key, p) for p in abspaths]
        return "".join(statements)
    def test_use(self):
        """Test generating module use statements."""
        if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
            # Test regular 'module use' statements
            expected = ''.join([
                'module use "/some/path"\n',
                'module use "/foo/bar/baz"\n',
            ])
            self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected)

            # Test guarded 'module use' statements using prefix
            expected = ''.join([
                'if { [ file isdirectory [ file join "/foo" "/some/path" ] ] } {\n',
                '    module use [ file join "/foo" "/some/path" ]\n',
                '}\n',
            ])
            self.assertEqual(self.modgen.use(["/some/path"], prefix=quote_str("/foo"), guarded=True), expected)
        else:
            # Test regular 'module use' statements
            expected = ''.join([
                'prepend_path("MODULEPATH", "/some/path")\n',
                'prepend_path("MODULEPATH", "/foo/bar/baz")\n',
            ])
            self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected)

            # Test guarded 'module use' statements using prefix
            expected = ''.join([
                'if isDir(pathJoin("/foo", "/some/path")) then\n',
                '    prepend_path("MODULEPATH", pathJoin("/foo", "/some/path"))\n',
                'end\n',
            ])
            self.assertEqual(self.modgen.use(["/some/path"], prefix=quote_str("/foo"), guarded=True), expected)
예제 #3
0
def stats_to_str(stats):
    """
    Pretty print build statistics to string.
    """
    if not isinstance(stats, (OrderedDict, dict)):
        raise EasyBuildError("Can only pretty print build stats in dictionary form, not of type %s", type(stats))

    txt = "{\n"
    pref = "    "
    for (k, v) in stats.items():
        txt += "%s%s: %s,\n" % (pref, quote_str(k), quote_str(v))
    txt += "}"
    return txt
 def set_environment(self, key, value, relpath=False):
     """
     Generate setenv statement for the given key/value pair.
     """
     # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
     if relpath:
         if value:
             val = quote_str(os.path.join("$root", value))
         else:
             val = '"$root"'
     else:
         val = quote_str(value)
     return "setenv\t%s\t\t%s\n" % (key, val)
예제 #5
0
def avail_easyconfig_params_rst(title, grouped_params):
    """
    Compose overview of available easyconfig parameters, in RST format.
    """
    # main title
    doc = [
        title,
        '=' * len(title),
        '',
    ]

    for grpname in grouped_params:
        # group section title
        title = "%s parameters" % grpname
        table_titles = ["**Parameter name**", "**Description**", "**Default value**"]
        table_values = [
            ['``%s``' % name for name in grouped_params[grpname].keys()],  # parameter name
            [x[0] for x in grouped_params[grpname].values()],  # description
            [str(quote_str(x[1])) for x in grouped_params[grpname].values()]  # default value
        ]

        doc.extend(rst_title_and_table(title, table_titles, table_values))
        doc.append('')

    return '\n'.join(doc)
예제 #6
0
def avail_easyconfig_params_rst(title, grouped_params):
    """
    Compose overview of available easyconfig parameters, in RST format.
    """
    # main title
    lines = [
        title,
        '=' * len(title),
        '',
    ]

    for grpname in grouped_params:
        # group section title
        lines.append("%s parameters" % grpname)
        lines.extend(['-' * len(lines[-1]), ''])

        titles = ["**Parameter name**", "**Description**", "**Default value**"]
        values = [
            ['``' + name + '``' for name in grouped_params[grpname].keys()],  # parameter name
            [x[0] for x in grouped_params[grpname].values()],  # description
            [str(quote_str(x[1])) for x in grouped_params[grpname].values()]  # default value
        ]

        lines.extend(mk_rst_table(titles, values))
        lines.append('')

    return '\n'.join(lines)
예제 #7
0
def avail_easyconfig_params_rst(title, grouped_params):
    """
    Compose overview of available easyconfig parameters, in RST format.
    """
    def det_col_width(entries, title):
        """Determine column width based on column title and list of entries."""
        return max(map(len, entries + [title]))

    # main title
    lines = [
        title,
        '=' * len(title),
        '',
    ]

    for grpname in grouped_params:
        # group section title
        lines.append("%s parameters" % grpname)
        lines.extend(['-' * len(lines[-1]), ''])

        name_title = "**Parameter name**"
        descr_title = "**Description**"
        dflt_title = "**Default value**"

        # figure out column widths
        nw = det_col_width(grouped_params[grpname].keys(), name_title) + 4  # +4 for raw format ("``foo``")
        dw = det_col_width([x[0] for x in grouped_params[grpname].values()], descr_title)
        dfw = det_col_width([str(quote_str(x[1])) for x in grouped_params[grpname].values()], dflt_title)

        # 3 columns (name, description, default value), left-aligned, {c} is fill char
        line_tmpl = "{0:{c}<%s}   {1:{c}<%s}   {2:{c}<%s}" % (nw, dw, dfw)
        table_line = line_tmpl.format('', '', '', c='=', nw=nw, dw=dw, dfw=dfw)

        # table header
        lines.append(table_line)
        lines.append(line_tmpl.format(name_title, descr_title, dflt_title, c=' '))
        lines.append(line_tmpl.format('', '', '', c='-'))

        # table rows by parameter
        for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
            rawname = '``%s``' % name
            lines.append(line_tmpl.format(rawname, descr, str(quote_str(dflt)), c=' '))
        lines.append(table_line)
        lines.append('')

    return '\n'.join(lines)
예제 #8
0
 def msg_on_load(self, msg):
     """
     Add a message that should be printed when loading the module.
     """
     # escape any (non-escaped) characters with special meaning by prefixing them with a backslash
     msg = re.sub(r'((?<!\\)[%s])'% ''.join(self.CHARS_TO_ESCAPE), r'\\\1', msg)
     print_cmd = "puts stderr %s" % quote_str(msg)
     return '\n'.join(['', self.conditional_statement("module-info mode load", print_cmd)])
    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
        """
        Generate prepend_path or append_path statements for the given list of paths

        :param key: environment variable to prepend/append paths to
        :param paths: list of paths to prepend/append
        :param prepend: whether to prepend (True) or append (False) paths
        :param allow_abs: allow providing of absolute paths
        :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
        """
        if prepend:
            update_type = 'prepend'
        else:
            update_type = 'append'

        if not self.define_env_var(key):
            self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key)
            return ''

        if isinstance(paths, basestring):
            self.log.debug("Wrapping %s into a list before using it to %s path %s", update_type, paths, key)
            paths = [paths]

        abspaths = []
        for path in paths:
            if os.path.isabs(path):
                if allow_abs:
                    abspaths.append(quote_str(path))
                else:
                    raise EasyBuildError("Absolute path %s passed to update_paths which only expects relative paths.",
                                         path)
            else:
                # use pathJoin for (non-empty) relative paths
                if path:
                    if expand_relpaths:
                        abspaths.append(self.PATH_JOIN_TEMPLATE % path)
                    else:
                        abspaths.append(quote_str(path))
                else:
                    abspaths.append('root')

        statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
        statements.append('')
        return '\n'.join(statements)
예제 #10
0
def stats_to_str(stats, isyeb=False):
    """
    Pretty print build statistics to string.
    """
    if not isinstance(stats, (OrderedDict, dict)):
        raise EasyBuildError("Can only pretty print build stats in dictionary form, not of type %s", type(stats))

    txt = "{\n"
    pref = "    "
    for key in sorted(stats):
        if isyeb:
            val = stats[key]
            if isinstance(val, tuple):
                val = list(val)
            key, val = quote_yaml_special_chars(key), quote_yaml_special_chars(val)
        else:
            key, val = quote_str(key), quote_str(stats[key])
        txt += "%s%s: %s,\n" % (pref, key, val)
    txt += "}"
    return txt
 def set_environment(self, key, value, relpath=False):
     """
     Generate a quoted setenv statement for the given key/value pair.
     """
     if relpath:
         if value:
             val = self.PATH_JOIN_TEMPLATE % value
         else:
             val = 'root'
     else:
         val = quote_str(value)
     return 'setenv("%s", %s)\n' % (key, val)
 def msg_on_load(self, msg):
     """
     Add a message that should be printed when loading the module.
     """
     # escape any (non-escaped) characters with special meaning by prefixing them with a backslash
     msg = re.sub(r'((?<!\\)[%s])' % ''.join(self.CHARS_TO_ESCAPE), r'\\\1',
                  msg)
     print_cmd = "puts stderr %s" % quote_str(msg)
     return '\n'.join([
         '',
         self.conditional_statement("module-info mode load", print_cmd)
     ])
예제 #13
0
    def set_environment(self, key, value, relpath=False):
        """
        Generate a quoted setenv statement for the given key/value pair.

        :param key: name of environment variable to define
        :param value: value to define environment variable with
        :param relpath: value is path relative to installation prefix
        """
        if not self.define_env_var(key):
            self.log.info("Not including statement to define environment variable $%s, as specified", key)
            return ''

        # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
        if relpath:
            if value:
                val = quote_str(os.path.join('$root', value))
            else:
                val = '"$root"'
        else:
            val = quote_str(value)
        return 'setenv\t%s\t\t%s\n' % (key, val)
 def set_environment(self, key, value, relpath=False):
     """
     Generate a quoted setenv statement for the given key/value pair.
     """
     if relpath:
         if value:
             val = self.PATH_JOIN_TEMPLATE % value
         else:
             val = "root"
     else:
         val = quote_str(value)
     return 'setenv("%s", %s)\n' % (key, val)
예제 #15
0
    def set_environment(self, key, value, relpath=False):
        """
        Generate a quoted setenv statement for the given key/value pair.

        :param key: name of environment variable to define
        :param value: value to define environment variable with
        :param relpath: value is path relative to installation prefix
        """
        if not self.define_env_var(key):
            self.log.info("Not including statement to define environment variable $%s, as specified", key)
            return ''

        # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
        if relpath:
            if value:
                val = quote_str(os.path.join('$root', value))
            else:
                val = '"$root"'
        else:
            val = quote_str(value)
        return 'setenv\t%s\t\t%s\n' % (key, val)
예제 #16
0
    def test_use(self):
        """Test generating module use statements."""
        if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
            # Test regular 'module use' statements
            expected = ''.join([
                'module use "/some/path"\n',
                'module use "/foo/bar/baz"\n',
            ])
            self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]),
                             expected)

            # Test guarded 'module use' statements using prefix
            expected = ''.join([
                'if { [ file isdirectory [ file join "/foo" "/some/path" ] ] } {\n',
                '    module use [ file join "/foo" "/some/path" ]\n',
                '}\n',
            ])
            self.assertEqual(
                self.modgen.use(["/some/path"],
                                prefix=quote_str("/foo"),
                                guarded=True), expected)
        else:
            # Test regular 'module use' statements
            expected = ''.join([
                'prepend_path("MODULEPATH", "/some/path")\n',
                'prepend_path("MODULEPATH", "/foo/bar/baz")\n',
            ])
            self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]),
                             expected)

            # Test guarded 'module use' statements using prefix
            expected = ''.join([
                'if isDir(pathJoin("/foo", "/some/path")) then\n',
                '    prepend_path("MODULEPATH", pathJoin("/foo", "/some/path"))\n',
                'end\n',
            ])
            self.assertEqual(
                self.modgen.use(["/some/path"],
                                prefix=quote_str("/foo"),
                                guarded=True), expected)
예제 #17
0
def stats_to_str(stats, isyeb=False):
    """
    Pretty print build statistics to string.
    """
    if not isinstance(stats, (OrderedDict, dict)):
        raise EasyBuildError(
            "Can only pretty print build stats in dictionary form, not of type %s",
            type(stats))

    txt = "{\n"
    pref = "    "
    for key in sorted(stats):
        if isyeb:
            val = stats[key]
            if isinstance(val, tuple):
                val = list(val)
            key, val = quote_yaml_special_chars(key), quote_yaml_special_chars(
                val)
        else:
            key, val = quote_str(key), quote_str(stats[key])
        txt += "%s%s: %s,\n" % (pref, key, val)
    txt += "}"
    return txt
    def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
        """
        Generate prepend-path statements for the given list of paths

        @param key: environment variable to prepend paths to
        @param paths: list of paths to prepend
        @param allow_abs: allow providing of absolute paths
        @param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
        """
        if isinstance(paths, basestring):
            self.log.debug(
                "Wrapping %s into a list before using it to prepend path %s",
                paths, key)
            paths = [paths]

        abspaths = []
        for path in paths:
            if os.path.isabs(path):
                if allow_abs:
                    abspaths.append(quote_str(path))
                else:
                    raise EasyBuildError(
                        "Absolute path %s passed to prepend_paths which only expects relative paths.",
                        path)
            else:
                # use pathJoin for (non-empty) relative paths
                if path:
                    if expand_relpaths:
                        abspaths.append(self.PATH_JOIN_TEMPLATE % path)
                    else:
                        abspaths.append(quote_str(path))
                else:
                    abspaths.append('root')

        statements = [self.PREPEND_PATH_TEMPLATE % (key, p) for p in abspaths]
        statements.append('')
        return '\n'.join(statements)
예제 #19
0
    def set_environment(self, key, value, relpath=False):
        """
        Generate a quoted setenv statement for the given key/value pair.

        :param key: name of environment variable to define
        :param value: value to define environment variable with
        :param relpath: value is path relative to installation prefix
        """
        if not self.define_env_var(key):
            self.log.info("Not including statement to define environment variable $%s, as specified", key)
            return ''

        if relpath:
            if value:
                val = self.PATH_JOIN_TEMPLATE % value
            else:
                val = 'root'
        else:
            val = quote_str(value)
        return 'setenv("%s", %s)\n' % (key, val)
예제 #20
0
 def test_deprecated_options(self):
     """Test whether deprecated options are handled correctly."""
     deprecated_options = [
         ('makeopts', 'buildopts', 'CC=foo'),
         ('premakeopts', 'prebuildopts', ['PATH=%(builddir)s/foo:$PATH', 'PATH=%(builddir)s/bar:$PATH']),
     ]
     clean_contents = [
         'name = "pi"',
         'version = "3.14"',
         'homepage = "http://example.com"',
         'description = "test easyconfig"',
         'toolchain = {"name": "dummy", "version": "dummy"}',
         'buildininstalldir = True',
     ]
     # alternative option is ready to use
     for depr_opt, new_opt, val in deprecated_options:
         self.contents = '\n'.join(clean_contents + ['%s = %s' % (depr_opt, quote_str(val))])
         self.prep()
         ec = EasyConfig(self.eb_file)
         self.assertEqual(ec[depr_opt], ec[new_opt])
예제 #21
0
    def set_environment(self, key, value, relpath=False):
        """
        Generate a quoted setenv statement for the given key/value pair.

        :param key: name of environment variable to define
        :param value: value to define environment variable with
        :param relpath: value is path relative to installation prefix
        """
        if not self.define_env_var(key):
            self.log.info("Not including statement to define environment variable $%s, as specified", key)
            return ''

        if relpath:
            if value:
                val = self.PATH_JOIN_TEMPLATE % value
            else:
                val = 'root'
        else:
            val = quote_str(value)
        return 'setenv("%s", %s)\n' % (key, val)
예제 #22
0
 def use(self, paths, prefix=None, guarded=False):
     """
     Generate module use statements for given list of module paths.
     :param paths: list of module path extensions to generate use statements for; paths will be quoted
     :param prefix: optional path prefix; not quoted, i.e., can be a statement
     :param guarded: use statements will be guarded to only apply if path exists
     """
     use_statements = []
     for path in paths:
         quoted_path = quote_str(path)
         if prefix:
             full_path = 'pathJoin(%s, %s)' % (prefix, quoted_path)
         else:
             full_path = quoted_path
         prepend_modulepath = self.UPDATE_PATH_TEMPLATE % ('prepend', 'MODULEPATH', full_path)
         if guarded:
             cond_statement = self.conditional_statement('isDir(%s)' % full_path, prepend_modulepath)
             use_statements.append(cond_statement)
         else:
             use_statements.append(prepend_modulepath + '\n')
     return ''.join(use_statements)
예제 #23
0
def avail_easyconfig_params_txt(title, grouped_params):
    """
    Compose overview of available easyconfig parameters, in plain text format.
    """
    # main title
    doc = ["%s:" % title, ""]

    for grpname in grouped_params:
        # group section title
        doc.append(grpname.upper())
        doc.append("-" * len(doc[-1]))

        # determine width of 'name' column, to left-align descriptions
        nw = max(map(len, grouped_params[grpname].keys()))

        # line by parameter
        for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
            doc.append("{0:<{nw}}   {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw))
        doc.append("")

    return "\n".join(doc)
예제 #24
0
 def use(self, paths, prefix=None, guarded=False):
     """
     Generate module use statements for given list of module paths.
     :param paths: list of module path extensions to generate use statements for; paths will be quoted
     :param prefix: optional path prefix; not quoted, i.e., can be a statement
     :param guarded: use statements will be guarded to only apply if path exists
     """
     use_statements = []
     for path in paths:
         quoted_path = quote_str(path)
         if prefix:
             full_path = 'pathJoin(%s, %s)' % (prefix, quoted_path)
         else:
             full_path = quoted_path
         if guarded:
             cond_statement = self.conditional_statement('isDir(%s)' % full_path,
                                                         self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', full_path))
             use_statements.append(cond_statement)
         else:
             use_statements.append(self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', full_path) + '\n')
     return ''.join(use_statements)
예제 #25
0
 def use(self, paths, prefix=None, guarded=False):
     """
     Generate module use statements for given list of module paths.
     :param paths: list of module path extensions to generate use statements for; paths will be quoted
     :param prefix: optional path prefix; not quoted, i.e., can be a statement
     :param guarded: use statements will be guarded to only apply if path exists
     """
     use_statements = []
     for path in paths:
         quoted_path = quote_str(path)
         if prefix:
             full_path = '[ file join %s %s ]' % (prefix, quoted_path)
         else:
             full_path = quoted_path
         if guarded:
             cond_statement = self.conditional_statement('file isdirectory %s' % full_path,
                                                         'module use %s' % full_path)
             use_statements.append(cond_statement)
         else:
             use_statements.append("module use %s\n" % full_path)
     return ''.join(use_statements)
예제 #26
0
 def use(self, paths, prefix=None, guarded=False):
     """
     Generate module use statements for given list of module paths.
     :param paths: list of module path extensions to generate use statements for; paths will be quoted
     :param prefix: optional path prefix; not quoted, i.e., can be a statement
     :param guarded: use statements will be guarded to only apply if path exists
     """
     use_statements = []
     for path in paths:
         quoted_path = quote_str(path)
         if prefix:
             full_path = '[ file join %s %s ]' % (prefix, quoted_path)
         else:
             full_path = quoted_path
         if guarded:
             cond_statement = self.conditional_statement('file isdirectory %s' % full_path,
                                                         'module use %s' % full_path)
             use_statements.append(cond_statement)
         else:
             use_statements.append("module use %s\n" % full_path)
     return ''.join(use_statements)
예제 #27
0
    def _inject_constants_dict(self, txt):
        """Inject constants so they are resolved when actually parsing the YAML text."""
        constants_dict = build_easyconfig_constants_dict()

        lines = txt.splitlines()

        # extract possible YAML header, for example
        # %YAML 1.2
        # ---
        yaml_header = []
        for i, line in enumerate(lines):
            if line.startswith(YAML_DIR):
                if lines[i+1].startswith(YAML_SEP):
                    yaml_header.extend([lines.pop(i), lines.pop(i)])

        injected_constants = ['__CONSTANTS__: ']
        for key, value in constants_dict.items():
            injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value)))

        full_txt = '\n'.join(yaml_header + injected_constants + lines)

        return full_txt
예제 #28
0
    def _inject_constants_dict(self, txt):
        """Inject constants so they are resolved when actually parsing the YAML text."""
        constants_dict = build_easyconfig_constants_dict()

        lines = txt.splitlines()

        # extract possible YAML header, for example
        # %YAML 1.2
        # ---
        yaml_header = []
        for i, line in enumerate(lines):
            if line.startswith(YAML_DIR):
                if lines[i+1].startswith(YAML_SEP):
                    yaml_header.extend([lines.pop(i), lines.pop(i)])

        injected_constants = ['__CONSTANTS__: ']
        for key, value in constants_dict.items():
            injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value)))

        full_txt = '\n'.join(yaml_header + injected_constants + lines)

        return full_txt
예제 #29
0
def avail_easyconfig_params_txt(title, grouped_params):
    """
    Compose overview of available easyconfig parameters, in plain text format.
    """
    # main title
    lines = [
        '%s:' % title,
        '',
    ]

    for grpname in grouped_params:
        # group section title
        lines.append(grpname.upper())
        lines.append('-' * len(lines[-1]))

        # determine width of 'name' column, to left-align descriptions
        nw = max(map(len, grouped_params[grpname].keys()))

        # line by parameter
        for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
            lines.append("{0:<{nw}}   {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw))
        lines.append('')

    return '\n'.join(lines)
    def prepend_paths(self, key, paths, allow_abs=False):
        """
        Generate prepend-path statements for the given list of paths
        """
        if isinstance(paths, basestring):
            self.log.debug("Wrapping %s into a list before using it to prepend path %s", paths, key)
            paths = [paths]

        for i, path in enumerate(paths):
            if os.path.isabs(path):
                if allow_abs:
                    paths[i] = quote_str(path)
                else:
                    raise EasyBuildError("Absolute path %s passed to prepend_paths which only expects relative paths.",
                                         path)
            else:
                # use pathJoin for (non-empty) relative paths
                if path:
                    paths[i] = self.PATH_JOIN_TEMPLATE % path
                else:
                    paths[i] = 'root'

        statements = [self.PREPEND_PATH_TEMPLATE % (key, p) for p in paths]
        return ''.join(statements)
예제 #31
0
def tweak(src_fn, target_fn, tweaks):
    """
    Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
    Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
    once and then use the 'version' variable to construct the list of sources, and possibly other
    parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)

    The tweaks should be specified in a dictionary, with parameters and keys that map to the values
    to be set.

    Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.

    If no target filename is provided, a target filepath is generated based on the contents of
    the tweaked easyconfig file.
    """

    # read easyconfig file
    ectxt = read_file(src_fn)

    _log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" %
               ectxt)
    # determine new toolchain if it's being changed
    keys = tweaks.keys()
    if 'toolchain_name' in keys or 'toolchain_version' in keys:

        tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)

        res = tc_regexp.search(ectxt)
        if not res:
            _log.error("No toolchain found in easyconfig file %s?" % src_fn)

        toolchain = eval(res.group(1))

        for key in ['name', 'version']:
            tc_key = "toolchain_%s" % key
            if tc_key in keys:
                toolchain.update({key: tweaks[tc_key]})
                tweaks.pop(tc_key)

        class TcDict(dict):
            """A special dict class that represents trivial toolchains properly."""
            def __repr__(self):
                return "{'name': '%(name)s', 'version': '%(version)s'}" % self

        tweaks.update({
            'toolchain':
            TcDict({
                'name': toolchain['name'],
                'version': toolchain['version']
            })
        })

        _log.debug("New toolchain constructed: %s" % tweaks['toolchain'])

    additions = []

    # we need to treat list values seperately, i.e. we prepend to the current value (if any)
    for (key, val) in tweaks.items():

        if isinstance(val, list):

            regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)

            res = regexp.search(ectxt)
            if res:
                newval = "%s + %s" % (val, res.group(1))
                ectxt = regexp.sub(
                    "%s = %s # tweaked by EasyBuild (was: %s)" %
                    (key, newval, res.group(1)), ectxt)
                _log.info("Tweaked %s list to '%s'" % (key, newval))
            else:
                additions.append("%s = %s # added by EasyBuild" % (key, val))

            tweaks.pop(key)

    # add parameters or replace existing ones
    for (key, val) in tweaks.items():

        regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)
        _log.debug("Regexp pattern for replacing '%s': %s" %
                   (key, regexp.pattern))

        res = regexp.search(ectxt)
        if res:
            # only tweak if the value is different
            diff = True
            try:
                _log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1))))
                diff = not eval(res.group(1)) == val
            except (NameError, SyntaxError):
                # if eval fails, just fall back to string comparison
                _log.debug(
                    "eval failed for \"%s\", falling back to string comparison against \"%s\"..."
                    % (res.group(1), val))
                diff = not res.group(1) == val

            if diff:
                ectxt = regexp.sub(
                    "%s = %s # tweaked by EasyBuild (was: %s)" %
                    (key, quote_str(val), res.group(1)), ectxt)
                _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
        else:
            additions.append("%s = %s" % (key, quote_str(val)))

    if additions:
        _log.info(
            "Adding additional parameters to tweaked easyconfig file: %s")
        ectxt += "\n\n# added by EasyBuild as dictated by command line options\n"
        ectxt += '\n'.join(additions) + '\n'

    _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)

    # come up with suiting file name for tweaked easyconfig file if none was specified
    if not target_fn:

        fn = None

        try:
            # obtain temporary filename
            fd, tmpfn = tempfile.mkstemp()
            os.close(fd)

            # write easyconfig to temporary file
            write_file(tmpfn, ectxt)

            # determine suiting filename
            fn = ec_filename_for(tmpfn)

            # get rid of temporary file
            os.remove(tmpfn)

        except OSError, err:
            _log.error(
                "Failed to determine suiting filename for tweaked easyconfig file: %s"
                % err)

        target_fn = os.path.join(tempfile.gettempdir(), fn)
        _log.debug("Generated file name for tweaked easyconfig file: %s" %
                   target_fn)
예제 #32
0
 def use(self, paths):
     """
     Generate module use statements for given list of module paths.
     @param paths: list of module path extensions to generate use statements for
     """
     return ''.join([self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', quote_str(p)) for p in paths])
예제 #33
0
def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params,
                                  doc_functions, all_blocks):
    """
    Compose overview of one easyblock given class object of the easyblock in rst format
    """
    classname = eb_class.__name__

    doc = [
        '.. _' + classname + ':',
        '',
        '``' + classname + '``',
        '=' * (len(classname) + 4),
        '',
    ]

    bases = []
    for b in eb_class.__bases__:
        base = ':ref:`' + b.__name__ + '`' if b in all_blocks else b.__name__
        bases.append(base)

    derived = '(derives from ' + ', '.join(bases) + ')'
    doc.extend([derived, ''])

    # Description (docstring)
    if eb_class.__doc__ is not None:
        doc.extend([eb_class.__doc__.strip(), ''])

    # Add extra options, if any
    if eb_class.extra_options():
        title = 'Extra easyconfig parameters specific to ``%s`` easyblock' % classname
        ex_opt = eb_class.extra_options()
        keys = sorted(ex_opt.keys())
        values = [ex_opt[k] for k in keys]

        table_titles = ['easyconfig parameter', 'description', 'default value']
        table_values = [
            ['``' + key + '``' for key in keys],  # parameter name
            [val[1] for val in values],  # description
            ['``' + str(quote_str(val[0])) + '``'
             for val in values]  # default value
        ]

        doc.extend(rst_title_and_table(title, table_titles, table_values))

    # Add commonly used parameters
    if classname in common_params:
        title = 'Commonly used easyconfig parameters with ``%s`` easyblock' % classname

        table_titles = ['easyconfig parameter', 'description']
        table_values = [
            [opt for opt in common_params[classname]],
            [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
        ]

        doc.extend(rst_title_and_table(title, table_titles, table_values))
        doc.append('')

    # Add docstring for custom steps
    custom = []
    inh = ''
    f = None
    for func in doc_functions:
        if func in eb_class.__dict__:
            f = eb_class.__dict__[func]

        if f.__doc__:
            custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh)

    if custom:
        title = 'Customised steps in ``' + classname + '`` easyblock'
        doc.extend([title, '-' * len(title)] + custom)
        doc.append('')

    # Add example if available
    if os.path.exists(os.path.join(path_to_examples, '%s.eb' % classname)):
        title = 'Example easyconfig for ``' + classname + '`` easyblock'
        doc.extend([title, '-' * len(title), '', '.. code::', ''])
        for line in read_file(os.path.join(path_to_examples,
                                           classname + '.eb')).split('\n'):
            doc.append(INDENT_4SPACES + line)
        doc.append('')  # empty line after literal block

    return doc
 def use(self, paths):
     """
     Generate module use statements for given list of module paths.
     @param paths: list of module path extensions to generate use statements for
     """
     return "".join([self.PREPEND_PATH_TEMPLATE % ("MODULEPATH", quote_str(p)) for p in paths])
예제 #35
0
def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks):
    """
    Compose overview of one easyblock given class object of the easyblock in rst format
    """
    classname = eb_class.__name__

    doc = [".. _" + classname + ":", "", "``" + classname + "``", "=" * (len(classname) + 4), ""]

    bases = []
    for b in eb_class.__bases__:
        base = ":ref:`" + b.__name__ + "`" if b in all_blocks else b.__name__
        bases.append(base)

    derived = "(derives from " + ", ".join(bases) + ")"
    doc.extend([derived, ""])

    # Description (docstring)
    doc.extend([eb_class.__doc__.strip(), ""])

    # Add extra options, if any
    if eb_class.extra_options():
        title = "Extra easyconfig parameters specific to ``%s`` easyblock" % classname
        ex_opt = eb_class.extra_options()

        table_titles = ["easyconfig parameter", "description", "default value"]
        table_values = [
            ["``" + key + "``" for key in ex_opt],  # parameter name
            [val[1] for val in ex_opt.values()],  # description
            ["``" + str(quote_str(val[0])) + "``" for val in ex_opt.values()],  # default value
        ]

        doc.extend(rst_title_and_table(title, table_titles, table_values))

    # Add commonly used parameters
    if classname in common_params:
        title = "Commonly used easyconfig parameters with ``%s`` easyblock" % classname

        table_titles = ["easyconfig parameter", "description"]
        table_values = [
            [opt for opt in common_params[classname]],
            [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
        ]

        doc.extend(rst_title_and_table(title, table_titles, table_values))
        doc.append("")

    # Add docstring for custom steps
    custom = []
    inh = ""
    f = None
    for func in doc_functions:
        if func in eb_class.__dict__:
            f = eb_class.__dict__[func]

        if f.__doc__:
            custom.append("* ``" + func + "`` - " + f.__doc__.strip() + inh)

    if custom:
        title = "Customised steps in ``" + classname + "`` easyblock"
        doc.extend([title, "-" * len(title)] + custom)
        doc.append("")

    # Add example if available
    if os.path.exists(os.path.join(path_to_examples, "%s.eb" % classname)):
        title = "Example easyconfig for ``" + classname + "`` easyblock"
        doc.extend([title, "-" * len(title), "", ".. code::", ""])
        for line in read_file(os.path.join(path_to_examples, classname + ".eb")).split("\n"):
            doc.append(INDENT_4SPACES + line)
        doc.append("")  # empty line after literal block

    return doc
예제 #36
0
def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks):
    """
    Compose overview of one easyblock given class object of the easyblock in rst format
    """
    classname = eb_class.__name__

    lines = [
        '.. _' + classname + ':',
        '',
        '``' + classname + '``',
        '=' * (len(classname)+4),
        '',
    ]

    bases = []
    for b in eb_class.__bases__:
        base = ':ref:`' + b.__name__ +'`' if b in all_blocks else b.__name__
        bases.append(base)

    derived = '(derives from ' + ', '.join(bases) + ')'
    lines.extend([derived, ''])

    # Description (docstring)
    lines.extend([eb_class.__doc__.strip(), ''])

    # Add extra options, if any
    if eb_class.extra_options():
        extra_parameters = 'Extra easyconfig parameters specific to ``' + classname + '`` easyblock'
        lines.extend([extra_parameters, '-' * len(extra_parameters), ''])
        ex_opt = eb_class.extra_options()

        titles = ['easyconfig parameter', 'description', 'default value']
        values = [
            ['``' + key + '``' for key in ex_opt],  # parameter name
            [val[1] for val in ex_opt.values()],  # description
            ['``' + str(quote_str(val[0])) + '``' for val in ex_opt.values()]  # default value
        ]

        lines.extend(mk_rst_table(titles, values))

    # Add commonly used parameters
    if classname in common_params:
        commonly_used = 'Commonly used easyconfig parameters with ``' + classname + '`` easyblock'
        lines.extend([commonly_used, '-' * len(commonly_used)])

        titles = ['easyconfig parameter', 'description']
        values = [
            [opt for opt in common_params[classname]],
            [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
        ]

        lines.extend(mk_rst_table(titles, values))

        lines.append('')

    # Add docstring for custom steps
    custom = []
    inh = ''
    f = None
    for func in doc_functions:
        if func in eb_class.__dict__:
            f = eb_class.__dict__[func]

        if f.__doc__:
            custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh)

    if custom:
        title = 'Customised steps in ``' + classname + '`` easyblock'
        lines.extend([title, '-' * len(title)] + custom)
        lines.append('')

    # Add example if available
    if os.path.exists(os.path.join(path_to_examples, '%s.eb' % classname)):
        title = 'Example for ``' + classname + '`` easyblock'
        lines.extend(['', title, '-' * len(title), '', '::', ''])
        for line in read_file(os.path.join(path_to_examples, classname+'.eb')).split('\n'):
            lines.append('    ' + line.strip())
        lines.append('') # empty line after literal block

    return '\n'.join(lines)
예제 #37
0
# add versionsuffix if Python is specified as a dependency
if any(dep[0] == 'Python' for dep in cfg.get('dependencies', [])):
    if not_found_yet('versionsuffix'):
        cfg['versionsuffix'] = '-Python-%(pyver)s'

# add empty sanity_check_paths
if 'sanity_check_paths' not in cfg:
    cfg['sanity_check_paths'] = {'files': [], 'dirs': []}

# enable use_pip & sanity_pip_check
if cfg.get('easyblock') in ['PythonBundle', 'PythonPackage']:
    cfg.update({
        'use_pip': True,
        'sanity_pip_check': True,
    })

# enable download_dep_fail
if cfg.get('easyblock') == 'PythonPackage':
    cfg['download_dep_fail'] = True

pprint.pprint(cfg)

ec_raw = '\n'.join("%s = %s" % (key, quote_str(cfg[key])) for key in cfg)
ec = EasyConfig(None, rawtxt=ec_raw)

full_ec_ver = det_full_ec_version(ec)
fn = os.path.join('%s-%s.eb' % (cfg['name'], full_ec_ver))

ec.dump(fn)
info("Easyconfig file created: %s" % fn)
예제 #38
0
    depstring = ''
    for dep in deps:
        _log.debug("The dep added looks like %s ", dep)
        dep_pkgname = package_naming_scheme.name(dep)
        depstring += " --depends '%s'" % dep_pkgname

    cmdlist = [
        PKG_TOOL_FPM,
        '--workdir', workdir,
        '--name', pkgname,
        '--provides', pkgname,
        '-t', pkgtype,  # target
        '-s', 'dir',  # source
        '--version', pkgver,
        '--iteration', pkgrel,
        '--description', quote_str(easyblock.cfg["description"]),
        '--url', easyblock.cfg["homepage"],
        depstring,
        easyblock.installdir,
        easyblock.module_generator.get_module_filepath(),
    ]
    cmd = ' '.join(cmdlist)
    _log.debug("The flattened cmdlist looks like: %s", cmd)
    run_cmd(cmd, log_all=True, simple=True)

    _log.info("Created %s package(s) in %s", pkgtype, workdir)

    try:
        os.chdir(origdir)
    except OSError, err:
        raise EasyBuildError("Failed to chdir back to %s: %s", origdir, err)
예제 #39
0
 def set_environment(self, key, value):
     """
     Generate setenv statement for the given key/value pair.
     """
     # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
     return 'setenv\t%s\t\t%s\n' % (key, quote_str(value))
예제 #40
0
 def set_alias(self, key, value):
     """
     Generate set-alias statement in modulefile for the given key/value pair.
     """
     # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
     return 'set_alias("%s", %s)\n' % (key, quote_str(value))
        _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern))

        res = regexp.search(ectxt)
        if res:
            # only tweak if the value is different
            diff = True
            try:
                _log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1))))
                diff = not eval(res.group(1)) == val
            except (NameError, SyntaxError):
                # if eval fails, just fall back to string comparison
                _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % (res.group(1), val))
                diff = not res.group(1) == val

            if diff:
                ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, quote_str(val), res.group(1)), ectxt)
                _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
        else:
            additions.append("%s = %s" % (key, quote_str(val)))

    if additions:
        _log.info("Adding additional parameters to tweaked easyconfig file: %s")
        ectxt += "\n\n# added by EasyBuild as dictated by command line options\n"
        ectxt += '\n'.join(additions) + '\n'

    _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)

    # come up with suiting file name for tweaked easyconfig file if none was specified
    if not target_fn:

        fn = None
 def set_alias(self, key, value):
     """
     Generate set-alias statement in modulefile for the given key/value pair.
     """
     # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
     return 'set-alias\t%s\t\t%s\n' % (key, quote_str(value))
예제 #43
0
    if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME:
        toolchain_dict = easyblock.toolchain.as_dict()
        deps.extend([toolchain_dict])

    deps.extend(easyblock.cfg.dependencies())

    _log.debug("The dependencies to be added to the package are: %s",
               pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies()))
    depstring = ''
    for dep in deps:
        if dep.get('external_module', False):
            _log.debug("Skipping dep marked as external module: %s", dep['name'])
        else:
            _log.debug("The dep added looks like %s ", dep)
            dep_pkgname = package_naming_scheme.name(dep)
            depstring += " --depends %s" % quote_str(dep_pkgname)

    # Excluding the EasyBuild logs and test reports that might be in the installdir
    exclude_files_glob = [
        os.path.join(log_path(), "*.log"),
        os.path.join(log_path(), "*.md"),
    ]
    # stripping off leading / to match expected glob in fpm
    exclude_files_glob = [
        '--exclude %s' % quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
        for x in exclude_files_glob
    ]
    _log.debug("The list of excluded files passed to fpm: %s", exclude_files_glob)
    cmdlist = [
        PKG_TOOL_FPM,
        '--workdir', workdir,
    if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME:
        toolchain_dict = easyblock.toolchain.as_dict()
        deps.extend([toolchain_dict])

    deps.extend(easyblock.cfg.dependencies())

    _log.debug("The dependencies to be added to the package are: %s",
               pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies()))
    depstring = ''
    for dep in deps:
        if dep.get('external_module', False):
            _log.debug("Skipping dep marked as external module: %s", dep['name'])
        else:
            _log.debug("The dep added looks like %s ", dep)
            dep_pkgname = package_naming_scheme.name(dep)
            depstring += " --depends %s" % quote_str(dep_pkgname)

    # Excluding the EasyBuild logs and test reports that might be in the installdir
    exclude_files_glob = [
        os.path.join(log_path(), "*.log"),
        os.path.join(log_path(), "*.md"),
    ]
    # stripping off leading / to match expected glob in fpm
    exclude_files_glob = [
        '--exclude %s' % quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
        for x in exclude_files_glob
    ]
    _log.debug("The list of excluded files passed to fpm: %s", exclude_files_glob)
    cmdlist = [
        PKG_TOOL_FPM,
        '--workdir', workdir,
예제 #45
0
def tweak(src_fn, target_fn, tweaks):
    """
    Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
    Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
    once and then use the 'version' variable to construct the list of sources, and possibly other
    parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)

    The tweaks should be specified in a dictionary, with parameters and keys that map to the values
    to be set.

    Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.

    If no target filename is provided, a target filepath is generated based on the contents of
    the tweaked easyconfig file.
    """

    # read easyconfig file
    ectxt = read_file(src_fn)

    _log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" % ectxt)
    # determine new toolchain if it's being changed
    keys = tweaks.keys()
    if 'toolchain_name' in keys or 'toolchain_version' in keys:

        tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)

        res = tc_regexp.search(ectxt)
        if not res:
            _log.error("No toolchain found in easyconfig file %s?" % src_fn)

        toolchain = eval(res.group(1))

        for key in ['name', 'version']:
            tc_key = "toolchain_%s" % key
            if tc_key in keys:
                toolchain.update({key: tweaks[tc_key]})
                tweaks.pop(tc_key)

        class TcDict(dict):
            """A special dict class that represents trivial toolchains properly."""
            def __repr__(self):
                return "{'name': '%(name)s', 'version': '%(version)s'}" % self

        tweaks.update({'toolchain': TcDict({'name': toolchain['name'], 'version': toolchain['version']})})

        _log.debug("New toolchain constructed: %s" % tweaks['toolchain'])

    additions = []

    # we need to treat list values seperately, i.e. we prepend to the current value (if any)
    for (key, val) in tweaks.items():

        if isinstance(val, list):

            regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)

            res = regexp.search(ectxt)
            if res:
                newval = "%s + %s" % (val, res.group(1))
                ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, newval, res.group(1)), ectxt)
                _log.info("Tweaked %s list to '%s'" % (key, newval))
            else:
                additions.append("%s = %s # added by EasyBuild" % (key, val))

            tweaks.pop(key)

    # add parameters or replace existing ones
    for (key, val) in tweaks.items():

        regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)
        _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern))

        res = regexp.search(ectxt)
        if res:
            # only tweak if the value is different
            diff = True
            try:
                _log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1))))
                diff = not eval(res.group(1)) == val
            except (NameError, SyntaxError):
                # if eval fails, just fall back to string comparison
                _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % (res.group(1), val))
                diff = not res.group(1) == val

            if diff:
                ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, quote_str(val), res.group(1)), ectxt)
                _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
        else:
            additions.append("%s = %s" % (key, quote_str(val)))

    if additions:
        _log.info("Adding additional parameters to tweaked easyconfig file: %s")
        ectxt += "\n\n# added by EasyBuild as dictated by command line options\n"
        ectxt += '\n'.join(additions) + '\n'

    _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)

    # come up with suiting file name for tweaked easyconfig file if none was specified
    if not target_fn:

        fn = None

        try:
            # obtain temporary filename
            fd, tmpfn = tempfile.mkstemp()
            os.close(fd)

            # write easyconfig to temporary file
            write_file(tmpfn, ectxt)

            # determine suiting filename
            fn = ec_filename_for(tmpfn)

            # get rid of temporary file
            os.remove(tmpfn)

        except OSError, err:
            _log.error("Failed to determine suiting filename for tweaked easyconfig file: %s" % err)

        target_fn = os.path.join(tempfile.gettempdir(), fn)
        _log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn)
예제 #46
0
def tweak_one(src_fn, target_fn, tweaks, targetdir=None):
    """
    Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
    Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
    once and then use the 'version' variable to construct the list of sources, and possibly other
    parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)

    The tweaks should be specified in a dictionary, with parameters and keys that map to the values
    to be set.

    Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.

    If no target filename is provided, a target filepath is generated based on the contents of
    the tweaked easyconfig file.
    """

    # read easyconfig file
    ectxt = read_file(src_fn)

    _log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" % ectxt)
    # determine new toolchain if it's being changed
    keys = tweaks.keys()
    if 'toolchain_name' in keys or 'toolchain_version' in keys:
        # note: this assumes that the toolchain spec is single-line
        tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)
        res = tc_regexp.search(ectxt)
        if not res:
            raise EasyBuildError("No toolchain found in easyconfig file %s: %s", src_fn, ectxt)

        toolchain = eval(res.group(1))
        for key in ['name', 'version']:
            tc_key = "toolchain_%s" % key
            if tc_key in keys:
                toolchain.update({key: tweaks[tc_key]})
                tweaks.pop(tc_key)

        class TcDict(dict):
            """A special dict class that represents trivial toolchains properly."""
            def __repr__(self):
                return "{'name': '%(name)s', 'version': '%(version)s'}" % self

        tweaks.update({'toolchain': TcDict({'name': toolchain['name'], 'version': toolchain['version']})})
        _log.debug("New toolchain constructed: %s" % tweaks['toolchain'])

    additions = []

    # automagically clear out list of checksums if software version is being tweaked 
    if 'version' in tweaks and 'checksums' not in tweaks:
        tweaks['checksums'] = []
        _log.warning("Tweaking version: checksums cleared, verification disabled.")

    # we need to treat list values seperately, i.e. we prepend to the current value (if any)
    for (key, val) in tweaks.items():

        if isinstance(val, list):
            regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>\[(.|\n)*\])\s*$" % key, re.M)
            res = regexp.search(ectxt)
            if res:
                fval = [x for x in val if x != '']  # filter out empty strings
                # determine to prepend/append or overwrite by checking first/last list item
                # - input ending with comma (empty tail list element) => prepend
                # - input starting with comma (empty head list element) => append
                # - no empty head/tail list element => overwrite
                if not val:
                    newval = '[]'
                    _log.debug("Clearing %s to empty list (was: %s)" % (key, res.group('val')))
                elif val[0] == '':
                    newval = "%s + %s" % (res.group('val'), fval)
                    _log.debug("Appending %s to %s" % (fval, key))
                elif val[-1] == '':
                    newval = "%s + %s" % (fval, res.group('val'))
                    _log.debug("Prepending %s to %s" % (fval, key))
                else:
                    newval = "%s" % fval
                    _log.debug("Overwriting %s with %s" % (key, fval))
                ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt)
                _log.info("Tweaked %s list to '%s'" % (key, newval))
            elif get_easyconfig_parameter_default(key) != val:
                additions.append("%s = %s" % (key, val))

            tweaks.pop(key)

    # add parameters or replace existing ones
    for (key, val) in tweaks.items():

        regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>.*)$" % key, re.M)
        _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern))
        res = regexp.search(ectxt)
        if res:
            # only tweak if the value is different
            diff = True
            try:
                _log.debug("eval(%s): %s" % (res.group('val'), eval(res.group('val'))))
                diff = eval(res.group('val')) != val
            except (NameError, SyntaxError):
                # if eval fails, just fall back to string comparison
                _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"...",
                           res.group('val'), val)
                diff = res.group('val') != val

            if diff:
                ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt)
                _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
        elif get_easyconfig_parameter_default(key) != val:
            additions.append("%s = %s" % (key, quote_str(val)))

    if additions:
        _log.info("Adding additional parameters to tweaked easyconfig file: %s" % additions)
        ectxt = '\n'.join([ectxt] + additions)

    _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)

    # come up with suiting file name for tweaked easyconfig file if none was specified
    if target_fn is None:
        fn = None
        try:
            # obtain temporary filename
            fd, tmpfn = tempfile.mkstemp()
            os.close(fd)

            # write easyconfig to temporary file
            write_file(tmpfn, ectxt)

            # determine suiting filename
            fn = ec_filename_for(tmpfn)

            # get rid of temporary file
            os.remove(tmpfn)
        except OSError, err:
            raise EasyBuildError("Failed to determine suiting filename for tweaked easyconfig file: %s", err)

        if targetdir is None:
            targetdir = tempfile.gettempdir()
        target_fn = os.path.join(targetdir, fn)
        _log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn)
예제 #47
0
    deps.extend(easyblock.cfg.dependencies())

    _log.debug(
        "The dependencies to be added to the package are: %s",
        pprint.pformat([easyblock.toolchain.as_dict()] +
                       easyblock.cfg.dependencies()))
    depstring = ''
    for dep in deps:
        if dep.get('external_module', False):
            _log.debug("Skipping dep marked as external module: %s",
                       dep['name'])
        else:
            _log.debug("The dep added looks like %s ", dep)
            dep_pkgname = package_naming_scheme.name(dep)
            depstring += " --depends %s" % quote_str(dep_pkgname)

    # Excluding the EasyBuild logs and test reports that might be in the installdir
    exclude_files_glob = [
        os.path.join(log_path(), "*.log"),
        os.path.join(log_path(), "*.md"),
    ]
    # stripping off leading / to match expected glob in fpm
    exclude_files_glob = [
        '--exclude %s' %
        quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
        for x in exclude_files_glob
    ]
    _log.debug("The list of excluded files passed to fpm: %s",
               exclude_files_glob)
    cmdlist = [
예제 #48
0
def tweak_one(src_fn, target_fn, tweaks, targetdir=None):
    """
    Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
    Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
    once and then use the 'version' variable to construct the list of sources, and possibly other
    parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)

    The tweaks should be specified in a dictionary, with parameters and keys that map to the values
    to be set.

    Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.

    If no target filename is provided, a target filepath is generated based on the contents of
    the tweaked easyconfig file.
    """

    # read easyconfig file
    ectxt = read_file(src_fn)

    _log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" %
               ectxt)
    # determine new toolchain if it's being changed
    keys = tweaks.keys()
    if 'toolchain_name' in keys or 'toolchain_version' in keys:
        tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)
        res = tc_regexp.search(ectxt)
        if not res:
            raise EasyBuildError("No toolchain found in easyconfig file %s?",
                                 src_fn)

        toolchain = eval(res.group(1))
        for key in ['name', 'version']:
            tc_key = "toolchain_%s" % key
            if tc_key in keys:
                toolchain.update({key: tweaks[tc_key]})
                tweaks.pop(tc_key)

        class TcDict(dict):
            """A special dict class that represents trivial toolchains properly."""
            def __repr__(self):
                return "{'name': '%(name)s', 'version': '%(version)s'}" % self

        tweaks.update({
            'toolchain':
            TcDict({
                'name': toolchain['name'],
                'version': toolchain['version']
            })
        })
        _log.debug("New toolchain constructed: %s" % tweaks['toolchain'])

    additions = []

    # automagically clear out list of checksums if software version is being tweaked
    if 'version' in tweaks and 'checksums' not in tweaks:
        tweaks['checksums'] = []
        _log.warning(
            "Tweaking version: checksums cleared, verification disabled.")

    # we need to treat list values seperately, i.e. we prepend to the current value (if any)
    for (key, val) in tweaks.items():

        if isinstance(val, list):
            regexp = re.compile(
                r"^(?P<key>\s*%s)\s*=\s*(?P<val>\[(.|\n)*\])\s*$" % key, re.M)
            res = regexp.search(ectxt)
            if res:
                fval = [x for x in val if x != '']  # filter out empty strings
                # determine to prepend/append or overwrite by checking first/last list item
                # - input ending with comma (empty tail list element) => prepend
                # - input starting with comma (empty head list element) => append
                # - no empty head/tail list element => overwrite
                if not val:
                    newval = '[]'
                    _log.debug("Clearing %s to empty list (was: %s)" %
                               (key, res.group('val')))
                elif val[0] == '':
                    newval = "%s + %s" % (res.group('val'), fval)
                    _log.debug("Appending %s to %s" % (fval, key))
                elif val[-1] == '':
                    newval = "%s + %s" % (fval, res.group('val'))
                    _log.debug("Prepending %s to %s" % (fval, key))
                else:
                    newval = "%s" % fval
                    _log.debug("Overwriting %s with %s" % (key, fval))
                ectxt = regexp.sub("%s = %s" % (res.group('key'), newval),
                                   ectxt)
                _log.info("Tweaked %s list to '%s'" % (key, newval))
            elif get_easyconfig_parameter_default(key) != val:
                additions.append("%s = %s" % (key, val))

            tweaks.pop(key)

    # add parameters or replace existing ones
    for (key, val) in tweaks.items():

        regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>.*)$" % key, re.M)
        _log.debug("Regexp pattern for replacing '%s': %s" %
                   (key, regexp.pattern))
        res = regexp.search(ectxt)
        if res:
            # only tweak if the value is different
            diff = True
            try:
                _log.debug("eval(%s): %s" %
                           (res.group('val'), eval(res.group('val'))))
                diff = eval(res.group('val')) != val
            except (NameError, SyntaxError):
                # if eval fails, just fall back to string comparison
                _log.debug(
                    "eval failed for \"%s\", falling back to string comparison against \"%s\"...",
                    res.group('val'), val)
                diff = res.group('val') != val

            if diff:
                ectxt = regexp.sub(
                    "%s = %s" % (res.group('key'), quote_str(val)), ectxt)
                _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
        elif get_easyconfig_parameter_default(key) != val:
            additions.append("%s = %s" % (key, quote_str(val)))

    if additions:
        _log.info(
            "Adding additional parameters to tweaked easyconfig file: %s" %
            additions)
        ectxt = '\n'.join([ectxt] + additions)

    _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)

    # come up with suiting file name for tweaked easyconfig file if none was specified
    if target_fn is None:
        fn = None
        try:
            # obtain temporary filename
            fd, tmpfn = tempfile.mkstemp()
            os.close(fd)

            # write easyconfig to temporary file
            write_file(tmpfn, ectxt)

            # determine suiting filename
            fn = ec_filename_for(tmpfn)

            # get rid of temporary file
            os.remove(tmpfn)
        except OSError, err:
            raise EasyBuildError(
                "Failed to determine suiting filename for tweaked easyconfig file: %s",
                err)

        if targetdir is None:
            targetdir = tempfile.gettempdir()
        target_fn = os.path.join(targetdir, fn)
        _log.debug("Generated file name for tweaked easyconfig file: %s" %
                   target_fn)