def get_easyblock_class(easyblock, name=None): """ Get class for a particular easyblock (or use default) """ def_class = get_easyconfig_parameter_default('easyblock') def_mod_path = get_module_path(def_class, generic=True) try: if easyblock: # something was specified, lets parse it es = easyblock.split('.') class_name = es.pop(-1) # figure out if full path was specified or not if es: modulepath = '.'.join(es) tup = (class_name, modulepath) _log.info("Assuming that full easyblock module path was specified (class: %s, modulepath: %s)" % tup) cls = get_class_for(modulepath, class_name) else: # if we only get the class name, most likely we're dealing with a generic easyblock try: modulepath = get_module_path(easyblock, generic=True) cls = get_class_for(modulepath, class_name) except ImportError, err: # we might be dealing with a non-generic easyblock, e.g. with --easyblock is used modulepath = get_module_path(easyblock) cls = get_class_for(modulepath, class_name) _log.info("Derived full easyblock module path for %s: %s" % (class_name, modulepath)) else:
def tweak_one(src_fn, target_fn, tweaks, targetdir=None): """ Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions. Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version once and then use the 'version' variable to construct the list of sources, and possibly other parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...) The tweaks should be specified in a dictionary, with parameters and keys that map to the values to be set. Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>. If no target filename is provided, a target filepath is generated based on the contents of the tweaked easyconfig file. """ # read easyconfig file ectxt = read_file(src_fn) _log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" % ectxt) # determine new toolchain if it's being changed keys = tweaks.keys() if 'toolchain_name' in keys or 'toolchain_version' in keys: tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M) res = tc_regexp.search(ectxt) if not res: raise EasyBuildError("No toolchain found in easyconfig file %s?", src_fn) toolchain = eval(res.group(1)) for key in ['name', 'version']: tc_key = "toolchain_%s" % key if tc_key in keys: toolchain.update({key: tweaks[tc_key]}) tweaks.pop(tc_key) class TcDict(dict): """A special dict class that represents trivial toolchains properly.""" def __repr__(self): return "{'name': '%(name)s', 'version': '%(version)s'}" % self tweaks.update({ 'toolchain': TcDict({ 'name': toolchain['name'], 'version': toolchain['version'] }) }) _log.debug("New toolchain constructed: %s" % tweaks['toolchain']) additions = [] # automagically clear out list of checksums if software version is being tweaked if 'version' in tweaks and 'checksums' not in tweaks: tweaks['checksums'] = [] _log.warning( "Tweaking version: checksums cleared, verification disabled.") # we need to treat list values seperately, i.e. we prepend to the current value (if any) for (key, val) in tweaks.items(): if isinstance(val, list): regexp = re.compile( r"^(?P<key>\s*%s)\s*=\s*(?P<val>\[(.|\n)*\])\s*$" % key, re.M) res = regexp.search(ectxt) if res: fval = [x for x in val if x != ''] # filter out empty strings # determine to prepend/append or overwrite by checking first/last list item # - input ending with comma (empty tail list element) => prepend # - input starting with comma (empty head list element) => append # - no empty head/tail list element => overwrite if not val: newval = '[]' _log.debug("Clearing %s to empty list (was: %s)" % (key, res.group('val'))) elif val[0] == '': newval = "%s + %s" % (res.group('val'), fval) _log.debug("Appending %s to %s" % (fval, key)) elif val[-1] == '': newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: newval = "%s" % fval _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) elif get_easyconfig_parameter_default(key) != val: additions.append("%s = %s" % (key, val)) tweaks.pop(key) # add parameters or replace existing ones for (key, val) in tweaks.items(): regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>.*)$" % key, re.M) _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern)) res = regexp.search(ectxt) if res: # only tweak if the value is different diff = True try: _log.debug("eval(%s): %s" % (res.group('val'), eval(res.group('val')))) diff = eval(res.group('val')) != val except (NameError, SyntaxError): # if eval fails, just fall back to string comparison _log.debug( "eval failed for \"%s\", falling back to string comparison against \"%s\"...", res.group('val'), val) diff = res.group('val') != val if diff: ectxt = regexp.sub( "%s = %s" % (res.group('key'), quote_str(val)), ectxt) _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) elif get_easyconfig_parameter_default(key) != val: additions.append("%s = %s" % (key, quote_str(val))) if additions: _log.info( "Adding additional parameters to tweaked easyconfig file: %s" % additions) ectxt = '\n'.join([ectxt] + additions) _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt) # come up with suiting file name for tweaked easyconfig file if none was specified if target_fn is None: fn = None try: # obtain temporary filename fd, tmpfn = tempfile.mkstemp() os.close(fd) # write easyconfig to temporary file write_file(tmpfn, ectxt) # determine suiting filename fn = ec_filename_for(tmpfn) # get rid of temporary file os.remove(tmpfn) except OSError, err: raise EasyBuildError( "Failed to determine suiting filename for tweaked easyconfig file: %s", err) if targetdir is None: targetdir = tempfile.gettempdir() target_fn = os.path.join(targetdir, fn) _log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn)
def tweak_one(src_fn, target_fn, tweaks, targetdir=None): """ Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions. Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version once and then use the 'version' variable to construct the list of sources, and possibly other parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...) The tweaks should be specified in a dictionary, with parameters and keys that map to the values to be set. Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>. If no target filename is provided, a target filepath is generated based on the contents of the tweaked easyconfig file. """ # read easyconfig file ectxt = read_file(src_fn) _log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" % ectxt) # determine new toolchain if it's being changed keys = tweaks.keys() if 'toolchain_name' in keys or 'toolchain_version' in keys: # note: this assumes that the toolchain spec is single-line tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M) res = tc_regexp.search(ectxt) if not res: raise EasyBuildError("No toolchain found in easyconfig file %s: %s", src_fn, ectxt) toolchain = eval(res.group(1)) for key in ['name', 'version']: tc_key = "toolchain_%s" % key if tc_key in keys: toolchain.update({key: tweaks[tc_key]}) tweaks.pop(tc_key) class TcDict(dict): """A special dict class that represents trivial toolchains properly.""" def __repr__(self): return "{'name': '%(name)s', 'version': '%(version)s'}" % self tweaks.update({'toolchain': TcDict({'name': toolchain['name'], 'version': toolchain['version']})}) _log.debug("New toolchain constructed: %s" % tweaks['toolchain']) additions = [] # automagically clear out list of checksums if software version is being tweaked if 'version' in tweaks and 'checksums' not in tweaks: tweaks['checksums'] = [] _log.warning("Tweaking version: checksums cleared, verification disabled.") # we need to treat list values seperately, i.e. we prepend to the current value (if any) for (key, val) in tweaks.items(): if isinstance(val, list): regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>\[(.|\n)*\])\s*$" % key, re.M) res = regexp.search(ectxt) if res: fval = [x for x in val if x != ''] # filter out empty strings # determine to prepend/append or overwrite by checking first/last list item # - input ending with comma (empty tail list element) => prepend # - input starting with comma (empty head list element) => append # - no empty head/tail list element => overwrite if not val: newval = '[]' _log.debug("Clearing %s to empty list (was: %s)" % (key, res.group('val'))) elif val[0] == '': newval = "%s + %s" % (res.group('val'), fval) _log.debug("Appending %s to %s" % (fval, key)) elif val[-1] == '': newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: newval = "%s" % fval _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) elif get_easyconfig_parameter_default(key) != val: additions.append("%s = %s" % (key, val)) tweaks.pop(key) # add parameters or replace existing ones for (key, val) in tweaks.items(): regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>.*)$" % key, re.M) _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern)) res = regexp.search(ectxt) if res: # only tweak if the value is different diff = True try: _log.debug("eval(%s): %s" % (res.group('val'), eval(res.group('val')))) diff = eval(res.group('val')) != val except (NameError, SyntaxError): # if eval fails, just fall back to string comparison _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"...", res.group('val'), val) diff = res.group('val') != val if diff: ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt) _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) elif get_easyconfig_parameter_default(key) != val: additions.append("%s = %s" % (key, quote_str(val))) if additions: _log.info("Adding additional parameters to tweaked easyconfig file: %s" % additions) ectxt = '\n'.join([ectxt] + additions) _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt) # come up with suiting file name for tweaked easyconfig file if none was specified if target_fn is None: fn = None try: # obtain temporary filename fd, tmpfn = tempfile.mkstemp() os.close(fd) # write easyconfig to temporary file write_file(tmpfn, ectxt) # determine suiting filename fn = ec_filename_for(tmpfn) # get rid of temporary file os.remove(tmpfn) except OSError, err: raise EasyBuildError("Failed to determine suiting filename for tweaked easyconfig file: %s", err) if targetdir is None: targetdir = tempfile.gettempdir() target_fn = os.path.join(targetdir, fn) _log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn)