def test_ordered_versop_expressions(self): """Given set of ranges, order them according to version/operator (most recent/specific first)""" # simple version ordering, all different versions ovop = OrderedVersionOperators() versop_exprs = [ '> 3.0.0', '>= 2.5.0', '> 2.0.0', '== 1.0.0', ] # add version expressions out of order intentionally ovop.add(versop_exprs[1]) ovop.add(versop_exprs[-1]) ovop.add(versop_exprs[0]) ovop.add(versop_exprs[2]) # verify whether order is what we expect it to be self.assertEqual(ovop.versops, [VersionOperator(x) for x in versop_exprs]) # more complex version ordering, identical/overlapping vesions ovop = OrderedVersionOperators() versop_exprs = [ '== 1.0.0', '> 1.0.0', '< 1.0.0', ] # add version expressions out of order intentionally ovop.add(versop_exprs[-1]) ovop.add(versop_exprs[1]) ovop.add(versop_exprs[0]) # verify whether order is what we expect it to be self.assertEqual(ovop.versops, [VersionOperator(x) for x in versop_exprs])
def test_ordered_versop_add_data(self): """Test the add and data handling""" ovop = OrderedVersionOperators() tests = [ ('> 1', '5'), ('> 2', { 'x': 3 }), ] for versop_txt, data in tests: versop = VersionOperator(versop_txt) ovop.add(versop) # no data was added, this is a new entry, mapper is initialised with None self.assertEqual(ovop.get_data(versop), None) ovop.add(versop, data) # test data self.assertEqual(ovop.get_data(versop), data) # new data for same versops tests = [ ('> 1', '6'), ('> 2', { 'x': 4 }), ] for versop_txt, data in tests: versop = VersionOperator(versop_txt) ovop.add(versop, data) # test updated data self.assertEqual(ovop.get_data(versop), data) # 'update' a value # the data for '> 1' has no .update() extra_data = {'y': 4} tests = [ ('> 2', extra_data), ] for versop_txt, data in tests: versop = VersionOperator(versop_txt) prevdata = copy.deepcopy(ovop.get_data(versop)) prevdata.update(extra_data) ovop.add(versop, data, update=True) # test updated data self.assertEqual(ovop.get_data(versop), prevdata) # use update=True on new element versop = VersionOperator('> 10000') new_data = {'new': 5} ovop.add(versop, new_data, update=True) # test updated data self.assertEqual(ovop.get_data(versop), new_data)
def _squash_versop(self, key, value, squashed, sanity, vt_tuple): """ Squash VERSION_OPERATOR_VALUE_TYPES value return None or new Squashed instance :param key: section key :param nested_dict: the nested_dict instance :param squashed: Squashed instance :param sanity: the sanity dict :param vt_tuple: version, tc_name, tc_version tuple """ version, tcname, tcversion = vt_tuple if key == 'toolchains': # remove any other toolchain from list self.log.debug("Filtering 'toolchains' key") matching_toolchains = [] tmp_tc_oversops = {} # temporary, only for conflict checking for tcversop in value: tc_overops = tmp_tc_oversops.setdefault(tcversop.tc_name, OrderedVersionOperators()) self.log.debug("Add tcversop %s to tc_overops %s tcname %s tcversion %s", tcversop, tc_overops, tcname, tcversion) tc_overops.add(tcversop) # test non-conflicting list if tcversop.test(tcname, tcversion): matching_toolchains.append(tcversop) if matching_toolchains: # does this have any use? self.log.debug('Matching toolchains %s found (but data not needed)' % matching_toolchains) else: self.log.debug('No matching toolchains, removing the whole current key %s' % key) return Squashed() elif key == 'versions': self.log.debug("Adding all versions %s from versions key" % value) matching_versions = [] tmp_versops = OrderedVersionOperators() # temporary, only for conflict checking for versop in value: tmp_versops.add(versop) # test non-conflicting list if versop.test(version): matching_versions.append(versop) if matching_versions: # does this have any use? self.log.debug('Matching versions %s found (but data not needed)' % matching_versions) else: self.log.debug('No matching versions, removing the whole current key %s' % key) return Squashed() else: raise EasyBuildError('Unexpected VERSION_OPERATOR_VALUE_TYPES key %s value %s', key, value) return None
def test_ordered_versop_add_data(self): """Test the add and data handling""" ovop = OrderedVersionOperators() tests = [ ('> 1', '5'), ('> 2', {'x': 3}), ] for versop_txt, data in tests: versop = VersionOperator(versop_txt) ovop.add(versop) # no data was added, this is a new entry, mapper is initialised with None self.assertEqual(ovop.get_data(versop), None) ovop.add(versop, data) # test data self.assertEqual(ovop.get_data(versop), data) # new data for same versops tests = [ ('> 1', '6'), ('> 2', {'x': 4}), ] for versop_txt, data in tests: versop = VersionOperator(versop_txt) ovop.add(versop, data) # test updated data self.assertEqual(ovop.get_data(versop), data) # 'update' a value # the data for '> 1' has no .update() extra_data = {'y': 4} tests = [ ('> 2', extra_data), ] for versop_txt, data in tests: versop = VersionOperator(versop_txt) prevdata = copy.deepcopy(ovop.get_data(versop)) prevdata.update(extra_data) ovop.add(versop, data, update=True) # test updated data self.assertEqual(ovop.get_data(versop), prevdata) # use update=True on new element versop = VersionOperator('> 10000') new_data = {'new': 5} ovop.add(versop, new_data, update=True) # test updated data self.assertEqual(ovop.get_data(versop), new_data)
class Squashed(object): """Class to ease the squashing of OrderedVersionOperators and OrderedToolchainVersionOperators""" def __init__(self): """Initialise Squashed instance""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) # OrderedVersionOperators instances to keep track of the data of the matching # version and toolchain version sections self.versions = OrderedVersionOperators() self.tcversions = OrderedVersionOperators() self.result = {} def add_toolchain(self, squashed): """ Add squashed instance from a toolchain section @param squashed: a Squashed instance """ # TODO unify with add_version, make one .add() # data from toolchain self.result.update(squashed.result) for versop in squashed.versions.versops: self.versions.add(versop, squashed.versions.get_data(versop), update=True) def add_version(self, section, squashed): """ Add squashed instance from version section @param section: the version section versionoperator instance @param squashed: a Squashed instance """ # TODO unify with add_toolchain, make one .add() # don't update res_sections # add this to a orderedversop that has matching versops. # data in this matching orderedversop must be updated to the res at the end for versop in squashed.versions.versops: self.versions.add(versop, squashed.versions.get_data(versop), update=True) self.versions.add(section, squashed.result, update=True) def final(self): """Final squashing of version and toolchainversion operators and return the result""" self.log.debug('Pre-final result %s' % self.result) self.log.debug('Pre-final versions %s with data %s' % (self.versions, self.versions.datamap)) self.log.debug('Pre-final tcversions %s with data %s' % (self.tcversions, self.tcversions.datamap)) # update self.result, most strict matching versionoperator should be first element # so update in reversed order # also update toolchain data before version data for vers in [self.tcversions, self.versions]: for versop in vers.versops[::-1]: self.result.update(vers.get_data(versop)) return self.result
class Squashed(object): """Class to ease the squashing of OrderedVersionOperators and OrderedToolchainVersionOperators""" def __init__(self): """Initialise Squashed instance""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) # OrderedVersionOperators instances to keep track of the data of the matching # version and toolchain version sections self.versions = OrderedVersionOperators() self.tcversions = OrderedVersionOperators() self.result = {} def add_toolchain(self, squashed): """ Add squashed instance from a toolchain section :param squashed: a Squashed instance """ # TODO unify with add_version, make one .add() # data from toolchain self.result.update(squashed.result) for versop in squashed.versions.versops: self.versions.add(versop, squashed.versions.get_data(versop), update=True) def add_version(self, section, squashed): """ Add squashed instance from version section :param section: the version section versionoperator instance :param squashed: a Squashed instance """ # TODO unify with add_toolchain, make one .add() # don't update res_sections # add this to a orderedversop that has matching versops. # data in this matching orderedversop must be updated to the res at the end for versop in squashed.versions.versops: self.versions.add(versop, squashed.versions.get_data(versop), update=True) self.versions.add(section, squashed.result, update=True) def final(self): """Final squashing of version and toolchainversion operators and return the result""" self.log.debug('Pre-final result %s' % self.result) self.log.debug('Pre-final versions %s with data %s' % (self.versions, self.versions.datamap)) self.log.debug('Pre-final tcversions %s with data %s' % (self.tcversions, self.tcversions.datamap)) # update self.result, most strict matching versionoperator should be first element # so update in reversed order # also update toolchain data before version data for vers in [self.tcversions, self.versions]: for versop in vers.versops[::-1]: self.result.update(vers.get_data(versop)) return self.result
class EBConfigObj(object): """ Enhanced ConfigObj, version/toolchain and other easyconfig specific aspects aware Given ConfigObj instance, make instance that represents a parser Mandatory/minimal (to mimic v1.0 behaviour); first version/toolchain is the default [SUPPORTED] versions=version_operator toolchains=toolchain_version_operator Optional [DEFAULT] ... [<operatorX> <versionX>] ... [<toolchainA> <operatorA> <versionA>] [[<operatorY> <versionY>]] ... ... """ SECTION_MARKER_DEFAULT = 'DEFAULT' SECTION_MARKER_DEPENDENCIES = 'DEPENDENCIES' SECTION_MARKER_SUPPORTED = 'SUPPORTED' # list of known marker types (except default) KNOWN_VERSION_MARKER_TYPES = [ToolchainVersionOperator, VersionOperator] # order matters, see parse_sections VERSION_OPERATOR_VALUE_TYPES = { # toolchains: comma-separated list of toolchain version operators 'toolchains': ToolchainVersionOperator, # versions: comma-separated list of version operators 'versions': VersionOperator, } def __init__(self, configobj=None): """ Initialise EBConfigObj instance @param configobj: ConfigObj instance """ self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.tcname = None self.default = {} # default section self.supported = {} # supported section self.sections = {} # all other sections self.unfiltered_sections = {} # unfiltered other sections self.versops = OrderedVersionOperators() self.tcversops = OrderedVersionOperators() if configobj is not None: self.parse(configobj) def parse_sections(self, configobj, toparse=None, parent=None, depth=0): """ Parse configobj instance; convert all supported sections, keys and values to their respective representations Returns a dict of (nested) Sections @param configobj: a ConfigObj instance, basically a dict of (unparsed) sections """ # note: configobj already converts comma-separated strings in lists # # list of supported keywords, all else will fail special_keys = self.VERSION_OPERATOR_VALUE_TYPES.keys() if parent is None: # no parent, so top sections parsed = {} else: # parent specified, so not a top section parsed = Section(parent=parent, depth=depth + 1, main=configobj) # start with full configobj initially, and then process subsections recursively if toparse is None: toparse = configobj for key, value in toparse.items(): if isinstance(value, Section): self.log.debug("Enter subsection key %s value %s" % (key, value)) # only supported types of section keys are: # * DEFAULT # * SUPPORTED # * dependencies # * VersionOperator or ToolchainVersionOperator (e.g. [> 2.0], [goolf > 1]) if key in [self.SECTION_MARKER_DEFAULT, self.SECTION_MARKER_SUPPORTED]: # parse value as a section, recursively new_value = self.parse_sections(configobj, toparse=value, parent=value.parent, depth=value.depth) self.log.debug('Converted %s section to new value %s' % (key, new_value)) parsed[key] = new_value elif key == self.SECTION_MARKER_DEPENDENCIES: new_key = 'dependencies' new_value = [] for dep_name, dep_val in value.items(): if isinstance(dep_val, Section): self.log.error("Unsupported nested section '%s' found in dependencies section" % dep_name) else: # FIXME: parse the dependency specification for version, toolchain, suffix, etc. dep = Dependency(dep_val, name=dep_name) if dep.name() is None or dep.version() is None: self.log.error("Failed to find name/version in parsed dependency: %s (dict: %s)" % (dep, dict(dep))) new_value.append(dep) self.log.debug('Converted %s section to %s, passed it to parent section (or default)' % (key, new_value)) if isinstance(parsed, Section): parsed.parent[new_key] = new_value else: parsed[self.SECTION_MARKER_DEFAULT].update({new_key: new_value}) else: # try parsing key as toolchain version operator first # try parsing as version operator if it's not a toolchain version operator for marker_type in self.KNOWN_VERSION_MARKER_TYPES: new_key = marker_type(key) if new_key: self.log.debug("'%s' was parsed as a %s section marker" % (key, marker_type.__name__)) break else: self.log.debug("Not a %s section marker" % marker_type.__name__) if not new_key: self.log.error("Unsupported section marker '%s'" % key) # parse value as a section, recursively new_value = self.parse_sections(configobj, toparse=value, parent=value.parent, depth=value.depth) self.log.debug('Converted key %s value %s in new key %s new value %s' % (key, value, new_key, new_value)) parsed[new_key] = new_value else: # simply pass down any non-special key-value items if not key in special_keys: self.log.debug('Passing down key %s with value %s' % (key, value)) new_value = value # parse individual key-value assignments elif key in self.VERSION_OPERATOR_VALUE_TYPES: value_type = self.VERSION_OPERATOR_VALUE_TYPES[key] # list of supported toolchains/versions # first one is default if isinstance(value, basestring): # so the split should be unnecessary # (if it's not a list already, it's just one value) # TODO this is annoying. check if we can force this in configobj value = value.split(',') # remove possible surrounding whitespace (some people add space after comma) new_value = [value_type(x.strip()) for x in value] if False in [x.is_valid() for x in new_value]: self.log.error("Failed to parse '%s' as list of %s" % (value, value_type.__name__)) else: tup = (key, value, type(value)) self.log.error('Bug: supported but unknown key %s with non-string value: %s, type %s' % tup) self.log.debug("Converted value '%s' for key '%s' into new value '%s'" % (value, key, new_value)) parsed[key] = new_value return parsed def validate_and_filter_by_toolchain(self, tcname, processed=None, filtered_sections=None, other_sections=None): """ Build the ordered version operator and toolchain version operator, ignoring all other toolchains @param tcname: toolchain name to keep @param processed: a processed dict of sections to filter @param path: list of keys to identify the path in the dict """ top_call = False if processed is None: processed = self.sections top_call = True if filtered_sections is None: filtered_sections = {} if other_sections is None: other_sections = {} # walk over dictionary of parsed sections, and check for marker conflicts (using .add()) # add section markers relevant to specified toolchain to self.tcversops for key, value in processed.items(): if isinstance(value, Section): if isinstance(key, ToolchainVersionOperator): if not key.tc_name == tcname: self.log.debug("Found marker for other toolchain '%s'" % key.tc_name) # also perform sanity check for other toolchains, make add check for conflicts tc_overops = other_sections.setdefault(key.tc_name, OrderedVersionOperators()) tc_overops.add(key) # nothing more to do here, just continue with other sections continue else: self.log.debug("Found marker for specified toolchain '%s': %s" % (tcname, key)) # add marker to self.tcversops (which triggers a conflict check) self.tcversops.add(key, value) filtered_sections[key] = value elif isinstance(key, VersionOperator): self.log.debug("Found marker for version '%s'" % key) # keep track of all version operators, and enforce conflict check self.versops.add(key, value) filtered_sections[key] = value else: self.log.error("Unhandled section marker '%s' (type '%s')" % (key, type(key))) # recursively go deeper for (relevant) sections self.validate_and_filter_by_toolchain(tcname, processed=value, filtered_sections=filtered_sections, other_sections=other_sections) elif key in self.VERSION_OPERATOR_VALUE_TYPES: self.log.debug("Found version operator key-value entry (%s)" % key) if key == 'toolchains': # remove any other toolchain from list filtered_sections[key] = [tcversop for tcversop in value if tcversop.tc_name == tcname] else: # retain all other values filtered_sections[key] = value else: self.log.debug("Found non-special key-value entry (key %s), skipping it" % key) if top_call: self.unfiltered_sections = self.sections self.sections = filtered_sections def parse(self, configobj): """ First parse the configobj instance Then build the structure to support the versionoperators and all other parts of the structure @param configobj: ConfigObj instance """ # keep reference to original (in case it's needed/wanted) self.configobj = configobj # process the configobj instance self.sections = self.parse_sections(self.configobj) # handle default section # no nesting # - add DEFAULT key-value entries to the root of self.sections # - key-value items from other sections will be deeper down # - deepest level is best match and wins, so defaults are on top level self.default = self.sections.pop(self.SECTION_MARKER_DEFAULT, {}) for key, value in self.default.items(): self.sections[key] = value # handle supported section # supported should only have 'versions' and 'toolchains' keys self.supported = self.sections.pop(self.SECTION_MARKER_SUPPORTED, {}) for key, value in self.supported.items(): if not key in self.VERSION_OPERATOR_VALUE_TYPES: self.log.error('Unsupported key %s in %s section' % (key, self.SECTION_MARKER_SUPPORTED)) self.sections['%s' % key] = value if 'versions' in self.supported: # first of list is special: it is the default self.default['version'] = self.supported['versions'][0].get_version_str() if 'toolchains' in self.supported: # first of list is special: it is the default self.default['toolchain'] = self.supported['toolchains'][0].as_dict() tup = (self.default, self.supported, self.sections) self.log.debug("(parse) default: %s; supported: %s, sections: %s" % tup) def get_specs_for(self, version=None, tcname=None, tcversion=None): """ Return dictionary with specifications listed in sections applicable for specified info. """ if isinstance(self.default, Section): cfg = self.default.dict() else: cfg = copy.deepcopy(self.default) # make sure that requested version/toolchain are supported by this easyconfig versions = [x.get_version_str() for x in self.supported['versions']] if version is None: self.log.debug("No version specified") elif version in versions: self.log.debug("Version '%s' is supported in easyconfig." % version) else: self.log.error("Version '%s' not supported in easyconfig (only %s)" % (version, versions)) tcnames = [tc.tc_name for tc in self.supported['toolchains']] if tcname is None: self.log.debug("Toolchain name not specified.") elif tcname in tcnames: self.log.debug("Toolchain '%s' is supported in easyconfig." % tcname) tcversions = [tc.get_version_str() for tc in self.supported['toolchains'] if tc.tc_name == tcname] if tcversion is None: self.log.debug("Toolchain version not specified.") elif tcversion in tcversions: self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion)) else: tup = (tcname, tcversion, tcversions) self.log.error("Toolchain '%s' version '%s' not supported in easyconfig (only %s)" % tup) else: self.log.error("Toolchain '%s' not supported in easyconfig (only %s)" % (tcname, tcnames)) # TODO: determine 'path' to take in sections based on version and toolchain version # SDW: ask the versionoperator self.log.debug("self.versops: %s" % self.versops) self.log.debug("self.tcversops: %s" % self.tcversops) return cfg