def __init__(self, config, datasetExpr, datasetNick, datasetID = 0): DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID) (self._path, self._events, selist) = utils.optSplit(datasetExpr, '|@') self._selist = utils.parseList(selist, delimeter = ',', onEmpty = None) if not (self._path and self._events): raise ConfigError('Invalid dataset expression!\nCorrect: /local/path/to/file|events[@SE1,SE2]')
def _parseFile(self, container, configFile, defaults = None, searchPaths = []): try: configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError) log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.')) log.log(logging.INFO1, 'Reading config file %s' % configFile) for line in map(lambda x: x.rstrip() + '=:', open(configFile, 'r').readlines()): if line.startswith('[') or line.lstrip().startswith(';'): continue # skip section and comment lines # Abort if non-indented line with ":" preceeding "=" was found if (line.lstrip() == line) and (line.find(":") < line.find("=")): raise ConfigError('Invalid config line:\n\t%s\nPlease use "key = value" syntax or indent values!' % line) # Parse with python config parser parser = ConfigParser.ConfigParser(defaults) parser.readfp(open(configFile, 'r')) # Parse include files if parser.has_option('global', 'include'): includeFiles = parser.get('global', 'include').split('#')[0].split(';')[0] for includeFile in utils.parseList(includeFiles, None): self._parseFile(container, includeFile, parser.defaults(), searchPaths + [os.path.dirname(configFile)]) # Store config settings for section in parser.sections(): for option in parser.options(section): if (section, option) != ('global', 'include'): value_list = parser.get(section, option).splitlines() # Strip comments value_list = map(lambda l: l.rsplit(';', 1)[0].strip(), value_list) value_list = filter(lambda l: l != '', value_list) container.setEntry(section, option, str.join('\n', value_list), configFile) except: raise RethrowError('Error while reading configuration file "%s"!' % configFile, ConfigError)
def getBlocksInternal(self): def doFilter(block): if self._filter: name = '/%s#%s#' % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, '')) return self._filter in name return True (blockinfo, commonMetadata) = (None, []) for line in open(self._filename, 'rb'): # Found start of block: line = line.strip() if line.startswith(';'): continue elif line.startswith('['): if blockinfo and doFilter(blockinfo): yield blockinfo blockinfo = { DataProvider.Locations: None, DataProvider.FileList: [] } blockname = line.lstrip('[').rstrip(']').split('#') if len(blockname) > 0: blockinfo[DataProvider.Dataset] = blockname[0] if len(blockname) > 1: blockinfo[DataProvider.BlockName] = blockname[1] commonprefix = self._forcePrefix commonMetadata = [] elif line != '': tmp = map(str.strip, QM('[' in line, line.split(' = ', 1), rsplit(line, '=', 1))) if len(tmp) != 2: raise ConfigError('Malformed entry in dataset file:\n%s' % line) key, value = tmp if key.lower() == 'nickname': blockinfo[DataProvider.Nickname] = value elif key.lower() == 'id': blockinfo[DataProvider.DatasetID] = int(value) elif key.lower() == 'events': blockinfo[DataProvider.NEntries] = int(value) elif key.lower() == 'metadata': blockinfo[DataProvider.Metadata] = eval(value) elif key.lower() == 'metadata common': commonMetadata = eval(value) elif key.lower() == 'se list': blockinfo[DataProvider.Locations] = utils.parseList(value) elif key.lower() == 'prefix': if not self._forcePrefix: commonprefix = value else: if commonprefix: key = '%s/%s' % (commonprefix, key) value = value.split(' ', 1) data = { DataProvider.URL: key, DataProvider.NEntries: int(value[0]) } if commonMetadata: data[DataProvider.Metadata] = commonMetadata if len(value) > 1: data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1]) blockinfo[DataProvider.FileList].append(data) else: if blockinfo and doFilter(blockinfo): yield blockinfo
def getBlocksInternal(self): def doFilter(block): if self._filter: name = "/%s#%s#" % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, "")) return self._filter in name return True (blockinfo, commonMetadata) = (None, []) for line in open(self._filename, "rb"): # Found start of block: line = line.strip() if line.startswith(";"): continue elif line.startswith("["): if blockinfo and doFilter(blockinfo): yield blockinfo blockinfo = {DataProvider.Locations: None, DataProvider.FileList: []} blockname = line.lstrip("[").rstrip("]").split("#") if len(blockname) > 0: blockinfo[DataProvider.Dataset] = blockname[0] if len(blockname) > 1: blockinfo[DataProvider.BlockName] = blockname[1] commonprefix = self._forcePrefix commonMetadata = [] elif line != "": tmp = map(str.strip, utils.QM("[" in line, line.split(" = ", 1), rsplit(line, "=", 1))) if len(tmp) != 2: raise ConfigError("Malformed entry in dataset file:\n%s" % line) key, value = tmp if key.lower() == "nickname": blockinfo[DataProvider.Nickname] = value elif key.lower() == "id": blockinfo[DataProvider.DatasetID] = int(value) elif key.lower() == "events": blockinfo[DataProvider.NEntries] = int(value) elif key.lower() == "metadata": blockinfo[DataProvider.Metadata] = eval(value) elif key.lower() == "metadata common": commonMetadata = eval(value) elif key.lower() == "se list": blockinfo[DataProvider.Locations] = utils.parseList(value) elif key.lower() == "prefix": if not self._forcePrefix: commonprefix = value else: if commonprefix: key = "%s/%s" % (commonprefix, key) value = value.split(" ", 1) data = {DataProvider.URL: key, DataProvider.NEntries: int(value[0])} if commonMetadata: data[DataProvider.Metadata] = commonMetadata if len(value) > 1: data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1]) blockinfo[DataProvider.FileList].append(data) else: if blockinfo and doFilter(blockinfo): yield blockinfo
def getClassList(self, *args, **kwargs): baseClass = kwargs.pop('cls') tags = kwargs.pop('tags', []) inherit = kwargs.pop('inherit', False) defaultName = kwargs.pop('defaultName', '') parseSingle = lambda value: ClassWrapper(baseClass, value, self, tags, inherit, defaultName) str2obj = lambda value: map(parseSingle, utils.parseList(value, None, onEmpty = [])) obj2str = lambda value: str.join('\n', map(str, value)) (selector, args, kwargs) = self._selectorFilter(*args, **kwargs) return self._getTyped('class', obj2str, str2obj, str2obj, selector, *args, **kwargs)
def getCompositePlugin(self, option, default = noDefault, default_compositor = noDefault, option_compositor = None, cls = Plugin, tags = [], inherit = False, defaultName = '', **kwargs): parseSingle = lambda value: ClassWrapper(cls, value, self, tags, inherit, defaultName, self._getPluginPaths()) str2obj = lambda value: map(parseSingle, utils.parseList(value, None, onEmpty = [])) obj2str = lambda value: str.join('\n', map(str, value)) clsList = self._getInternal('plugins', obj2str, str2obj, str2obj, option, default, **kwargs) if len(clsList) == 1: return clsList[0] if not option_compositor: option_compositor = option + ' manager' clsCompositor = self.getPlugin(option_compositor, default_compositor, cls, tags, inherit, defaultName, **kwargs) return CompositedClassWrapper(clsCompositor, clsList)
def getPaths(self, option, default = noDefault, mustExist = True, **kwargs): def patlist2pathlist(value, mustExist): try: for pattern in value: for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError): yield fn except Exception: raise ConfigError('Error resolving pattern %s' % pattern) str2obj = lambda value: list(patlist2pathlist(utils.parseList(value, None, onEmpty = []), mustExist)) obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False)) return self._getInternal('paths', obj2str, str2obj, None, option, default, **kwargs)
def getPaths(self, *args, **kwargs): def patlist2pathlist(value, mustExist): try: for pattern in value: for fn in utils.resolvePaths(pattern, [self._pathBase], mustExist, ConfigError): yield fn except: raise RethrowError('Error resolving pattern %s' % pattern, ConfigError) mustExist = kwargs.pop('mustExist', True) str2obj = lambda value: list(patlist2pathlist(utils.parseList(value, None, onEmpty = []), mustExist)) obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False)) (selector, args, kwargs) = self._selectorFilter(*args, **kwargs) return self._getTyped('paths', obj2str, str2obj, None, selector, *args, **kwargs)
def getList(self, option, default = noDefault, parseItem = lambda x: x, **kwargs): obj2str = lambda value: '\n' + str.join('\n', map(str, value)) str2obj = lambda value: map(parseItem, utils.parseList(value, None)) return self._getInternal('list', obj2str, str2obj, None, option, default, **kwargs)
def __init__(self, config, name, userOpt, itemName, discoverFun): Broker.__init__(self, config, name, userOpt, itemName, discoverFun) self._storageDict = config.getDict('%s storage access' % userOpt, {}, onChange = None, parser = lambda x: utils.parseList(x, ' '), strfun = lambda x: str.join(' ', x))[0]
def getFlatList(section, option): for (opt, value, s) in filter(lambda (opt, v, s): opt == option, tmpConfigContent.get(section, [])): for entry in utils.parseList(value, None): yield entry
def getList(self, *args, **kwargs): parseItem = kwargs.pop('parseItem', lambda x: x) obj2str = lambda value: '\n' + str.join('\n', map(str, value)) str2obj = lambda value: map(parseItem, utils.parseList(value, None)) (selector, args, kwargs) = self._selectorFilter(*args, **kwargs) return self._getTyped('list', obj2str, str2obj, None, selector, *args, **kwargs)