def __init__(self, config, datasetExpr, datasetNick, datasetID = 0):
DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID)
(self._path, self._events, selist) = utils.optSplit(datasetExpr, '|@')
self._selist = utils.parseList(selist, delimeter = ',', onEmpty = None)
if not (self._path and self._events):
raise ConfigError('Invalid dataset expression!\nCorrect: /local/path/to/file|events[@SE1,SE2]')
def _parseFile(self, container, configFile, defaults = None, searchPaths = []):
try:
configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.'))
log.log(logging.INFO1, 'Reading config file %s' % configFile)
for line in map(lambda x: x.rstrip() + '=:', open(configFile, 'r').readlines()):
if line.startswith('[') or line.lstrip().startswith(';'):
continue # skip section and comment lines
# Abort if non-indented line with ":" preceeding "=" was found
if (line.lstrip() == line) and (line.find(":") < line.find("=")):
raise ConfigError('Invalid config line:\n\t%s\nPlease use "key = value" syntax or indent values!' % line)
# Parse with python config parser
parser = ConfigParser.ConfigParser(defaults)
parser.readfp(open(configFile, 'r'))
# Parse include files
if parser.has_option('global', 'include'):
includeFiles = parser.get('global', 'include').split('#')[0].split(';')[0]
for includeFile in utils.parseList(includeFiles, None):
self._parseFile(container, includeFile, parser.defaults(),
searchPaths + [os.path.dirname(configFile)])
# Store config settings
for section in parser.sections():
for option in parser.options(section):
if (section, option) != ('global', 'include'):
value_list = parser.get(section, option).splitlines() # Strip comments
value_list = map(lambda l: l.rsplit(';', 1)[0].strip(), value_list)
value_list = filter(lambda l: l != '', value_list)
container.setEntry(section, option, str.join('\n', value_list), configFile)
except:
raise RethrowError('Error while reading configuration file "%s"!' % configFile, ConfigError)
def getBlocksInternal(self):
def doFilter(block):
if self._filter:
name = '/%s#%s#' % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, ''))
return self._filter in name
return True
(blockinfo, commonMetadata) = (None, [])
for line in open(self._filename, 'rb'):
# Found start of block:
line = line.strip()
if line.startswith(';'):
continue
elif line.startswith('['):
if blockinfo and doFilter(blockinfo):
yield blockinfo
blockinfo = { DataProvider.Locations: None, DataProvider.FileList: [] }
blockname = line.lstrip('[').rstrip(']').split('#')
if len(blockname) > 0:
blockinfo[DataProvider.Dataset] = blockname[0]
if len(blockname) > 1:
blockinfo[DataProvider.BlockName] = blockname[1]
commonprefix = self._forcePrefix
commonMetadata = []
elif line != '':
tmp = map(str.strip, QM('[' in line, line.split(' = ', 1), rsplit(line, '=', 1)))
if len(tmp) != 2:
raise ConfigError('Malformed entry in dataset file:\n%s' % line)
key, value = tmp
if key.lower() == 'nickname':
blockinfo[DataProvider.Nickname] = value
elif key.lower() == 'id':
blockinfo[DataProvider.DatasetID] = int(value)
elif key.lower() == 'events':
blockinfo[DataProvider.NEntries] = int(value)
elif key.lower() == 'metadata':
blockinfo[DataProvider.Metadata] = eval(value)
elif key.lower() == 'metadata common':
commonMetadata = eval(value)
elif key.lower() == 'se list':
blockinfo[DataProvider.Locations] = utils.parseList(value)
elif key.lower() == 'prefix':
if not self._forcePrefix:
commonprefix = value
else:
if commonprefix:
key = '%s/%s' % (commonprefix, key)
value = value.split(' ', 1)
data = { DataProvider.URL: key, DataProvider.NEntries: int(value[0]) }
if commonMetadata:
data[DataProvider.Metadata] = commonMetadata
if len(value) > 1:
data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1])
blockinfo[DataProvider.FileList].append(data)
else:
if blockinfo and doFilter(blockinfo):
yield blockinfo
def getBlocksInternal(self):
def doFilter(block):
if self._filter:
name = "/%s#%s#" % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, ""))
return self._filter in name
return True
(blockinfo, commonMetadata) = (None, [])
for line in open(self._filename, "rb"):
# Found start of block:
line = line.strip()
if line.startswith(";"):
continue
elif line.startswith("["):
if blockinfo and doFilter(blockinfo):
yield blockinfo
blockinfo = {DataProvider.Locations: None, DataProvider.FileList: []}
blockname = line.lstrip("[").rstrip("]").split("#")
if len(blockname) > 0:
blockinfo[DataProvider.Dataset] = blockname[0]
if len(blockname) > 1:
blockinfo[DataProvider.BlockName] = blockname[1]
commonprefix = self._forcePrefix
commonMetadata = []
elif line != "":
tmp = map(str.strip, utils.QM("[" in line, line.split(" = ", 1), rsplit(line, "=", 1)))
if len(tmp) != 2:
raise ConfigError("Malformed entry in dataset file:\n%s" % line)
key, value = tmp
if key.lower() == "nickname":
blockinfo[DataProvider.Nickname] = value
elif key.lower() == "id":
blockinfo[DataProvider.DatasetID] = int(value)
elif key.lower() == "events":
blockinfo[DataProvider.NEntries] = int(value)
elif key.lower() == "metadata":
blockinfo[DataProvider.Metadata] = eval(value)
elif key.lower() == "metadata common":
commonMetadata = eval(value)
elif key.lower() == "se list":
blockinfo[DataProvider.Locations] = utils.parseList(value)
elif key.lower() == "prefix":
if not self._forcePrefix:
commonprefix = value
else:
if commonprefix:
key = "%s/%s" % (commonprefix, key)
value = value.split(" ", 1)
data = {DataProvider.URL: key, DataProvider.NEntries: int(value[0])}
if commonMetadata:
data[DataProvider.Metadata] = commonMetadata
if len(value) > 1:
data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1])
blockinfo[DataProvider.FileList].append(data)
else:
if blockinfo and doFilter(blockinfo):
yield blockinfo
def getClassList(self, *args, **kwargs):
baseClass = kwargs.pop('cls')
tags = kwargs.pop('tags', [])
inherit = kwargs.pop('inherit', False)
defaultName = kwargs.pop('defaultName', '')
parseSingle = lambda value: ClassWrapper(baseClass, value, self, tags, inherit, defaultName)
str2obj = lambda value: map(parseSingle, utils.parseList(value, None, onEmpty = []))
obj2str = lambda value: str.join('\n', map(str, value))
(selector, args, kwargs) = self._selectorFilter(*args, **kwargs)
return self._getTyped('class', obj2str, str2obj, str2obj, selector, *args, **kwargs)
def getCompositePlugin(self, option, default = noDefault, default_compositor = noDefault, option_compositor = None,
cls = Plugin, tags = [], inherit = False, defaultName = '', **kwargs):
parseSingle = lambda value: ClassWrapper(cls, value, self, tags, inherit, defaultName, self._getPluginPaths())
str2obj = lambda value: map(parseSingle, utils.parseList(value, None, onEmpty = []))
obj2str = lambda value: str.join('\n', map(str, value))
clsList = self._getInternal('plugins', obj2str, str2obj, str2obj, option, default, **kwargs)
if len(clsList) == 1:
return clsList[0]
if not option_compositor:
option_compositor = option + ' manager'
clsCompositor = self.getPlugin(option_compositor, default_compositor, cls, tags, inherit, defaultName, **kwargs)
return CompositedClassWrapper(clsCompositor, clsList)
def getPaths(self, option, default = noDefault, mustExist = True, **kwargs):
def patlist2pathlist(value, mustExist):
try:
for pattern in value:
for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError):
yield fn
except Exception:
raise ConfigError('Error resolving pattern %s' % pattern)
str2obj = lambda value: list(patlist2pathlist(utils.parseList(value, None, onEmpty = []), mustExist))
obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False))
return self._getInternal('paths', obj2str, str2obj, None, option, default, **kwargs)
def getPaths(self, *args, **kwargs):
def patlist2pathlist(value, mustExist):
try:
for pattern in value:
for fn in utils.resolvePaths(pattern, [self._pathBase], mustExist, ConfigError):
yield fn
except:
raise RethrowError('Error resolving pattern %s' % pattern, ConfigError)
mustExist = kwargs.pop('mustExist', True)
str2obj = lambda value: list(patlist2pathlist(utils.parseList(value, None, onEmpty = []), mustExist))
obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False))
(selector, args, kwargs) = self._selectorFilter(*args, **kwargs)
return self._getTyped('paths', obj2str, str2obj, None, selector, *args, **kwargs)
def getList(self, option, default = noDefault, parseItem = lambda x: x, **kwargs):
obj2str = lambda value: '\n' + str.join('\n', map(str, value))
str2obj = lambda value: map(parseItem, utils.parseList(value, None))
return self._getInternal('list', obj2str, str2obj, None, option, default, **kwargs)
def __init__(self, config, name, userOpt, itemName, discoverFun):
Broker.__init__(self, config, name, userOpt, itemName, discoverFun)
self._storageDict = config.getDict('%s storage access' % userOpt, {}, onChange = None,
parser = lambda x: utils.parseList(x, ' '), strfun = lambda x: str.join(' ', x))[0]
def getFlatList(section, option): for (opt, value, s) in filter(lambda (opt, v, s): opt == option, tmpConfigContent.get(section, [])): for entry in utils.parseList(value, None): yield entry
def getList(self, *args, **kwargs):
parseItem = kwargs.pop('parseItem', lambda x: x)
obj2str = lambda value: '\n' + str.join('\n', map(str, value))
str2obj = lambda value: map(parseItem, utils.parseList(value, None))
(selector, args, kwargs) = self._selectorFilter(*args, **kwargs)
return self._getTyped('list', obj2str, str2obj, None, selector, *args, **kwargs)
