def _readBlockFromConfig(self, ds_config, datasetExpr, datasetNick, datasetID):
metadata_keys = parseJSON(ds_config.get('metadata', '[]', onChange = None))
common_metadata = parseJSON(ds_config.get('metadata common', '[]', onChange = None))
if len(common_metadata) > len(metadata_keys):
raise DatasetError('Unable to set %d common metadata items with %d metadata keys' % (len(common_metadata), len(metadata_keys)))
common_prefix = ds_config.get('prefix', '', onChange = None)
file_list = []
has_events = False
has_se_list = False
for url in ds_config.getOptions():
if url == 'se list':
has_se_list = True
elif url == 'events':
has_events = True
elif url not in ['dataset hash', 'id', 'metadata', 'metadata common', 'nickname', 'prefix']:
file_list.append(self._readFileFromConfig(ds_config, url, metadata_keys, common_metadata, common_prefix))
if not file_list:
raise DatasetError('There are no dataset files specified for dataset %r' % datasetExpr)
result = {
DataProvider.Nickname: ds_config.get('nickname', datasetNick, onChange = None),
DataProvider.DatasetID: ds_config.getInt('id', datasetID, onChange = None),
DataProvider.Dataset: datasetExpr,
DataProvider.FileList: sorted(file_list, key = lambda fi: fi[DataProvider.URL]),
}
if metadata_keys:
result[DataProvider.Metadata] = metadata_keys
if has_events:
result[DataProvider.NEntries] = ds_config.getInt('events', -1, onChange = None)
if has_se_list:
result[DataProvider.Locations] = parseList(ds_config.get('se list', '', onChange = None), ',')
return result
def __init__(self, config, datasetExpr, datasetNick = None, datasetID = 0):
DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID)
(self._path, self._events, selist) = utils.optSplit(datasetExpr, '|@')
self._selist = parseList(selist, ',') or None
if not (self._path and self._events):
raise ConfigError('Invalid dataset expression!\nCorrect: /local/path/to/file|events[@SE1,SE2]')
def __init__(self, config, datasetExpr, datasetNick = None):
DataProvider.__init__(self, config, datasetExpr, datasetNick)
(self._path, self._events, selist) = utils.optSplit(datasetExpr, '|@')
self._selist = parseList(selist, ',') or None
if not (self._path and self._events):
raise ConfigError('Invalid dataset expression!\nCorrect: /local/path/to/file|events[@SE1,SE2]')
def _readBlockFromConfig(self, config, datasetExpr, datasetNick, datasetID):
common_metadata = parseJSON(config.get('metadata common', '[]', onChange = None))
common_prefix = config.get('prefix', '', onChange = None)
file_list = []
has_events = False
has_se_list = False
for url in config.getOptions():
if url == 'se list':
has_se_list = True
elif url == 'events':
has_events = True
elif url not in ['dataset hash', 'id', 'metadata', 'metadata common', 'nickname', 'prefix']:
info = config.get(url, onChange = None)
tmp = info.split(' ', 1)
fi = {DataProvider.URL: common_prefix + url, DataProvider.NEntries: int(tmp[0])}
if common_metadata:
fi[DataProvider.Metadata] = common_metadata
if len(tmp) == 2:
fi[DataProvider.Metadata] = fi.get(DataProvider.Metadata, []) + parseJSON(tmp[1])
file_list.append(fi)
if not file_list:
raise DatasetError('There are no dataset files specified for dataset %r' % datasetExpr)
result = {
DataProvider.Nickname: config.get('nickname', datasetNick, onChange = None),
DataProvider.DatasetID: config.getInt('id', datasetID, onChange = None),
DataProvider.Dataset: datasetExpr,
DataProvider.Metadata: parseJSON(config.get('metadata', '[]', onChange = None)),
DataProvider.FileList: sorted(file_list, key = lambda fi: fi[DataProvider.URL]),
}
if has_events:
result[DataProvider.NEntries] = config.getInt('events', -1, onChange = None)
if has_se_list:
result[DataProvider.Locations] = parseList(config.get('se list', '', onChange = None), ',')
return result
def __init__(self, config, name, userOpt, itemName, discoverFun):
Broker.__init__(self, config, name, userOpt, itemName, discoverFun)
self._storageDict = config.getLookup(
'%s storage access' % userOpt, {},
onChange=None,
parser=lambda x: parseList(x, ' '),
strfun=lambda x: str.join(' ', x))
def getFlatList(section, option):
for (opt, value, src) in tmpConfigContent.get(section, []):
try:
if opt == option:
for entry in parseList(value, None):
yield entry
except Exception:
raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))
def getFlatList(section, option):
for (opt, value, src) in tmpConfigContent.get(section, []):
try:
if opt == option:
for entry in parseList(value, None):
yield entry
except Exception:
raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))
def _readBlockFromConfig(self, config, datasetExpr, datasetNick,
datasetID):
common_metadata = parseJSON(
config.get('metadata common', '[]', onChange=None))
common_prefix = config.get('prefix', '', onChange=None)
file_list = []
has_events = False
has_se_list = False
for url in config.getOptions():
if url == 'se list':
has_se_list = True
elif url == 'events':
has_events = True
elif url not in [
'dataset hash', 'id', 'metadata', 'metadata common',
'nickname', 'prefix'
]:
info = config.get(url, onChange=None)
tmp = info.split(' ', 1)
fi = {
DataProvider.URL: common_prefix + url,
DataProvider.NEntries: int(tmp[0])
}
if common_metadata:
fi[DataProvider.Metadata] = common_metadata
if len(tmp) == 2:
fi[DataProvider.Metadata] = fi.get(DataProvider.Metadata,
[]) + parseJSON(tmp[1])
file_list.append(fi)
if not file_list:
raise DatasetError(
'There are no dataset files specified for dataset %r' %
datasetExpr)
result = {
DataProvider.Nickname:
config.get('nickname', datasetNick, onChange=None),
DataProvider.DatasetID:
config.getInt('id', datasetID, onChange=None),
DataProvider.Dataset:
datasetExpr,
DataProvider.Metadata:
parseJSON(config.get('metadata', '[]', onChange=None)),
DataProvider.FileList:
sorted(file_list, key=lambda fi: fi[DataProvider.URL]),
}
if has_events:
result[DataProvider.NEntries] = config.getInt('events',
-1,
onChange=None)
if has_se_list:
result[DataProvider.Locations] = parseList(
config.get('se list', '', onChange=None), ',')
return result
def _readBlockFromConfig(self, ds_config, datasetExpr, datasetNick,
datasetID):
metadata_keys = parseJSON(
ds_config.get('metadata', '[]', onChange=None))
common_metadata = parseJSON(
ds_config.get('metadata common', '[]', onChange=None))
if len(common_metadata) > len(metadata_keys):
raise DatasetError(
'Unable to set %d common metadata items with %d metadata keys'
% (len(common_metadata), len(metadata_keys)))
common_prefix = ds_config.get('prefix', '', onChange=None)
file_list = []
has_events = False
has_se_list = False
for url in ds_config.getOptions():
if url == 'se list':
has_se_list = True
elif url == 'events':
has_events = True
elif url not in [
'dataset hash', 'id', 'metadata', 'metadata common',
'nickname', 'prefix'
]:
file_list.append(
self._readFileFromConfig(ds_config, url, metadata_keys,
common_metadata, common_prefix))
if not file_list:
raise DatasetError(
'There are no dataset files specified for dataset %r' %
datasetExpr)
result = {
DataProvider.Nickname:
ds_config.get('nickname', datasetNick, onChange=None),
DataProvider.DatasetID:
ds_config.getInt('id', datasetID, onChange=None),
DataProvider.Dataset:
datasetExpr,
DataProvider.FileList:
sorted(file_list, key=lambda fi: fi[DataProvider.URL]),
}
if metadata_keys:
result[DataProvider.Metadata] = metadata_keys
if has_events:
result[DataProvider.NEntries] = ds_config.getInt('events',
-1,
onChange=None)
if has_se_list:
result[DataProvider.Locations] = parseList(
ds_config.get('se list', '', onChange=None), ',')
return result
def getPaths(self, option, default = noDefault, mustExist = True, **kwargs):
def patlist2pathlist(value, mustExist):
ec = ExceptionCollector()
for pattern in value:
try:
for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError):
yield fn
except Exception:
ec.collect()
ec.raise_any(ConfigError('Error resolving paths'))
str2obj = lambda value: list(patlist2pathlist(parseList(value, None), mustExist))
obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False))
return self._getInternal('paths', obj2str, str2obj, None, option, default, **kwargs)
def getPaths(self, option, default = noDefault, mustExist = True, **kwargs):
def patlist2pathlist(value, mustExist):
ec = ExceptionCollector()
for pattern in value:
try:
for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError):
yield fn
except Exception:
ec.collect()
ec.raise_any(ConfigError('Error resolving paths'))
str2obj = lambda value: list(patlist2pathlist(parseList(value, None), mustExist))
obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False))
return self._getInternal('paths', obj2str, str2obj, None, option, default, **kwargs)
def __init__(self, config, datasetExpr, datasetNick = None, datasetID = 0):
DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID)
self._CommonPrefix = max(self.enumValues) + 1
self._CommonMetadata = max(self.enumValues) + 2
self._handleEntry = {
'events': (DataProvider.NEntries, int, 'block entry counter'),
'id': (DataProvider.DatasetID, int, 'dataset ID'),
'metadata': (DataProvider.Metadata, parseJSON, 'metadata description'),
'metadata common': (self._CommonMetadata, parseJSON, 'common metadata'),
'nickname': (DataProvider.Nickname, str, 'dataset nickname'),
'prefix': (self._CommonPrefix, str, 'common prefix'),
'se list': (DataProvider.Locations, lambda value: parseList(value, ','), 'block location'),
}
(path, self._forcePrefix, self._filter) = utils.optSplit(datasetExpr, '@%')
self._filename = config.resolvePath(path, True, 'Error resolving dataset file: %s' % path)
def __init__(self, config, datasetExpr, datasetNick = None):
DataProvider.__init__(self, config, datasetExpr, datasetNick)
self._CommonPrefix = max(self.enumValues) + 1
self._CommonMetadata = max(self.enumValues) + 2
self._handleEntry = {
'events': (DataProvider.NEntries, int, 'block entry counter'),
'id': (None, None, 'dataset ID'), # legacy key - skip
'metadata': (DataProvider.Metadata, parseJSON, 'metadata description'),
'metadata common': (self._CommonMetadata, parseJSON, 'common metadata'),
'nickname': (DataProvider.Nickname, str, 'dataset nickname'),
'prefix': (self._CommonPrefix, str, 'common prefix'),
'se list': (DataProvider.Locations, lambda value: parseList(value, ','), 'block location'),
}
(path, self._forcePrefix, self._filter) = utils.optSplit(datasetExpr, '@%')
self._filename = config.resolvePath(path, True, 'Error resolving dataset file: %s' % path)
def __init__(self, path):
activity = Activity('Reading dataset partition file')
self._lock = GCLock()
self._fmt = utils.DictFormat()
self._tar = tarfile.open(path, 'r:')
(self._cacheKey, self._cacheTar) = (None, None)
metadata = self._fmt.parse(self._tar.extractfile('Metadata').readlines(), keyParser = {None: str})
self.maxJobs = metadata.pop('MaxJobs')
self.classname = metadata.pop('ClassName')
self.metadata = {'dataset': dict(ifilter(lambda k_v: not k_v[0].startswith('['), metadata.items()))}
for (k, v) in ifilter(lambda k_v: k_v[0].startswith('['), metadata.items()):
self.metadata.setdefault('dataset %s' % k.split(']')[0].lstrip('['), {})[k.split(']')[1].strip()] = v
activity.finish()
self._parserMap = { None: str, DataSplitter.NEntries: int, DataSplitter.Skipped: int,
DataSplitter.DatasetID: int, DataSplitter.Invalid: parseBool,
DataSplitter.Locations: lambda x: parseList(x, ','),
DataSplitter.MetadataHeader: parseJSON,
DataSplitter.Metadata: lambda x: parseJSON(x.strip("'")) }
def getList(self, option, default=noDefault, parseItem=identity, **kwargs):
obj2str = lambda value: '\n' + str.join('\n', imap(str, value))
str2obj = lambda value: lmap(parseItem, parseList(value, None))
return self._getInternal('list', obj2str, str2obj, None, option,
default, **kwargs)
def getList(self, option, default = noDefault, parseItem = identity, **kwargs):
obj2str = lambda value: '\n' + str.join('\n', imap(str, value))
str2obj = lambda value: lmap(parseItem, parseList(value, None))
return self._getInternal('list', obj2str, str2obj, None, option, default, **kwargs)
def getFlatList(section, option):
for (opt, value, _) in tmpConfigContent.get(section, []):
if opt == option:
for entry in parseList(value, None):
yield entry
