def _readBlockFromConfig(self, ds_config, datasetExpr, datasetNick, datasetID): metadata_keys = parseJSON(ds_config.get('metadata', '[]', onChange = None)) common_metadata = parseJSON(ds_config.get('metadata common', '[]', onChange = None)) if len(common_metadata) > len(metadata_keys): raise DatasetError('Unable to set %d common metadata items with %d metadata keys' % (len(common_metadata), len(metadata_keys))) common_prefix = ds_config.get('prefix', '', onChange = None) file_list = [] has_events = False has_se_list = False for url in ds_config.getOptions(): if url == 'se list': has_se_list = True elif url == 'events': has_events = True elif url not in ['dataset hash', 'id', 'metadata', 'metadata common', 'nickname', 'prefix']: file_list.append(self._readFileFromConfig(ds_config, url, metadata_keys, common_metadata, common_prefix)) if not file_list: raise DatasetError('There are no dataset files specified for dataset %r' % datasetExpr) result = { DataProvider.Nickname: ds_config.get('nickname', datasetNick, onChange = None), DataProvider.DatasetID: ds_config.getInt('id', datasetID, onChange = None), DataProvider.Dataset: datasetExpr, DataProvider.FileList: sorted(file_list, key = lambda fi: fi[DataProvider.URL]), } if metadata_keys: result[DataProvider.Metadata] = metadata_keys if has_events: result[DataProvider.NEntries] = ds_config.getInt('events', -1, onChange = None) if has_se_list: result[DataProvider.Locations] = parseList(ds_config.get('se list', '', onChange = None), ',') return result
def __init__(self, config, datasetExpr, datasetNick = None, datasetID = 0): DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID) (self._path, self._events, selist) = utils.optSplit(datasetExpr, '|@') self._selist = parseList(selist, ',') or None if not (self._path and self._events): raise ConfigError('Invalid dataset expression!\nCorrect: /local/path/to/file|events[@SE1,SE2]')
def __init__(self, config, datasetExpr, datasetNick = None): DataProvider.__init__(self, config, datasetExpr, datasetNick) (self._path, self._events, selist) = utils.optSplit(datasetExpr, '|@') self._selist = parseList(selist, ',') or None if not (self._path and self._events): raise ConfigError('Invalid dataset expression!\nCorrect: /local/path/to/file|events[@SE1,SE2]')
def _readBlockFromConfig(self, config, datasetExpr, datasetNick, datasetID): common_metadata = parseJSON(config.get('metadata common', '[]', onChange = None)) common_prefix = config.get('prefix', '', onChange = None) file_list = [] has_events = False has_se_list = False for url in config.getOptions(): if url == 'se list': has_se_list = True elif url == 'events': has_events = True elif url not in ['dataset hash', 'id', 'metadata', 'metadata common', 'nickname', 'prefix']: info = config.get(url, onChange = None) tmp = info.split(' ', 1) fi = {DataProvider.URL: common_prefix + url, DataProvider.NEntries: int(tmp[0])} if common_metadata: fi[DataProvider.Metadata] = common_metadata if len(tmp) == 2: fi[DataProvider.Metadata] = fi.get(DataProvider.Metadata, []) + parseJSON(tmp[1]) file_list.append(fi) if not file_list: raise DatasetError('There are no dataset files specified for dataset %r' % datasetExpr) result = { DataProvider.Nickname: config.get('nickname', datasetNick, onChange = None), DataProvider.DatasetID: config.getInt('id', datasetID, onChange = None), DataProvider.Dataset: datasetExpr, DataProvider.Metadata: parseJSON(config.get('metadata', '[]', onChange = None)), DataProvider.FileList: sorted(file_list, key = lambda fi: fi[DataProvider.URL]), } if has_events: result[DataProvider.NEntries] = config.getInt('events', -1, onChange = None) if has_se_list: result[DataProvider.Locations] = parseList(config.get('se list', '', onChange = None), ',') return result
def __init__(self, config, name, userOpt, itemName, discoverFun): Broker.__init__(self, config, name, userOpt, itemName, discoverFun) self._storageDict = config.getLookup( '%s storage access' % userOpt, {}, onChange=None, parser=lambda x: parseList(x, ' '), strfun=lambda x: str.join(' ', x))
def getFlatList(section, option): for (opt, value, src) in tmpConfigContent.get(section, []): try: if opt == option: for entry in parseList(value, None): yield entry except Exception: raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))
def getFlatList(section, option): for (opt, value, src) in tmpConfigContent.get(section, []): try: if opt == option: for entry in parseList(value, None): yield entry except Exception: raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))
def _readBlockFromConfig(self, config, datasetExpr, datasetNick, datasetID): common_metadata = parseJSON( config.get('metadata common', '[]', onChange=None)) common_prefix = config.get('prefix', '', onChange=None) file_list = [] has_events = False has_se_list = False for url in config.getOptions(): if url == 'se list': has_se_list = True elif url == 'events': has_events = True elif url not in [ 'dataset hash', 'id', 'metadata', 'metadata common', 'nickname', 'prefix' ]: info = config.get(url, onChange=None) tmp = info.split(' ', 1) fi = { DataProvider.URL: common_prefix + url, DataProvider.NEntries: int(tmp[0]) } if common_metadata: fi[DataProvider.Metadata] = common_metadata if len(tmp) == 2: fi[DataProvider.Metadata] = fi.get(DataProvider.Metadata, []) + parseJSON(tmp[1]) file_list.append(fi) if not file_list: raise DatasetError( 'There are no dataset files specified for dataset %r' % datasetExpr) result = { DataProvider.Nickname: config.get('nickname', datasetNick, onChange=None), DataProvider.DatasetID: config.getInt('id', datasetID, onChange=None), DataProvider.Dataset: datasetExpr, DataProvider.Metadata: parseJSON(config.get('metadata', '[]', onChange=None)), DataProvider.FileList: sorted(file_list, key=lambda fi: fi[DataProvider.URL]), } if has_events: result[DataProvider.NEntries] = config.getInt('events', -1, onChange=None) if has_se_list: result[DataProvider.Locations] = parseList( config.get('se list', '', onChange=None), ',') return result
def _readBlockFromConfig(self, ds_config, datasetExpr, datasetNick, datasetID): metadata_keys = parseJSON( ds_config.get('metadata', '[]', onChange=None)) common_metadata = parseJSON( ds_config.get('metadata common', '[]', onChange=None)) if len(common_metadata) > len(metadata_keys): raise DatasetError( 'Unable to set %d common metadata items with %d metadata keys' % (len(common_metadata), len(metadata_keys))) common_prefix = ds_config.get('prefix', '', onChange=None) file_list = [] has_events = False has_se_list = False for url in ds_config.getOptions(): if url == 'se list': has_se_list = True elif url == 'events': has_events = True elif url not in [ 'dataset hash', 'id', 'metadata', 'metadata common', 'nickname', 'prefix' ]: file_list.append( self._readFileFromConfig(ds_config, url, metadata_keys, common_metadata, common_prefix)) if not file_list: raise DatasetError( 'There are no dataset files specified for dataset %r' % datasetExpr) result = { DataProvider.Nickname: ds_config.get('nickname', datasetNick, onChange=None), DataProvider.DatasetID: ds_config.getInt('id', datasetID, onChange=None), DataProvider.Dataset: datasetExpr, DataProvider.FileList: sorted(file_list, key=lambda fi: fi[DataProvider.URL]), } if metadata_keys: result[DataProvider.Metadata] = metadata_keys if has_events: result[DataProvider.NEntries] = ds_config.getInt('events', -1, onChange=None) if has_se_list: result[DataProvider.Locations] = parseList( ds_config.get('se list', '', onChange=None), ',') return result
def getPaths(self, option, default = noDefault, mustExist = True, **kwargs): def patlist2pathlist(value, mustExist): ec = ExceptionCollector() for pattern in value: try: for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError): yield fn except Exception: ec.collect() ec.raise_any(ConfigError('Error resolving paths')) str2obj = lambda value: list(patlist2pathlist(parseList(value, None), mustExist)) obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False)) return self._getInternal('paths', obj2str, str2obj, None, option, default, **kwargs)
def getPaths(self, option, default = noDefault, mustExist = True, **kwargs): def patlist2pathlist(value, mustExist): ec = ExceptionCollector() for pattern in value: try: for fn in utils.resolvePaths(pattern, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError): yield fn except Exception: ec.collect() ec.raise_any(ConfigError('Error resolving paths')) str2obj = lambda value: list(patlist2pathlist(parseList(value, None), mustExist)) obj2str = lambda value: '\n' + str.join('\n', patlist2pathlist(value, False)) return self._getInternal('paths', obj2str, str2obj, None, option, default, **kwargs)
def __init__(self, config, datasetExpr, datasetNick = None, datasetID = 0): DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID) self._CommonPrefix = max(self.enumValues) + 1 self._CommonMetadata = max(self.enumValues) + 2 self._handleEntry = { 'events': (DataProvider.NEntries, int, 'block entry counter'), 'id': (DataProvider.DatasetID, int, 'dataset ID'), 'metadata': (DataProvider.Metadata, parseJSON, 'metadata description'), 'metadata common': (self._CommonMetadata, parseJSON, 'common metadata'), 'nickname': (DataProvider.Nickname, str, 'dataset nickname'), 'prefix': (self._CommonPrefix, str, 'common prefix'), 'se list': (DataProvider.Locations, lambda value: parseList(value, ','), 'block location'), } (path, self._forcePrefix, self._filter) = utils.optSplit(datasetExpr, '@%') self._filename = config.resolvePath(path, True, 'Error resolving dataset file: %s' % path)
def __init__(self, config, datasetExpr, datasetNick = None): DataProvider.__init__(self, config, datasetExpr, datasetNick) self._CommonPrefix = max(self.enumValues) + 1 self._CommonMetadata = max(self.enumValues) + 2 self._handleEntry = { 'events': (DataProvider.NEntries, int, 'block entry counter'), 'id': (None, None, 'dataset ID'), # legacy key - skip 'metadata': (DataProvider.Metadata, parseJSON, 'metadata description'), 'metadata common': (self._CommonMetadata, parseJSON, 'common metadata'), 'nickname': (DataProvider.Nickname, str, 'dataset nickname'), 'prefix': (self._CommonPrefix, str, 'common prefix'), 'se list': (DataProvider.Locations, lambda value: parseList(value, ','), 'block location'), } (path, self._forcePrefix, self._filter) = utils.optSplit(datasetExpr, '@%') self._filename = config.resolvePath(path, True, 'Error resolving dataset file: %s' % path)
def __init__(self, path): activity = Activity('Reading dataset partition file') self._lock = GCLock() self._fmt = utils.DictFormat() self._tar = tarfile.open(path, 'r:') (self._cacheKey, self._cacheTar) = (None, None) metadata = self._fmt.parse(self._tar.extractfile('Metadata').readlines(), keyParser = {None: str}) self.maxJobs = metadata.pop('MaxJobs') self.classname = metadata.pop('ClassName') self.metadata = {'dataset': dict(ifilter(lambda k_v: not k_v[0].startswith('['), metadata.items()))} for (k, v) in ifilter(lambda k_v: k_v[0].startswith('['), metadata.items()): self.metadata.setdefault('dataset %s' % k.split(']')[0].lstrip('['), {})[k.split(']')[1].strip()] = v activity.finish() self._parserMap = { None: str, DataSplitter.NEntries: int, DataSplitter.Skipped: int, DataSplitter.DatasetID: int, DataSplitter.Invalid: parseBool, DataSplitter.Locations: lambda x: parseList(x, ','), DataSplitter.MetadataHeader: parseJSON, DataSplitter.Metadata: lambda x: parseJSON(x.strip("'")) }
def getList(self, option, default=noDefault, parseItem=identity, **kwargs): obj2str = lambda value: '\n' + str.join('\n', imap(str, value)) str2obj = lambda value: lmap(parseItem, parseList(value, None)) return self._getInternal('list', obj2str, str2obj, None, option, default, **kwargs)
def getList(self, option, default = noDefault, parseItem = identity, **kwargs): obj2str = lambda value: '\n' + str.join('\n', imap(str, value)) str2obj = lambda value: lmap(parseItem, parseList(value, None)) return self._getInternal('list', obj2str, str2obj, None, option, default, **kwargs)
def getFlatList(section, option): for (opt, value, _) in tmpConfigContent.get(section, []): if opt == option: for entry in parseList(value, None): yield entry