def processBlock(self, block):
if block[DataProvider.Locations] is not None:
sites = self._locationfilter.filterList(block[DataProvider.Locations])
if (sites is not None) and (len(sites) == 0) and (len(block[DataProvider.FileList]) != 0):
if not len(block[DataProvider.Locations]):
self._log.warning('Block %s is not available at any site!', DataProvider.bName(block))
elif not len(sites):
self._log.warning('Block %s is not available at any selected site!', DataProvider.bName(block))
block[DataProvider.Locations] = sites
return block
def processBlock(self, block):
if block[DataProvider.Locations] is not None:
sites = self._locationfilter.filterList(
block[DataProvider.Locations])
if (sites is not None) and (len(sites) == 0) and (len(
block[DataProvider.FileList]) != 0):
if not len(block[DataProvider.Locations]):
self._log.warning('Block %s is not available at any site!',
DataProvider.bName(block))
elif not len(sites):
self._log.warning(
'Block %s is not available at any selected site!',
DataProvider.bName(block))
block[DataProvider.Locations] = sites
return block
def processBlock(self, block):
# Check entry consistency
events = sum(imap(lambda x: x[DataProvider.NEntries], block[DataProvider.FileList]))
if block.setdefault(DataProvider.NEntries, events) != events:
self._handleError('Inconsistency in block %s: Number of events doesn\'t match (b:%d != f:%d)' % (
DataProvider.bName(block), block[DataProvider.NEntries], events), self._mode)
return block
def processBlock(self, block):
# Check entry consistency
events = sum(
imap(lambda x: x[DataProvider.NEntries],
block[DataProvider.FileList]))
if block.setdefault(DataProvider.NEntries, events) != events:
self._handleError(
'Inconsistency in block %s: Number of events doesn\'t match (b:%d != f:%d)'
% (DataProvider.bName(block), block[DataProvider.NEntries],
events), self._mode)
return block
def processBlock(self, block):
# Check uniqueness of URLs
recordedBlockURL = []
if self._checkURL != DatasetUniqueMode.ignore:
def processFI(fiList):
for fi in fiList:
urlHash = md5_hex(
repr((fi[DataProvider.URL], fi[DataProvider.NEntries],
fi.get(DataProvider.Metadata))))
if urlHash in self._recordedURL:
msg = 'Multiple occurences of URL: %r!' % fi[
DataProvider.URL]
msg += ' (This check can be configured with %r)' % 'dataset check unique url'
if self._checkURL == DatasetUniqueMode.warn:
self._log.warning(msg)
elif self._checkURL == DatasetUniqueMode.abort:
raise DatasetError(msg)
elif self._checkURL == DatasetUniqueMode.skip:
continue
self._recordedURL.add(urlHash)
recordedBlockURL.append(urlHash)
yield fi
block[DataProvider.FileList] = list(
processFI(block[DataProvider.FileList]))
recordedBlockURL.sort()
# Check uniqueness of blocks
if self._checkBlock != DatasetUniqueMode.ignore:
blockHash = md5_hex(
repr((block.get(DataProvider.Dataset),
block[DataProvider.BlockName], recordedBlockURL,
block[DataProvider.NEntries],
block[DataProvider.Locations],
block.get(DataProvider.Metadata))))
if blockHash in self._recordedBlock:
msg = 'Multiple occurences of block: "%s"!' % DataProvider.bName(
block)
msg += ' (This check can be configured with %r)' % 'dataset check unique block'
if self._checkBlock == DatasetUniqueMode.warn:
self._log.warning(msg)
elif self._checkBlock == DatasetUniqueMode.abort:
raise DatasetError(msg)
elif self._checkBlock == DatasetUniqueMode.skip:
return None
self._recordedBlock.add(blockHash)
return block
def processBlock(self, block): # Check uniqueness of URLs recordedBlockURL = [] if self._checkURL != DatasetUniqueMode.ignore: def processFI(fiList): for fi in fiList: urlHash = md5_hex(repr((fi[DataProvider.URL], fi[DataProvider.NEntries], fi.get(DataProvider.Metadata)))) if urlHash in self._recordedURL: msg = 'Multiple occurences of URL: %r!' % fi[DataProvider.URL] msg += ' (This check can be configured with %r)' % 'dataset check unique url' if self._checkURL == DatasetUniqueMode.warn: self._log.warning(msg) elif self._checkURL == DatasetUniqueMode.abort: raise DatasetError(msg) elif self._checkURL == DatasetUniqueMode.skip: continue self._recordedURL.add(urlHash) recordedBlockURL.append(urlHash) yield fi block[DataProvider.FileList] = list(processFI(block[DataProvider.FileList])) recordedBlockURL.sort() # Check uniqueness of blocks if self._checkBlock != DatasetUniqueMode.ignore: blockHash = md5_hex(repr((block.get(DataProvider.Dataset), block[DataProvider.BlockName], recordedBlockURL, block[DataProvider.NEntries], block[DataProvider.Locations], block.get(DataProvider.Metadata)))) if blockHash in self._recordedBlock: msg = 'Multiple occurences of block: "%s"!' % DataProvider.bName(block) msg += ' (This check can be configured with %r)' % 'dataset check unique block' if self._checkBlock == DatasetUniqueMode.warn: self._log.warning(msg) elif self._checkBlock == DatasetUniqueMode.abort: raise DatasetError(msg) elif self._checkBlock == DatasetUniqueMode.skip: return None self._recordedBlock.add(blockHash) return block
