def check(self, job):
t = getLogger('Ganga.GangaLHCb.Lib.LHCbDataset')
old = t.level
_set_log_level(t, 'WARNING')
bkmd = job.inputdata.bkMetadata()
t.setLevel(old)
if bkmd['Failed']:
logger.warning('Could not get the bookeeping metadata.')
return self.failure
n_expected = sum(
bkMetadataStat(v) for v in bkmd['Successful'].values())
try:
n_processed = job.metadata['events']['input']
except KeyError:
#raise PostProcessException("The metadata value ['events']['input'] was not defined")
logger.warning(
"The metadata value ['events']['input'] was not defined")
return self.failure
if n_processed != n_expected:
diff = n_expected - n_processed
reldiff = float(diff) / float(n_expected)
logger.info(
'Job {}: Number of processed events ({}) differs from the '
'expected number of input events ({}) by {}'.format(
job.fqid, n_processed, n_expected, diff))
if self.maxAbsDiff >= 0 and diff > self.maxAbsDiff:
logger.warning(
'Job {}: Absolute difference ({}) is more than the maximum allowed ({})'
.format(job.fqid, diff, self.maxAbsDiff))
elif self.maxRelDiff >= 0.0 and reldiff > self.maxRelDiff:
logger.warning(
'Job {}: Relative difference ({}) is more than the maximum allowed ({})'
.format(job.fqid, reldiff, self.maxRelDiff))
else:
return self.success
return self.failure
return self.success
from GangaCore.GPIDev.Base import GangaObject
from GangaCore.GPIDev.Adapters.IPostProcessor import PostProcessException, IPostProcessor
from GangaCore.GPIDev.Base.Proxy import GPIProxyObject
from GangaCore.GPIDev.Adapters.IChecker import IChecker
from GangaCore.GPIDev.Schema import ComponentItem, FileItem, Schema, SimpleItem, Version
from GangaCore.Utility.Config import makeConfig, ConfigError, getConfig
from GangaCore.Utility.Plugin import allPlugins
from GangaCore.Utility.logging import getLogger, log_user_exception
logger = getLogger()
class LHCbCompleteChecker(IChecker):
"""
Checks that all input files were fully processed (using the metadata).
Example:
job.postprocessors.append(LHCbCompleteChecker())
"""
_schema = IChecker._schema.inherit_copy()
_category = 'postprocessor'
_name = 'LHCbCompleteChecker'
def check(self, job):
"""Checks metadata of job is within a certain range."""
try:
nfullfiles = len(job.metadata['xmldatafiles'].get('full', []))
except KeyError:
raise PostProcessException(
"The metadata value 'xmldatanumbers' was not defined")
try:
from GangaCore.GPIDev.Base import GangaObject
from GangaCore.GPIDev.Adapters.IPostProcessor import PostProcessException, IPostProcessor
from GangaCore.GPIDev.Base.Proxy import GPIProxyObject
from GangaCore.GPIDev.Adapters.IChecker import IChecker
from GangaCore.GPIDev.Schema import ComponentItem, FileItem, Schema, SimpleItem, Version
from GangaCore.Utility.Config import makeConfig, ConfigError, getConfig
from GangaCore.Utility.Plugin import allPlugins
from GangaCore.Utility.logging import getLogger, log_user_exception
logger = getLogger()
class LHCbCompleteChecker(IChecker):
"""
Checks that all input files were fully processed (using the metadata).
Example:
job.postprocessors.append(LHCbCompleteChecker())
"""
_schema = IChecker._schema.inherit_copy()
_category = 'postprocessor'
_name = 'LHCbCompleteChecker'
def check(self, job):
"""Checks metadata of job is within a certain range."""
try:
nfullfiles = len(job.metadata['xmldatafiles'].get('full', []))
except KeyError:
raise PostProcessException("The metadata value 'xmldatanumbers' was not defined")
try:
nskipped = len(job.metadata['xmlskippedfiles'])