def validate_single(args, child=True):
if child:
from cStringIO import StringIO
sys.stdout = out = StringIO()
sys.stderr = out
name = args[0]
info = args[1]
complete = True
try:
dirs = info.dirs
root_files = []
for dir in dirs:
root_files += glob.glob(os.path.join(dir, info.file_pattern))
events = 0
for fname in root_files:
try:
with root_open(fname) as rfile:
try: # skimmed dataset
events += int(rfile.cutflow_event[0])
except DoesNotExist: # unskimmed dataset
tree = rfile.tau
events += tree.GetEntries()
except IOError:
log.warning("Currupt file: %s" % fname)
pass
# determine events in original ntuples
# use first dir
ds_name = info.ds
log.info('NTUP: ' + ds_name)
ds_info = get_dataset_info(amiclient, ds_name)
ntuple_events = int(ds_info.info['totalEvents'])
try:
# determine events in AODs
prov = get_provenance(amiclient, ds_name, type='AOD')
AOD_ds = prov.values()[0][0].replace('recon', 'merge')
log.info('AOD: ' + AOD_ds)
AOD_events = int(get_datasets(amiclient, AOD_ds, fields='events',
flatten=True)[0][0])
except IndexError:
log.info('AOD: UNKNOWN')
AOD_events = ntuple_events
log.info(name)
log.info("\tevts\tNTUP\tAOD")
log.info("\t%i\t%i\t%i" % (events, ntuple_events, AOD_events))
if events != ntuple_events:
log.warning("NTUP MISMATCH")
if events != AOD_events:
log.warning("AOD MISMATCH")
if events != ntuple_events and (events != AOD_events or AOD_events == 0):
log.warning("MISSING EVENTS")
complete = False
if child:
return out.getvalue(), complete
return complete
except Exception, e:
import traceback
log.warning("dataset %s exception" % name)
traceback.print_exception(*sys.exc_info())
if child:
return out.getvalue(), False
return False
def validate_single(args, child=True):
if child:
from cStringIO import StringIO
sys.stdout = out = StringIO()
sys.stderr = out
name = args[0]
info = args[1]
complete = True
try:
dirs = info.dirs
root_files = []
for dir in dirs:
root_files += glob.glob(os.path.join(dir, info.file_pattern))
events = 0
for fname in root_files:
try:
with root_open(fname) as rfile:
try: # skimmed dataset
events += int(rfile.cutflow_event[0])
except DoesNotExist: # unskimmed dataset
tree = rfile.tau
events += tree.GetEntries()
except IOError:
log.warning("Currupt file: %s" % fname)
pass
# determine events in original ntuples
# use first dir
ds_name = info.ds
log.info('NTUP: ' + ds_name)
ds_info = get_dataset_info(amiclient, ds_name)
ntuple_events = int(ds_info.info['totalEvents'])
try:
# determine events in AODs
prov = get_provenance(amiclient, ds_name, type='AOD')
AOD_ds = prov.values()[0][0].replace('recon', 'merge')
log.info('AOD: ' + AOD_ds)
AOD_events = int(
get_datasets(amiclient, AOD_ds, fields='events',
flatten=True)[0][0])
except IndexError:
log.info('AOD: UNKNOWN')
AOD_events = ntuple_events
log.info(name)
log.info("\tevts\tNTUP\tAOD")
log.info("\t%i\t%i\t%i" % (events, ntuple_events, AOD_events))
if events != ntuple_events:
log.warning("NTUP MISMATCH")
if events != AOD_events:
log.warning("AOD MISMATCH")
if events != ntuple_events and (events != AOD_events
or AOD_events == 0):
log.warning("MISSING EVENTS")
complete = False
if child:
return out.getvalue(), complete
return complete
except Exception, e:
import traceback
log.warning("dataset %s exception" % name)
traceback.print_exception(*sys.exc_info())
if child:
return out.getvalue(), False
return False