def checkFile_lhe(self, f):
size = os.path.getsize(f)
if size == 0:
self.count += 1
print 'file size is 0, job is bad'
return -1, False
filecounting = 'filecounting'
if os.path.isdir(filecounting) == False:
os.system('mkdir %s' % filecounting)
cmd = 'cp %s %s' % (f, filecounting)
outputCMD = ut.getCommandOutput(cmd)
fcount = '%s/%s' % (filecounting, f.split('/')[-1])
if os.path.isfile(fcount):
cmd = 'gunzip %s' % (fcount)
outputCMD = ut.getCommandOutput(cmd)
stderr = outputCMD["stderr"]
if len(stderr) > 0:
print 'can not unzip the file, try again (count %i)' % self.count
self.count += 1
os.system('rm %s' % (fcount))
return -1, False
cmd = 'grep \"<event>\" %s | wc -l' % (fcount.replace('.gz', ''))
outputCMD = ut.getCommandOutput(cmd)
stdoutplit = outputCMD["stdout"].split(' ')
nevts = int(stdoutplit[0])
if nevts == 0:
print 'no events in the file, job is bad'
os.system('rm %s' % (fcount.replace('.gz', '')))
return 0, False
else:
print '%i events in file %s, job is good' % (nevts, f)
os.system('rm %s' % (fcount.replace('.gz', '')))
return nevts, True
else:
print 'file not properly copied... try again (count %i)' % self.count
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
self.count += 1
return -1, False
yamldir=para.yamldir+'lhe/'
statfile=para.lhe_stat
elif args.reco:
indir='%s%s'%(para.delphes_dir,version)
fext=para.delphes_ext
yamldir=para.yamldir+version+'/'
statfile=para.delphes_stat.replace('VERSION',version)
print 'Running reco production system with version %s'%version
else:
print 'problem, need to specify --reco or --LHE'
sys.exit(3)
import EventProducer.common.utils as ut
if not ut.testeos(para.eostest,para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
if args.check:
print 'running the check'
if args.process!='':
print 'using a specific process ',args.process
if args.reco and args.process[0:3]=='mg_': args.process='mgp8_'+args.process[3:]
if args.reco and args.process[0:3]=='ch_': args.process='chp8_'+args.process[3:]
import EventProducer.common.checker_yaml as chky
print args.process
checker=chky.checker_yaml(indir, para, fext, args.process, yamldir)
checker.check(args.force, statfile)
def check(self, force, statfile):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indir))[1]
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if ut.yamlcheck(self.yamlcheck, l) and not force: continue
print '--------------------- ', l
process = l
All_files = glob.glob("%s/%s/events_*%s" %
(self.indir, l, self.fext))
print 'number of files ', len(All_files)
if len(All_files) == 0: continue
if l == 'lhe' or l == "__restored_files__": continue
print 'process from the input directory ', process
outdir = self.makeyamldir(self.yamldir + process)
hasbeenchecked = False
nevents_tot = 0
njobsdone_tot = 0
njobsbad_tot = 0
for f in All_files:
self.count = 0
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
jobid = f.split('_')[-1]
jobid = jobid.replace(self.fext, '')
userid = ut.find_owner(f)
outfile = '%sevents_%s.yaml' % (outdir, jobid)
if ut.file_exist(
outfile) and ut.getsize(outfile) > 100 and not force:
continue
hasbeenchecked = True
print '-----------', f
if '.root' in self.fext:
nevts, check = self.checkFile_root(f, self.para.treename)
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
elif '.lhe.gz' in self.fext:
nevts, check = self.checkFile_lhe(f)
while nevts == -1 and not check:
nevts, check = self.checkFile_lhe(f)
if self.count == 10:
print 'can not copy or unzip the file, declare it wrong'
break
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
else:
print 'not correct file extension %s' % self.fext
if hasbeenchecked:
ut.yamlstatus(self.yamlcheck, process, False)
cmdp = 'date=%s <span class="espace"/> time=%s <span class="espace"/> njobs=%i <span class="espace"/> nevents=%i <span class="espace"/> njobbad=%i <span class="espace"/> process=%s <br>\n' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
stat_exist = ut.file_exist(statfile)
with open(statfile, "a") as myfile:
if not stat_exist:
myfile.write(
'<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n'
)
myfile.write(
'<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n'
)
myfile.write(cmdp)
print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
def check(self, force, statfile):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indir))[1]
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if l == 'BADPYTHIA' or l == 'lhe' or l == "__restored_files__" or l == "backup":
continue
print '%s/%s/check' % (self.yamldir, l)
if not ut.file_exist('%s/%s/check' %
(self.yamldir, l)) and not force:
continue
print '--------------------- ', l
process = l
All_files = glob.glob("%s/%s/events_*%s" %
(self.indir, l, self.fext))
print 'number of files ', len(All_files)
if len(All_files) == 0: continue
print 'process from the input directory ', process
outdir = self.makeyamldir(self.yamldir + process)
hasbeenchecked = False
nevents_tot = 0
njobsdone_tot = 0
njobsbad_tot = 0
for f in All_files:
self.count = 0
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
jobid = f.split('_')[-1]
jobid = jobid.replace(self.fext, '')
userid = ut.find_owner(f)
outfile = '%sevents_%s.yaml' % (outdir, jobid)
if ut.getsize(outfile) == 0:
cmd = "rm %s" % (outfile)
print 'file size 0, remove and continue ', cmd
os.system(cmd)
continue
if ut.file_exist(
outfile) and ut.getsize(outfile) > 100 and not force:
doc = None
with open(outfile) as ftmp:
try:
doc = yaml.load(ftmp)
except yaml.YAMLError as exc:
print(exc)
except IOError as exc:
print "I/O error({0}): {1}".format(
exc.errno, exc.strerror)
print "outfile ", outfile
try:
if doc != None: value = doc['processing']['status']
if value == 'DONE': continue
except KeyError, e:
print 'status %s does not exist' % str(e)
hasbeenchecked = True
print '-----------', f
if '.root' in self.fext:
nevts, check = self.checkFile_root(f, self.para.treename)
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
try:
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
except IOError as exc:
print "I/O error({0}): {1}".format(
exc.errno, exc.strerror)
print "outfile ", outfile
time.sleep(10)
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
elif '.lhe.gz' in self.fext:
nevts, check = self.checkFile_lhe(f)
while nevts == -1 and not check:
nevts, check = self.checkFile_lhe(f)
if self.count == 10:
print 'can not copy or unzip the file, declare it wrong'
break
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
else:
print 'not correct file extension %s' % self.fext
if hasbeenchecked:
cmdp = '<pre>date=%s \t time=%s njobs=%i \t nevents=%i \t njobbad=%i \t process=%s </pre>\n' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
stat_exist = ut.file_exist(statfile)
with open(statfile, "a") as myfile:
if not stat_exist:
myfile.write(
'<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n'
)
myfile.write(
'<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n'
)
myfile.write(cmdp)
print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
def check(self, para):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indireos))[1]
if not ut.testeos(para.eostest, para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
dic = {}
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if l == 'BADPYTHIA' or l == 'lhe' or l == "__restored_files__" or l == "backup":
continue
print '--------------------- ', l
proc = l
nfileseos = 0
if os.path.isdir('%s/%s' % (self.indireos, proc)):
listeos = [
x for x in os.listdir('%s/%s' % (self.indireos, proc))
if 'events' in x
]
nfileseos = len(listeos)
if nfileseos == 0: continue
nfilesmerged = 0
mergefile = self.indirafs + '/' + l + '/merge.yaml'
if not ut.file_exist(mergefile):
if not ut.dir_exist('%s/%s' % (self.indirafs, proc)):
os.system('mkdir %s/%s' % (self.indirafs, proc))
self.touch('%s/%s/check' % (self.indirafs, proc))
continue
if not os.path.isdir(self.indirafs):
os.system('mkdir %s' % self.indirafs)
tmpf = None
with open(mergefile, 'r') as stream:
try:
tmpf = yaml.load(stream)
except yaml.YAMLError as exc:
print(exc)
bad_tot = tmpf['merge']['nbad']
files_tot = tmpf['merge']['ndone']
ntot_files = bad_tot + files_tot
print "tot files ", ntot_files, " files eos ", nfileseos
dic[proc] = {'neos': nfileseos, 'nmerged': ntot_files}
print '%s/%s/check' % (self.indirafs, proc)
if ntot_files < nfileseos:
self.touch('%s/%s/check' % (self.indirafs, proc))
elif ntot_files > nfileseos:
os.system('rm %s/%s/events*.yaml' % (self.indirafs, proc))
os.system('rm %s/%s/merge.yaml' % (self.indirafs, proc))
else:
if ut.file_exist('%s/%s/check' % (self.indirafs, proc)):
os.system('rm %s/%s/check' % (self.indirafs, proc))
outfile = self.indirafs + '/files.yaml'
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)