def clean(self):
nfailed=0
All_files = []
if self.process=='':All_files = glob.glob("%s/*/merge.yaml"%self.yamldir)
else:All_files = glob.glob("%s/merge.yaml"%self.yamldir)
print "========================= %s/merge.yaml"%self.yamldir
for f in All_files:
print '===================== ',f
with open(f, 'r') as stream:
try:
tmpf = yaml.load(stream)
if tmpf['merge']['nbad']==0:continue
nfailed+=tmpf['merge']['nbad']
for r in tmpf['merge']['outfilesbad']:
cmd="rm %s/%s"%(tmpf['merge']['outdir'],r)
print 'remove file %s from process %s'%(r, tmpf['merge']['process'])
os.system(cmd)
if self.process=='':
cmd="rm %s/%s/%s"%(self.yamldir,tmpf['merge']['process'],r.replace('.lhe.gz','.yaml').replace('.root','.yaml'))
os.system(cmd)
else:
cmd="rm %s/%s"%(self.yamldir,r.replace('.lhe.gz','.yaml').replace('.root','.yaml'))
os.system(cmd)
ut.yamlstatus(self.yamlcheck,tmpf['merge']['process'] , False)
except yaml.YAMLError as exc:
print(exc)
print 'removed %i files'%nfailed
nfileseos=0
if self.isLHE:
nfileseos=len(os.listdir('%s%s'%(self.para.lhe_dir,proc)))
else:
if os.path.isdir('%s%s/%s'%(self.para.delphes_dir,self.version,process)):
nfileseos=len(os.listdir('%s%s/%s'%(self.para.delphes_dir,self.version,process)))
print 'nevents : %i'%events_tot
print 'nfiles on eos/checked : %i/%i'%(nfileseos,files_tot)
marked_b=''
marked_e=''
if nfileseos>files_tot+bad_tot:
ut.yamlstatus(yamlcheck, process, False)
marked_b='<h2><mark>'
marked_e='</mark></h2>'
cmd=''
if not self.matching and not ispythiaonly:
cmd='%s,,%s,,%s%i%s,,%i,,%s%i%s,,%.2f,,%s,,%s,,%s,,%s,,%s\n'%(process,self.comma_me(str(events_tot)),
marked_b,files_tot,marked_e,bad_tot,
marked_b,nfileseos,marked_e,size_tot,
tmpf['merge']['outdir'],
self.para.gridpacklist[proc][0],self.para.gridpacklist[proc][1],
self.para.gridpacklist[proc][2],self.para.gridpacklist[proc][3])
elif self.matching and not ispythiaonly:
cmd='%s,,%s,,%s,,%s%i%s,,%i,,%s%i%s,,%.2f,,%s,,%s,,%s,,%s,,%s,,%s\n'%(process,self.comma_me(str(events_tot)),self.comma_me(str(sumw_tot)),
marked_b,files_tot,marked_e,bad_tot,
def merge(self, force):
ldir = next(os.walk(self.indir))[1]
print self.indir, ' ==== ', self.process
#ldir=[x[0] for x in os.walk(self.indir)]
for l in ldir:
if self.process != '' and self.process != l:
continue
outfile = self.indir + '/' + l + '/merge.yaml'
totsize = 0
totevents = 0
process = None
outfiles = []
outfilesbad = []
outdir = None
ndone = 0
nbad = 0
All_files = glob.glob("%s/%s/events_*.yaml" % (self.indir, l))
if len(All_files) == 0: continue
#continue if process has been checked
if ut.yamlcheck(self.yamlcheck, l) and not force: continue
print 'merging process %s %i files' % (l, len(All_files))
for f in All_files:
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
with open(f, 'r') as stream:
try:
tmpf = yaml.load(stream)
if tmpf['processing']['status'] == 'sending': continue
if tmpf['processing']['status'] == 'BAD':
nbad += 1
outfilesbad.append(
tmpf['processing']['out'].split('/')[-1])
outdir = tmpf['processing']['out'].replace(
tmpf['processing']['out'].split('/')[-1], '')
continue
totsize += tmpf['processing']['size']
totevents += tmpf['processing']['nevents']
process = tmpf['processing']['process']
tmplist = [
tmpf['processing']['out'].split('/')[-1],
tmpf['processing']['nevents']
]
outfiles.append(tmplist)
outdir = tmpf['processing']['out'].replace(
tmpf['processing']['out'].split('/')[-1], '')
ndone += 1
except yaml.YAMLError as exc:
print(exc)
dic = {
'merge': {
'process': process,
'nevents': totevents,
'outfiles': outfiles,
'outdir': outdir,
'size': totsize,
'ndone': ndone,
'nbad': nbad,
'outfilesbad': outfilesbad,
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
if ndone + nbad == len(All_files):
ut.yamlstatus(self.yamlcheck, process, True)
def check(self, force, statfile):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indir))[1]
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if ut.yamlcheck(self.yamlcheck, l) and not force: continue
print '--------------------- ', l
process = l
All_files = glob.glob("%s/%s/events_*%s" %
(self.indir, l, self.fext))
print 'number of files ', len(All_files)
if len(All_files) == 0: continue
if l == 'lhe' or l == "__restored_files__": continue
print 'process from the input directory ', process
outdir = self.makeyamldir(self.yamldir + process)
hasbeenchecked = False
nevents_tot = 0
njobsdone_tot = 0
njobsbad_tot = 0
for f in All_files:
self.count = 0
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
jobid = f.split('_')[-1]
jobid = jobid.replace(self.fext, '')
userid = ut.find_owner(f)
outfile = '%sevents_%s.yaml' % (outdir, jobid)
if ut.file_exist(
outfile) and ut.getsize(outfile) > 100 and not force:
continue
hasbeenchecked = True
print '-----------', f
if '.root' in self.fext:
nevts, check = self.checkFile_root(f, self.para.treename)
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
elif '.lhe.gz' in self.fext:
nevts, check = self.checkFile_lhe(f)
while nevts == -1 and not check:
nevts, check = self.checkFile_lhe(f)
if self.count == 10:
print 'can not copy or unzip the file, declare it wrong'
break
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
else:
print 'not correct file extension %s' % self.fext
if hasbeenchecked:
ut.yamlstatus(self.yamlcheck, process, False)
cmdp = 'date=%s <span class="espace"/> time=%s <span class="espace"/> njobs=%i <span class="espace"/> nevents=%i <span class="espace"/> njobbad=%i <span class="espace"/> process=%s <br>\n' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
stat_exist = ut.file_exist(statfile)
with open(statfile, "a") as myfile:
if not stat_exist:
myfile.write(
'<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n'
)
myfile.write(
'<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n'
)
myfile.write(cmdp)
print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
print 'preparing to send lhe jobs from madgraph gridpacks for process {}'.format(args.process)
import EventProducer.bin.send_lhe as slhe
sendlhe=slhe.send_lhe(args.numJobs,args.numEvents, args.process, args.lsf, args.queue, para)
sendlhe.send()
elif args.reco:
if sendOpt=='lhep8':
print 'preparing to send FCCSW jobs from lhe'
import EventProducer.bin.send_lhep8 as slhep8
sendlhep8=slhep8.send_lhep8(args.numJobs,args.numEvents, args.process, args.lsf, args.queue, para, version, args.decay)
sendlhep8.send()
elif sendOpt=='p8':
print 'preparing to send FCCSW jobs from pythia8 directly'
import EventProducer.bin.send_p8 as sp8
sendp8=sp8.send_p8(args.numJobs,args.numEvents, args.process, args.lsf, args.queue, para, version)
sendp8.send()
ut.yamlstatus(yamlcheck, args.process, False)
elif args.web:
if args.LHE:
print 'create web page for LHE'
import EventProducer.common.printer as prt
printdic=prt.printer(yamldir,para.lhe_web, False, True, para)
printdic.run()
elif args.reco:
print 'create web page for reco version %s'%version
webpage=para.delphes_web.replace('VERSION',version)
import EventProducer.common.printer as prt
printdic=prt.printer(yamldir, webpage, True, False, para, version)
printdic.run()