def clean(self): nfailed=0 All_files = [] if self.process=='':All_files = glob.glob("%s/*/merge.yaml"%self.yamldir) else:All_files = glob.glob("%s/merge.yaml"%self.yamldir) print "========================= %s/merge.yaml"%self.yamldir for f in All_files: print '===================== ',f with open(f, 'r') as stream: try: tmpf = yaml.load(stream) if tmpf['merge']['nbad']==0:continue nfailed+=tmpf['merge']['nbad'] for r in tmpf['merge']['outfilesbad']: cmd="rm %s/%s"%(tmpf['merge']['outdir'],r) print 'remove file %s from process %s'%(r, tmpf['merge']['process']) os.system(cmd) if self.process=='': cmd="rm %s/%s/%s"%(self.yamldir,tmpf['merge']['process'],r.replace('.lhe.gz','.yaml').replace('.root','.yaml')) os.system(cmd) else: cmd="rm %s/%s"%(self.yamldir,r.replace('.lhe.gz','.yaml').replace('.root','.yaml')) os.system(cmd) ut.yamlstatus(self.yamlcheck,tmpf['merge']['process'] , False) except yaml.YAMLError as exc: print(exc) print 'removed %i files'%nfailed
nfileseos=0 if self.isLHE: nfileseos=len(os.listdir('%s%s'%(self.para.lhe_dir,proc))) else: if os.path.isdir('%s%s/%s'%(self.para.delphes_dir,self.version,process)): nfileseos=len(os.listdir('%s%s/%s'%(self.para.delphes_dir,self.version,process))) print 'nevents : %i'%events_tot print 'nfiles on eos/checked : %i/%i'%(nfileseos,files_tot) marked_b='' marked_e='' if nfileseos>files_tot+bad_tot: ut.yamlstatus(yamlcheck, process, False) marked_b='<h2><mark>' marked_e='</mark></h2>' cmd='' if not self.matching and not ispythiaonly: cmd='%s,,%s,,%s%i%s,,%i,,%s%i%s,,%.2f,,%s,,%s,,%s,,%s,,%s\n'%(process,self.comma_me(str(events_tot)), marked_b,files_tot,marked_e,bad_tot, marked_b,nfileseos,marked_e,size_tot, tmpf['merge']['outdir'], self.para.gridpacklist[proc][0],self.para.gridpacklist[proc][1], self.para.gridpacklist[proc][2],self.para.gridpacklist[proc][3]) elif self.matching and not ispythiaonly: cmd='%s,,%s,,%s,,%s%i%s,,%i,,%s%i%s,,%.2f,,%s,,%s,,%s,,%s,,%s,,%s\n'%(process,self.comma_me(str(events_tot)),self.comma_me(str(sumw_tot)), marked_b,files_tot,marked_e,bad_tot,
def merge(self, force): ldir = next(os.walk(self.indir))[1] print self.indir, ' ==== ', self.process #ldir=[x[0] for x in os.walk(self.indir)] for l in ldir: if self.process != '' and self.process != l: continue outfile = self.indir + '/' + l + '/merge.yaml' totsize = 0 totevents = 0 process = None outfiles = [] outfilesbad = [] outdir = None ndone = 0 nbad = 0 All_files = glob.glob("%s/%s/events_*.yaml" % (self.indir, l)) if len(All_files) == 0: continue #continue if process has been checked if ut.yamlcheck(self.yamlcheck, l) and not force: continue print 'merging process %s %i files' % (l, len(All_files)) for f in All_files: if not os.path.isfile(f): print 'file does not exists... %s' % f continue with open(f, 'r') as stream: try: tmpf = yaml.load(stream) if tmpf['processing']['status'] == 'sending': continue if tmpf['processing']['status'] == 'BAD': nbad += 1 outfilesbad.append( tmpf['processing']['out'].split('/')[-1]) outdir = tmpf['processing']['out'].replace( tmpf['processing']['out'].split('/')[-1], '') continue totsize += tmpf['processing']['size'] totevents += tmpf['processing']['nevents'] process = tmpf['processing']['process'] tmplist = [ tmpf['processing']['out'].split('/')[-1], tmpf['processing']['nevents'] ] outfiles.append(tmplist) outdir = tmpf['processing']['out'].replace( tmpf['processing']['out'].split('/')[-1], '') ndone += 1 except yaml.YAMLError as exc: print(exc) dic = { 'merge': { 'process': process, 'nevents': totevents, 'outfiles': outfiles, 'outdir': outdir, 'size': totsize, 'ndone': ndone, 'nbad': nbad, 'outfilesbad': outfilesbad, } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) if ndone + nbad == len(All_files): ut.yamlstatus(self.yamlcheck, process, True)
def check(self, force, statfile): #ldir=[x[0] for x in os.walk(self.indir)] ldir = next(os.walk(self.indir))[1] if not ut.testeos(self.para.eostest, self.para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) for l in ldir: if self.process != '' and self.process != l: continue #continue if process has been checked if ut.yamlcheck(self.yamlcheck, l) and not force: continue print '--------------------- ', l process = l All_files = glob.glob("%s/%s/events_*%s" % (self.indir, l, self.fext)) print 'number of files ', len(All_files) if len(All_files) == 0: continue if l == 'lhe' or l == "__restored_files__": continue print 'process from the input directory ', process outdir = self.makeyamldir(self.yamldir + process) hasbeenchecked = False nevents_tot = 0 njobsdone_tot = 0 njobsbad_tot = 0 for f in All_files: self.count = 0 if not os.path.isfile(f): print 'file does not exists... %s' % f continue jobid = f.split('_')[-1] jobid = jobid.replace(self.fext, '') userid = ut.find_owner(f) outfile = '%sevents_%s.yaml' % (outdir, jobid) if ut.file_exist( outfile) and ut.getsize(outfile) > 100 and not force: continue hasbeenchecked = True print '-----------', f if '.root' in self.fext: nevts, check = self.checkFile_root(f, self.para.treename) status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue elif '.lhe.gz' in self.fext: nevts, check = self.checkFile_lhe(f) while nevts == -1 and not check: nevts, check = self.checkFile_lhe(f) if self.count == 10: print 'can not copy or unzip the file, declare it wrong' break status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue else: print 'not correct file extension %s' % self.fext if hasbeenchecked: ut.yamlstatus(self.yamlcheck, process, False) cmdp = 'date=%s <span class="espace"/> time=%s <span class="espace"/> njobs=%i <span class="espace"/> nevents=%i <span class="espace"/> njobbad=%i <span class="espace"/> process=%s <br>\n' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process) stat_exist = ut.file_exist(statfile) with open(statfile, "a") as myfile: if not stat_exist: myfile.write( '<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n' ) myfile.write( '<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n' ) myfile.write(cmdp) print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process)
print 'preparing to send lhe jobs from madgraph gridpacks for process {}'.format(args.process) import EventProducer.bin.send_lhe as slhe sendlhe=slhe.send_lhe(args.numJobs,args.numEvents, args.process, args.lsf, args.queue, para) sendlhe.send() elif args.reco: if sendOpt=='lhep8': print 'preparing to send FCCSW jobs from lhe' import EventProducer.bin.send_lhep8 as slhep8 sendlhep8=slhep8.send_lhep8(args.numJobs,args.numEvents, args.process, args.lsf, args.queue, para, version, args.decay) sendlhep8.send() elif sendOpt=='p8': print 'preparing to send FCCSW jobs from pythia8 directly' import EventProducer.bin.send_p8 as sp8 sendp8=sp8.send_p8(args.numJobs,args.numEvents, args.process, args.lsf, args.queue, para, version) sendp8.send() ut.yamlstatus(yamlcheck, args.process, False) elif args.web: if args.LHE: print 'create web page for LHE' import EventProducer.common.printer as prt printdic=prt.printer(yamldir,para.lhe_web, False, True, para) printdic.run() elif args.reco: print 'create web page for reco version %s'%version webpage=para.delphes_web.replace('VERSION',version) import EventProducer.common.printer as prt printdic=prt.printer(yamldir, webpage, True, False, para, version) printdic.run()