def checkFile_lhe(self, f): size = os.path.getsize(f) if size == 0: self.count += 1 print 'file size is 0, job is bad' return -1, False filecounting = 'filecounting' if os.path.isdir(filecounting) == False: os.system('mkdir %s' % filecounting) cmd = 'cp %s %s' % (f, filecounting) outputCMD = ut.getCommandOutput(cmd) fcount = '%s/%s' % (filecounting, f.split('/')[-1]) if os.path.isfile(fcount): cmd = 'gunzip %s' % (fcount) outputCMD = ut.getCommandOutput(cmd) stderr = outputCMD["stderr"] if len(stderr) > 0: print 'can not unzip the file, try again (count %i)' % self.count self.count += 1 os.system('rm %s' % (fcount)) return -1, False cmd = 'grep \"<event>\" %s | wc -l' % (fcount.replace('.gz', '')) outputCMD = ut.getCommandOutput(cmd) stdoutplit = outputCMD["stdout"].split(' ') nevts = int(stdoutplit[0]) if nevts == 0: print 'no events in the file, job is bad' os.system('rm %s' % (fcount.replace('.gz', ''))) return 0, False else: print '%i events in file %s, job is good' % (nevts, f) os.system('rm %s' % (fcount.replace('.gz', ''))) return nevts, True else: print 'file not properly copied... try again (count %i)' % self.count if not ut.testeos(self.para.eostest, self.para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) self.count += 1 return -1, False
yamldir=para.yamldir+'lhe/' statfile=para.lhe_stat elif args.reco: indir='%s%s'%(para.delphes_dir,version) fext=para.delphes_ext yamldir=para.yamldir+version+'/' statfile=para.delphes_stat.replace('VERSION',version) print 'Running reco production system with version %s'%version else: print 'problem, need to specify --reco or --LHE' sys.exit(3) import EventProducer.common.utils as ut if not ut.testeos(para.eostest,para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) if args.check: print 'running the check' if args.process!='': print 'using a specific process ',args.process if args.reco and args.process[0:3]=='mg_': args.process='mgp8_'+args.process[3:] if args.reco and args.process[0:3]=='ch_': args.process='chp8_'+args.process[3:] import EventProducer.common.checker_yaml as chky print args.process checker=chky.checker_yaml(indir, para, fext, args.process, yamldir) checker.check(args.force, statfile)
def check(self, force, statfile): #ldir=[x[0] for x in os.walk(self.indir)] ldir = next(os.walk(self.indir))[1] if not ut.testeos(self.para.eostest, self.para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) for l in ldir: if self.process != '' and self.process != l: continue #continue if process has been checked if ut.yamlcheck(self.yamlcheck, l) and not force: continue print '--------------------- ', l process = l All_files = glob.glob("%s/%s/events_*%s" % (self.indir, l, self.fext)) print 'number of files ', len(All_files) if len(All_files) == 0: continue if l == 'lhe' or l == "__restored_files__": continue print 'process from the input directory ', process outdir = self.makeyamldir(self.yamldir + process) hasbeenchecked = False nevents_tot = 0 njobsdone_tot = 0 njobsbad_tot = 0 for f in All_files: self.count = 0 if not os.path.isfile(f): print 'file does not exists... %s' % f continue jobid = f.split('_')[-1] jobid = jobid.replace(self.fext, '') userid = ut.find_owner(f) outfile = '%sevents_%s.yaml' % (outdir, jobid) if ut.file_exist( outfile) and ut.getsize(outfile) > 100 and not force: continue hasbeenchecked = True print '-----------', f if '.root' in self.fext: nevts, check = self.checkFile_root(f, self.para.treename) status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue elif '.lhe.gz' in self.fext: nevts, check = self.checkFile_lhe(f) while nevts == -1 and not check: nevts, check = self.checkFile_lhe(f) if self.count == 10: print 'can not copy or unzip the file, declare it wrong' break status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue else: print 'not correct file extension %s' % self.fext if hasbeenchecked: ut.yamlstatus(self.yamlcheck, process, False) cmdp = 'date=%s <span class="espace"/> time=%s <span class="espace"/> njobs=%i <span class="espace"/> nevents=%i <span class="espace"/> njobbad=%i <span class="espace"/> process=%s <br>\n' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process) stat_exist = ut.file_exist(statfile) with open(statfile, "a") as myfile: if not stat_exist: myfile.write( '<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n' ) myfile.write( '<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n' ) myfile.write(cmdp) print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process)
def check(self, force, statfile): #ldir=[x[0] for x in os.walk(self.indir)] ldir = next(os.walk(self.indir))[1] if not ut.testeos(self.para.eostest, self.para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) for l in ldir: if self.process != '' and self.process != l: continue #continue if process has been checked if l == 'BADPYTHIA' or l == 'lhe' or l == "__restored_files__" or l == "backup": continue print '%s/%s/check' % (self.yamldir, l) if not ut.file_exist('%s/%s/check' % (self.yamldir, l)) and not force: continue print '--------------------- ', l process = l All_files = glob.glob("%s/%s/events_*%s" % (self.indir, l, self.fext)) print 'number of files ', len(All_files) if len(All_files) == 0: continue print 'process from the input directory ', process outdir = self.makeyamldir(self.yamldir + process) hasbeenchecked = False nevents_tot = 0 njobsdone_tot = 0 njobsbad_tot = 0 for f in All_files: self.count = 0 if not os.path.isfile(f): print 'file does not exists... %s' % f continue jobid = f.split('_')[-1] jobid = jobid.replace(self.fext, '') userid = ut.find_owner(f) outfile = '%sevents_%s.yaml' % (outdir, jobid) if ut.getsize(outfile) == 0: cmd = "rm %s" % (outfile) print 'file size 0, remove and continue ', cmd os.system(cmd) continue if ut.file_exist( outfile) and ut.getsize(outfile) > 100 and not force: doc = None with open(outfile) as ftmp: try: doc = yaml.load(ftmp) except yaml.YAMLError as exc: print(exc) except IOError as exc: print "I/O error({0}): {1}".format( exc.errno, exc.strerror) print "outfile ", outfile try: if doc != None: value = doc['processing']['status'] if value == 'DONE': continue except KeyError, e: print 'status %s does not exist' % str(e) hasbeenchecked = True print '-----------', f if '.root' in self.fext: nevts, check = self.checkFile_root(f, self.para.treename) status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } try: with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue except IOError as exc: print "I/O error({0}): {1}".format( exc.errno, exc.strerror) print "outfile ", outfile time.sleep(10) with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue elif '.lhe.gz' in self.fext: nevts, check = self.checkFile_lhe(f) while nevts == -1 and not check: nevts, check = self.checkFile_lhe(f) if self.count == 10: print 'can not copy or unzip the file, declare it wrong' break status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue else: print 'not correct file extension %s' % self.fext if hasbeenchecked: cmdp = '<pre>date=%s \t time=%s njobs=%i \t nevents=%i \t njobbad=%i \t process=%s </pre>\n' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process) stat_exist = ut.file_exist(statfile) with open(statfile, "a") as myfile: if not stat_exist: myfile.write( '<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n' ) myfile.write( '<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n' ) myfile.write(cmdp) print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process)
def check(self, para): #ldir=[x[0] for x in os.walk(self.indir)] ldir = next(os.walk(self.indireos))[1] if not ut.testeos(para.eostest, para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) dic = {} for l in ldir: if self.process != '' and self.process != l: continue #continue if process has been checked if l == 'BADPYTHIA' or l == 'lhe' or l == "__restored_files__" or l == "backup": continue print '--------------------- ', l proc = l nfileseos = 0 if os.path.isdir('%s/%s' % (self.indireos, proc)): listeos = [ x for x in os.listdir('%s/%s' % (self.indireos, proc)) if 'events' in x ] nfileseos = len(listeos) if nfileseos == 0: continue nfilesmerged = 0 mergefile = self.indirafs + '/' + l + '/merge.yaml' if not ut.file_exist(mergefile): if not ut.dir_exist('%s/%s' % (self.indirafs, proc)): os.system('mkdir %s/%s' % (self.indirafs, proc)) self.touch('%s/%s/check' % (self.indirafs, proc)) continue if not os.path.isdir(self.indirafs): os.system('mkdir %s' % self.indirafs) tmpf = None with open(mergefile, 'r') as stream: try: tmpf = yaml.load(stream) except yaml.YAMLError as exc: print(exc) bad_tot = tmpf['merge']['nbad'] files_tot = tmpf['merge']['ndone'] ntot_files = bad_tot + files_tot print "tot files ", ntot_files, " files eos ", nfileseos dic[proc] = {'neos': nfileseos, 'nmerged': ntot_files} print '%s/%s/check' % (self.indirafs, proc) if ntot_files < nfileseos: self.touch('%s/%s/check' % (self.indirafs, proc)) elif ntot_files > nfileseos: os.system('rm %s/%s/events*.yaml' % (self.indirafs, proc)) os.system('rm %s/%s/merge.yaml' % (self.indirafs, proc)) else: if ut.file_exist('%s/%s/check' % (self.indirafs, proc)): os.system('rm %s/%s/check' % (self.indirafs, proc)) outfile = self.indirafs + '/files.yaml' with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False)