class send_lhep8(): #__________________________________________________________ def __init__(self, njobs, events, process, islsf, iscondor, queue, para, version, decay): self.njobs = njobs self.events = -1 self.process = process self.islsf = islsf self.iscondor = iscondor self.queue = queue self.para = para self.version = version self.decay = decay self.user = os.environ['USER'] #__________________________________________________________ def send(self, force): Dir = os.getcwd() gplist = self.para.gridpacklist outdir = '%s%s/' % (self.para.delphes_dir, self.version) try: gplist[self.process] except KeyError, e: print 'process %s does not exist as gridpack' % self.process sys.exit(3) delphescards_mmr = '%s%s/%s' % (self.para.delphescards_dir, self.version, self.para.delphescard_mmr) if ut.file_exist( delphescards_mmr ) == False and self.version != 'cms' and 'helhc' not in self.version: print 'delphes card does not exist: ', delphescards_mmr sys.exit(3) delphescards_mr = '%s%s/%s' % (self.para.delphescards_dir, self.version, self.para.delphescard_mr) if ut.file_exist( delphescards_mr ) == False and self.version != 'cms' and 'helhc' not in self.version: print 'delphes card does not exist: ', delphescards_mr sys.exit(3) delphescards_base = '%s%s/%s' % (self.para.delphescards_dir, self.version, self.para.delphescard_base) if ut.file_exist(delphescards_base) == False: print 'delphes card does not exist: ', delphescards_base sys.exit(3) fccconfig = '%s%s' % (self.para.fccconfig_dir, self.para.fccconfig) if ut.file_exist(fccconfig) == False: print 'fcc config file does not exist: ', fccconfig sys.exit(3) print '======================================', self.process pythiacard = '%s%s.cmd' % ( self.para.pythiacards_dir, self.process.replace( 'mg_pp', 'p8_pp').replace('mg_gg', 'p8_gg')) if self.decay != '': pythiacard = '%s%s_%s.cmd' % ( self.para.pythiacards_dir, self.process.replace('mg_pp', 'p8_pp').replace( 'mg_gg', 'p8_gg'), self.decay) if ut.file_exist(pythiacard) == False and not force: print 'pythia card does not exist: ', pythiacard timeout = 60 print "do you want to use the default pythia card [y/n] (60sec to reply)" rlist, _, _ = select([sys.stdin], [], [], timeout) if rlist: s = sys.stdin.readline() if s == "y\n": print 'use default card' pythiacard = '%sp8_pp_default.cmd' % ( self.para.pythiacards_dir) else: print 'exit' sys.exit(3) else: print "timeout, use default card" pythiacard = '%sp8_pp_default.cmd' % ( self.para.pythiacards_dir) elif ut.file_exist(pythiacard) == False and force: print "force argument, use default card" pythiacard = '%sp8_pp_default.cmd' % (self.para.pythiacards_dir) pr_noht = '' if '_HT_' in self.process: ssplit = self.process.split('_') stest = '' for s in xrange(0, len(ssplit) - 3): stest += ssplit[s] + '_' pr_noht = stest[0:len(stest) - 1] #check that the specified decay exists if self.process in self.para.decaylist and self.decay != '' and '_HT_' not in self.process: if self.decay not in self.para.decaylist[self.process]: print 'decay ==%s== does not exist for process ==%s==' % ( self.decay, self.process) sys.exit(3) #check that the specified decay exists if pr_noht in self.para.decaylist and self.decay != '' and '_HT_' in self.process: if self.decay not in self.para.decaylist[pr_noht]: print 'decay ==%s== does not exist for process ==%s==' % ( self.decay, self.process) sys.exit(3) pr_decay = self.process if self.decay != '': pr_decay = self.process + '_' + self.decay print '====', pr_decay, '====' processp8 = pr_decay.replace('mg_pp', 'mgp8_pp').replace('mg_gg', 'mgp8_gg') acctype = 'FCC' if 'HELHC' in self.para.module_name: acctype = 'HELHC' logdir = Dir + "/BatchOutputs/%s/%s/%s/" % (acctype, self.version, processp8) if not ut.dir_exist(logdir): os.system("mkdir -p %s" % logdir) yamldir = '%s/%s/%s' % (self.para.yamldir, self.version, processp8) if not ut.dir_exist(yamldir): os.system("mkdir -p %s" % yamldir) yamllhedir = '%s/lhe/%s' % (self.para.yamldir, self.process) All_files = glob.glob("%s/events_*.yaml" % yamllhedir) if len(All_files) == 0: print 'there is no LHE files checked for process %s exit' % self.process sys.exit(3) if len(All_files) < self.njobs: print 'only %i LHE file exists, will not run all the jobs requested' % len( All_files) nbjobsSub = 0 ntmp = 0 if self.islsf == False and self.iscondor == False: print "Submit issue : LSF nor CONDOR flag defined !!!" sys.exit(3) condor_file_str = '' for i in xrange(len(All_files)): if nbjobsSub == self.njobs: break tmpf = None with open(All_files[i], 'r') as stream: try: tmpf = yaml.load(stream) if ut.getsize(All_files[i]) == 0: continue if tmpf['processing']['status'] != 'DONE': continue except yaml.YAMLError as exc: print(exc) jobid = tmpf['processing']['jobid'] myyaml = my.makeyaml(yamldir, jobid) if not myyaml: print 'job %s already exists' % jobid continue outfile = '%s/%s/events_%s.root' % (outdir, processp8, jobid) if ut.file_exist(outfile): print 'outfile already exist, continue ', outfile frunname = 'job%s.sh' % (jobid) frunfull = '%s/%s' % (logdir, frunname) frun = None try: frun = open(frunfull, 'w') except IOError as e: print "I/O error({0}): {1}".format(e.errno, e.strerror) time.sleep(10) frun = open(frunfull, 'w') commands.getstatusoutput('chmod 777 %s' % (frunfull)) frun.write('#!/bin/bash\n') frun.write('unset LD_LIBRARY_PATH\n') frun.write('unset PYTHONHOME\n') frun.write('unset PYTHONPATH\n') frun.write('source %s\n' % (self.para.stack)) frun.write('mkdir job%s_%s\n' % (jobid, processp8)) frun.write('cd job%s_%s\n' % (jobid, processp8)) frun.write('export EOS_MGM_URL=\"root://eospublic.cern.ch\"\n') frun.write('mkdir -p %s%s/%s\n' % (self.para.delphes_dir, self.version, processp8)) frun.write( 'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n' % (tmpf['processing']['out'])) frun.write('gunzip -c %s > events.lhe\n' % tmpf['processing']['out'].split('/')[-1]) frun.write( 'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n' % (delphescards_base)) if 'fcc' in self.version: frun.write( 'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n' % (delphescards_mmr)) frun.write( 'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n' % (delphescards_mr)) frun.write( 'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s config.py \n' % (fccconfig)) frun.write( 'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s card.cmd\n' % (pythiacard)) frun.write('echo "Beams:LHEF = events.lhe" >> card.cmd\n') frun.write('echo "Random:seed = %s" >> card.cmd\n' % jobid) if 'helhc' in self.version: frun.write('echo " Beams:eCM = 27000." >> card.cmd\n') frun.write( '%s/run fccrun.py config.py --delphescard=card.tcl --inputfile=card.cmd --outputfile=events_%s.root --nevents=%i\n' % (self.para.fccsw, jobid, self.events)) frun.write( 'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py events_%s.root %s\n' % (jobid, outfile)) frun.write('cd ..\n') frun.write('rm -rf job%s_%s\n' % (jobid, processp8)) frun.close() if self.islsf == True: cmdBatch = "bsub -M 3000000 -R \"pool=40000\" -q %s -o %s -cwd %s %s" % ( self.queue, logdir + '/job%s/' % (jobid), logdir + '/job%s/' % (jobid), frunfull) batchid = -1 job, batchid = ut.SubmitToLsf( cmdBatch, 10, "%i/%i" % (nbjobsSub, self.njobs)) nbjobsSub += job elif self.iscondor == True: condor_file_str += frunfull + " " nbjobsSub += 1 if self.iscondor == True: # clean string condor_file_str = condor_file_str.replace("//", "/") # frunname_condor = 'job_desc_lhep8.cfg' frunfull_condor = '%s/%s' % (logdir, frunname_condor) frun_condor = None try: frun_condor = open(frunfull_condor, 'w') except IOError as e: print "I/O error({0}): {1}".format(e.errno, e.strerror) time.sleep(10) frun_condor = open(frunfull_condor, 'w') commands.getstatusoutput('chmod 777 %s' % frunfull_condor) # frun_condor.write('executable = $(filename)\n') frun_condor.write( 'Log = %s/condor_job.%s.$(ClusterId).$(ProcId).log\n' % (logdir, str(jobid))) frun_condor.write( 'Output = %s/condor_job.%s.$(ClusterId).$(ProcId).out\n' % (logdir, str(jobid))) frun_condor.write( 'Error = %s/condor_job.%s.$(ClusterId).$(ProcId).error\n' % (logdir, str(jobid))) frun_condor.write('getenv = True\n') frun_condor.write('environment = "LS_SUBCWD=%s"\n' % logdir) # not sure frun_condor.write('request_memory = 4G\n') # frun_condor.write('requirements = ( (OpSysAndVer =?= "CentOS7") && (Machine =!= LastRemoteHost) )\n') frun_condor.write( 'requirements = ( (OpSysAndVer =?= "SLCern6") && (Machine =!= LastRemoteHost) )\n' ) frun_condor.write( 'on_exit_remove = (ExitBySignal == False) && (ExitCode == 0)\n' ) frun_condor.write('max_retries = 3\n') frun_condor.write('+JobFlavour = "%s"\n' % self.queue) frun_condor.write('+AccountingGroup = "group_u_FCC.local_gen"\n') frun_condor.write('queue filename matching files %s\n' % condor_file_str) frun_condor.close() # nbjobsSub = 0 cmdBatch = "condor_submit %s" % frunfull_condor print cmdBatch job = ut.SubmitToCondor(cmdBatch, 10, "%i/%i" % (nbjobsSub, self.njobs)) nbjobsSub += job print 'succesfully sent %i job(s)' % nbjobsSub
def cleanoldjobs(self): ldir = [] if self.process == '': ldir = next(os.walk(self.yamldir))[1] else: ldir = [self.process] #ldir=[x[0] for x in os.walk(self.yamldir)] print ldir for l in ldir: All_files = [] if self.process == '': All_files = glob.glob("%s/%s/events_*.yaml" % (self.yamldir, l)) else: All_files = glob.glob("%s/events_*.yaml" % (self.yamldir)) print "%s/events_*.yaml" % (self.yamldir) if len(All_files) == 0: continue process = l if self.process != '' and self.process != process: print process continue print 'process from the input directory ', process for f in All_files: if not os.path.isfile(f): print 'file does not exists... %s' % f continue if ut.getsize(f) == 0: print 'file size 0 %s will delete the yaml' % (f) cmd = "rm %s" % (f) print cmd os.system(cmd) continue with open(f, 'r') as stream: try: tmpf = yaml.load(stream) if tmpf['processing']['status'] == 'sending': #from datetime import datetime #import time #ts = time.time() #print(ts) #ds=str(tmpf['processing']['timestamp']) #d = datetime(int(ds[0:4]), int(ds[5:6]), int(ds[7:8]), int(ds[9:10]), int(ds[11:12])) #print int(ds[0:4]), int(ds[4:6]), int(ds[6:8]), int(ds[8:10]), int(ds[10:12]) #dt=datetime.timestamp() #print dt if ut.gettimestamp( ) - tmpf['processing']['timestamp'] > 18000: print 'job %s is running since too long %i , will delete the yaml' % ( f, ut.gettimestamp() - tmpf['processing']['timestamp']) cmd = "rm %s" % (f) print cmd os.system(cmd) except yaml.YAMLError as exc: print(exc) except IOError as e: print(e)
def check(self, force, statfile): #ldir=[x[0] for x in os.walk(self.indir)] ldir = next(os.walk(self.indir))[1] if not ut.testeos(self.para.eostest, self.para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) for l in ldir: if self.process != '' and self.process != l: continue #continue if process has been checked if ut.yamlcheck(self.yamlcheck, l) and not force: continue print '--------------------- ', l process = l All_files = glob.glob("%s/%s/events_*%s" % (self.indir, l, self.fext)) print 'number of files ', len(All_files) if len(All_files) == 0: continue if l == 'lhe' or l == "__restored_files__": continue print 'process from the input directory ', process outdir = self.makeyamldir(self.yamldir + process) hasbeenchecked = False nevents_tot = 0 njobsdone_tot = 0 njobsbad_tot = 0 for f in All_files: self.count = 0 if not os.path.isfile(f): print 'file does not exists... %s' % f continue jobid = f.split('_')[-1] jobid = jobid.replace(self.fext, '') userid = ut.find_owner(f) outfile = '%sevents_%s.yaml' % (outdir, jobid) if ut.file_exist( outfile) and ut.getsize(outfile) > 100 and not force: continue hasbeenchecked = True print '-----------', f if '.root' in self.fext: nevts, check = self.checkFile_root(f, self.para.treename) status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue elif '.lhe.gz' in self.fext: nevts, check = self.checkFile_lhe(f) while nevts == -1 and not check: nevts, check = self.checkFile_lhe(f) if self.count == 10: print 'can not copy or unzip the file, declare it wrong' break status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue else: print 'not correct file extension %s' % self.fext if hasbeenchecked: ut.yamlstatus(self.yamlcheck, process, False) cmdp = 'date=%s <span class="espace"/> time=%s <span class="espace"/> njobs=%i <span class="espace"/> nevents=%i <span class="espace"/> njobbad=%i <span class="espace"/> process=%s <br>\n' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process) stat_exist = ut.file_exist(statfile) with open(statfile, "a") as myfile: if not stat_exist: myfile.write( '<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n' ) myfile.write( '<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n' ) myfile.write(cmdp) print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process)
def check(self, force, statfile): #ldir=[x[0] for x in os.walk(self.indir)] ldir = next(os.walk(self.indir))[1] if not ut.testeos(self.para.eostest, self.para.eostest_size): print 'eos seems to have problems, should check, will exit' sys.exit(3) for l in ldir: if self.process != '' and self.process != l: continue #continue if process has been checked if l == 'BADPYTHIA' or l == 'lhe' or l == "__restored_files__" or l == "backup": continue print '%s/%s/check' % (self.yamldir, l) if not ut.file_exist('%s/%s/check' % (self.yamldir, l)) and not force: continue print '--------------------- ', l process = l All_files = glob.glob("%s/%s/events_*%s" % (self.indir, l, self.fext)) print 'number of files ', len(All_files) if len(All_files) == 0: continue print 'process from the input directory ', process outdir = self.makeyamldir(self.yamldir + process) hasbeenchecked = False nevents_tot = 0 njobsdone_tot = 0 njobsbad_tot = 0 for f in All_files: self.count = 0 if not os.path.isfile(f): print 'file does not exists... %s' % f continue jobid = f.split('_')[-1] jobid = jobid.replace(self.fext, '') userid = ut.find_owner(f) outfile = '%sevents_%s.yaml' % (outdir, jobid) if ut.getsize(outfile) == 0: cmd = "rm %s" % (outfile) print 'file size 0, remove and continue ', cmd os.system(cmd) continue if ut.file_exist( outfile) and ut.getsize(outfile) > 100 and not force: doc = None with open(outfile) as ftmp: try: doc = yaml.load(ftmp) except yaml.YAMLError as exc: print(exc) except IOError as exc: print "I/O error({0}): {1}".format( exc.errno, exc.strerror) print "outfile ", outfile try: if doc != None: value = doc['processing']['status'] if value == 'DONE': continue except KeyError, e: print 'status %s does not exist' % str(e) hasbeenchecked = True print '-----------', f if '.root' in self.fext: nevts, check = self.checkFile_root(f, self.para.treename) status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } try: with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue except IOError as exc: print "I/O error({0}): {1}".format( exc.errno, exc.strerror) print "outfile ", outfile time.sleep(10) with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue elif '.lhe.gz' in self.fext: nevts, check = self.checkFile_lhe(f) while nevts == -1 and not check: nevts, check = self.checkFile_lhe(f) if self.count == 10: print 'can not copy or unzip the file, declare it wrong' break status = 'DONE' if not check: status = 'BAD' if status == 'DONE': nevents_tot += nevts njobsdone_tot += 1 else: njobsbad_tot += 1 dic = { 'processing': { 'process': process, 'jobid': jobid, 'nevents': nevts, 'status': status, 'out': f, 'size': os.path.getsize(f), 'user': userid } } with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) continue else: print 'not correct file extension %s' % self.fext if hasbeenchecked: cmdp = '<pre>date=%s \t time=%s njobs=%i \t nevents=%i \t njobbad=%i \t process=%s </pre>\n' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process) stat_exist = ut.file_exist(statfile) with open(statfile, "a") as myfile: if not stat_exist: myfile.write( '<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n' ) myfile.write( '<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n' ) myfile.write(cmdp) print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % ( ut.getdate_str(), ut.gettime_str(), njobsdone_tot, nevents_tot, njobsbad_tot, process)
def merge(self, force): ldir = next(os.walk(self.indir))[1] print self.indir, ' ==== ', self.process #ldir=[x[0] for x in os.walk(self.indir)] for l in ldir: if self.process != '' and self.process != l: continue outfile = self.indir + '/' + l + '/merge.yaml' totsize = 0 totevents = 0 process = None outfiles = [] outfilesbad = [] outdir = None ndone = 0 nbad = 0 All_files = glob.glob("%s/%s/events_*.yaml" % (self.indir, l)) print 'ypuhfwegwegwgwe', len(All_files) print "%s/%s/events_*.yaml" % (self.indir, l) if len(All_files) == 0: if os.path.isfile("%s/%s/merge.yaml" % (self.indir, l)): os.system("rm %s/%s/merge.yaml" % (self.indir, l)) continue #continue if process has been checked print '%s/%s/check' % (self.indir, l) if not ut.file_exist('%s/%s/check' % (self.indir, l)) and not force: continue print 'merging process %s %i files' % (l, len(All_files)) for f in All_files: if not os.path.isfile(f): print 'file does not exists... %s' % f continue with open(f, 'r') as stream: try: tmpf = yaml.load(stream) if ut.getsize(f) == 0: continue if tmpf['processing']['status'] == 'sending': continue if tmpf['processing']['status'] == 'BAD': nbad += 1 outfilesbad.append( tmpf['processing']['out'].split('/')[-1]) outdir = tmpf['processing']['out'].replace( tmpf['processing']['out'].split('/')[-1], '') process = tmpf['processing']['process'] continue totsize += tmpf['processing']['size'] totevents += tmpf['processing']['nevents'] process = tmpf['processing']['process'] tmplist = [ tmpf['processing']['out'].split('/')[-1], tmpf['processing']['nevents'] ] outfiles.append(tmplist) outdir = tmpf['processing']['out'].replace( tmpf['processing']['out'].split('/')[-1], '') ndone += 1 except yaml.YAMLError as exc: print(exc) except IOError as exc: print "I/O error({0}): {1}".format( exc.errno, exc.strerror) print "outfile ", f dic = { 'merge': { 'process': process, 'nevents': totevents, 'outfiles': outfiles, 'outdir': outdir, 'size': totsize, 'ndone': ndone, 'nbad': nbad, 'outfilesbad': outfilesbad, } } try: with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False) except IOError as exc: print "I/O error({0}): {1}".format(exc.errno, exc.strerror) print "outfile ", outfile time.sleep(10) with open(outfile, 'w') as outyaml: yaml.dump(dic, outyaml, default_flow_style=False)