class send_lhep8():
#__________________________________________________________
def __init__(self, njobs, events, process, islsf, iscondor, queue, para,
version, decay):
self.njobs = njobs
self.events = -1
self.process = process
self.islsf = islsf
self.iscondor = iscondor
self.queue = queue
self.para = para
self.version = version
self.decay = decay
self.user = os.environ['USER']
#__________________________________________________________
def send(self, force):
Dir = os.getcwd()
gplist = self.para.gridpacklist
outdir = '%s%s/' % (self.para.delphes_dir, self.version)
try:
gplist[self.process]
except KeyError, e:
print 'process %s does not exist as gridpack' % self.process
sys.exit(3)
delphescards_mmr = '%s%s/%s' % (self.para.delphescards_dir,
self.version,
self.para.delphescard_mmr)
if ut.file_exist(
delphescards_mmr
) == False and self.version != 'cms' and 'helhc' not in self.version:
print 'delphes card does not exist: ', delphescards_mmr
sys.exit(3)
delphescards_mr = '%s%s/%s' % (self.para.delphescards_dir,
self.version, self.para.delphescard_mr)
if ut.file_exist(
delphescards_mr
) == False and self.version != 'cms' and 'helhc' not in self.version:
print 'delphes card does not exist: ', delphescards_mr
sys.exit(3)
delphescards_base = '%s%s/%s' % (self.para.delphescards_dir,
self.version,
self.para.delphescard_base)
if ut.file_exist(delphescards_base) == False:
print 'delphes card does not exist: ', delphescards_base
sys.exit(3)
fccconfig = '%s%s' % (self.para.fccconfig_dir, self.para.fccconfig)
if ut.file_exist(fccconfig) == False:
print 'fcc config file does not exist: ', fccconfig
sys.exit(3)
print '======================================', self.process
pythiacard = '%s%s.cmd' % (
self.para.pythiacards_dir, self.process.replace(
'mg_pp', 'p8_pp').replace('mg_gg', 'p8_gg'))
if self.decay != '':
pythiacard = '%s%s_%s.cmd' % (
self.para.pythiacards_dir,
self.process.replace('mg_pp', 'p8_pp').replace(
'mg_gg', 'p8_gg'), self.decay)
if ut.file_exist(pythiacard) == False and not force:
print 'pythia card does not exist: ', pythiacard
timeout = 60
print "do you want to use the default pythia card [y/n] (60sec to reply)"
rlist, _, _ = select([sys.stdin], [], [], timeout)
if rlist:
s = sys.stdin.readline()
if s == "y\n":
print 'use default card'
pythiacard = '%sp8_pp_default.cmd' % (
self.para.pythiacards_dir)
else:
print 'exit'
sys.exit(3)
else:
print "timeout, use default card"
pythiacard = '%sp8_pp_default.cmd' % (
self.para.pythiacards_dir)
elif ut.file_exist(pythiacard) == False and force:
print "force argument, use default card"
pythiacard = '%sp8_pp_default.cmd' % (self.para.pythiacards_dir)
pr_noht = ''
if '_HT_' in self.process:
ssplit = self.process.split('_')
stest = ''
for s in xrange(0, len(ssplit) - 3):
stest += ssplit[s] + '_'
pr_noht = stest[0:len(stest) - 1]
#check that the specified decay exists
if self.process in self.para.decaylist and self.decay != '' and '_HT_' not in self.process:
if self.decay not in self.para.decaylist[self.process]:
print 'decay ==%s== does not exist for process ==%s==' % (
self.decay, self.process)
sys.exit(3)
#check that the specified decay exists
if pr_noht in self.para.decaylist and self.decay != '' and '_HT_' in self.process:
if self.decay not in self.para.decaylist[pr_noht]:
print 'decay ==%s== does not exist for process ==%s==' % (
self.decay, self.process)
sys.exit(3)
pr_decay = self.process
if self.decay != '':
pr_decay = self.process + '_' + self.decay
print '====', pr_decay, '===='
processp8 = pr_decay.replace('mg_pp',
'mgp8_pp').replace('mg_gg', 'mgp8_gg')
acctype = 'FCC'
if 'HELHC' in self.para.module_name: acctype = 'HELHC'
logdir = Dir + "/BatchOutputs/%s/%s/%s/" % (acctype, self.version,
processp8)
if not ut.dir_exist(logdir):
os.system("mkdir -p %s" % logdir)
yamldir = '%s/%s/%s' % (self.para.yamldir, self.version, processp8)
if not ut.dir_exist(yamldir):
os.system("mkdir -p %s" % yamldir)
yamllhedir = '%s/lhe/%s' % (self.para.yamldir, self.process)
All_files = glob.glob("%s/events_*.yaml" % yamllhedir)
if len(All_files) == 0:
print 'there is no LHE files checked for process %s exit' % self.process
sys.exit(3)
if len(All_files) < self.njobs:
print 'only %i LHE file exists, will not run all the jobs requested' % len(
All_files)
nbjobsSub = 0
ntmp = 0
if self.islsf == False and self.iscondor == False:
print "Submit issue : LSF nor CONDOR flag defined !!!"
sys.exit(3)
condor_file_str = ''
for i in xrange(len(All_files)):
if nbjobsSub == self.njobs: break
tmpf = None
with open(All_files[i], 'r') as stream:
try:
tmpf = yaml.load(stream)
if ut.getsize(All_files[i]) == 0: continue
if tmpf['processing']['status'] != 'DONE': continue
except yaml.YAMLError as exc:
print(exc)
jobid = tmpf['processing']['jobid']
myyaml = my.makeyaml(yamldir, jobid)
if not myyaml:
print 'job %s already exists' % jobid
continue
outfile = '%s/%s/events_%s.root' % (outdir, processp8, jobid)
if ut.file_exist(outfile):
print 'outfile already exist, continue ', outfile
frunname = 'job%s.sh' % (jobid)
frunfull = '%s/%s' % (logdir, frunname)
frun = None
try:
frun = open(frunfull, 'w')
except IOError as e:
print "I/O error({0}): {1}".format(e.errno, e.strerror)
time.sleep(10)
frun = open(frunfull, 'w')
commands.getstatusoutput('chmod 777 %s' % (frunfull))
frun.write('#!/bin/bash\n')
frun.write('unset LD_LIBRARY_PATH\n')
frun.write('unset PYTHONHOME\n')
frun.write('unset PYTHONPATH\n')
frun.write('source %s\n' % (self.para.stack))
frun.write('mkdir job%s_%s\n' % (jobid, processp8))
frun.write('cd job%s_%s\n' % (jobid, processp8))
frun.write('export EOS_MGM_URL=\"root://eospublic.cern.ch\"\n')
frun.write('mkdir -p %s%s/%s\n' %
(self.para.delphes_dir, self.version, processp8))
frun.write(
'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n'
% (tmpf['processing']['out']))
frun.write('gunzip -c %s > events.lhe\n' %
tmpf['processing']['out'].split('/')[-1])
frun.write(
'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n'
% (delphescards_base))
if 'fcc' in self.version:
frun.write(
'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n'
% (delphescards_mmr))
frun.write(
'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s .\n'
% (delphescards_mr))
frun.write(
'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s config.py \n'
% (fccconfig))
frun.write(
'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py %s card.cmd\n'
% (pythiacard))
frun.write('echo "Beams:LHEF = events.lhe" >> card.cmd\n')
frun.write('echo "Random:seed = %s" >> card.cmd\n' % jobid)
if 'helhc' in self.version:
frun.write('echo " Beams:eCM = 27000." >> card.cmd\n')
frun.write(
'%s/run fccrun.py config.py --delphescard=card.tcl --inputfile=card.cmd --outputfile=events_%s.root --nevents=%i\n'
% (self.para.fccsw, jobid, self.events))
frun.write(
'python /afs/cern.ch/work/h/helsens/public/FCCutils/eoscopy.py events_%s.root %s\n'
% (jobid, outfile))
frun.write('cd ..\n')
frun.write('rm -rf job%s_%s\n' % (jobid, processp8))
frun.close()
if self.islsf == True:
cmdBatch = "bsub -M 3000000 -R \"pool=40000\" -q %s -o %s -cwd %s %s" % (
self.queue, logdir + '/job%s/' %
(jobid), logdir + '/job%s/' % (jobid), frunfull)
batchid = -1
job, batchid = ut.SubmitToLsf(
cmdBatch, 10, "%i/%i" % (nbjobsSub, self.njobs))
nbjobsSub += job
elif self.iscondor == True:
condor_file_str += frunfull + " "
nbjobsSub += 1
if self.iscondor == True:
# clean string
condor_file_str = condor_file_str.replace("//", "/")
#
frunname_condor = 'job_desc_lhep8.cfg'
frunfull_condor = '%s/%s' % (logdir, frunname_condor)
frun_condor = None
try:
frun_condor = open(frunfull_condor, 'w')
except IOError as e:
print "I/O error({0}): {1}".format(e.errno, e.strerror)
time.sleep(10)
frun_condor = open(frunfull_condor, 'w')
commands.getstatusoutput('chmod 777 %s' % frunfull_condor)
#
frun_condor.write('executable = $(filename)\n')
frun_condor.write(
'Log = %s/condor_job.%s.$(ClusterId).$(ProcId).log\n'
% (logdir, str(jobid)))
frun_condor.write(
'Output = %s/condor_job.%s.$(ClusterId).$(ProcId).out\n'
% (logdir, str(jobid)))
frun_condor.write(
'Error = %s/condor_job.%s.$(ClusterId).$(ProcId).error\n'
% (logdir, str(jobid)))
frun_condor.write('getenv = True\n')
frun_condor.write('environment = "LS_SUBCWD=%s"\n' %
logdir) # not sure
frun_condor.write('request_memory = 4G\n')
# frun_condor.write('requirements = ( (OpSysAndVer =?= "CentOS7") && (Machine =!= LastRemoteHost) )\n')
frun_condor.write(
'requirements = ( (OpSysAndVer =?= "SLCern6") && (Machine =!= LastRemoteHost) )\n'
)
frun_condor.write(
'on_exit_remove = (ExitBySignal == False) && (ExitCode == 0)\n'
)
frun_condor.write('max_retries = 3\n')
frun_condor.write('+JobFlavour = "%s"\n' % self.queue)
frun_condor.write('+AccountingGroup = "group_u_FCC.local_gen"\n')
frun_condor.write('queue filename matching files %s\n' %
condor_file_str)
frun_condor.close()
#
nbjobsSub = 0
cmdBatch = "condor_submit %s" % frunfull_condor
print cmdBatch
job = ut.SubmitToCondor(cmdBatch, 10,
"%i/%i" % (nbjobsSub, self.njobs))
nbjobsSub += job
print 'succesfully sent %i job(s)' % nbjobsSub
def cleanoldjobs(self):
ldir = []
if self.process == '':
ldir = next(os.walk(self.yamldir))[1]
else:
ldir = [self.process]
#ldir=[x[0] for x in os.walk(self.yamldir)]
print ldir
for l in ldir:
All_files = []
if self.process == '':
All_files = glob.glob("%s/%s/events_*.yaml" %
(self.yamldir, l))
else:
All_files = glob.glob("%s/events_*.yaml" % (self.yamldir))
print "%s/events_*.yaml" % (self.yamldir)
if len(All_files) == 0: continue
process = l
if self.process != '' and self.process != process:
print process
continue
print 'process from the input directory ', process
for f in All_files:
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
if ut.getsize(f) == 0:
print 'file size 0 %s will delete the yaml' % (f)
cmd = "rm %s" % (f)
print cmd
os.system(cmd)
continue
with open(f, 'r') as stream:
try:
tmpf = yaml.load(stream)
if tmpf['processing']['status'] == 'sending':
#from datetime import datetime
#import time
#ts = time.time()
#print(ts)
#ds=str(tmpf['processing']['timestamp'])
#d = datetime(int(ds[0:4]), int(ds[5:6]), int(ds[7:8]), int(ds[9:10]), int(ds[11:12]))
#print int(ds[0:4]), int(ds[4:6]), int(ds[6:8]), int(ds[8:10]), int(ds[10:12])
#dt=datetime.timestamp()
#print dt
if ut.gettimestamp(
) - tmpf['processing']['timestamp'] > 18000:
print 'job %s is running since too long %i , will delete the yaml' % (
f, ut.gettimestamp() -
tmpf['processing']['timestamp'])
cmd = "rm %s" % (f)
print cmd
os.system(cmd)
except yaml.YAMLError as exc:
print(exc)
except IOError as e:
print(e)
def check(self, force, statfile):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indir))[1]
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if ut.yamlcheck(self.yamlcheck, l) and not force: continue
print '--------------------- ', l
process = l
All_files = glob.glob("%s/%s/events_*%s" %
(self.indir, l, self.fext))
print 'number of files ', len(All_files)
if len(All_files) == 0: continue
if l == 'lhe' or l == "__restored_files__": continue
print 'process from the input directory ', process
outdir = self.makeyamldir(self.yamldir + process)
hasbeenchecked = False
nevents_tot = 0
njobsdone_tot = 0
njobsbad_tot = 0
for f in All_files:
self.count = 0
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
jobid = f.split('_')[-1]
jobid = jobid.replace(self.fext, '')
userid = ut.find_owner(f)
outfile = '%sevents_%s.yaml' % (outdir, jobid)
if ut.file_exist(
outfile) and ut.getsize(outfile) > 100 and not force:
continue
hasbeenchecked = True
print '-----------', f
if '.root' in self.fext:
nevts, check = self.checkFile_root(f, self.para.treename)
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
elif '.lhe.gz' in self.fext:
nevts, check = self.checkFile_lhe(f)
while nevts == -1 and not check:
nevts, check = self.checkFile_lhe(f)
if self.count == 10:
print 'can not copy or unzip the file, declare it wrong'
break
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
else:
print 'not correct file extension %s' % self.fext
if hasbeenchecked:
ut.yamlstatus(self.yamlcheck, process, False)
cmdp = 'date=%s <span class="espace"/> time=%s <span class="espace"/> njobs=%i <span class="espace"/> nevents=%i <span class="espace"/> njobbad=%i <span class="espace"/> process=%s <br>\n' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
stat_exist = ut.file_exist(statfile)
with open(statfile, "a") as myfile:
if not stat_exist:
myfile.write(
'<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n'
)
myfile.write(
'<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n'
)
myfile.write(cmdp)
print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
def check(self, force, statfile):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indir))[1]
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if l == 'BADPYTHIA' or l == 'lhe' or l == "__restored_files__" or l == "backup":
continue
print '%s/%s/check' % (self.yamldir, l)
if not ut.file_exist('%s/%s/check' %
(self.yamldir, l)) and not force:
continue
print '--------------------- ', l
process = l
All_files = glob.glob("%s/%s/events_*%s" %
(self.indir, l, self.fext))
print 'number of files ', len(All_files)
if len(All_files) == 0: continue
print 'process from the input directory ', process
outdir = self.makeyamldir(self.yamldir + process)
hasbeenchecked = False
nevents_tot = 0
njobsdone_tot = 0
njobsbad_tot = 0
for f in All_files:
self.count = 0
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
jobid = f.split('_')[-1]
jobid = jobid.replace(self.fext, '')
userid = ut.find_owner(f)
outfile = '%sevents_%s.yaml' % (outdir, jobid)
if ut.getsize(outfile) == 0:
cmd = "rm %s" % (outfile)
print 'file size 0, remove and continue ', cmd
os.system(cmd)
continue
if ut.file_exist(
outfile) and ut.getsize(outfile) > 100 and not force:
doc = None
with open(outfile) as ftmp:
try:
doc = yaml.load(ftmp)
except yaml.YAMLError as exc:
print(exc)
except IOError as exc:
print "I/O error({0}): {1}".format(
exc.errno, exc.strerror)
print "outfile ", outfile
try:
if doc != None: value = doc['processing']['status']
if value == 'DONE': continue
except KeyError, e:
print 'status %s does not exist' % str(e)
hasbeenchecked = True
print '-----------', f
if '.root' in self.fext:
nevts, check = self.checkFile_root(f, self.para.treename)
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
try:
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
except IOError as exc:
print "I/O error({0}): {1}".format(
exc.errno, exc.strerror)
print "outfile ", outfile
time.sleep(10)
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
elif '.lhe.gz' in self.fext:
nevts, check = self.checkFile_lhe(f)
while nevts == -1 and not check:
nevts, check = self.checkFile_lhe(f)
if self.count == 10:
print 'can not copy or unzip the file, declare it wrong'
break
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
else:
print 'not correct file extension %s' % self.fext
if hasbeenchecked:
cmdp = '<pre>date=%s \t time=%s njobs=%i \t nevents=%i \t njobbad=%i \t process=%s </pre>\n' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
stat_exist = ut.file_exist(statfile)
with open(statfile, "a") as myfile:
if not stat_exist:
myfile.write(
'<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n'
)
myfile.write(
'<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n'
)
myfile.write(cmdp)
print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
def merge(self, force):
ldir = next(os.walk(self.indir))[1]
print self.indir, ' ==== ', self.process
#ldir=[x[0] for x in os.walk(self.indir)]
for l in ldir:
if self.process != '' and self.process != l:
continue
outfile = self.indir + '/' + l + '/merge.yaml'
totsize = 0
totevents = 0
process = None
outfiles = []
outfilesbad = []
outdir = None
ndone = 0
nbad = 0
All_files = glob.glob("%s/%s/events_*.yaml" % (self.indir, l))
print 'ypuhfwegwegwgwe', len(All_files)
print "%s/%s/events_*.yaml" % (self.indir, l)
if len(All_files) == 0:
if os.path.isfile("%s/%s/merge.yaml" % (self.indir, l)):
os.system("rm %s/%s/merge.yaml" % (self.indir, l))
continue
#continue if process has been checked
print '%s/%s/check' % (self.indir, l)
if not ut.file_exist('%s/%s/check' %
(self.indir, l)) and not force:
continue
print 'merging process %s %i files' % (l, len(All_files))
for f in All_files:
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
with open(f, 'r') as stream:
try:
tmpf = yaml.load(stream)
if ut.getsize(f) == 0: continue
if tmpf['processing']['status'] == 'sending': continue
if tmpf['processing']['status'] == 'BAD':
nbad += 1
outfilesbad.append(
tmpf['processing']['out'].split('/')[-1])
outdir = tmpf['processing']['out'].replace(
tmpf['processing']['out'].split('/')[-1], '')
process = tmpf['processing']['process']
continue
totsize += tmpf['processing']['size']
totevents += tmpf['processing']['nevents']
process = tmpf['processing']['process']
tmplist = [
tmpf['processing']['out'].split('/')[-1],
tmpf['processing']['nevents']
]
outfiles.append(tmplist)
outdir = tmpf['processing']['out'].replace(
tmpf['processing']['out'].split('/')[-1], '')
ndone += 1
except yaml.YAMLError as exc:
print(exc)
except IOError as exc:
print "I/O error({0}): {1}".format(
exc.errno, exc.strerror)
print "outfile ", f
dic = {
'merge': {
'process': process,
'nevents': totevents,
'outfiles': outfiles,
'outdir': outdir,
'size': totsize,
'ndone': ndone,
'nbad': nbad,
'outfilesbad': outfilesbad,
}
}
try:
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
except IOError as exc:
print "I/O error({0}): {1}".format(exc.errno, exc.strerror)
print "outfile ", outfile
time.sleep(10)
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)