def createtarball(self):
mkdir_p(self.workdir)
with KeepWhileOpenFile(self.tmptarball + ".tmp") as kwof:
if not kwof:
return "another process is already copying the tarball"
if not os.path.exists(self.originaltarball):
return "original tarball does not exist"
if datetime.datetime.fromtimestamp(
os.path.getmtime(
self.originaltarball)) <= self.modifiedafter:
return "original tarball is an older version than we want"
mkdir_p(os.path.dirname(self.foreostarball))
if self.patchkwargs:
kwargs = self.patchkwargs
for _ in "oldfilename", "newfilename", "sample":
assert _ not in kwargs, _
with cdtemp():
kwargs["oldfilename"] = self.originaltarball
kwargs["newfilename"] = os.path.abspath(
os.path.basename(self.originaltarball))
#kwargs["sample"] = self #???
patches.dopatch(**kwargs)
shutil.move(os.path.basename(self.originaltarball),
self.foreostarball)
else:
shutil.copy(self.originaltarball, self.foreostarball)
return "gridpack is copied from " + self.originaltarball + " to this folder, to be copied to eos"
def findmatchefficiency(self):
#figure out the filter efficiency
if not self.hasfilter:
self.matchefficiency = 1
return "filter efficiency is set to 1 +/- 0"
else:
if not self.implementsfilter: raise ValueError("Can't find match efficiency for {.__name__} which doesn't implement filtering!".format(type(self)))
mkdir_p(self.workdir)
jobsrunning = False
eventsprocessed = eventsaccepted = 0
with cd(self.workdir):
for i in range(100):
mkdir_p(str(i))
with cd(str(i)), KeepWhileOpenFile("runningfilterjob.tmp", message=LSB_JOBID(), deleteifjobdied=True) as kwof:
if not kwof:
jobsrunning = True
continue
if not os.path.exists(self.filterresultsfile):
if not LSB_JOBID():
submitLSF(self.filterefficiencyqueue)
jobsrunning = True
continue
if not queuematches(self.filterefficiencyqueue):
jobsrunning = True
continue
self.dofilterjob(i)
processed, accepted = self.getfilterresults(i)
eventsprocessed += processed
eventsaccepted += accepted
if jobsrunning: return "some filter efficiency jobs are still running"
self.matchefficiency = uncertainties.ufloat(1.0*eventsaccepted / eventsprocessed, (1.0*eventsaccepted * (eventsprocessed-eventsaccepted) / eventsprocessed**3) ** .5)
#shutil.rmtree(self.workdir)
return "match efficiency is measured to be {}".format(self.matchefficiency)
def patchtarball(self):
if os.path.exists(self.cvmfstarball) or os.path.exists(self.eostarball) or os.path.exists(self.foreostarball): return
if not self.needspatch: assert False
mkdir_p(self.workdir)
with KeepWhileOpenFile(self.tmptarball+".tmp", message=LSB_JOBID()) as kwof:
if not kwof:
return "job to patch the tarball is already running"
kwargs = self.needspatch
if isinstance(kwargs, int):
kwargs = self.patchkwargs
kwargs["oldtarballversion"] = self.needspatch
if "oldfilename" in kwargs or "newfilename" in kwargs or "sample" in kwargs: assert False, kwargs
kwargs["oldfilename"] = self.cvmfstarball_anyversion(version=kwargs.pop("oldtarballversion"))
kwargs["newfilename"] = self.foreostarball
mkdir_p(os.path.dirname(self.foreostarball))
patches.dopatch(**kwargs)
if not os.path.exists(self.foreostarball): raise RuntimeError("Patching failed, gridpack doesn't exist")
if self.timeperevent is not None:
del self.timeperevent
self.needspatch = False
return "tarball is patched and the new version is in this directory to be copied to eos"
def checkandfixtarball(self):
mkdir_p(self.workdir)
with KeepWhileOpenFile(os.path.join(self.workdir,
self.prepid + '.tmp'),
message=LSB_JOBID(),
deleteifjobdied=True) as kwof:
if not kwof: return " check in progress"
if not LSB_JOBID():
self.submitLSF()
return "Check if the tarball needs fixing"
with cdtemp():
subprocess.call(['cp', self.cvmfstarball, '.'])
subprocess.check_call(['tar', 'xzvf', self.cvmfstarball])
subprocess.call(['cp', 'readInput.DAT', 'readInput.DAT_bak'])
os.system('chmod 755 runcmsgrid.sh')
try:
output = subprocess.check_output(
['bash', 'runcmsgrid.sh', '1', '31313', '12'],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
output = e.output
for line in output.split('\n'):
if not 'Reading in vegas grid from' in line: continue
else:
line = line.split()[-2]
internalgridname = line.split('CMS_')[1]
internalgridname = str(internalgridname)
print "internal tarball name: " + internalgridname
if self.datasetname + '_grid' == internalgridname:
with open(os.path.join(self.workdir, 'INTACT'),
'w') as fout:
fout.write(LSB_JOBID())
return str(self.identifiers) + "'s gridpack is intact"
else:
os.system('cp ' + self.datasetname + '_grid ' +
internalgridname)
os.system('mv readInput.DAT_bak readInput.DAT')
os.system('rm -r *tgz CMSSW*')
curdirpath = subprocess.check_output(['pwd'])
os.system('tar cvaf ' + self.tmptarball + ' ./*')
if os.path.exists(self.tmptarball):
with open(os.path.join(self.workdir, 'FIXED'),
'w') as fout:
fout.write(LSB_JOBID())
def getsizeandtime(self):
mkdir_p(self.workdir)
with KeepWhileOpenFile(os.path.join(self.workdir, self.prepid+".tmp"), message=LSB_JOBID(), deleteifjobdied=True) as kwof:
if not kwof: return "job to get the size and time is already running"
if not LSB_JOBID(): return "need to get time and size per event, submitting to LSF" if submitLSF(self.timepereventqueue) else "need to get time and size per event, job is pending on LSF"
if not queuematches(self.timepereventqueue): return "need to get time and size per event, but on the wrong queue"
with cdtemp():
wget(os.path.join("https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_test/", self.prepid, str(self.neventsfortest) if self.neventsfortest else "").rstrip("/"), output=self.prepid)
with open(self.prepid) as f:
testjob = f.read()
with open(self.prepid, "w") as newf:
newf.write(eval(testjob))
os.chmod(self.prepid, os.stat(self.prepid).st_mode | stat.S_IEXEC)
subprocess.check_call(["./"+self.prepid], stderr=subprocess.STDOUT)
with open(self.prepid+"_rt.xml") as f:
nevents = totalsize = None
for line in f:
line = line.strip()
match = re.match('<TotalEvents>([0-9]*)</TotalEvents>', line)
if match: nevents = int(match.group(1))
match = re.match('<Metric Name="Timing-tstoragefile-write-totalMegabytes" Value="([0-9.]*)"/>', line)
if match: totalsize = float(match.group(1))
if self.year >= 2017:
match = re.match('<Metric Name="EventThroughput" Value="([0-9.eE+-]*)"/>', line)
if match: self.timeperevent = 1/float(match.group(1))
else:
match = re.match('<Metric Name="AvgEventTime" Value="([0-9.eE+-]*)"/>', line)
if match: self.timeperevent = float(match.group(1))
if nevents is not None is not totalsize:
self.sizeperevent = totalsize * 1024 / nevents
shutil.rmtree(self.workdir)
if not (self.sizeperevent and self.timeperevent):
return "failed to get the size and time"
if LSB_JOBID(): return "size and time per event are found to be {} and {}, run locally to send to McM".format(self.sizeperevent, self.timeperevent)
self.updaterequest()
return "size and time per event are found to be {} and {}, sent it to McM".format(self.sizeperevent, self.timeperevent)
def createtarball(self):
if os.path.exists(self.cvmfstarball) or os.path.exists(self.eostarball) or os.path.exists(self.foreostarball): return
mkdir_p(self.workdir)
with cd(self.workdir), KeepWhileOpenFile(self.tmptarball+".tmp", message=LSB_JOBID()) as kwof:
if not kwof:
with open(self.tmptarball+".tmp") as f:
try:
jobid = int(f.read().strip())
except ValueError:
return "try running again, probably you just got really bad timing"
if jobended(str(jobid)):
if self.makinggridpacksubmitsjob:
os.remove(self.tmptarball+".tmp")
return "job died at a very odd time, cleaned it up. Try running again."
for _ in os.listdir("."): #--> delete everything in the folder, except the tarball if that exists
if os.path.basename(_) != os.path.basename(self.tmptarball) and os.path.basename(_) != os.path.basename(self.tmptarball)+".tmp":
try:
os.remove(_)
except OSError:
shutil.rmtree(_)
os.remove(os.path.basename(self.tmptarball)+".tmp") #remove that last
return "gridpack job died, cleaned it up. run makegridpacks.py again."
else:
return "job to make the tarball is already running"
if self.gridpackjobsrunning:
return "job to make the tarball is already running"
if not os.path.exists(self.tmptarball):
if not self.inthemiddleofmultistepgridpackcreation:
for _ in os.listdir("."):
if not _.endswith(".tmp"):
try:
os.remove(_)
except OSError:
shutil.rmtree(_)
if not self.makinggridpacksubmitsjob and self.creategridpackqueue is not None:
if not LSB_JOBID(): return "need to create the gridpack, submitting to LSF" if submitLSF(self.creategridpackqueue) else "need to create the gridpack, job is pending on LSF"
if not queuematches(self.creategridpackqueue): return "need to create the gridpack, but on the wrong queue"
for filename in self.makegridpackscriptstolink:
os.symlink(filename, os.path.basename(filename))
makinggridpacksubmitsjob = self.makinggridpacksubmitsjob
#https://stackoverflow.com/a/17698359/5228524
makegridpackstdout = ""
pipe = subprocess.Popen(self.makegridpackcommand, stdout=subprocess.PIPE, bufsize=1)
with pipe.stdout:
for line in iter(pipe.stdout.readline, b''):
print line,
makegridpackstdout += line
self.processmakegridpackstdout(makegridpackstdout)
if makinggridpacksubmitsjob:
return "submitted the gridpack creation job"
if self.inthemiddleofmultistepgridpackcreation:
return "ran one step of gridpack creation, run again to continue"
mkdir_p(os.path.dirname(self.foreostarball))
if self.patchkwargs:
kwargs = self.patchkwargs
for _ in "oldfilename", "newfilename", "sample": assert _ not in kwargs, _
with cdtemp():
kwargs["oldfilename"] = self.tmptarball
kwargs["newfilename"] = os.path.abspath(os.path.basename(self.tmptarball))
#kwargs["sample"] = self #???
patches.dopatch(**kwargs)
shutil.move(os.path.basename(self.tmptarball), self.tmptarball)
if self.timeperevent is not None:
del self.timeperevent
shutil.move(self.tmptarball, self.foreostarball)
shutil.rmtree(os.path.dirname(self.tmptarball))
return "tarball is created and moved to this folder, to be copied to eos"