def cardsurl(self):
icard = "VBF_"
if self.signalbkgbsi == "SIG" or self.signalbkgbsi == "BSI":
icard += "H125"
if self.signalbkgbsi == "BKG" or self.signalbkgbsi == "BSI":
icard += "ZZcont"
icard += "_NNPDF31_13TeV_"
icard += {
"4e": "ee_ee_",
"4mu": "mumu_mumu_",
"2e2mu": "ee_mumu_",
"2e2nue": "ee_veve_",
"2e2num": "ee_vmvm_",
"2e2nut": "ee_vtvt_",
"2mu2nue": "mumu_veve_",
"2mu2num": "mumu_vmvm_",
"2mu2nut": "mumu_vtvt_",
}[self.finalstate]
icard += ".py"
card = os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/",
self.genproductionscommit,
"bin/Phantom/cards/production/13TeV/HZZ_VBFoffshell_Phantom",
icard)
with cdtemp():
wget(card)
with open(os.path.basename(card)) as f:
gitcardcontents = f.read()
with cdtemp():
subprocess.check_output(["tar", "xvaf", self.cvmfstarball])
if glob.glob("core.*"):
raise ValueError(
"There is a core dump in the tarball\n{}".format(self))
cardnameintarball = icard
try:
with open(cardnameintarball) as f:
cardcontents = f.read()
except IOError:
raise ValueError("no " + cardnameintarball +
" in the tarball\n{}".format(self))
if cardcontents != gitcardcontents:
with cd(here):
with open("cardcontents", "w") as f:
f.write(cardcontents)
with open("powheggitcard", "w") as f:
f.write(gitcardcontents)
raise ValueError(
"cardcontents != gitcardcontents\n{}\nSee ./cardcontents and ./gitcardcontents"
.format(self))
return card
def filterjobscript(self, jobindex):
olddir = os.getcwd()
with cdtemp():
wget(os.path.join("https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_test/", self.prepid, str(self.neventsfortest) if self.neventsfortest else "").rstrip("/"), output=self.prepid)
with open(self.prepid) as f:
testjob = f.read()
try:
testjob = eval(testjob) #sometimes it's a string within a string
except SyntaxError:
pass #sometimes it's not
lines = testjob.split("\n")
cmsdriverindex = {i for i, line in enumerate(lines) if "cmsDriver.py" in line}
assert len(cmsdriverindex) == 1, cmsdriverindex
cmsdriverindex = cmsdriverindex.pop()
lines.insert(cmsdriverindex+1, 'sed -i "/Services/aprocess.RandomNumberGeneratorService.externalLHEProducer.initialSeed = {}" *_cfg.py'.format(abs(hash(self))%900000000 + jobindex)) #The CLHEP::HepJamesRandom engine seed should be in the range 0 to 900000000.
return "\n".join(lines)
def dofilterjob(self, jobindex):
olddir = os.getcwd()
with cdtemp():
wget(os.path.join(
"https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_test/",
self.prepid,
str(self.neventsfortest)
if self.neventsfortest else "").rstrip("/"),
output=self.prepid)
with open(self.prepid) as f:
testjob = f.read()
with open(self.prepid, "w") as newf:
newf.write(eval(testjob))
os.chmod(self.prepid, os.stat(self.prepid).st_mode | stat.S_IEXEC)
subprocess.check_call(["./" + self.prepid],
stderr=subprocess.STDOUT)
shutil.move(self.prepid + "_rt.xml", olddir)
def getsizeandtime(self):
mkdir_p(self.workdir)
with KeepWhileOpenFile(os.path.join(self.workdir, self.prepid+".tmp"), message=LSB_JOBID(), deleteifjobdied=True) as kwof:
if not kwof: return "job to get the size and time is already running"
if not LSB_JOBID(): return "need to get time and size per event, submitting to LSF" if submitLSF(self.timepereventqueue) else "need to get time and size per event, job is pending on LSF"
if not queuematches(self.timepereventqueue): return "need to get time and size per event, but on the wrong queue"
with cdtemp():
wget(os.path.join("https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_test/", self.prepid, str(self.neventsfortest) if self.neventsfortest else "").rstrip("/"), output=self.prepid)
with open(self.prepid) as f:
testjob = f.read()
with open(self.prepid, "w") as newf:
newf.write(eval(testjob))
os.chmod(self.prepid, os.stat(self.prepid).st_mode | stat.S_IEXEC)
subprocess.check_call(["./"+self.prepid], stderr=subprocess.STDOUT)
with open(self.prepid+"_rt.xml") as f:
nevents = totalsize = None
for line in f:
line = line.strip()
match = re.match('<TotalEvents>([0-9]*)</TotalEvents>', line)
if match: nevents = int(match.group(1))
match = re.match('<Metric Name="Timing-tstoragefile-write-totalMegabytes" Value="([0-9.]*)"/>', line)
if match: totalsize = float(match.group(1))
if self.year >= 2017:
match = re.match('<Metric Name="EventThroughput" Value="([0-9.eE+-]*)"/>', line)
if match: self.timeperevent = 1/float(match.group(1))
else:
match = re.match('<Metric Name="AvgEventTime" Value="([0-9.eE+-]*)"/>', line)
if match: self.timeperevent = float(match.group(1))
if nevents is not None is not totalsize:
self.sizeperevent = totalsize * 1024 / nevents
shutil.rmtree(self.workdir)
if not (self.sizeperevent and self.timeperevent):
return "failed to get the size and time"
if LSB_JOBID(): return "size and time per event are found to be {} and {}, run locally to send to McM".format(self.sizeperevent, self.timeperevent)
self.updaterequest()
return "size and time per event are found to be {} and {}, sent it to McM".format(self.sizeperevent, self.timeperevent)
def cardsurl(self):
commit = self.genproductionscommit
if self.powhegcardusesscript:
powhegdir, powhegcard = os.path.split(self.powhegcard)
powhegscript = os.path.join(powhegdir, "makecards.py")
powhegscript = os.path.join("https://raw.githubusercontent.com/cms-sw/genproductions/", commit, powhegscript.split("genproductions/")[-1])
result = ( powhegscript + "\n"
+ "# " + powhegcard)
else:
powhegcard = os.path.join("https://raw.githubusercontent.com/cms-sw/genproductions/", commit, self.powhegcard.split("genproductions/")[-1])
result = powhegcard
with cdtemp():
if self.powhegcardusesscript:
wget(powhegscript)
wget(os.path.join(os.path.dirname(powhegscript), powhegcard.replace("M{}".format(self.mass), "template").replace("Wplus", "W").replace("Wminus", "W")))
subprocess.check_call(["python", "makecards.py"])
else:
wget(powhegcard)
with open(os.path.basename(powhegcard)) as f:
powheggitcard = f.read()
powheggitcardlines = [re.sub(" *([#!].*)?$", "", line) for line in powheggitcard.split("\n")]
powheggitcardlines = [re.sub("(iseed|ncall2|fakevirt) *", r"\1 ", line) for line in powheggitcardlines
if line and all(_ not in line for _ in
("pdfreweight", "storeinfo_rwgt", "withnegweights", "rwl_", "lhapdf6maxsets", "xgriditeration", "fakevirt")
)]
powheggitcard = "\n".join(line for line in powheggitcardlines)
with cdtemp():
subprocess.check_output(["tar", "xvaf", self.cvmfstarball])
if glob.glob("core.*") and self.cvmfstarball != "/cvmfs/cms.cern.ch/phys_generator/gridpacks/2017/13TeV/powheg/V2/HJJ_M125_13TeV/HJJ_slc6_amd64_gcc630_CMSSW_9_3_0_HJJ_NNPDF31_13TeV_M125.tgz":
raise ValueError("There is a core dump in the tarball\n{}".format(self))
try:
with open("powheg.input") as f:
powhegcard = f.read()
powhegcardlines = [re.sub(" *([#!].*)?$", "", line) for line in powhegcard.split("\n")]
powhegcardlines = [re.sub("(iseed|ncall2|fakevirt) *", r"\1 ", line) for line in powhegcardlines
if line and all(_ not in line for _ in
("pdfreweight", "storeinfo_rwgt", "withnegweights", "rwl_", "lhapdf6maxsets", "xgriditeration", "fakevirt")
)]
powhegcard = "\n".join(line for line in powhegcardlines)
except IOError:
raise ValueError("no powheg.input in the tarball\n{}".format(self))
if powhegcard != powheggitcard:
with cd(here):
with open("powhegcard", "w") as f:
f.write(powhegcard)
with open("powheggitcard", "w") as f:
f.write(powheggitcard)
raise ValueError("powhegcard != powheggitcard\n{}\nSee ./powhegcard and ./powheggitcard".format(self))
return result
def cardsurl(self):
commit = self.genproductionscommit
productioncardurl = os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/", commit,
self.productioncard.split("genproductions/")[-1])
mdatascript = os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/", commit,
"bin/MCFM/ACmdataConfig.py")
with cdtemp():
with contextlib.closing(urllib.urlopen(productioncardurl)) as f:
productiongitcard = f.read()
with cdtemp():
subprocess.check_output(["tar", "xvaf", self.cvmfstarball])
if glob.glob("core.*"):
raise ValueError(
"There is a core dump in the tarball\n{}".format(self))
# for root, dirs, files in os.walk("."):
# for ifile in files:
# try:
# os.stat(ifile)
# except Exception as e:
# if e.args == 'No such file or directory': continue
# print ifile
# print e.message, e.args
# raise ValueError("There is a broken symlink in the tarball\n{}".format(self))
try:
with open("readInput.DAT") as f:
productioncard = f.read()
except IOError:
raise ValueError(
"no readInput.DAT in the tarball\n{}".format(self))
try:
with open("src/User/mdata.f") as f:
mdatacard = f.read()
except IOError:
raise ValueError(
"no src/User/mdata.f in the tarball\n{}".format(self))
if differentproductioncards(
productioncard,
productiongitcard) and not 'BKG' in self.identifiers:
with cd(here):
with open("productioncard", "w") as f:
f.write(productioncard)
with open("productiongitcard", "w") as f:
f.write(productiongitcard)
raise ValueError(
"productioncard != productiongitcard\n{}\nSee ./productioncard and ./productiongitcard"
.format(self))
with contextlib.closing(
urllib.urlopen(
os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/"
+ commit + "/bin/MCFM/run_mcfm_AC.py"))) as f:
infunction = False
for line in f:
if re.match(r"^\s*def .*", line): infunction = False
if re.match(r"^\s*def downloadmcfm.*", line): infunction = True
if not infunction: continue
match = re.search(r"git checkout ([\w.]*)", line)
if match: mcfmcommit = match.group(1)
with cdtemp():
mkdir_p("src/User")
with cd("src/User"):
wget(
os.path.join(
"https://raw.githubusercontent.com/usarica/MCFM-7.0_JHUGen",
mcfmcommit, "src/User/mdata.f"))
wget(mdatascript)
subprocess.check_call([
"python",
os.path.basename(mdatascript), "--coupling", self.coupling,
"--mcfmdir", ".", "--bsisigbkg", self.signalbkgbsi
])
with open("src/User/mdata.f") as f:
mdatagitcard = f.read()
if mdatacard != mdatagitcard and not 'BKG' in self.identifiers:
with cd(here):
with open("mdatacard", "w") as f:
f.write(mdatacard)
with open("mdatagitcard", "w") as f:
f.write(mdatagitcard)
raise ValueError(
"mdatacard != mdatagitcard\n{}\nSee ./mdatacard and ./mdatagitcard"
.format(self))
result = (productioncardurl + "\n" + "# " + mdatascript + "\n" +
"# --coupling " + self.coupling + " --bsisigbkg " +
self.signalbkgbsi)
return result
def cardsurl(self):
def getcontents(f):
contents = ""
for line in f:
line = line.split("!")[0]
line = line.split("#")[0]
line = line.strip()
line = re.sub(" *= *", " = ", line)
if not line: continue
if line.startswith("define p = "): continue
if line.startswith("define j = "): continue
contents += line+"\n"
return contents
gitcardcontents = []
if self.madgraphcardscript is None:
cardurls = tuple(
os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/",
self.genproductionscommit,
(_[0] if len(_) == 2 else _).replace(genproductions+"/", "")
) for _ in self.madgraphcards
)
with cdtemp():
for cardurl in cardurls:
wget(cardurl)
with open(os.path.basename(cardurl)) as f:
gitcardcontents.append(getcontents(f))
else:
scripturls = tuple(
os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/",
self.genproductionscommit,
_.replace(genproductions+"/", "")
) for _ in self.madgraphcardscript
)
with cdtemp():
wget(scripturls[0])
for _ in scripturls[1:]:
relpath = os.path.relpath(os.path.dirname(_), os.path.dirname(scripturls[0]))
assert ".." not in relpath, relpath
mkdir_p(relpath)
with cd(relpath):
wget(_)
subprocess.check_call(["chmod", "u+x", os.path.basename(scripturls[0])])
try:
subprocess.check_output(["./"+os.path.basename(scripturls[0])], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print e.output
raise
for _ in self.madgraphcards:
if len(_) == 2: _ = _[0]
with open(_) as f:
gitcardcontents.append(getcontents(f))
with cdtemp():
subprocess.check_output(["tar", "xvaf", self.cvmfstarball])
if glob.glob("core.*"):
raise ValueError("There is a core dump in the tarball\n{}".format(self))
cardnamesintarball = tuple(
os.path.join("InputCards", os.path.basename(_[1] if len(_) == 2 else _))
for _ in self.madgraphcards
)
cardcontents = []
for cardnameintarball in cardnamesintarball:
try:
with open(cardnameintarball) as f:
cardcontents.append(getcontents(f))
except IOError:
raise ValueError("no "+cardnameintarball+" in the tarball\n{}".format(self))
for _ in glob.iglob("InputCards/*"):
if _ not in cardnamesintarball and not _.endswith(".tar.gz") and _ not in self.otherthingsininputcards:
raise ValueError("unknown thing "+_+" in InputCards\n{}".format(self))
for name, cc, gcc in itertools.izip(cardnamesintarball, cardcontents, gitcardcontents):
_, suffix = os.path.splitext(os.path.basename(name))
if cc != gcc:
with cd(here):
with open("cardcontents"+suffix, "w") as f:
f.write(cc)
with open("gitcardcontents"+suffix, "w") as f:
f.write(gcc)
raise ValueError(name + " in tarball != " + name + " in git\n{}\nSee ./cardcontents{} and ./gitcardcontents{}".format(self, suffix, suffix))
if self.madgraphcardscript:
return "\n# ".join((scripturls[0],) + tuple(self.madgraphcards))
else:
return "\n# ".join(cardurls)
def cardsurl(self):
commit = self.genproductionscommit
if self.powhegcardusesscript:
powhegdir, powhegcard = os.path.split(self.powhegcard)
powhegscript = os.path.join(powhegdir, "makecards.py")
powhegscript = os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/",
commit,
powhegscript.split("genproductions/")[-1])
result = (powhegscript + "\n" + "# " + powhegcard)
else:
powhegcard = os.path.join(
"https://raw.githubusercontent.com/cms-sw/genproductions/",
commit,
self.powhegcard.split("genproductions/")[-1])
result = powhegcard
with cdtemp():
if self.powhegcardusesscript:
wget(powhegscript)
wget(
os.path.join(
os.path.dirname(powhegscript),
powhegcard.replace("M{}".format(self.mass),
"template").replace("Wplus",
"W").replace(
"Wminus",
"W")))
subprocess.check_call(["python", "makecards.py"])
else:
wget(powhegcard)
with open(os.path.basename(powhegcard)) as f:
powheggitcard = f.read()
powheggitcardlines = [
re.sub(" *([#!].*)?$", "", line)
for line in powheggitcard.split("\n")
]
powheggitcardlines = [
re.sub("(iseed|ncall2|fakevirt) *", r"\1 ", line)
for line in powheggitcardlines if line and all(
_ not in line
for _ in ("pdfreweight", "storeinfo_rwgt",
"withnegweights", "rwl_", "lhapdf6maxsets",
"xgriditeration", "fakevirt"))
]
powheggitcard = "\n".join(line for line in powheggitcardlines)
try:
with open("powheg.input") as f:
powhegcard = f.read()
powhegcardlines = [
re.sub(" *([#!].*)?$", "", line)
for line in powhegcard.split("\n")
]
powhegcardlines = [
re.sub("(iseed|ncall2|fakevirt) *", r"\1 ", line)
for line in powhegcardlines if line and all(
_ not in line
for _ in ("pdfreweight", "storeinfo_rwgt",
"withnegweights", "rwl_", "lhapdf6maxsets",
"xgriditeration", "fakevirt"))
]
powhegcard = "\n".join(line for line in powhegcardlines)
except IOError:
raise ValueError("no powheg.input in the tarball\n{}".format(self))
if powhegcard != powheggitcard:
with cd(here):
with open("powhegcard", "w") as f:
f.write(powhegcard)
with open("powheggitcard", "w") as f:
f.write(powheggitcard)
raise ValueError(
"powhegcard != powheggitcard\n{}\nSee ./powhegcard and ./powheggitcard"
.format(self))
moreresult = super(POWHEGMCSample, self).cardsurl
if moreresult: result += "\n# " + moreresult
return result