def fillDAG_cmp( jobsub, tag, date, xsec_a_path, eventdir, reportdir, main_tune, tunes, regretags, regredir ):
# check if job is done already
if resultsExist ( tag, date, reportdir ):
msg.warning ("MINERvA comparisons plots found in " + reportdir + " ... " + msg.BOLD + "skipping minerva:fillDAG_cmp\n", 1)
return
# not done, add jobs to dag
msg.info ("\tAdding MINERvA comparisons (plots) jobs\n")
# in serial mode
jobsub.add ("<serial>")
config = "global-minerva-cfg-" + tag + "_" + date + ".xml"
plotfile = "genie_" + tag + "-minerva.pdf"
tablechi2 = "genie_" + tag + "-minerva-summary-chi2.txt"
tableks = "genie_" + tag + "-minerva-summary-KS.txt"
cmd = "gvld_general_comparison --no-root-output --global-config input/" + config + " -o " + plotfile
cmd = cmd + " --summary-chi2-table " + tablechi2
cmd = cmd + " --summary-KS-table " + tableks
# add the command to dag
# --> old format --> inputs = reportdir + "/*.xml " + eventdir + "/*.ghep.root "
inputs = reportdir + "/*.xml " + xsec_a_path + "/xsec-vA-" + tag + ".root " + eventdir + "/*.ghep.root "
if not (main_tune is None):
inputs = reportdir + "/*.xml " + xsec_a_path + "/" + main_tune + "-xsec-vA-" + tag + ".root " + eventdir + "/*.ghep.root "
if not (tunes is None):
for tn in range(len(tunes)):
# --> old format --> inputs = " " + inputs + eventdir + "/" + tunes[tn] + "/*.ghep.root "
inputs = " " + inputs + xsec_a_path + "/" + tunes[tn] + "/" + tunes[tn] + "-xsec-vA-" + tag + ".root " \
+ eventdir + "/" + tunes[tn] + "/*.ghep.root "
logfile = "gvld_general_comparison.log"
regre = None
if not (regretags is None):
# --> now we need XSec --> regreOK = commonFunctions.regreInputOK( "minerva", regretags, regredir, len(data_struct), None, None )
regreOK = commonFunctions.regreInputOK( "minerva", regretags, regredir, len(data_struct), "vA", "/xsec/nuA" )
if regreOK:
regre = ""
for rt in range(len(regretags)):
# NOTE: no need to fetch rtune because we don't get xsec, otherwise it's part of regretags
# regre = regre + regredir + "/" + regretags[rt] + "/events/minerva/*.ghep.root "
# NOTE (11/12/19): now we DO NEED to fetch xsec...
rversion, rtune = regretags[rt].split("/")
regre = regre + regredir + "/" + regretags[rt] + "/xsec/nuA/" + rtune + "-xsec-vA-" + rversion + ".root "
regre = regre + regredir + "/" + regretags[rt] + "/events/minerva/*.ghep.root "
else:
msg.info( "\t\tNO input for regression will be copied over \n" )
regre = None
jobsub.addJob ( inputs, reportdir, logfile, cmd, regre )
# done
jobsub.add ("</serial>")
def createFileList (tag, date, xsec_a_path, outEvent, outRep, main_tune, tunes, regretags, regredir):
# create xml file with the file list in the format as src/scripts/production/misc/make_genie_sim_file_list.pl
xmlFile = outRep + "/file_list-" + tag + "-" + date + ".xml"
try: os.remove (xmlFile)
except OSError: pass
xml = open (xmlFile, 'w');
print >>xml, '<?xml version="1.0" encoding="ISO-8859-1"?>'
print >>xml, '<genie_simulation_outputs>'
if (main_tune is None):
print >>xml, '\t<model name="' + tag + '-' + date + ':default:world' '">'
else:
print >>xml, '\t<model name="' + tag + '-' + date + ':' + main_tune + ':world' '">'
for key in nuPDG.iterkeys():
print >>xml, '\t\t<evt_file format="ghep"> input/gntp.' + key + '.ghep.root </evt_file>'
if (main_tune is None):
print >>xml, '\t\t<xsec_file> input/xsec-vA-' + tag + '.root </xsec_file>'
else:
print >>xml, '\t\t<xsec_file> input/' + main_tune + '-xsec-vA-' + tag + '.root </xsec_file>'
print >>xml, '\t</model>'
# same for tunes if specified
if not (tunes is None):
for tn in range(len(tunes)):
print >>xml, '\t<model name="' + tag + '-' + date + ':' + tunes[tn] + ':world' '">'
for key in nuPDG.iterkeys():
print >>xml, '\t\t<evt_file format="ghep"> input/' + tunes[tn] + '-gntp.' + key + '.ghep.root </evt_file>'
print >>xml, '\t\t<xsec_file> input/' + tunes[tn] + '-xsec-vA-' + tag + '.root </xsec_file>'
print >>xml, '\t</model>'
regreOK = commonFunctions.regreInputOK( "xsec_validation", regretags, regredir, len(nuPDG), "vA", "/xsec/nuA/" )
# --> if not (regretags is None):
if regreOK:
# need to fetch date stamp for the regression from the leading path
# assume that regredir is always /leading/path/to/TIMESTAMP/Index
# NOTE: redirect output of split(...) to a separate array;
# otherwise len(...) will be the length of regredir, not the length of array after splitting
regredir_tmp = regredir.split("/")
rdate = regredir_tmp[len(regredir_tmp)-2] # i.e. one before the last
for rt in range(len(regretags)):
rversion, rtune = regretags[rt].split("/")
# print >>xml, '\t<model name="' + regretags[rt] + ':default:world' '">'
print >>xml, '\t<model name="' + rversion + "-" + rdate + ':' + rtune + ':world' '">'
for key in nuPDG.iterkeys():
print >>xml, '\t\t<evt_file format="ghep"> input/regre/' + rdate + '/' + regretags[rt] +'/gntp.' + key + '.ghep.root </evt_file>'
print >>xml, '\t\t<xsec_file> input/regre/' + rdate + '/' + regretags[rt] + '/' + rtune + '-xsec-vA-' + rversion + '.root </xsec_file>'
print >>xml, '\t</model>'
else:
msg.info( "\t\tNO REGRESSION due to missing/incorrect input files \n" )
print >>xml, '</genie_simulation_outputs>'
xml.close()
def fillDAG_cmp (jobsub, tag, date, xsec_a_path, outEvents, outRep, main_tune, tunes, regretags, regredir):
# check if job is done already
if isDoneData (tag, date, outRep):
msg.warning ("xsec validation plots found in " + outRep + " ... " + msg.BOLD + "skipping xsecval:fillDAG_data\n", 1)
return
# not done, add jobs to dag
msg.info ("\tAdding xsec validation (data) jobs\n")
inputs = outRep + "/*.xml " + xsec_a_path + "/xsec-vA-" + tag + ".root " + outEvents + "/*.ghep.root "
if not (main_tune is None):
inputs = outRep + "/*.xml " + xsec_a_path + "/" + main_tune +"-xsec-vA-" + tag + ".root " + outEvents + "/*.ghep.root "
if not ( tunes is None):
for tn in range(len(tunes)):
inputs = inputs + " " + xsec_a_path + "/" + tunes[tn] + "/" + tunes[tn] + "-xsec-vA-" + tag + ".root " + \
outEvents + "/" + tunes[tn] + "/*.ghep.root"
# regression test if requested
regre = None
if not (regretags is None):
regreOK = commonFunctions.regreInputOK( "xsec_validation", regretags, regredir, len(nuPDG), "vA", "/xsec/nuA/" )
if regreOK:
regre = ""
# NOTE: no need to fetch rdate here; it'll just be part of regre_dir "leading" path
for rt in range(len(regretags)):
rversion, rtune = regretags[rt].split("/")
regre = regre + regredir + "/" + regretags[rt] + "/xsec/nuA/" + rtune + "-xsec-vA-" + rversion + ".root "
regre = regre + regredir + "/" + regretags[rt] + "/events/xsec_validation/*.ghep.root "
else:
msg.info( "\t\tNO input for regression will be copied over \n" )
regre = None
# in parallel mode
jobsub.add ("<parallel>")
for comp in comparisons:
inFile = comp + "-" + tag + "-" + date + ".xml"
outFile = "genie_" + tag + "_" + comparisons[comp]['outprefix'] + comp
tablechi2 = "genie_" + tag + "_" + comparisons[comp]['outprefix'] + comp + "-summary-chi2.txt"
tableks = "genie_" + tag + "_" + comparisons[comp]['outprefix'] + comp + "-summary-KS.txt"
cmd = "gvld_general_comparison --no-root-output --global-config input/" + inFile + " -o " + outFile
cmd = cmd + " --summary-chi2-table " + tablechi2
cmd = cmd + " --summary-KS-table " + tableks
logFile = "gvld_nu_xsec_" + comp + ".log"
jobsub.addJob (inputs, outRep, logFile, cmd, regre)
# done
jobsub.add ("</parallel>")
def fillDAG_data(jobsub, tag, date, xsec_n_path, outEvents, outRep, main_tune,
tunes, regretags, regredir):
# check if job is done already
if isDoneData(tag, date, outRep):
msg.warning(
"hadronization test plots found in " + outRep + " ... " +
msg.BOLD + "skipping hadronization:fillDAG_data\n", 1)
return
# not done, add jobs to dag
msg.info("\tAdding hadronization test (plots) jobs\n")
# in serial mode
jobsub.add("<serial>")
inFile = "file_list-" + tag + "-" + date + ".xml"
cmd = "gvld_hadronization -g input/" + inFile
# add the command to dag
inputs = outRep + "/" + inFile + " " + xsec_n_path + "/xsec-vN-" + tag + ".root " + outEvents + "/*.ghep.root"
if not (main_tune is None):
inputs = outRep + "/" + inFile + " " + xsec_n_path + "/" + main_tune + "-xsec-vN-" + tag + ".root " + outEvents + "/*.ghep.root"
if not (tunes is None):
for tn in range(len(tunes)):
inputs = inputs + " " + xsec_n_path + "/" + tunes[tn] + "/" + tunes[tn] + "-xsec-vN-" + tag + ".root " + \
outEvents + "/" + tunes[tn] + "/*.ghep.root"
logFile = "gvld_hadronz_test.log"
regre = None
if not (regretags is None):
# check if input for regression is OK
regreOK = commonFunctions.regreInputOK("hadronization", regretags,
regredir, len(nuPDG), "vN",
"/xsec/nuN/")
if regreOK:
# NOTE: no need to fetch rdate here; it'll just be part of regre_dir "leading" path
regre = ""
for rt in range(len(regretags)):
rversion, rtune = regretags[rt].split("/")
regre = regre + regredir + "/" + regretags[
rt] + "/xsec/nuN/" + rtune + "-xsec-vN-" + rversion + ".root "
regre = regre + regredir + "/" + regretags[
rt] + "/events/hadronization/*.ghep.root "
else:
msg.info("\t\tNO input for regression will be copied over \n")
regre = None
jobsub.addJob(inputs, outRep, logFile, cmd, regre)
# done
jobsub.add("</serial>")
def createCmpConfigs( tag, date, reportdir, main_tune, tunes, regretags, regredir ):
msg.info( "\tCreate configuration XML for MINERvA test \n" )
# start GLOBAL CMP CONFIG
gcfg = reportdir + "/global-minerva-cfg-" + tag + "_" + date + ".xml"
try: os.remove(gcfg)
except OSError: pass
gxml = open( gcfg, 'w' )
print >>gxml, '<?xml version="1.0" encoding="ISO-8859-1"?>'
print >>gxml, '<config>'
print >>gxml, '\t<experiment name="MINERvA">'
print >>gxml, '\t\t<paths_relative_to_geniecmp_topdir> false </paths_relative_to_geniecmp_topdir>'
regreOK = commonFunctions.regreInputOK( "minerva", regretags, regredir, len(data_struct), None, None )
# in the loop, create GSim cfg files and also put their names in the global cfg
for key in data_struct.iterkeys():
gsimfile = "/gsimfile-" + tag + "-" + date + "-minerva-" + key + ".xml"
# ---> xmlfile = reportdir + "/gsimfile-" + tag + "-" + date + "-minerva-" + key + ".xml"
xmlfile = reportdir + gsimfile
try: os.remove(xmlfile)
except OSError: pass
xml = open( xmlfile, 'w' )
print >>xml, '<?xml version="1.0" encoding="ISO-8859-1"?>'
print >>xml, '<genie_simulation_outputs>'
if (main_tune is None):
print >>xml, '\t<model name="GENIE_' + tag + "-" + date + ':default:' + data_struct[key]['releaselabel'] + '">'
else:
print >>xml, '\t<model name="GENIE_' + tag + "-" + date + ':' + main_tune + ':' + data_struct[key]['releaselabel'] + '">'
print >>xml, '\t\t<evt_file format="ghep"> input/gntp.' + key + '-' + data_struct[key]['releaselabel'] + '.ghep.root </evt_file>'
print >>xml, '\t</model>'
# tunes if specied
if not (tunes is None):
for tn in range(len(tunes)):
print >>xml, '\t<model name="GENIE_' + tag + "-" + date + ':' + tunes[tn] + ':' + data_struct[key]['releaselabel'] + '">'
print >>xml, '\t\t<evt_file format="ghep"> input/' + tunes[tn] + '-gntp.' + key + "-" + data_struct[key]['releaselabel'] + '.ghep.root </evt_file>'
print >>xml, '\t</model>'
# regression if specified (but not for COH pion since regression files seems corrupted)
# backward compatibility repaired for COH (and perhaps others); so resume "full" regression
# if key.find("CoherentPi") == -1:
# --> if not (regretags is None):
if regreOK:
# need to fetch date stamp for the regression from the leading path
# assume that regredir is always /leading/path/to/TIMESTAMP/Index
# NOTE: redirect output of split(...) to a separate array;
# otherwise len(...) will be the length of regredir, not the length of array after splitting
regredir_tmp = regredir.split("/")
rdate = regredir_tmp[len(regredir_tmp)-2] # i.e. one before the last
for rt in range(len(regretags)):
rversion, rtune = regretags[rt].split("/")
print >>xml, '\t<model name="GENIE_' + rversion + '-' + rdate + ':' + rtune + ':' + data_struct[key]['releaselabel'] + '">'
print >>xml, '\t\t<evt_file format="ghep"> input/regre/' + rdate + '/' + regretags[rt] + '/gntp.' + key + '-' + data_struct[key]['releaselabel'] + '.ghep.root </evt_file>'
print >>xml, '\t</model>'
else:
msg.info( "\t\tNO REGRESSION due to missing/incorrect input files \n" )
print >>xml, '</genie_simulation_outputs>'
xml.close()
print >>gxml, '\t\t\t<comparison>'
for i in range( len( data_struct[key]['datafiles'] ) ):
print >>gxml, '\t\t\t\t<spec>'
print >>gxml, '\t\t\t\t\t<path2data> data/measurements/vA/minerva/' + data_struct[key]['datafiles'][i] + ' </path2data>'
print >>gxml, '\t\t\t\t\t<dataclass> MINERvAExData </dataclass>'
print >>gxml, '\t\t\t\t\t<predictionclass> ' + data_struct[key]['mcpredictions'][i] + ' </predictionclass>'
print >>gxml, '\t\t\t\t</spec>'
print >>gxml, '\t\t\t\t<genie> input' + gsimfile + ' </genie>'
print >>gxml, '\t\t\t</comparison>'
# now finish up and close global config
print >>gxml, '\t</experiment>'
print >>gxml, '</config>'
gxml.close()