def fillDAG_cmp( jobsub, tag, date, xsec_a_path, eventdir, reportdir, main_tune, tunes, regretags, regredir ): # check if job is done already if resultsExist ( tag, date, reportdir ): msg.warning ("MINERvA comparisons plots found in " + reportdir + " ... " + msg.BOLD + "skipping minerva:fillDAG_cmp\n", 1) return # not done, add jobs to dag msg.info ("\tAdding MINERvA comparisons (plots) jobs\n") # in serial mode jobsub.add ("<serial>") config = "global-minerva-cfg-" + tag + "_" + date + ".xml" plotfile = "genie_" + tag + "-minerva.pdf" tablechi2 = "genie_" + tag + "-minerva-summary-chi2.txt" tableks = "genie_" + tag + "-minerva-summary-KS.txt" cmd = "gvld_general_comparison --no-root-output --global-config input/" + config + " -o " + plotfile cmd = cmd + " --summary-chi2-table " + tablechi2 cmd = cmd + " --summary-KS-table " + tableks # add the command to dag # --> old format --> inputs = reportdir + "/*.xml " + eventdir + "/*.ghep.root " inputs = reportdir + "/*.xml " + xsec_a_path + "/xsec-vA-" + tag + ".root " + eventdir + "/*.ghep.root " if not (main_tune is None): inputs = reportdir + "/*.xml " + xsec_a_path + "/" + main_tune + "-xsec-vA-" + tag + ".root " + eventdir + "/*.ghep.root " if not (tunes is None): for tn in range(len(tunes)): # --> old format --> inputs = " " + inputs + eventdir + "/" + tunes[tn] + "/*.ghep.root " inputs = " " + inputs + xsec_a_path + "/" + tunes[tn] + "/" + tunes[tn] + "-xsec-vA-" + tag + ".root " \ + eventdir + "/" + tunes[tn] + "/*.ghep.root " logfile = "gvld_general_comparison.log" regre = None if not (regretags is None): # --> now we need XSec --> regreOK = commonFunctions.regreInputOK( "minerva", regretags, regredir, len(data_struct), None, None ) regreOK = commonFunctions.regreInputOK( "minerva", regretags, regredir, len(data_struct), "vA", "/xsec/nuA" ) if regreOK: regre = "" for rt in range(len(regretags)): # NOTE: no need to fetch rtune because we don't get xsec, otherwise it's part of regretags # regre = regre + regredir + "/" + regretags[rt] + "/events/minerva/*.ghep.root " # NOTE (11/12/19): now we DO NEED to fetch xsec... rversion, rtune = regretags[rt].split("/") regre = regre + regredir + "/" + regretags[rt] + "/xsec/nuA/" + rtune + "-xsec-vA-" + rversion + ".root " regre = regre + regredir + "/" + regretags[rt] + "/events/minerva/*.ghep.root " else: msg.info( "\t\tNO input for regression will be copied over \n" ) regre = None jobsub.addJob ( inputs, reportdir, logfile, cmd, regre ) # done jobsub.add ("</serial>")
def createFileList (tag, date, xsec_a_path, outEvent, outRep, main_tune, tunes, regretags, regredir): # create xml file with the file list in the format as src/scripts/production/misc/make_genie_sim_file_list.pl xmlFile = outRep + "/file_list-" + tag + "-" + date + ".xml" try: os.remove (xmlFile) except OSError: pass xml = open (xmlFile, 'w'); print >>xml, '<?xml version="1.0" encoding="ISO-8859-1"?>' print >>xml, '<genie_simulation_outputs>' if (main_tune is None): print >>xml, '\t<model name="' + tag + '-' + date + ':default:world' '">' else: print >>xml, '\t<model name="' + tag + '-' + date + ':' + main_tune + ':world' '">' for key in nuPDG.iterkeys(): print >>xml, '\t\t<evt_file format="ghep"> input/gntp.' + key + '.ghep.root </evt_file>' if (main_tune is None): print >>xml, '\t\t<xsec_file> input/xsec-vA-' + tag + '.root </xsec_file>' else: print >>xml, '\t\t<xsec_file> input/' + main_tune + '-xsec-vA-' + tag + '.root </xsec_file>' print >>xml, '\t</model>' # same for tunes if specified if not (tunes is None): for tn in range(len(tunes)): print >>xml, '\t<model name="' + tag + '-' + date + ':' + tunes[tn] + ':world' '">' for key in nuPDG.iterkeys(): print >>xml, '\t\t<evt_file format="ghep"> input/' + tunes[tn] + '-gntp.' + key + '.ghep.root </evt_file>' print >>xml, '\t\t<xsec_file> input/' + tunes[tn] + '-xsec-vA-' + tag + '.root </xsec_file>' print >>xml, '\t</model>' regreOK = commonFunctions.regreInputOK( "xsec_validation", regretags, regredir, len(nuPDG), "vA", "/xsec/nuA/" ) # --> if not (regretags is None): if regreOK: # need to fetch date stamp for the regression from the leading path # assume that regredir is always /leading/path/to/TIMESTAMP/Index # NOTE: redirect output of split(...) to a separate array; # otherwise len(...) will be the length of regredir, not the length of array after splitting regredir_tmp = regredir.split("/") rdate = regredir_tmp[len(regredir_tmp)-2] # i.e. one before the last for rt in range(len(regretags)): rversion, rtune = regretags[rt].split("/") # print >>xml, '\t<model name="' + regretags[rt] + ':default:world' '">' print >>xml, '\t<model name="' + rversion + "-" + rdate + ':' + rtune + ':world' '">' for key in nuPDG.iterkeys(): print >>xml, '\t\t<evt_file format="ghep"> input/regre/' + rdate + '/' + regretags[rt] +'/gntp.' + key + '.ghep.root </evt_file>' print >>xml, '\t\t<xsec_file> input/regre/' + rdate + '/' + regretags[rt] + '/' + rtune + '-xsec-vA-' + rversion + '.root </xsec_file>' print >>xml, '\t</model>' else: msg.info( "\t\tNO REGRESSION due to missing/incorrect input files \n" ) print >>xml, '</genie_simulation_outputs>' xml.close()
def fillDAG_cmp (jobsub, tag, date, xsec_a_path, outEvents, outRep, main_tune, tunes, regretags, regredir): # check if job is done already if isDoneData (tag, date, outRep): msg.warning ("xsec validation plots found in " + outRep + " ... " + msg.BOLD + "skipping xsecval:fillDAG_data\n", 1) return # not done, add jobs to dag msg.info ("\tAdding xsec validation (data) jobs\n") inputs = outRep + "/*.xml " + xsec_a_path + "/xsec-vA-" + tag + ".root " + outEvents + "/*.ghep.root " if not (main_tune is None): inputs = outRep + "/*.xml " + xsec_a_path + "/" + main_tune +"-xsec-vA-" + tag + ".root " + outEvents + "/*.ghep.root " if not ( tunes is None): for tn in range(len(tunes)): inputs = inputs + " " + xsec_a_path + "/" + tunes[tn] + "/" + tunes[tn] + "-xsec-vA-" + tag + ".root " + \ outEvents + "/" + tunes[tn] + "/*.ghep.root" # regression test if requested regre = None if not (regretags is None): regreOK = commonFunctions.regreInputOK( "xsec_validation", regretags, regredir, len(nuPDG), "vA", "/xsec/nuA/" ) if regreOK: regre = "" # NOTE: no need to fetch rdate here; it'll just be part of regre_dir "leading" path for rt in range(len(regretags)): rversion, rtune = regretags[rt].split("/") regre = regre + regredir + "/" + regretags[rt] + "/xsec/nuA/" + rtune + "-xsec-vA-" + rversion + ".root " regre = regre + regredir + "/" + regretags[rt] + "/events/xsec_validation/*.ghep.root " else: msg.info( "\t\tNO input for regression will be copied over \n" ) regre = None # in parallel mode jobsub.add ("<parallel>") for comp in comparisons: inFile = comp + "-" + tag + "-" + date + ".xml" outFile = "genie_" + tag + "_" + comparisons[comp]['outprefix'] + comp tablechi2 = "genie_" + tag + "_" + comparisons[comp]['outprefix'] + comp + "-summary-chi2.txt" tableks = "genie_" + tag + "_" + comparisons[comp]['outprefix'] + comp + "-summary-KS.txt" cmd = "gvld_general_comparison --no-root-output --global-config input/" + inFile + " -o " + outFile cmd = cmd + " --summary-chi2-table " + tablechi2 cmd = cmd + " --summary-KS-table " + tableks logFile = "gvld_nu_xsec_" + comp + ".log" jobsub.addJob (inputs, outRep, logFile, cmd, regre) # done jobsub.add ("</parallel>")
def fillDAG_data(jobsub, tag, date, xsec_n_path, outEvents, outRep, main_tune, tunes, regretags, regredir): # check if job is done already if isDoneData(tag, date, outRep): msg.warning( "hadronization test plots found in " + outRep + " ... " + msg.BOLD + "skipping hadronization:fillDAG_data\n", 1) return # not done, add jobs to dag msg.info("\tAdding hadronization test (plots) jobs\n") # in serial mode jobsub.add("<serial>") inFile = "file_list-" + tag + "-" + date + ".xml" cmd = "gvld_hadronization -g input/" + inFile # add the command to dag inputs = outRep + "/" + inFile + " " + xsec_n_path + "/xsec-vN-" + tag + ".root " + outEvents + "/*.ghep.root" if not (main_tune is None): inputs = outRep + "/" + inFile + " " + xsec_n_path + "/" + main_tune + "-xsec-vN-" + tag + ".root " + outEvents + "/*.ghep.root" if not (tunes is None): for tn in range(len(tunes)): inputs = inputs + " " + xsec_n_path + "/" + tunes[tn] + "/" + tunes[tn] + "-xsec-vN-" + tag + ".root " + \ outEvents + "/" + tunes[tn] + "/*.ghep.root" logFile = "gvld_hadronz_test.log" regre = None if not (regretags is None): # check if input for regression is OK regreOK = commonFunctions.regreInputOK("hadronization", regretags, regredir, len(nuPDG), "vN", "/xsec/nuN/") if regreOK: # NOTE: no need to fetch rdate here; it'll just be part of regre_dir "leading" path regre = "" for rt in range(len(regretags)): rversion, rtune = regretags[rt].split("/") regre = regre + regredir + "/" + regretags[ rt] + "/xsec/nuN/" + rtune + "-xsec-vN-" + rversion + ".root " regre = regre + regredir + "/" + regretags[ rt] + "/events/hadronization/*.ghep.root " else: msg.info("\t\tNO input for regression will be copied over \n") regre = None jobsub.addJob(inputs, outRep, logFile, cmd, regre) # done jobsub.add("</serial>")
def createCmpConfigs( tag, date, reportdir, main_tune, tunes, regretags, regredir ): msg.info( "\tCreate configuration XML for MINERvA test \n" ) # start GLOBAL CMP CONFIG gcfg = reportdir + "/global-minerva-cfg-" + tag + "_" + date + ".xml" try: os.remove(gcfg) except OSError: pass gxml = open( gcfg, 'w' ) print >>gxml, '<?xml version="1.0" encoding="ISO-8859-1"?>' print >>gxml, '<config>' print >>gxml, '\t<experiment name="MINERvA">' print >>gxml, '\t\t<paths_relative_to_geniecmp_topdir> false </paths_relative_to_geniecmp_topdir>' regreOK = commonFunctions.regreInputOK( "minerva", regretags, regredir, len(data_struct), None, None ) # in the loop, create GSim cfg files and also put their names in the global cfg for key in data_struct.iterkeys(): gsimfile = "/gsimfile-" + tag + "-" + date + "-minerva-" + key + ".xml" # ---> xmlfile = reportdir + "/gsimfile-" + tag + "-" + date + "-minerva-" + key + ".xml" xmlfile = reportdir + gsimfile try: os.remove(xmlfile) except OSError: pass xml = open( xmlfile, 'w' ) print >>xml, '<?xml version="1.0" encoding="ISO-8859-1"?>' print >>xml, '<genie_simulation_outputs>' if (main_tune is None): print >>xml, '\t<model name="GENIE_' + tag + "-" + date + ':default:' + data_struct[key]['releaselabel'] + '">' else: print >>xml, '\t<model name="GENIE_' + tag + "-" + date + ':' + main_tune + ':' + data_struct[key]['releaselabel'] + '">' print >>xml, '\t\t<evt_file format="ghep"> input/gntp.' + key + '-' + data_struct[key]['releaselabel'] + '.ghep.root </evt_file>' print >>xml, '\t</model>' # tunes if specied if not (tunes is None): for tn in range(len(tunes)): print >>xml, '\t<model name="GENIE_' + tag + "-" + date + ':' + tunes[tn] + ':' + data_struct[key]['releaselabel'] + '">' print >>xml, '\t\t<evt_file format="ghep"> input/' + tunes[tn] + '-gntp.' + key + "-" + data_struct[key]['releaselabel'] + '.ghep.root </evt_file>' print >>xml, '\t</model>' # regression if specified (but not for COH pion since regression files seems corrupted) # backward compatibility repaired for COH (and perhaps others); so resume "full" regression # if key.find("CoherentPi") == -1: # --> if not (regretags is None): if regreOK: # need to fetch date stamp for the regression from the leading path # assume that regredir is always /leading/path/to/TIMESTAMP/Index # NOTE: redirect output of split(...) to a separate array; # otherwise len(...) will be the length of regredir, not the length of array after splitting regredir_tmp = regredir.split("/") rdate = regredir_tmp[len(regredir_tmp)-2] # i.e. one before the last for rt in range(len(regretags)): rversion, rtune = regretags[rt].split("/") print >>xml, '\t<model name="GENIE_' + rversion + '-' + rdate + ':' + rtune + ':' + data_struct[key]['releaselabel'] + '">' print >>xml, '\t\t<evt_file format="ghep"> input/regre/' + rdate + '/' + regretags[rt] + '/gntp.' + key + '-' + data_struct[key]['releaselabel'] + '.ghep.root </evt_file>' print >>xml, '\t</model>' else: msg.info( "\t\tNO REGRESSION due to missing/incorrect input files \n" ) print >>xml, '</genie_simulation_outputs>' xml.close() print >>gxml, '\t\t\t<comparison>' for i in range( len( data_struct[key]['datafiles'] ) ): print >>gxml, '\t\t\t\t<spec>' print >>gxml, '\t\t\t\t\t<path2data> data/measurements/vA/minerva/' + data_struct[key]['datafiles'][i] + ' </path2data>' print >>gxml, '\t\t\t\t\t<dataclass> MINERvAExData </dataclass>' print >>gxml, '\t\t\t\t\t<predictionclass> ' + data_struct[key]['mcpredictions'][i] + ' </predictionclass>' print >>gxml, '\t\t\t\t</spec>' print >>gxml, '\t\t\t\t<genie> input' + gsimfile + ' </genie>' print >>gxml, '\t\t\t</comparison>' # now finish up and close global config print >>gxml, '\t</experiment>' print >>gxml, '</config>' gxml.close()