def transfer_file_to_EOS_area(sourceFile, targetDirectory):
sourceFileName = (sourceFile.split("/"))[-1]
command = "xrdcp --verbose --force --path --streams 15 {sF} {tD}/{sFN}".format(
sF=sourceFile, tD=targetDirectory, sFN=sourceFileName)
stealthEnv.execute_in_env(commandToRun=command,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
def produce_STComparisons(dataPath, outputFilePrefix_STComparisons,
analyzeSignalBins):
command_produceComparisons = "./plotSTDistributionComparisons.py --inputFilePath {dP} --outputDirectory_plots {aOD}/publicationPlots --outputDirectory_dataSystematics {aOD}/dataSystematics --outputFilePrefix {oFP}".format(
dP=dataPath,
aOD=analysisOutputDirectory,
oFP=outputFilePrefix_STComparisons)
if not (analyzeSignalBins):
command_produceComparisons += " --nJetsMaxPlot 3"
stealthEnv.execute_in_env(commandToRun=command_produceComparisons,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
def run_combine_chain(eventProgenitor, path_dataSystematics_signal,
path_dataSystematics_signal_loose,
path_dataSystematics_control,
path_dataSystematics_scaling_signal,
path_dataSystematics_scaling_signal_loose,
path_dataSystematics_scaling_control,
path_dataObservedEventCounters_signal,
path_dataObservedEventCounters_signal_loose,
path_dataObservedEventCounters_control,
path_dataExpectedEventCounters_signal,
path_dataExpectedEventCounters_signal_loose,
path_dataExpectedEventCounters_control, MCPrefix_signal,
MCPrefix_signal_loose, MCPrefix_control, outputPrefix):
command_submitCombineJobs = (
"./submitCombineToolJobs.py --dataCardsPrefix {oP} --outputDirectory {eP}/{cREOSOD}/ --eventProgenitor {eP2} --MCTemplatePath {MTP} --minNeutralinoMass {mNM} --crossSectionsFileName {cSFP} --path_dataSystematics_signal {pDSS} --path_dataSystematics_signal_loose {pDSSL} --path_dataSystematics_control {pDSC} --path_dataSystematics_scaling_signal {pDSSS} --path_dataSystematics_scaling_signal_loose {pDSSSL} --path_dataSystematics_scaling_control {pDSSC} --path_dataObservedEventCounters_signal {pDOECS} --path_dataObservedEventCounters_signal_loose {pDOECSL} --path_dataObservedEventCounters_control {pDOECC} --path_dataExpectedEventCounters_signal {pDEECS} --path_dataExpectedEventCounters_signal_loose {pDEECSL} --path_dataExpectedEventCounters_control {pDEECC} --MCHistogramsSignal {EAA}/{pS}_savedObjects.root --MCHistogramsSignalLoose {EAA}/{pSL}_savedObjects.root --MCHistogramsControl {EAA}/{pC}_savedObjects.root --MCUncertaintiesSignal {EAA}/{pS}_MCUncertainties_savedObjects.root --MCUncertaintiesSignalLoose {EAA}/{pSL}_MCUncertainties_savedObjects.root --MCUncertaintiesControl {EAA}/{pC}_MCUncertainties_savedObjects.root --luminosityUncertainty {lU} --EOSAnalysisArea {EAA}"
.format(oP=outputPrefix,
eP=stealthEnv.EOSPrefix,
sER=stealthEnv.stealthEOSRoot,
MTP=MCTemplatesForProgenitor[eventProgenitor],
mNM=minNeutralinoMassToPlot[eventProgenitor],
eP2=eventProgenitor,
cREOSOD=combineResultsEOSOutputDirectory,
cSFP=crossSectionsForProgenitor[eventProgenitor],
pDSS=path_dataSystematics_signal,
pDSSL=path_dataSystematics_signal_loose,
pDSC=path_dataSystematics_control,
pDSSS=path_dataSystematics_scaling_signal,
pDSSSL=path_dataSystematics_scaling_signal_loose,
pDSSC=path_dataSystematics_scaling_control,
pDOECS=path_dataObservedEventCounters_signal,
pDOECSL=path_dataObservedEventCounters_signal_loose,
pDOECC=path_dataObservedEventCounters_control,
pDEECS=path_dataExpectedEventCounters_signal,
pDEECSL=path_dataExpectedEventCounters_signal_loose,
pDEECC=path_dataExpectedEventCounters_control,
pS=MCPrefix_signal,
pSL=MCPrefix_signal_loose,
pC=MCPrefix_control,
lU=lumi_uncertainty,
EAA="{eP}/{aEOD}".format(eP=stealthEnv.EOSPrefix,
aEOD=analysisEOSOutputDirectory)))
if (inputArguments.optionalIdentifier != ""):
command_submitCombineJobs += " --optionalIdentifier {oI}".format(
oI=inputArguments.optionalIdentifier
) # Just "inputArguments.optionalIdentifier", without the underscore
if (inputArguments.runUnblinded):
command_submitCombineJobs += " --runUnblinded"
if not (inputArguments.noLooseSignal):
command_submitCombineJobs += " --addLooseSignal"
if (inputArguments.isDryRun): command_submitCombineJobs += " --isDryRun"
stealthEnv.execute_in_env(commandToRun=command_submitCombineJobs,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
def plot_limits(eventProgenitor):
command_plotLimits = "condor_q && ./plotLimits.py --crossSectionsFile {cSFP} --MCTemplatePath {MTP} --eventProgenitor {eP2} --combineResultsDirectory {eP}/{cREOSOD} --combineOutputPrefix {eP2} --outputDirectory_rawOutput {aOD}/limits --outputDirectory_plots {aOD}/publicationPlots --outputSuffix {eP2} --minNeutralinoMass {mNM}".format(
eP=stealthEnv.EOSPrefix,
MTP=MCTemplatesForProgenitor[eventProgenitor],
cREOSOD=combineResultsEOSOutputDirectory,
eP2=eventProgenitor,
cSFP=crossSectionsForProgenitor[eventProgenitor],
aOD=analysisOutputDirectory,
mNM=minNeutralinoMassToPlot[eventProgenitor])
if (inputArguments.runUnblinded): command_plotLimits += " --plotObserved"
stealthEnv.execute_in_env(commandToRun=command_plotLimits,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
def produce_ancillary_plots_control(eventProgenitor, path_data_expectedNEvents,
path_data_observedNEvents,
path_MC_weightedNEvents,
path_dataSystematics):
command_controlSTDistributions_dataAndSignal = "./plotSTDistributionsWithErrors.py --eventProgenitor {eP} --path_data_expectedNEvents {pDENE} --path_data_observedNEvents {pDONE} --path_MC_weightedNEvents {pMCWNE} --path_dataSystematics {pDS} --outputDirectory {aOD}/publicationPlots/ --outputFilePrefix {oFP} --plotObservedData".format(
eP=eventProgenitor,
pDENE=path_data_expectedNEvents,
pDONE=path_data_observedNEvents,
pMCWNE=path_MC_weightedNEvents,
pDS=path_dataSystematics,
aOD=analysisOutputDirectory,
oFP="STDistributions_{eP}_control".format(eP=eventProgenitor))
stealthEnv.execute_in_env(
commandToRun=command_controlSTDistributions_dataAndSignal,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
def checkAndEstablishLock(
): # Make sure that at most one instance is running at a time
global multiProcessLauncher
if (os.path.isfile(
"{aLD}/analysisRunning.lock".format(aLD=analysisLogsDirectory))):
sys.exit(
"ERROR: only one instance of analysis running script can run at a time! If you're sure this is not a problem, remove this file: {aLD}/analysisRunning.lock"
.format(aLD=analysisLogsDirectory))
else:
command_createAnalysisParentDirectory = "mkdir -p {aOD}".format(
aOD=analysisOutputDirectory)
stealthEnv.execute_in_env(
commandToRun=command_createAnalysisParentDirectory,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
for outputSubdirectory in [
"dataEventHistograms", "dataSystematics", "MCEventHistograms",
"MCSystematics", "signalContamination", "publicationPlots",
"limits", "analysisLogs"
]:
command_createAnalysisSubdirectory = "mkdir -p {aOD}/{oS}".format(
aOD=analysisOutputDirectory, oS=outputSubdirectory)
stealthEnv.execute_in_env(
commandToRun=command_createAnalysisSubdirectory,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
command_createEOSAnalysisArea = ("eos {eP} mkdir -p {aEOD}".format(
eP=stealthEnv.EOSPrefix, aEOD=analysisEOSOutputDirectory))
stealthEnv.execute_in_env(
commandToRun=command_createEOSAnalysisArea,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
os.system("touch {aLD}/analysisRunning.lock".format(
aLD=analysisLogsDirectory))
multiProcessLauncher = tmMultiProcessLauncher.tmMultiProcessLauncher(
logOutputFolder=analysisLogsDirectory, printDebug=True)
analyzeSignalBins=inputArguments.runUnblinded)
if (inputArguments.isDryRun):
print("Not spawning due to dry run flag: {sC_s}".format(
sC_s=shellCommands_signal))
else:
multiProcessLauncher.spawn(
shellCommands=shellCommands_signal,
optionalEnvSetup="cd {sR} && source setupEnv.sh".format(
sR=stealthEnv.stealthRoot),
logFileName="step1_data_{sT}.log".format(sT=signalType),
printDebug=True)
multiProcessLauncher.monitorToCompletion()
elif (step == "MC"):
command_update = ("cd getMCSystematics && make && cd ..")
stealthEnv.execute_in_env(
commandToRun=command_update,
isDryRun=inputArguments.isDryRun,
functionToCallIfCommandExitsWithError=removeLock)
for eventProgenitor in eventProgenitors:
crossSectionsPath = crossSectionsForProgenitor[eventProgenitor]
for signalType in (list_signalTypes + ["control"]):
MCPathMain = ""
dataPUSourceMain = ""
HLTEfficienciesPathMain = ""
PUWeightsOutputPathMain = ""
lumiMain = ""
MCPathsAux = []
dataPUSourcesAux = []
PUWeightsOutputPathsAux = []
HLTEfficienciesPathsAux = []
lumisAux = []
if (inputArguments.year == "all"):
# DAS query for MC_QCD: dataset dataset=/QCD_HT*_TuneCP5_13TeV-madgraph-pythia8/RunIIFall17MiniAODv2-PU2017_12Apr2018_94X_mc2017_realistic_v14-v*/MINIAODSIM
"MC_QCD1": 322600.0/arbitraryFactor_QCD,
"MC_QCD_singlemedium1": 322600.0/arbitraryFactor_QCD,
"MC_QCD2": 29980.0/arbitraryFactor_QCD,
"MC_QCD_singlemedium2": 29980.0/arbitraryFactor_QCD,
"MC_QCD3": 6334.0/arbitraryFactor_QCD,
"MC_QCD_singlemedium3": 6334.0/arbitraryFactor_QCD,
"MC_QCD4": 1088.0/arbitraryFactor_QCD,
"MC_QCD_singlemedium4": 1088.0/arbitraryFactor_QCD,
"MC_QCD5": 99.11/arbitraryFactor_QCD,
"MC_QCD_singlemedium5": 99.11/arbitraryFactor_QCD,
"MC_QCD6": 20.23/arbitraryFactor_QCD,
"MC_QCD_singlemedium6": 20.23/arbitraryFactor_QCD
}
stealthEnv.execute_in_env(commandToRun="eos {eP} mkdir -p {sER}/selections/combined_DoublePhoton{oI}".format(eP=stealthEnv.EOSPrefix, sER=stealthEnv.stealthEOSRoot, oI=optional_identifier), functionToCallIfCommandExitsWithError=removeLock)
stealthEnv.execute_in_env(commandToRun="eos {eP} mkdir -p {sER}/statistics/combined_DoublePhoton{oI}".format(eP=stealthEnv.EOSPrefix, sER=stealthEnv.stealthEOSRoot, oI=optional_identifier), functionToCallIfCommandExitsWithError=removeLock)
# Step 1 merge: sufficient for everything except datasets with different event weights
filesToCleanup = []
for selectionType in selectionTypesToRun:
isMC = True
isMCString = "true"
if (("data" in selectionType)
or (selectionType == "MC_EMEnrichedQCD")
or (bool(re.match(r"^MC_GJet[0-9]*$", selectionType)))
or (bool(re.match(r"^MC_GJet_singlemedium[0-9]*$", selectionType)))
or (bool(re.match(r"^MC_QCD[0-9]*$", selectionType)))
or (bool(re.match(r"^MC_QCD_singlemedium[0-9]*$", selectionType)))
or (selectionType == "MC_hgg")
):