def create_job(inputData, saveName, outputDir, dontPromptMe):
slcioFile = saveName + '.slcio'
rootFile = saveName + '.root'
if check_file_existence(outputDir, slcioFile, dontPromptMe):
remove_file(outputDir, slcioFile, dontPromptMe)
if check_file_existence(outputDir, rootFile, dontPromptMe):
remove_file(outputDir, rootFile, dontPromptMe)
dIlc = DiracILC()
job = UserJob()
job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml'])
if SAVE_SLCIO:
job.setOutputData([slcioFile, rootFile],
OutputPath=outputDir,
OutputSE=STORAGE_SE)
else:
job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE)
job.setJobGroup('myMarlinRun1')
job.setName('MyMarlinJob1')
# job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk'])
job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz'])
marl = Marlin()
marl.setVersion('ILCSoft-2017-11-15_gcc62')
marl.setInputFile(inputData)
job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData)))
# marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio'])
marl.setSteeringFile('marlin/FullChain.xml')
# marl.setSteeringFile('marlin/FullChainNewDetModel.xml')
marl.setOutputFile(slcioFile)
gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear'
if not os.path.isfile(gearFile):
print('Error: gear file', gearFile,
'does not exist! Abort submission.')
return
marl.setGearFile(gearFile)
marl.setExtraCLIArguments(
"--MyNtupleMaker.OutputFileName={rootOutfile}".format(
rootOutfile=rootFile))
# marl.setNumberOfEvents(1000)
job.append(marl)
if dontPromptMe:
job.dontPromptMe()
job.submit(dIlc)
return False
def defMarlin(outputFile, rootFile, **kwargs):
'''Defines Marlin job'''
##Jet Settings
jetAlgo = kwargs.pop('jetAlgo', 'ValenciaPlugin 1.5 1 1')
jetCluster = kwargs.pop('jetCluster', 'ExclusiveNJets 2')
jetRecomb = kwargs.pop('jetRecomb', 'E_scheme')
##Top Tagger Settings
deltaR = kwargs.pop('deltaR', 0.01)
deltaP = kwargs.pop('deltaP', 0.01)
cos_theta_W_max = kwargs.pop('cos_theta_W_max', 1.0)
##Sixfermion sample
sixFermionSample = kwargs.pop('sixFermionSample', True)
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
ma = Marlin()
ma.setVersion('ILCSoft-2017-05-30_gcc62')
ma.setSteeringFile(
"/home/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/topasymmetry_template_new.xml"
)
ma.setGearFile("/home/lstroem/clicdp/gear/clic_ild_cdr.gear")
ma.setOutputFile(outputFile, rootFile)
ma.setExtraCLIArguments(
"--MyTopTaggerProcessorJ2.algorithm=\"{algorithm}\"\
--MyTopTaggerProcessorJ2.clusteringMode=\"{clusteringMode}\"\
--MyTopTaggerProcessorJ2.recombinationScheme={recombinationScheme}\
--MyTopTaggerProcessorJ2.deltaR={deltaR:.2f}\
--MyTopTaggerProcessorJ2.deltaP={deltaP:.2f}\
--MyTopTaggerProcessorJ2.cos_theta_W_max={cos_theta_W_max:.2f}\
--MyTopTaggerProcessorJ2wTrimming.algorithm=\"{algorithm}\"\
--MyTopTaggerProcessorJ2wTrimming.clusteringMode=\"{clusteringMode}\"\
--MyTopTaggerProcessorJ2wTrimming.recombinationScheme={recombinationScheme}\
--MyTopTaggerProcessorJ2wTrimming.deltaR={deltaR:.2f}\
--MyTopTaggerProcessorJ2wTrimming.deltaP={deltaP:.2f}\
--MyTopTaggerProcessorJ2wTrimming.cos_theta_W_max={cos_theta_W_max:.2f}\
--MyTopAsymmetryStoreProcessor.useSixFermionSample={sixFermionSample}\
--MyTopAsymmetryStoreProcessor.outFileName={outfile}"
.format(algorithm=jetAlgo,
clusteringMode=jetCluster,
recombinationScheme=jetRecomb,
deltaR=deltaR,
deltaP=deltaP,
cos_theta_W_max=cos_theta_W_max,
sixFermionSample=sixFermionSample,
outfile=rootFile))
return ma
def createMarlinApplication(self, energy, cliReco, over):
"""Create Marlin application with or without overlay."""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
marlin.setDebug()
marlin.setVersion( self.softwareVersion )
marlin.setDetectorModel( self.detectorModel )
marlin.detectortype = self.detectorModel
marlin.setKeepRecFile(False)
if over:
self.addOverlayOptionsToMarlin( energy )
self.cliReco = ' '.join([self.cliRecoOption, self.cliReco, cliReco]).strip()
marlin.setExtraCLIArguments(self.cliReco)
self.cliReco = ''
marlin.setSteeringFile(self.marlinSteeringFile)
self._setApplicationOptions("Marlin", marlin)
return marlin
def createMarlinApplication( self, energy ):
""" create Marlin Application without overlay """
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
marlin.setDebug()
marlin.setVersion( self.softwareVersion )
marlin.setDetectorModel( self.detectorModel )
marlin.detectortype = self.detectorModel
if self._flags.over:
self.addOverlayOptionsToMarlin( energy )
marlin.setExtraCLIArguments( self.cliReco )
steeringFile = {
350. : "clicReconstruction.xml",
380. : "clicReconstruction.xml",
420. : "clicReconstruction.xml",
1400.: "clicReconstruction.xml",
3000.: "clicReconstruction.xml",
}.get( energy, 'clicReconstruction.xml' )
marlin.setSteeringFile( steeringFile )
return marlin
print jobname
#lcoutputSIM = simOutfile + "_SIM_%s.slcio"%(indx)
lcoutputSIM = simOutfile + "_%s.SIM.slcio"%(indx)
lcinputREC = "/ilc/user/k/kurca/MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" + ireq +"/sim/" + lcoutputSIM
print lcinputREC
# Marlin
lcoutputDST = recOutfile + "_%s.DST.slcio"%(indx)
lcoutputREC = recOutfile + "_%s.REC.slcio"%(indx)
ma = Marlin()
ma.setVersion(ILCSoftVer)
ma.setDetectorModel(detectorModel)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments(" --constant.DetectorModel=%s "%(detectorModel) )
ma.setLogFile("marlin.log")
#ma.getInputFromApp(sim)
ma.setInputFile([lcoutputSIM])
ma.setEnergy(energy)
ma.setNumberOfEvents(evtsPerRun)
ma.setOutputDstFile(lcoutputDST)
ma.setOutputRecFile(lcoutputREC)
RECoutput.append(lcoutputDST)
RECoutput.append(lcoutputREC)
job = UserJob()
job.setName(jobname)
job.setJobGroup(jobGrName)
job.setILDConfig(ILDConfigVer)
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/clic_ild_cdr.gear",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/lib.tar.gz"
])
job.setInputData(
"LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
i)
job.setOutputSandbox(
["*.log", "*.sh", "*.py", "*.out", "*.xml", "*.steer "])
job.setJobGroup("myRoot")
job.setName("root_aa_%d" % i)
marl = Marlin()
marl.setVersion('ILCSoft-2016-09-27_gcc48')
marl.setInputFile([
"LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
i
])
# marl.setNumberOfEvents(10)
marl.setSteeringFile("steering.xml")
marl.setGearFile("clic_ild_cdr.gear")
marl.setExtraCLIArguments("--MyProcessor.RootFilename=%s" % lcoutput)
res = job.append(marl)
if not res['OK']:
print res['Message']
quit() #do something, like quit
job.setOutputData(lcoutput, "hgamgam/Marlin/rootFiles/aa/workingVersion",
"CERN-SRM")
print lcoutput
job.dontPromptMe()
print job.submit(dIlc)
time.sleep(0.5)
def subOverlay():
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = _clip.isLocal
nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
nbevts = 0 # To analize all input events
outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix
outputDir = _clip.outputDir
inputFile = _clip.inputFile
if inputFile == "":
gLogger.error("Input file for ddsim does not given.")
exit(-1)
recfile = outputFilePrefix + ".rec.slcio"
dstfile = outputFilePrefix + ".dst.slcio"
detector_model = "ILD_l5_o1_v02"
key = detector_model.split('_')
sim_detectorModel = "_".join([key[0], key[1], key[3]])
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myoverlayjob")
job.setName("myoverlay")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
# job.setBannedSites([]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
# Create Overlay application
ovldata = [{
"ProcessorName": "BgOverlayWW",
"evttype": "aa_lowpt_WW",
"ProdID": 10237,
"expBG": 0.211,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayWB",
"evttype": "aa_lowpt_WB",
"ProdID": 10241,
"expBG": 0.24605,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBW",
"evttype": "aa_lowpt_BW",
"ProdID": 10239,
"expBG": 0.243873,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBB",
"evttype": "aa_lowpt_BB",
"ProdID": 10235,
"expBG": 0.35063,
"subdir": "000"
}, {
"ProcessorName": "PairBgOverlay",
"evttype": "seeablepairs",
"ProdID": 10233,
"expBG": 1.0,
"subdir": "100"
}]
BXOverlay = 1
NbSigEvtsPerJob = 100
numberOfSignalEvents = NbSigEvtsPerJob
basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws"
energy = "500"
for ovl in ovldata:
print "### OverlayInput ... " + ovl["ProcessorName"]
ovlapp = OverlayInput()
ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \
( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] )
print " OverlayPath ... " + ovlpath
ovlapp.setMachine("ilc_dbd")
# ovlapp.setEnergy(energy)
# ovlapp.setDetectorModel(sim_detectorModel)
ovlapp.setProcessorName(ovl["ProcessorName"])
ovlapp.setBkgEvtType(ovl["evttype"])
ovlapp.setPathToFiles(ovlpath)
ovlapp.setGGToHadInt(ovl["expBG"])
ovlapp.setBXOverlay(BXOverlay)
ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob)
ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents)
res = job.append(ovlapp)
if not res['OK']:
print res['Message']
exit(1)
# Create Marlin application
marlin = Marlin()
marlin.setVersion("ILCSoft-02-00-02_gcc49")
marlin.setDetectorModel(detector_model)
marlin.setSteeringFile("MarlinStdReco.xml")
marlin.setInputFile(inputFile)
marlin.setNumberOfEvents(nbevts)
marlin.setOutputDstFile(dstfile)
marlin.setOutputRecFile(recfile)
extraCLIArguments = " --constant.DetectorModel=%s " % detector_model
extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str(
energy)
extraCLIArguments += " --global.Verbosity=MESSAGE "
marlin.setExtraCLIArguments(extraCLIArguments)
job.append(marlin)
if outputDir != "":
job.setOutputData([dstfile, recfile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
def main(argv):
# Input arguments
ildconfig_version = "$ILDCONFIGVER"
ilcsoft_version = "$ILCSOFTVER"
evts_per_run = $EVTSPERRUN
detector_model = "$DETECTOR"
sim_input = "$SIMINPUT"
process_name = "$PROCESS"
index = $IND
sim_input = diracpath_from_pnfspath( sim_input )
sim_detector_model = detector_model_wo_option( detector_model )
job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model
dirac = DiracILC(True,job_group+".rep")
# outputs to be saved onto grid SE
RECoutput = []
# DDSim
evtStart = (index-1)*evts_per_run
evtEnd = index*evts_per_run - 1
RandSeed = random.randrange(11623, 99999)
lcinputSIM = "LFN:" + sim_input
lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd))
sim = DDSim()
sim.setVersion(ilcsoft_version)
sim.setDetectorModel(sim_detector_model)
sim.setInputFile(lcinputSIM)
sim.setSteeringFile("ddsim_steer.py")
sim.setNumberOfEvents(evts_per_run)
sim.setRandomSeed(RandSeed)
sim.setEnergy(1000)
sim.setStartFrom(evtStart)
sim.setOutputFile(lcoutputSIM)
# Marlin
lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd))
ma = Marlin()
ma.setVersion(ilcsoft_version)
ma.setDetectorModel(detector_model)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) )
ma.setLogFile("marlin.log")
ma.getInputFromApp(sim)
ma.setEnergy(1000)
ma.setOutputDstFile(lcoutputDST)
RECoutput.append(lcoutputDST)
# ILCDirac user job
job = UserJob()
job.setName("user_sim_reco")
job.setJobGroup(job_group)
job.setILDConfig(ildconfig_version)
job.setCPUTime(86400)
tmp_file_name = process_name + "_sim_reco_job_tmp.py"
job.setInputSandbox([tmp_file_name])
job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])
job.dontPromptMe()
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])
# run simulation job
simres = job.append(sim)
if not simres['OK']:
print simres['Not ok appending ddsim to job']
quit()
# run Malrin reco jobs
mares = job.append(ma)
if not mares['OK']:
print mares['Not ok appending Marlin to job']
quit()
job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM")
print RECoutput
submit_output = job.submit(dirac)
print submit_output