def createOverlayApplication( self, energy ):
""" create Overlay Application """
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
overlay.setEnergy(energy)
overlay.setBackgroundType(self.overlayEventType)
overlay.setDetectorModel(self.detectorModel)
try:
overlayEnergy = energyToInt( self.overlayEvents ) if self.overlayEvents else energy
self.setOverlayParameters(overlayEnergy, self._machine, overlay)
except KeyError:
raise RuntimeError("No overlay parameters defined for %r GeV and %s " % (energy, self.overlayEventType))
if self.overlayEvents:
overlay.setUseEnergyForFileLookup( False )
self._setApplicationOptions("Overlay", overlay)
return overlay
def setUp(self):
"""set up the objects"""
self.olin = OverlayInput( {} )
def getJob(dirac, jobid, jobpara):
iser=jobid+100
outdir = "/ilc/user/a/amiyamoto/myprod2/test/"
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
dstlfn = outdir+"dst/"+outdst
reclfn = outdir+"rec/"+outrec
outsrm = "CERN-SRM"
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
# Upload files to different directories
upload_script="upload%i.sh"%iser
upload = GenericApplication()
# Create a script to upload files.
shfile = open(upload_script,"w")
shfile.write("#!/bin/bash\n")
shfile.write("/bin/ls -l \n")
shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n")
shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n")
shfile.close()
os.chmod(upload_script,0755)
upload.setScript(upload_script)
res = j.append(upload)
if not res['OK'] :
print res['Message']
exit(1)
# j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setInputSandbox([ setting_file, upload_script ] )
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
res = j.submit(dirac)
if not res["OK"] :
print "Failed submit job, jobid=%s" %jobid
print res
os.remove(upload_script)
return j
slicpandora = SLICPandora()
slicpandora.setVersion(pandoraVers)
slicpandora.setDetectorModel(detectorName)
slicpandora.setPandoraSettings(pandoraSettings)
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion(lcsimVers)
lcsim_postpandora.getInputFromApp(slicpandora)
lcsim_postpandora.setSteeringFile(lcsimSteering2)
lcsim_postpandora.setTrackingStrategy(strategies)
lcsim_postpandora.setDetectorModel(detectorName)
### Now with overlay
overlay = OverlayInput()
overlay.setBXOverlay(60)
overlay.setGGToHadInt(1.3)##When running at 1.4TeV
overlay.setDetectorModel("CLIC_SID_CDR")
overlay.setBkgEvtType("gghad")
lcsim_prepandora_ov = LCSIM()
lcsim_prepandora_ov.setVersion(lcsimVers)
lcsim_prepandora_ov.setSteeringFile(lcsimSteering1_ov)
lcsim_prepandora_ov.setTrackingStrategy(strategies)
lcsim_prepandora_ov.setDetectorModel(detectorName)
lcsim_prepandora_ov.setOutputFile("prePandora.slcio")
slicpandora_ov = SLICPandora()
slicpandora_ov.setVersion(pandoraVers)
slicpandora_ov.setDetectorModel(detectorName)
def getOverlay(self, nbevts):
""" Create an overlay step
"""
pathToFiles = None
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
if self.energy==350:
if self.detectorModel=="ILD_o1_v05":
pathToFiles="/ilc/user/s/sailer/testFiles/overlay/ild_350/"
if pathToFiles:
overlay.setPathToFiles(pathToFiles)
else:
self.log.warn("better define pathToFiles for this overlay: %s, %s, %s" %
(self.energy, self.machine, self.backgroundType) )
overlay.setMachine(self.machine)
overlay.setEnergy(self.energy)
overlay.setDetectorModel(self.detectorModel)
overlay.setBkgEvtType(self.backgroundType)
overlay.setBXOverlay(60)
overlay.setGGToHadInt(0.3)
overlay.setNumberOfSignalEventsPerJob(nbevts)
return overlay
##Simulation ILD
ddsim = None
if UseDD4hepGeometry:
ddsim = DDSim()
ddsim.setVersion(DDSimVer) ###SET HERE YOUR MOKKA VERSION, the software will come from the ILDConfig
ddsim.setDetectorModel(detectorModel)
ddsim.setSteeringFile("ddsim_steer.py")
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
##Define the overlay
overlay = OverlayInput()
overlay.setMachine("ilc_dbd") #Don't touch, this is how the system knows what files to get
overlay.setEnergy(energy) #Don't touch, this is how the system knows what files to get
overlay.setDetectorModel(detectorModel) #Don't touch, this is how the system knows what files to get
if energy==500.: #here you chose the overlay parameters as this determines how many files you need
#it does NOT affect the content of the marlin steering file whatsoever, you need to make sure the values
#there are correct. Only the file names are handled properly so that you don't need to care
overlay.setBXOverlay(BXOverlay)
overlay.setGGToHadInt(GGToHadInt500)
overlay.setBkgEvtType("aa_lowpt2") ## lowpt2: correct number of events (500),
## not increased to 2500 to reduce number
## of downloaded files
elif energy == 1000.:
overlay.setBXOverlay(BXOverlay)
overlay.setGGToHadInt(GGToHadInt1000)
overlay.setBkgEvtType("aa_lowpt")
mo.setSteeringFile("ild_00.steer")
##Simulation SID
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
overlay = OverlayInput()
overlay.setMachine("clic_cdr")
overlay.setEnergy(energy)
overlay.setBkgEvtType("gghad")
if energy==500.:
overlay.setBXOverlay(300)
overlay.setGGToHadInt(0.3)##When running at 500geV
overlay.setDetectorModel("CLIC_ILD_CDR500")
elif energy == 350.:
overlay.setBXOverlay(300)
overlay.setGGToHadInt(0.0464)##When running at 350geV
overlay.setDetectorModel("CLIC_ILD_CDR500")
elif energy == 3000.:
overlay.setBXOverlay(60)
overlay.setGGToHadInt(3.2)##When running at 3TeV
overlay.setDetectorModel("CLIC_ILD_CDR")
mo.setOutputFile("MyFile.slcio")
mo.setNbEvts(n_evts_per_job)
res = j.append(mo)
if not res['OK']:
print res['Message']
break
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.getInputFromApp(mo)
ma.setOutputDstFile("mydst_no_ov_%s.slcio" % i)
res = j.append(ma)
if not res['OK']:
print res['Message']
break
ov = OverlayInput()
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(n_evts_per_job)
ov.setBkgEvtType("gghad")
ov.setDetectorModel("CLIC_ILD_CDR")
res = j.append(ov)
if not res['OK']:
print res['Message']
break
ma2 = Marlin()
ma2.setVersion("v0111Prod")
ma2.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma2.getInputFromApp(mo)
ma2.setOutputDstFile("mydst_ov_%s.slcio" % i)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
meta={}
meta["ProdID"]=651
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print("Found no files")
exit()
lfns = res['Value']
print("found %s files" % len(lfns))
ovi = OverlayInput()
ovi.setEnergy(500.)
ovi.setBXOverlay(300)
ovi.setGGToHadInt(0.3)
ovi.setNbSigEvtsPerJob(10)
ovi.setBkgEvtType("gghad")
ovi.setDetectorModel("CLIC_ILD_CDR")
overlay = [True,False]
for ov in overlay:
d = DiracILC(True,"repo_overlay_%s.rep"%ov)
for lfn in lfns:
j = UserJob()
steeringf = "clic_ild_cdr_steering.xml"
if ov:
def getJob(jobid, jobpara):
iser=jobid+1
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_ovl_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
return j
def setUp(self):
"""set up the objects"""
self.olin = OverlayInput({})
class OverlayInputTestCase(unittest.TestCase):
""" Base class for the OverlayInput test cases
"""
def setUp(self):
"""set up the objects"""
self.olin = OverlayInput({})
def test_setters(self):
assertEqualsImproved(
(self.olin.machine, self.olin.prodID,
self.olin.useEnergyForFileLookup, self.olin.detectorModel,
self.olin.backgroundEventType), ('clic_cdr', 0, True, '', ''),
self)
self.assertFalse(self.olin._errorDict)
self.olin.setMachine('mytestMachine')
assertDiracSucceeds(self.olin.setProdID(14983), self)
self.olin.setUseEnergyForFileLookup(False)
self.olin.setDetectorModel('mytestDetector')
self.olin.setBackgroundType('myBackgroundParticle.Testme')
assertEqualsImproved(
(self.olin.machine, self.olin.prodID,
self.olin.useEnergyForFileLookup, self.olin.detectorModel,
self.olin.backgroundEventType),
('mytestMachine', 14983, False, 'mytestDetector',
'myBackgroundParticle.Testme'), self)
self.assertFalse(self.olin._errorDict)
def test_setters_checks(self):
self.olin.setMachine({'138rj': True})
self.olin.setProdID([])
self.olin.setUseEnergyForFileLookup('False')
self.olin.setDetectorModel(9024)
self.olin.setBackgroundType(8914)
assertEqualsImproved(len(self.olin._errorDict['_checkArgs']), 5, self)
def test_userjobmodules(self):
with patch.object(self.olin,
'_setApplicationModuleAndParameters',
new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.olin._userjobmodules(Mock()), self)
def test_userjobmodules_fails(self):
with patch.object(self.olin,
'_setApplicationModuleAndParameters',
new=Mock(return_value=S_ERROR('bla'))):
assertDiracFailsWith(self.olin._userjobmodules(Mock()),
'userjobmodules failed', self)
def test_prodjobmodules(self):
with patch.object(self.olin,
'_setApplicationModuleAndParameters',
new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.olin._prodjobmodules(Mock()), self)
def test_prodjobmodules_fails(self):
with patch.object(self.olin,
'_setApplicationModuleAndParameters',
new=Mock(return_value=S_ERROR('bla'))):
assertDiracFailsWith(self.olin._prodjobmodules(Mock()),
'prodjobmodules failed', self)
def test_addparameterstostep_fails(self):
with patch.object(self.olin,
'_addBaseParameters',
new=Mock(return_value=S_ERROR('bla'))):
assertDiracFailsWith(self.olin._addParametersToStep(Mock()),
'failed to set base param', self)
def test_addparameterstostep(self):
with patch.object(self.olin,
'_addBaseParameters',
new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.olin._addParametersToStep(Mock()), self)
def test_checkconsistency(self):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 1345
self.olin.backgroundEventType = 'myTestBGEvt'
self.olin._jobtype = 'notUser'
assertDiracSucceeds(self.olin._checkConsistency(), self)
assertEqualsImproved((self.olin.prodparameters['detectorModel'],
self.olin.prodparameters['BXOverlay'],
self.olin.prodparameters['GGtoHadInt']),
('', True, 1345), self)
def test_checkconsistency_nofilesinpath(self):
self.olin.pathToOverlayFiles = '/my/path/overlay.files'
with patch(
'DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient.findFilesByMetadata',
new=Mock(return_value=S_OK([]))) as fcc_mock:
assertDiracFailsWith(self.olin._checkConsistency(),
'no files in that path', self)
fcc_mock.assert_called_once_with({}, '/my/path/overlay.files')
def test_checkconsistency_no_bunchcrossings(self):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = None
assertDiracFailsWith(self.olin._checkConsistency(),
'number of overlay bunch crossings not defined',
self)
def test_checkconsistency_no_backgroundevts(self):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 0
assertDiracFailsWith(
self.olin._checkConsistency(),
'background events per bunch crossing is not defined', self)
def test_checkconsistency_no_bgevttype(self):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 1345
self.olin.backgroundEventType = ''
assertDiracFailsWith(self.olin._checkConsistency(),
'event type is not defined', self)
def test_checkconsistency_nosignalevtperjob(self):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 1345
self.olin.backgroundEventType = 'gghad'
self.olin._jobtype = 'User'
self.olin.numberOfSignalEventsPerJob = 0
assertDiracFailsWith(self.olin._checkConsistency(),
'signal event per job is not defined', self)
def test_checkfinalconsistency_simple(self):
self.olin.pathToOverlayFiles = 'some/path'
assertDiracSucceeds(self.olin._checkFinalConsistency(), self)
def test_checkfinalconsistency_noenergy(self):
self.olin.energy = 0
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'energy must be specified', self)
def test_checkfinalconsistency_ops_fails(self):
ops_mock = Mock()
ops_mock.getSections.return_value = S_ERROR('some_ops_error')
self.olin._ops = ops_mock
self.olin.energy = 198
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'could not resolve the cs path', self)
def test_checkfinalconsistency_machinemissing(self):
ops_mock = Mock()
ops_mock.getSections.return_value = S_OK(
['allowed_machine_1', 'other_mach'])
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
self.olin.energy = 198
assertDiracFailsWith(
self.olin._checkFinalConsistency(),
'machine mytestmachineveryrare does not have overlay data', self)
def test_checkfinalconsistency_energynotfound(self):
self.olin.energy = 824
section_dict = {
'/Overlay': ['myTestMachineVeryRare'],
'/Overlay/myTestMachineVeryRare': ['other_energy_tev', '824tev']
}
ops_mock = Mock()
ops_mock.getSections.side_effect = lambda path: S_OK(section_dict[path]
)
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'no overlay files corresponding to 824gev', self)
def test_checkfinalconsistency_nodetectormodels(self):
self.olin.energy = 1000.0
section_dict = {
'/Overlay': S_OK(['myTestMachineVeryRare']),
'/Overlay/myTestMachineVeryRare':
S_OK(['other_energy_tev', '1tev']),
'/Overlay/myTestMachineVeryRare/1tev': S_ERROR('bla')
}
ops_mock = Mock()
ops_mock.getSections.side_effect = lambda path: section_dict[path]
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'could not find the detector models', self)
def test_checkfinalconsistency_detectormodelnotfound(self):
self.olin.energy = 981324
section_dict = {
'/Overlay': ['myTestMachineVeryRare'],
'/Overlay/myTestMachineVeryRare': ['other_energy_tev', '981.3gev'],
'/Overlay/myTestMachineVeryRare/981.3tev':
['some_other_detector_model', 'neither_this']
}
ops_mock = Mock()
ops_mock.getSections.side_effect = lambda path: S_OK(section_dict[path]
)
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'no overlay files corresponding to 981.3tev',
self)
def test_checkfinalconsistency_nobkg(self):
self.olin.detectorModel = 'testDetMod'
self.olin.energy = 981000
section_dict = {
'/Overlay': ['myTestMachineVeryRare'],
'/Overlay/myTestMachineVeryRare': ['other_energy_tev', '981tev'],
'/Overlay/myTestMachineVeryRare/981tev': ['testDetMod']
}
ops_mock = Mock()
ops_mock.getSections.side_effect = lambda path: S_OK(section_dict[path]
)
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
with patch.object(inspect.getmodule(OverlayInput),
'allowedBkg',
new=Mock(return_value=S_ERROR('bkg_test_err'))):
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'bkg_test_err', self)
def test_checkfinalconsistency_negativeprodid(self):
self.olin.detectorModel = 'testDetMod'
self.olin.energy = 981000
section_dict = {
'/Overlay': ['myTestMachineVeryRare'],
'/Overlay/myTestMachineVeryRare': ['other_energy_tev', '981tev'],
'/Overlay/myTestMachineVeryRare/981tev': ['testDetMod']
}
ops_mock = Mock()
ops_mock.getSections.side_effect = lambda path: S_OK(section_dict[path]
)
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
with patch.object(inspect.getmodule(OverlayInput),
'allowedBkg',
new=Mock(return_value=S_OK(-147))):
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'no proper production id found', self)
def test_checkfinalconsistency(self):
self.olin.detectorModel = 'testDetMod'
self.olin.energy = 981000
section_dict = {
'/Overlay': ['myTestMachineVeryRare'],
'/Overlay/myTestMachineVeryRare': ['other_energy_tev', '981tev'],
'/Overlay/myTestMachineVeryRare/981tev': ['testDetMod']
}
ops_mock = Mock()
ops_mock.getSections.side_effect = lambda path: S_OK(section_dict[path]
)
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
with patch.object(inspect.getmodule(OverlayInput),
'allowedBkg',
new=Mock(return_value=S_OK(13987))):
assertDiracSucceeds(self.olin._checkFinalConsistency(), self)
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
print args.chain[0]
# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]
check_events_arguments(args.events, args.split)
detector = args.detector
alias_properties(detector)
outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
inputSandbox, outputSandbox = setup_sandboxes(args.macFile)
dirac = DiracILC(True, repoName)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(outputBase)
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ---> ", fileNumber
################## SLIC ##################################################
if 1 in args.chain:
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
slic.setDetectorModel(detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Overlay ###############################################
if 2 in args.chain:
'''
#Add the gghad background overlay.
gghad = OverlayInput()
#gghad.setProdID(1767)
gghad.setEnergy(500.0)
gghad.setBXOverlay('args.bunches')
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setMachine('ilc_dbd')
gghad.setDetectorModel('sidloi3')
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
#Add the pair background overlay.
pairs = OverlayInput()
pairs.setProdID(2)
pairs.setEnergy(500.0)
pairs.setBXOverlay('args.bunches')
pairs.setGGToHadInt(1.)
pairs.setNbSigEvtsPerJob(nOutputEvents)
pairs.setMachine('ilc_dbd')
pairs.setDetectorModel('sidloi3')
pairs.setBkgEvtType('eepairs')
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
gghad = OverlayInput()
gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
gghad.setBXOverlay(int(args.bunches))
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
pairs = OverlayInput()
pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
pairs.setBXOverlay(int(args.bunches))
pairs.setGGToHadInt(1.)
pairs.setBkgEvtType('eepairs')
pairs.setNbSigEvtsPerJob(nOutputEvents)
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
################## lcsim (digitization and tracking) #####################
if 3 in args.chain:
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
if 1 in args.chain:
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
lcsim.setAliasProperties('steeringFiles/alias.properties')
lcsim.setDetectorModel(detector+".zip")
#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
if 4 in args.chain:
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
if 5 in args.chain:
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/' + detector + '.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
lcsimDst.setAliasProperties('steeringFiles/alias.properties')
lcsimDst.setDetectorModel(detector+".zip")
lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if 6 in args.chain:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/' + detector + '.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
outputLevel = max(args.chain)
if outputLevel == 1:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 3:
#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 4:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 5:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 6:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')
job.setOutputSandbox(outputSandbox)
job.setInputData(args.Input)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
BXOverlay=1
outdst = "toto-ovl.dst.slcio" #% i
outrec = "toto-ovl.rec.slcio" #% i
d= DiracILC(True,"repo.rep")
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayExample")#%i)
j.setInputSandbox([setting_file])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(n_events_per_job)
ov.setBXOverlay(BXOverlay)
ov.setGGToHadInt(GGToHadInt500)
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
meta={}
meta["ProdID"]=651
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print "Found no files"
exit()
lfns = res['Value']
print "found %s files"%len(lfns)
ovi = OverlayInput()
ovi.setEnergy(500.)
ovi.setBXOverlay(300)
ovi.setGGToHadInt(0.3)
ovi.setNbSigEvtsPerJob(10)
ovi.setDetectorType("ILD")
overlay = [True,False]
for ov in overlay:
d = DiracILC(True,"repo_overlay_%s.rep"%ov)
for lfn in lfns:
j = UserJob()
steeringf = "clic_ild_cdr_steering.xml"
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
slicpandora = SLICPandora()
slicpandora.setVersion(pandoraVers)
slicpandora.setDetectorModel(detectorName)
slicpandora.setPandoraSettings(pandoraSettings)
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion(lcsimVers)
lcsim_postpandora.getInputFromApp(slicpandora)
lcsim_postpandora.setSteeringFile(lcsimSteering2)
lcsim_postpandora.setTrackingStrategy(strategies)
lcsim_postpandora.setDetectorModel(detectorName)
### Now with overlay
overlay = OverlayInput()
overlay.setBXOverlay(60)
overlay.setGGToHadInt(1.3)##When running at 1.4TeV
overlay.setDetectorType("SID")
lcsim_prepandora_ov = LCSIM()
lcsim_prepandora_ov.setVersion(lcsimVers)
lcsim_prepandora_ov.setSteeringFile(lcsimSteering1_ov)
lcsim_prepandora_ov.setTrackingStrategy(strategies)
lcsim_prepandora_ov.setDetectorModel(detectorName)
lcsim_prepandora_ov.setOutputFile("prePandora.slcio")
slicpandora_ov = SLICPandora()
slicpandora_ov.setVersion(pandoraVers)
slicpandora_ov.setDetectorModel(detectorName)
slicpandora_ov.setPandoraSettings(pandoraSettings)
dexit(1)
mo = Mokka()
mo.getInputFromApp(wh)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setNbEvts(1)
mo.setOutputFile("somefile.slcio")
res = j.append(mo)
if not res['OK']:
print res['Message']
dexit(1)
ov = OverlayInput()
ov.setDetectorModel("CLIC_ILD_CDR")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
ov.setBkgEvtType("gghad")
res = j.append(ov)
if not res['OK']:
print res['Message']
dexit(1)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
mo.setOutputFile("MyFile.slcio")
mo.setNbEvts(n_evts_per_job)
res = j.append(mo)
if not res['OK']:
print res['Message']
break
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.getInputFromApp(mo)
ma.setOutputDstFile("mydst_no_ov_%s.slcio"%i)
res = j.append(ma)
if not res['OK']:
print res['Message']
break
ov = OverlayInput()
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(n_evts_per_job)
ov.setDetectorType("ILD")
res = j.append(ov)
if not res['OK']:
print res['Message']
break
ma2 = Marlin()
ma2.setVersion("v0111Prod")
ma2.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma2.getInputFromApp(mo)
ma2.setOutputDstFile("mydst_ov_%s.slcio"%i)
res = j.append(ma2)
dexit(1)
mo = Mokka()
mo.getInputFromApp(wh)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setNbEvts(1)
mo.setOutputFile("somefile.slcio")
res = j.append(mo)
if not res["OK"]:
print res["Message"]
dexit(1)
ov = OverlayInput()
ov.setDetectorType("ILD")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
res = j.append(ov)
if not res["OK"]:
print res["Message"]
dexit(1)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
# mo.setSteeringFile("dssimsteering.py")
# else:
# print 'SteeringFile Model for DDSim undefined for this energy'
dd.setDetectorModel(detectormodel)
##Simulation SID
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
overlay = OverlayInput()
if detectormodel in CLICDetectorModels:
overlay.setMachine("clic_cdr")
overlay.setEnergy(energy)
overlay.setBkgEvtType("gghad")
if energy == 500.:
overlay.setBXOverlay(300)
overlay.setGGToHadInt(0.3) ##When running at 500geV
overlay.setDetectorModel("CLIC_ILD_CDR500")
elif energy == 420.:
overlay.setBXOverlay(300)
overlay.setGGToHadInt(0.17) ##When running at 420geV
overlay.setDetectorModel("CLIC_ILD_CDR500")
elif energy == 350.:
overlay.setBXOverlay(300)
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion(MokkaVer) ###SET HERE YOUR MOKKA VERSION, the software will come from the ILDConfig
mo.setDetectorModel(detectorModel)
mo.setSteeringFile("bbudsc_3evt.steer")
### Do not include '.tgz'
mo.setDbSlice(dbslice)
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
##Define the overlay
overlay = OverlayInput()
overlay.setMachine("ilc_dbd") #Don't touch, this is how the system knows what files to get
overlay.setEnergy(energy) #Don't touch, this is how the system knows what files to get
overlay.setDetectorModel(detectorModel) #Don't touch, this is how the system knows what files to get
if energy == 500.: #here you chose the overlay parameters as this determines how many files you need
#it does NOT affect the content of the marlin steering file whatsoever, you need to make sure the values
#there are correct. Only the file names are handled properly so that you don't need to care
overlay.setBXOverlay(BXOverlay)
overlay.setGGToHadInt(GGToHadInt500)
overlay.setBkgEvtType("aa_lowpt2") ## lowpt2: correct number of events (500),
## not increased to 2500 to reduce number
## of downloaded files
elif energy == 1000.:
overlay.setBXOverlay(BXOverlay)
overlay.setGGToHadInt(GGToHadInt1000)
overlay.setBkgEvtType("aa_lowpt")
def subOverlay():
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = _clip.isLocal
nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
nbevts = 0 # To analize all input events
outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix
outputDir = _clip.outputDir
inputFile = _clip.inputFile
if inputFile == "":
gLogger.error("Input file for ddsim does not given.")
exit(-1)
recfile = outputFilePrefix + ".rec.slcio"
dstfile = outputFilePrefix + ".dst.slcio"
detector_model = "ILD_l5_o1_v02"
key = detector_model.split('_')
sim_detectorModel = "_".join([key[0], key[1], key[3]])
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myoverlayjob")
job.setName("myoverlay")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
# job.setBannedSites([]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
# Create Overlay application
ovldata = [{
"ProcessorName": "BgOverlayWW",
"evttype": "aa_lowpt_WW",
"ProdID": 10237,
"expBG": 0.211,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayWB",
"evttype": "aa_lowpt_WB",
"ProdID": 10241,
"expBG": 0.24605,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBW",
"evttype": "aa_lowpt_BW",
"ProdID": 10239,
"expBG": 0.243873,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBB",
"evttype": "aa_lowpt_BB",
"ProdID": 10235,
"expBG": 0.35063,
"subdir": "000"
}, {
"ProcessorName": "PairBgOverlay",
"evttype": "seeablepairs",
"ProdID": 10233,
"expBG": 1.0,
"subdir": "100"
}]
BXOverlay = 1
NbSigEvtsPerJob = 100
numberOfSignalEvents = NbSigEvtsPerJob
basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws"
energy = "500"
for ovl in ovldata:
print "### OverlayInput ... " + ovl["ProcessorName"]
ovlapp = OverlayInput()
ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \
( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] )
print " OverlayPath ... " + ovlpath
ovlapp.setMachine("ilc_dbd")
# ovlapp.setEnergy(energy)
# ovlapp.setDetectorModel(sim_detectorModel)
ovlapp.setProcessorName(ovl["ProcessorName"])
ovlapp.setBkgEvtType(ovl["evttype"])
ovlapp.setPathToFiles(ovlpath)
ovlapp.setGGToHadInt(ovl["expBG"])
ovlapp.setBXOverlay(BXOverlay)
ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob)
ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents)
res = job.append(ovlapp)
if not res['OK']:
print res['Message']
exit(1)
# Create Marlin application
marlin = Marlin()
marlin.setVersion("ILCSoft-02-00-02_gcc49")
marlin.setDetectorModel(detector_model)
marlin.setSteeringFile("MarlinStdReco.xml")
marlin.setInputFile(inputFile)
marlin.setNumberOfEvents(nbevts)
marlin.setOutputDstFile(dstfile)
marlin.setOutputRecFile(recfile)
extraCLIArguments = " --constant.DetectorModel=%s " % detector_model
extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str(
energy)
extraCLIArguments += " --global.Verbosity=MESSAGE "
marlin.setExtraCLIArguments(extraCLIArguments)
job.append(marlin)
if outputDir != "":
job.setOutputData([dstfile, recfile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
slicpandora = SLICPandora()
slicpandora.setVersion(pandoraVers)
slicpandora.setDetectorModel(detectorName)
slicpandora.setPandoraSettings(pandoraSettings)
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion(lcsimVers)
lcsim_postpandora.getInputFromApp(slicpandora)
lcsim_postpandora.setSteeringFile(lcsimSteering2)
lcsim_postpandora.setTrackingStrategy(strategies)
lcsim_postpandora.setDetectorModel(detectorName)
### Now with overlay
overlay = OverlayInput()
overlay.setBXOverlay(60)
overlay.setGGToHadInt(1.3) ##When running at 1.4TeV
overlay.setDetectorModel("CLIC_SID_CDR")
overlay.setBkgEvtType("gghad")
lcsim_prepandora_ov = LCSIM()
lcsim_prepandora_ov.setVersion(lcsimVers)
lcsim_prepandora_ov.setSteeringFile(lcsimSteering1_ov)
lcsim_prepandora_ov.setTrackingStrategy(strategies)
lcsim_prepandora_ov.setDetectorModel(detectorName)
lcsim_prepandora_ov.setOutputFile("prePandora.slcio")
slicpandora_ov = SLICPandora()
slicpandora_ov.setVersion(pandoraVers)
slicpandora_ov.setDetectorModel(detectorName)
def getOverlay(self, nbevts):
""" Create an overlay step
"""
pathToFiles = None
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
if self.energy == 350:
if self.detectorModel == "ILD_o1_v05":
pathToFiles = "/ilc/user/s/sailer/testFiles/overlay/ild_350/"
if pathToFiles:
overlay.setPathToFiles(pathToFiles)
else:
self.log.warn(
"better define pathToFiles for this overlay: %s, %s, %s" %
(self.energy, self.machine, self.backgroundType))
overlay.setMachine(self.machine)
overlay.setEnergy(self.energy)
overlay.setDetectorModel(self.detectorModel)
overlay.setBkgEvtType(self.backgroundType)
overlay.setBXOverlay(60)
overlay.setGGToHadInt(0.3)
overlay.setNumberOfSignalEventsPerJob(nbevts)
return overlay
def createOverlayApplication( self, energy ):
""" create Overlay Application """
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
overlay.setMachine( 'clic_opt' )
overlay.setEnergy( energy )
overlay.setBackgroundType( self._overlayEventType )
overlay.setDetectorModel( self.detectorModel )
try:
overlayEnergy = energyToInt( self.overlayEvents ) if self.overlayEvents else energy
self.overlayParameterDict().get( overlayEnergy ) ( overlay )
except TypeError:
raise RuntimeError( "No overlay parameters defined for %r GeV and %s " % ( energy, self._overlayEventType ) )
if self.overlayEvents:
overlay.setUseEnergyForFileLookup( False )
return overlay
##Simulation ILD
mo = Mokka()
mo.setVersion('0706P08') ###SET HERE YOUR MOKKA VERSION
mo.setDetectorModel(detectorModel)
if energy in [500.]: ##YOU COULD HAVE THE SAME STEERING FILE FOR DIFFERENT ENERGIES
mo.setSteeringFile("clic_ild_cdr500.steer") ## define the prod steering file
else:
mo.setSteeringFile("clic_ild_cdr.steer")## define the prod steering file
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
##Define the overlay
overlay = OverlayInput()
overlay.setMachine("ilc_dbd") #Don't touch, this is how the system knows what files to get
overlay.setEnergy(energy)#Don't touch, this is how the system knows what files to get
overlay.setDetectorModel(detectorModel)#Don't touch, this is how the system knows what files to get
if energy==500.: #here you chose the overlay parameters as this determines how many files you need
#it does NOT affect the content of the marlin steering file whatsoever, you need to make sure the values
#there are correct. Only the file names are handled properly so that you don't need to care
overlay.setBXOverlay(300)
overlay.setGGToHadInt(0.3)##When running at 500geV
elif energy == 1000.:
overlay.setBXOverlay(60)
overlay.setGGToHadInt(1.3)##When running at 1tev
else:
print "Overlay ILD: No overlay parameters defined for this energy"
##Reconstruction ILD with overlay
lcfivertex = Marlin()
lcfivertex.setVersion()
lcfivertex.getInputFromApp(slicpandora)
lcfivertex.setSteeringFile()
lcfivertex.setOutputFile("lcfivertex.slcio")#NEVER CHANGE THIS, this file is not stored in any case
#Final step. Outpufile is resolved automatically
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion('CLIC_CDR')#This will change only once
lcsim_postpandora.getInputFromApp(lcfivertex)
lcsim_postpandora.setSteeringFile("clic_cdr_postPandoraOverlay.lcsim")#This will change only once
lcsim_postpandora.setTrackingStrategy(tracking_strategy)
#lcsim_postpandora.setDetectorModel(detector_model)
## SID Reco w/o overlay
overlay_sid_gg = OverlayInput()
overlay_sid_gg.setBkgEvtType("gghad")#DO NOT TOUCH THIS
overlay_sid_gg.setEnergy(energy)
if energy == 1000.:
overlay_sid_gg.setBXOverlay(1)
overlay_sid_gg.setGGToHadInt(4.1)##When running at 3TeV
else:
print("Overlay SID: No overlay parameters defined for this energy")
overlay_sid_gg.setDetectorModel("sidloi3")#DO NOT TOUCH THIS
overlay_sid_pairs = OverlayInput()
overlay_sid_pairs.setBkgEvtType("pairs")#DO NOT TOUCH THIS
overlay_sid_pairs.setEnergy(energy)
if energy == 1000.:
overlay_sid_pairs.setBXOverlay(1)
overlay_sid_pairs.setGGToHadInt(1)##When running at 3TeV
lcfivertex.setSteeringFile()
lcfivertex.setOutputFile(
"lcfivertex.slcio"
) #NEVER CHANGE THIS, this file is not stored in any case
#Final step. Outpufile is resolved automatically
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion('CLIC_CDR') #This will change only once
lcsim_postpandora.getInputFromApp(lcfivertex)
lcsim_postpandora.setSteeringFile(
"clic_cdr_postPandoraOverlay.lcsim") #This will change only once
lcsim_postpandora.setTrackingStrategy(tracking_strategy)
#lcsim_postpandora.setDetectorModel(detector_model)
## SID Reco w/o overlay
overlay_sid_gg = OverlayInput()
overlay_sid_gg.setBkgEvtType("gghad") #DO NOT TOUCH THIS
overlay_sid_gg.setEnergy(energy)
if energy == 1000.:
overlay_sid_gg.setBXOverlay(1)
overlay_sid_gg.setGGToHadInt(4.1) ##When running at 3TeV
else:
print "Overlay SID: No overlay parameters defined for this energy"
overlay_sid_gg.setDetectorModel("sidloi3") #DO NOT TOUCH THIS
overlay_sid_pairs = OverlayInput()
overlay_sid_pairs.setBkgEvtType("pairs") #DO NOT TOUCH THIS
overlay_sid_pairs.setEnergy(energy)
if energy == 1000.:
overlay_sid_pairs.setBXOverlay(1)
overlay_sid_pairs.setGGToHadInt(1) ##When running at 3TeV
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
meta = {}
meta["ProdID"] = 651
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print "Found no files"
exit()
lfns = res['Value']
print "found %s files" % len(lfns)
ovi = OverlayInput()
ovi.setEnergy(500.)
ovi.setBXOverlay(300)
ovi.setGGToHadInt(0.3)
ovi.setNbSigEvtsPerJob(10)
ovi.setBkgEvtType("gghad")
ovi.setDetectorModel("CLIC_ILD_CDR")
overlay = [True, False]
for ov in overlay:
d = DiracILC(True, "repo_overlay_%s.rep" % ov)
for lfn in lfns:
j = UserJob()
steeringf = "clic_ild_cdr_steering.xml"
if ov:
class OverlayInputTestCase( unittest.TestCase ):
""" Base class for the OverlayInput test cases
"""
def setUp(self):
"""set up the objects"""
self.olin = OverlayInput( {} )
def test_setters( self ):
assertEqualsImproved( ( self.olin.machine, self.olin.prodID, self.olin.useEnergyForFileLookup,
self.olin.detectorModel, self.olin.backgroundEventType ),
( 'clic_cdr', 0, True, '', '' ), self )
self.assertFalse( self.olin._errorDict )
self.olin.setMachine( 'mytestMachine' )
assertDiracSucceeds( self.olin.setProdID( 14983 ), self )
self.olin.setUseEnergyForFileLookup( False )
self.olin.setDetectorModel( 'mytestDetector' )
self.olin.setBackgroundType( 'myBackgroundParticle.Testme' )
assertEqualsImproved( ( self.olin.machine, self.olin.prodID, self.olin.useEnergyForFileLookup,
self.olin.detectorModel, self.olin.backgroundEventType ),
( 'mytestMachine', 14983, False, 'mytestDetector', 'myBackgroundParticle.Testme' ),
self )
self.assertFalse( self.olin._errorDict )
def test_setters_checks( self ):
self.olin.setMachine( { '138rj' : True } )
self.olin.setProdID( [] )
self.olin.setUseEnergyForFileLookup( 'False' )
self.olin.setDetectorModel( 9024 )
self.olin.setBackgroundType( 8914 )
assertEqualsImproved( len( self.olin._errorDict['_checkArgs'] ), 5, self )
def test_userjobmodules( self ):
with patch.object(self.olin, '_setApplicationModuleAndParameters', new=Mock(return_value=S_OK())):
assertDiracSucceeds( self.olin._userjobmodules( Mock() ), self )
def test_userjobmodules_fails( self ):
with patch.object(self.olin, '_setApplicationModuleAndParameters', new=Mock(return_value=S_ERROR('bla'))):
assertDiracFailsWith( self.olin._userjobmodules( Mock() ), 'userjobmodules failed', self )
def test_prodjobmodules( self ):
with patch.object(self.olin, '_setApplicationModuleAndParameters', new=Mock(return_value=S_OK())):
assertDiracSucceeds( self.olin._prodjobmodules( Mock() ), self )
def test_prodjobmodules_fails( self ):
with patch.object(self.olin, '_setApplicationModuleAndParameters', new=Mock(return_value=S_ERROR('bla'))):
assertDiracFailsWith( self.olin._prodjobmodules( Mock() ), 'prodjobmodules failed', self )
def test_addparameterstostep_fails( self ):
with patch.object(self.olin, '_addBaseParameters', new=Mock(return_value=S_ERROR('bla'))):
assertDiracFailsWith( self.olin._addParametersToStep( Mock() ), 'failed to set base param', self )
def test_addparameterstostep( self ):
with patch.object(self.olin, '_addBaseParameters', new=Mock(return_value=S_OK())):
assertDiracSucceeds( self.olin._addParametersToStep( Mock() ), self )
def test_checkconsistency( self ):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 1345
self.olin.backgroundEventType = 'myTestBGEvt'
self.olin._jobtype = 'notUser'
assertDiracSucceeds( self.olin._checkConsistency(), self )
assertEqualsImproved(
( self.olin.prodparameters['detectorModel'], self.olin.prodparameters['BXOverlay'],
self.olin.prodparameters['GGtoHadInt'] ),
( '', True, 1345 ), self )
def test_checkconsistency_nofilesinpath( self ):
self.olin.pathToOverlayFiles = '/my/path/overlay.files'
with patch('DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient.findFilesByMetadata', new=Mock(return_value=S_OK([]))) as fcc_mock:
assertDiracFailsWith( self.olin._checkConsistency(), 'no files in that path', self )
fcc_mock.assert_called_once_with( {}, '/my/path/overlay.files' )
def test_checkconsistency_no_bunchcrossings( self ):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = None
assertDiracFailsWith( self.olin._checkConsistency(), 'number of overlay bunch crossings not defined', self )
def test_checkconsistency_no_backgroundevts( self ):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 0
assertDiracFailsWith( self.olin._checkConsistency(), 'background events per bunch crossing is not defined',
self )
def test_checkconsistency_no_bgevttype( self ):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 1345
self.olin.backgroundEventType = ''
assertDiracFailsWith( self.olin._checkConsistency(), 'event type is not defined', self )
def test_checkconsistency_nosignalevtperjob( self ):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 1345
self.olin.backgroundEventType = 'gghad'
self.olin._jobtype = 'User'
self.olin.numberOfSignalEventsPerJob = 0
assertDiracFailsWith( self.olin._checkConsistency(), 'signal event per job is not defined',
self )
def test_checkfinalconsistency_simple( self ):
self.olin.pathToOverlayFiles = 'some/path'
assertDiracSucceeds( self.olin._checkFinalConsistency(), self )
def test_checkfinalconsistency_noenergy( self ):
self.olin.energy = 0
assertDiracFailsWith( self.olin._checkFinalConsistency(), 'energy must be specified', self )
def test_checkfinalconsistency_ops_fails( self ):
ops_mock = Mock()
ops_mock.getSections.return_value=S_ERROR('some_ops_error')
self.olin._ops = ops_mock
self.olin.energy = 198
assertDiracFailsWith( self.olin._checkFinalConsistency(), 'could not resolve the cs path', self )
def test_checkfinalconsistency_machinemissing( self ):
ops_mock = Mock()
ops_mock.getSections.return_value=S_OK( [ 'allowed_machine_1', 'other_mach' ] )
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
self.olin.energy = 198
assertDiracFailsWith( self.olin._checkFinalConsistency(),
'machine mytestmachineveryrare does not have overlay data', self )
def test_checkfinalconsistency_energynotfound( self ):
self.olin.energy = 824
section_dict = { '/Overlay' : [ 'myTestMachineVeryRare' ],
'/Overlay/myTestMachineVeryRare' : [ 'other_energy_tev', '824tev' ] }
ops_mock = Mock()
ops_mock.getSections.side_effect=lambda path: S_OK( section_dict[path] )
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
assertDiracFailsWith( self.olin._checkFinalConsistency(),
'no overlay files corresponding to 824gev', self )
def test_checkfinalconsistency_nodetectormodels( self ):
self.olin.energy = 1000.0
section_dict = { '/Overlay' : S_OK( [ 'myTestMachineVeryRare' ] ),
'/Overlay/myTestMachineVeryRare' : S_OK( [ 'other_energy_tev', '1tev' ] ),
'/Overlay/myTestMachineVeryRare/1tev' : S_ERROR('bla') }
ops_mock = Mock()
ops_mock.getSections.side_effect=lambda path: section_dict[path]
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
assertDiracFailsWith( self.olin._checkFinalConsistency(),
'could not find the detector models', self )
def test_checkfinalconsistency_detectormodelnotfound( self ):
self.olin.energy = 981324
section_dict = { '/Overlay' : [ 'myTestMachineVeryRare' ],
'/Overlay/myTestMachineVeryRare' : [ 'other_energy_tev', '981.3gev' ],
'/Overlay/myTestMachineVeryRare/981.3tev' : [ 'some_other_detector_model',
'neither_this' ] }
ops_mock = Mock()
ops_mock.getSections.side_effect=lambda path: S_OK( section_dict[path] )
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
assertDiracFailsWith( self.olin._checkFinalConsistency(),
'no overlay files corresponding to 981.3tev', self )
def test_checkfinalconsistency_nobkg( self ):
self.olin.detectorModel = 'testDetMod'
self.olin.energy = 981000
section_dict = { '/Overlay' : [ 'myTestMachineVeryRare' ],
'/Overlay/myTestMachineVeryRare' : [ 'other_energy_tev', '981tev' ],
'/Overlay/myTestMachineVeryRare/981tev' : [ 'testDetMod' ] }
ops_mock = Mock()
ops_mock.getSections.side_effect=lambda path: S_OK( section_dict[path] )
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
with patch.object(inspect.getmodule(OverlayInput), 'allowedBkg', new=Mock(return_value=S_ERROR('bkg_test_err'))):
assertDiracFailsWith( self.olin._checkFinalConsistency(), 'bkg_test_err', self )
def test_checkfinalconsistency_negativeprodid( self ):
self.olin.detectorModel = 'testDetMod'
self.olin.energy = 981000
section_dict = { '/Overlay' : [ 'myTestMachineVeryRare' ],
'/Overlay/myTestMachineVeryRare' : [ 'other_energy_tev', '981tev' ],
'/Overlay/myTestMachineVeryRare/981tev' : [ 'testDetMod' ] }
ops_mock = Mock()
ops_mock.getSections.side_effect=lambda path: S_OK( section_dict[path] )
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
with patch.object(inspect.getmodule(OverlayInput), 'allowedBkg', new=Mock(return_value=S_OK(-147))):
assertDiracFailsWith( self.olin._checkFinalConsistency(), 'no proper production id found', self )
def test_checkfinalconsistency( self ):
self.olin.detectorModel = 'testDetMod'
self.olin.energy = 981000
section_dict = { '/Overlay' : [ 'myTestMachineVeryRare' ],
'/Overlay/myTestMachineVeryRare' : [ 'other_energy_tev', '981tev' ],
'/Overlay/myTestMachineVeryRare/981tev' : [ 'testDetMod' ] }
ops_mock = Mock()
ops_mock.getSections.side_effect=lambda path: S_OK( section_dict[path] )
self.olin._ops = ops_mock
self.olin.machine = 'myTestMachineVeryRare'
with patch.object(inspect.getmodule(OverlayInput), 'allowedBkg', new=Mock(return_value=S_OK(13987))):
assertDiracSucceeds( self.olin._checkFinalConsistency(), self )