def getMarlin( self ):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def getMarlin(self):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def getJob(jobid, jobpara):
iser=jobid+1
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_ovl_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
return j
steeringf = "clic_ild_cdr_steering.xml"
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print res['Message']
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:" + lfn)
ma.setNbEvts(10)
ma.setEnergy(500.)
ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov)
ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit()
j.setCPUTime(86400)
j.setOutputData("myrec_overlay_%s.slcio" % ov, "some/path")
j.setName("SomeName")
j.setJobGroup("SomeGroup")
res = d.checkparams(j)
if not res['OK']:
print res['Message']
exit()
j.submit(d)
# Marlin
lcoutputDST = recOutfile + "_%s.DST.slcio"%(indx)
lcoutputREC = recOutfile + "_%s.REC.slcio"%(indx)
ma = Marlin()
ma.setVersion(ILCSoftVer)
ma.setDetectorModel(detectorModel)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments(" --constant.DetectorModel=%s "%(detectorModel) )
ma.setLogFile("marlin.log")
#ma.getInputFromApp(sim)
ma.setInputFile([lcoutputSIM])
ma.setEnergy(energy)
ma.setNumberOfEvents(evtsPerRun)
ma.setOutputDstFile(lcoutputDST)
ma.setOutputRecFile(lcoutputREC)
RECoutput.append(lcoutputDST)
RECoutput.append(lcoutputREC)
job = UserJob()
job.setName(jobname)
job.setJobGroup(jobGrName)
job.setILDConfig(ILDConfigVer)
job.setCPUTime(86400)
job.setInputData([lcinputREC])
job.setInputSandbox(["runRecoSplit_all_Tmp.py"])
job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])
#job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py ","*.root"])
#job.setDestinationCE('lyogrid07.in2p3.fr')
class MarlinTestCase( unittest.TestCase ):
""" Base class for the Marlin test cases
"""
def setUp( self ):
"""set up the objects"""
self.mar = Marlin( {} )
def test_setgear( self ):
self.mar.setGearFile( 'lfn:/my/gear/file.txt' )
self.assertFalse( self.mar._errorDict )
self.assertIn( 'lfn:/my/gear/file.txt', self.mar.inputSB )
def test_setKeepRecFile(self):
"""Tests for behaviour with KeepRecFile True/False."""
self.assertTrue(self.mar.keepRecFile)
self.mar.setKeepRecFile(False)
self.assertFalse(self.mar._errorDict)
self.mar.setKeepRecFile(True)
self.assertTrue(self.mar.keepRecFile)
self.assertFalse(self.mar._errorDict)
self.mar.setKeepRecFile(123)
self.assertIn('val = 123', str(self.mar._errorDict))
def test_setoutputrec( self ):
self.mar.setOutputRecFile( 'my/file.outputrec', 'mytestPath' )
assertEqualsImproved( self.mar.outputRecPath, 'mytestPath', self )
self.assertFalse( self.mar._errorDict )
def test_setoutputdst( self ):
self.mar.setOutputDstFile( 'my/file.outputdst', 'mytestPath' )
assertEqualsImproved( self.mar.outputDstPath, 'mytestPath', self )
self.assertFalse( self.mar._errorDict )
def test_setproclist( self ):
self.mar.setProcessorsToUse( [ 'proc1', 'proc2' ] )
self.assertFalse( self.mar._errorDict )
def test_setexcludeproclist( self ):
self.mar.setProcessorsToExclude( [ 'proc1', 'proc2' ] )
self.assertFalse( self.mar._errorDict )
def test_userjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.mar._userjobmodules( module_mock ), self )
def test_prodjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )
def test_prodjobmodules_outputpath( self ):
module_mock = Mock()
self.mar.outputPath = 'aef'
assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )
self.assertIn( { 'OutputFile' : '@{OutputFile}', 'outputPath' : '@{OutputPath}',
'outputDataSE' : '@{OutputSE}'}, self.mar._listofoutput )
def test_userjobmodules_fails( self ):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith( self.mar._userjobmodules( None ),
'userjobmodules failed', self )
def test_prodjobmodules_fails( self ):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith( self.mar._prodjobmodules( None ),
'prodjobmodules failed', self )
def test_checkconsistency( self ):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = None
self.mar._jobtype = 'notUser'
self.mar.outputFile = None
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
assertDiracSucceeds( self.mar._checkConsistency(), self )
self.assertIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
self.assertIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
self.assertIn( keyword, self.mar.prodparameters )
assertEqualsImproved( self.mar.gearFile, None, self )
def test_checkconsistency_noversion( self ):
self.mar.version = None
assertDiracFailsWith( self.mar._checkConsistency(), 'version not set', self )
def test_checkconsistency_invalidxml( self ):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
assertDiracFailsWith( self.mar._checkConsistency(), 'supplied steering file cannot be read with xml', self)
def test_checkconsistency_othercase( self ):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'notUser'
self.mar.outputFile = 'myoutput.test'
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds( self.mar._checkConsistency(), self )
self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
self.assertIn( keyword, self.mar.prodparameters )
assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )
def test_checkconsistency_lastcase( self ):
self.mar.version = '13'
self.mar.steeringFile = None
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'User'
self.mar.outputFile = 'myoutput.test'
assertDiracSucceeds( self.mar._checkConsistency(), self )
self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
self.assertNotIn( keyword, self.mar.prodparameters )
assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )
def test_resolvelinkedparams(self):
"""Test _resolveLinkedStepParameters with something happening."""
step_mock = Mock()
input_mock = Mock()
input_mock.getType.return_value = {'abc': False}
self.mar._linkedidx = 3
self.mar._jobsteps = [None, None, None, input_mock]
assertDiracSucceeds(self.mar._resolveLinkedStepParameters(step_mock), self)
step_mock.setLink.assert_called_once_with('InputFile', {'abc': False}, 'OutputFile')
def test_resolvelinkedparams_noinputstep(self):
"""Call _resolveLinkedStep function, which does nothing."""
self.mar._linkedidx = None
self.mar._inputappstep = []
assertDiracSucceeds(self.mar._resolveLinkedStepParameters(None), self)
def getJob(dirac, jobid, jobpara):
iser=jobid+100
outdir = "/ilc/user/a/amiyamoto/myprod2/test/"
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
dstlfn = outdir+"dst/"+outdst
reclfn = outdir+"rec/"+outrec
outsrm = "CERN-SRM"
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
# Upload files to different directories
upload_script="upload%i.sh"%iser
upload = GenericApplication()
# Create a script to upload files.
shfile = open(upload_script,"w")
shfile.write("#!/bin/bash\n")
shfile.write("/bin/ls -l \n")
shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n")
shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n")
shfile.close()
os.chmod(upload_script,0755)
upload.setScript(upload_script)
res = j.append(upload)
if not res['OK'] :
print res['Message']
exit(1)
# j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setInputSandbox([ setting_file, upload_script ] )
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
res = j.submit(dirac)
if not res["OK"] :
print "Failed submit job, jobid=%s" %jobid
print res
os.remove(upload_script)
return j
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)
ma = Marlin()
ma.setDebug()
# ma.setLogLevel("verbose")
# ma.setILDConfig("v01-16-p05_500")
ma.setVersion("v01-16-02")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile(gearfile)
ma.setInputFile([simfile, pandoraLikelihoodData, bg_aver])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh"])
j.dontPromptMe()
j.submit(d)
class MarlinTestCase(unittest.TestCase):
""" Base class for the Marlin test cases
"""
def setUp(self):
"""set up the objects"""
self.mar = Marlin({})
def test_setgear(self):
self.mar.setGearFile('lfn:/my/gear/file.txt')
self.assertFalse(self.mar._errorDict)
self.assertIn('lfn:/my/gear/file.txt', self.mar.inputSB)
def test_setoutputrec(self):
self.mar.setOutputRecFile('my/file.outputrec', 'mytestPath')
assertEqualsImproved(self.mar.outputRecPath, 'mytestPath', self)
self.assertFalse(self.mar._errorDict)
def test_setoutputdst(self):
self.mar.setOutputDstFile('my/file.outputdst', 'mytestPath')
assertEqualsImproved(self.mar.outputDstPath, 'mytestPath', self)
self.assertFalse(self.mar._errorDict)
def test_setproclist(self):
self.mar.setProcessorsToUse(['proc1', 'proc2'])
self.assertFalse(self.mar._errorDict)
def test_setexcludeproclist(self):
self.mar.setProcessorsToExclude(['proc1', 'proc2'])
self.assertFalse(self.mar._errorDict)
def test_userjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.mar._userjobmodules(module_mock), self)
def test_prodjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)
def test_prodjobmodules_outputpath(self):
module_mock = Mock()
self.mar.outputPath = 'aef'
assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)
self.assertIn(
{
'OutputFile': '@{OutputFile}',
'outputPath': '@{OutputPath}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
def test_userjobmodules_fails(self):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith(self.mar._userjobmodules(None),
'userjobmodules failed', self)
def test_prodjobmodules_fails(self):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith(self.mar._prodjobmodules(None),
'prodjobmodules failed', self)
def test_checkconsistency(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = None
self.mar._jobtype = 'notUser'
self.mar.outputFile = None
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, None, self)
def test_checkconsistency_noversion(self):
self.mar.version = None
assertDiracFailsWith(self.mar._checkConsistency(), 'version not set',
self)
def test_checkconsistency_invalidxml(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
assertDiracFailsWith(
self.mar._checkConsistency(),
'supplied steering file cannot be read with xml', self)
def test_checkconsistency_othercase(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'notUser'
self.mar.outputFile = 'myoutput.test'
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertNotIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertNotIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)
def test_checkconsistency_lastcase(self):
self.mar.version = '13'
self.mar.steeringFile = None
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'User'
self.mar.outputFile = 'myoutput.test'
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertNotIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertNotIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertNotIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)
mo = Mokka()
mo.setEnergy(3000)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setMacFile("particlegun_electron.mac")
mo.setOutputFile("MyFile.slcio")
mo.setNbEvts(n_evts_per_job)
res = j.append(mo)
if not res['OK']:
print res['Message']
break
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.getInputFromApp(mo)
ma.setOutputDstFile("mydst_no_ov_%s.slcio" % i)
res = j.append(ma)
if not res['OK']:
print res['Message']
break
ov = OverlayInput()
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(n_evts_per_job)
ov.setBkgEvtType("gghad")
ov.setDetectorModel("CLIC_ILD_CDR")
res = j.append(ov)
if not res['OK']:
print res['Message']
break
def subOverlay():
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = _clip.isLocal
nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
nbevts = 0 # To analize all input events
outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix
outputDir = _clip.outputDir
inputFile = _clip.inputFile
if inputFile == "":
gLogger.error("Input file for ddsim does not given.")
exit(-1)
recfile = outputFilePrefix + ".rec.slcio"
dstfile = outputFilePrefix + ".dst.slcio"
detector_model = "ILD_l5_o1_v02"
key = detector_model.split('_')
sim_detectorModel = "_".join([key[0], key[1], key[3]])
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myoverlayjob")
job.setName("myoverlay")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
# job.setBannedSites([]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
# Create Overlay application
ovldata = [{
"ProcessorName": "BgOverlayWW",
"evttype": "aa_lowpt_WW",
"ProdID": 10237,
"expBG": 0.211,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayWB",
"evttype": "aa_lowpt_WB",
"ProdID": 10241,
"expBG": 0.24605,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBW",
"evttype": "aa_lowpt_BW",
"ProdID": 10239,
"expBG": 0.243873,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBB",
"evttype": "aa_lowpt_BB",
"ProdID": 10235,
"expBG": 0.35063,
"subdir": "000"
}, {
"ProcessorName": "PairBgOverlay",
"evttype": "seeablepairs",
"ProdID": 10233,
"expBG": 1.0,
"subdir": "100"
}]
BXOverlay = 1
NbSigEvtsPerJob = 100
numberOfSignalEvents = NbSigEvtsPerJob
basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws"
energy = "500"
for ovl in ovldata:
print "### OverlayInput ... " + ovl["ProcessorName"]
ovlapp = OverlayInput()
ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \
( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] )
print " OverlayPath ... " + ovlpath
ovlapp.setMachine("ilc_dbd")
# ovlapp.setEnergy(energy)
# ovlapp.setDetectorModel(sim_detectorModel)
ovlapp.setProcessorName(ovl["ProcessorName"])
ovlapp.setBkgEvtType(ovl["evttype"])
ovlapp.setPathToFiles(ovlpath)
ovlapp.setGGToHadInt(ovl["expBG"])
ovlapp.setBXOverlay(BXOverlay)
ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob)
ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents)
res = job.append(ovlapp)
if not res['OK']:
print res['Message']
exit(1)
# Create Marlin application
marlin = Marlin()
marlin.setVersion("ILCSoft-02-00-02_gcc49")
marlin.setDetectorModel(detector_model)
marlin.setSteeringFile("MarlinStdReco.xml")
marlin.setInputFile(inputFile)
marlin.setNumberOfEvents(nbevts)
marlin.setOutputDstFile(dstfile)
marlin.setOutputRecFile(recfile)
extraCLIArguments = " --constant.DetectorModel=%s " % detector_model
extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str(
energy)
extraCLIArguments += " --global.Verbosity=MESSAGE "
marlin.setExtraCLIArguments(extraCLIArguments)
job.append(marlin)
if outputDir != "":
job.setOutputData([dstfile, recfile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
mo = Mokka()
mo.setEnergy(3000)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setMacFile("particlegun_electron.mac")
mo.setOutputFile("MyFile.slcio")
mo.setNbEvts(n_evts_per_job)
res = j.append(mo)
if not res['OK']:
print res['Message']
break
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.getInputFromApp(mo)
ma.setOutputDstFile("mydst_no_ov_%s.slcio"%i)
res = j.append(ma)
if not res['OK']:
print res['Message']
break
ov = OverlayInput()
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(n_evts_per_job)
ov.setDetectorType("ILD")
res = j.append(ov)
if not res['OK']:
print res['Message']
break
ma2 = Marlin()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
d= DiracILC(True,"repo.rep")
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("example")#%i)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.setInputFile("LFN:/ilc/prod/clic/3tev/gghad/ILD/SIM/00000187/000/gghad_sim_187_97.slcio")
outdst = "toto.dst.slcio" #% i
outrec = "toto.rec.slcio" #% i
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"some/path","KEK-SRM")
j.setOutputSandbox("*.log")
j.dontPromptMe()
j.submit(d)
steeringf = "clic_ild_cdr_steering.xml"
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print(res['Message'])
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:"+lfn)
ma.setNbEvts(10)
ma.setEnergy(500.)
ma.setOutputRecFile("myrec_overlay_%s.slcio"%ov)
ma.setOutputDstFile("mydst_overlay_%s.slcio"%ov)
res =j.append(ma)
if not res['OK']:
print(res['Message'])
exit()
j.setCPUTime(86400)
j.setOutputData("myrec_overlay_%s.slcio"%ov,"some/path")
j.setName("SomeName")
j.setJobGroup("SomeGroup")
res = d.checkparams(j)
if not res['OK']:
print(res['Message'])
exit()
j.submit(d)
def main(argv):
# Input arguments
ildconfig_version = "$ILDCONFIGVER"
ilcsoft_version = "$ILCSOFTVER"
evts_per_run = $EVTSPERRUN
detector_model = "$DETECTOR"
sim_input = "$SIMINPUT"
process_name = "$PROCESS"
index = $IND
sim_input = diracpath_from_pnfspath( sim_input )
sim_detector_model = detector_model_wo_option( detector_model )
job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model
dirac = DiracILC(True,job_group+".rep")
# outputs to be saved onto grid SE
RECoutput = []
# DDSim
evtStart = (index-1)*evts_per_run
evtEnd = index*evts_per_run - 1
RandSeed = random.randrange(11623, 99999)
lcinputSIM = "LFN:" + sim_input
lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd))
sim = DDSim()
sim.setVersion(ilcsoft_version)
sim.setDetectorModel(sim_detector_model)
sim.setInputFile(lcinputSIM)
sim.setSteeringFile("ddsim_steer.py")
sim.setNumberOfEvents(evts_per_run)
sim.setRandomSeed(RandSeed)
sim.setEnergy(1000)
sim.setStartFrom(evtStart)
sim.setOutputFile(lcoutputSIM)
# Marlin
lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd))
ma = Marlin()
ma.setVersion(ilcsoft_version)
ma.setDetectorModel(detector_model)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) )
ma.setLogFile("marlin.log")
ma.getInputFromApp(sim)
ma.setEnergy(1000)
ma.setOutputDstFile(lcoutputDST)
RECoutput.append(lcoutputDST)
# ILCDirac user job
job = UserJob()
job.setName("user_sim_reco")
job.setJobGroup(job_group)
job.setILDConfig(ildconfig_version)
job.setCPUTime(86400)
tmp_file_name = process_name + "_sim_reco_job_tmp.py"
job.setInputSandbox([tmp_file_name])
job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])
job.dontPromptMe()
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])
# run simulation job
simres = job.append(sim)
if not simres['OK']:
print simres['Not ok appending ddsim to job']
quit()
# run Malrin reco jobs
mares = job.append(ma)
if not mares['OK']:
print mares['Not ok appending Marlin to job']
quit()
job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM")
print RECoutput
submit_output = job.submit(dirac)
print submit_output
