def getLCSIM(self, prepandora=True):
""" Get some LCSIM
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM
lcsim = LCSIM()
lcsim.setVersion('CLIC_CDR')
lcsim.setDetectorModel('clic_sid_cdr.zip')
if prepandora:
lcsim.setSteeringFile(self.lcsimPreSteeringFile)
lcsim.setOutputFile("testlcsim.slcio")
else:
lcsim.setSteeringFile(self.lcsimPostSteeringFile)
#lcsim.setOutputFile("testlcsimfinal.slcio")
lcsim.setOutputDstFile("testlcsimDST.slcio")
lcsim.setOutputRecFile("testlcsimREC.slcio")
lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
return lcsim
def getLCSIM(self, prepandora = True):
""" Get some LCSIM
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM
lcsim = LCSIM()
lcsim.setVersion('CLIC_CDR')
lcsim.setDetectorModel('clic_sid_cdr.zip')
if prepandora:
lcsim.setSteeringFile(self.lcsimPreSteeringFile)
lcsim.setOutputFile("testlcsim.slcio")
else:
lcsim.setSteeringFile(self.lcsimPostSteeringFile)
#lcsim.setOutputFile("testlcsimfinal.slcio")
lcsim.setOutputDstFile("testlcsimDST.slcio")
lcsim.setOutputRecFile("testlcsimREC.slcio")
lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
return lcsim
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions)
slicOutput=outputFiles['slicOutput']
prePandoraOutput=outputFiles['prePandoraOutput']
pandoraOutput=outputFiles['pandoraOutput']
vertexingOutput=outputFiles['vertexingOutput']
lcsimRecOutput=outputFiles['lcsimRecOutput']
lcsimDstOutput=outputFiles['lcsimDstOutput']
flavortagOutput=outputFiles['flavortagOutput']
diracOutput=outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us',])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
"0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True,
setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent / nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector,
fileNumber, args.outputPath,
softVersions)
slicOutput = outputFiles['slicOutput']
prePandoraOutput = outputFiles['prePandoraOutput']
pandoraOutput = outputFiles['pandoraOutput']
vertexingOutput = outputFiles['vertexingOutput']
lcsimRecOutput = outputFiles['lcsimRecOutput']
lcsimDstOutput = outputFiles['lcsimDstOutput']
flavortagOutput = outputFiles['flavortagOutput']
diracOutput = outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(
'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
) # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy(
'steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites([
'LCG.IN2P3-CC.fr',
'LCG.RAL-LCG2.uk',
'LCG.DESY-HH.de',
'LCG.DESYZN.de',
'LCG.KEK.jp',
'OSG.PNNL.us',
])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
print args.chain[0]
# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]
check_events_arguments(args.events, args.split)
detector = args.detector
alias_properties(detector)
outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
inputSandbox, outputSandbox = setup_sandboxes(args.macFile)
dirac = DiracILC(True, repoName)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(outputBase)
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ---> ", fileNumber
################## SLIC ##################################################
if 1 in args.chain:
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
slic.setDetectorModel(detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Overlay ###############################################
if 2 in args.chain:
'''
#Add the gghad background overlay.
gghad = OverlayInput()
#gghad.setProdID(1767)
gghad.setEnergy(500.0)
gghad.setBXOverlay('args.bunches')
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setMachine('ilc_dbd')
gghad.setDetectorModel('sidloi3')
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
#Add the pair background overlay.
pairs = OverlayInput()
pairs.setProdID(2)
pairs.setEnergy(500.0)
pairs.setBXOverlay('args.bunches')
pairs.setGGToHadInt(1.)
pairs.setNbSigEvtsPerJob(nOutputEvents)
pairs.setMachine('ilc_dbd')
pairs.setDetectorModel('sidloi3')
pairs.setBkgEvtType('eepairs')
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
gghad = OverlayInput()
gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
gghad.setBXOverlay(int(args.bunches))
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
pairs = OverlayInput()
pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
pairs.setBXOverlay(int(args.bunches))
pairs.setGGToHadInt(1.)
pairs.setBkgEvtType('eepairs')
pairs.setNbSigEvtsPerJob(nOutputEvents)
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
################## lcsim (digitization and tracking) #####################
if 3 in args.chain:
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
if 1 in args.chain:
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
lcsim.setAliasProperties('steeringFiles/alias.properties')
lcsim.setDetectorModel(detector+".zip")
#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
if 4 in args.chain:
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
if 5 in args.chain:
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/' + detector + '.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
lcsimDst.setAliasProperties('steeringFiles/alias.properties')
lcsimDst.setDetectorModel(detector+".zip")
lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if 6 in args.chain:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/' + detector + '.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
outputLevel = max(args.chain)
if outputLevel == 1:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 3:
#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 4:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 5:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 6:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')
job.setOutputSandbox(outputSandbox)
job.setInputData(args.Input)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
class LCSIMTestCase(unittest.TestCase):
""" Base class for the LCSIM test cases
"""
def setUp(self):
"""set up the objects"""
self.lcs = LCSIM()
def test_setoutputrecfile(self):
self.assertFalse(self.lcs._errorDict)
self.lcs.setOutputRecFile('myTestRECfile.rec', 'test/path/rec')
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.outputRecFile, 'myTestRECfile.rec', self)
assertEqualsImproved(self.lcs.outputRecPath, 'test/path/rec', self)
def test_setoutputrecfile_argcheck_fails(self):
self.assertFalse(self.lcs._errorDict)
self.lcs.setOutputRecFile(123)
self.assertTrue(self.lcs._errorDict)
def test_setoutputdstfile(self):
self.assertFalse(self.lcs._errorDict)
self.lcs.setOutputDstFile('myTestDSTfile.dst', 'test/path/dst')
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.outputDstFile, 'myTestDSTfile.dst', self)
assertEqualsImproved(self.lcs.outputDstPath, 'test/path/dst', self)
def test_setoutputdstfile_argcheck_fails(self):
self.assertFalse(self.lcs._errorDict)
self.lcs.setOutputDstFile(123)
self.assertTrue(self.lcs._errorDict)
def test_setaliasproperties(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setAliasProperties('myAlias.Properties.Test')
assertEqualsImproved(self.lcs.aliasProperties,
'myAlias.Properties.Test', self)
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB, [], self)
def test_setaliasproperties_argcheck_fails(self):
with patch('os.path.exists', new=Mock(return_value=True)):
self.lcs.setAliasProperties(8934)
assertEqualsImproved(self.lcs.aliasProperties, 8934, self)
self.assertTrue(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB, [8934], self)
def test_setaliasproperties_lfn(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setAliasProperties('LFN:/myAlias.Properties.Test')
assertEqualsImproved(self.lcs.aliasProperties,
'LFN:/myAlias.Properties.Test', self)
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB,
['LFN:/myAlias.Properties.Test'], self)
def test_setdetectormodel(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setDetectorModel('Test_Detectorv103.clic')
assertEqualsImproved(self.lcs.detectorModel, 'Test_Detectorv103.clic',
self)
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB, [], self)
def test_setdetectormodel_argcheck_fails(self):
with patch('os.path.exists', new=Mock(return_value=True)):
self.lcs.setDetectorModel(2489)
assertEqualsImproved(self.lcs.detectorModel, 2489, self)
self.assertTrue(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB, [2489], self)
def test_setdetectormodel_lfn(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setDetectorModel(
'LFN:/dir/other_dir/DetectorTestmeModel.ilc')
assertEqualsImproved(self.lcs.detectorModel,
'LFN:/dir/other_dir/DetectorTestmeModel.ilc',
self)
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB,
['LFN:/dir/other_dir/DetectorTestmeModel.ilc'],
self)
def test_settrackingstrategy(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setTrackingStrategy('Test_Strategy.trackme')
assertEqualsImproved(self.lcs.trackingStrategy,
'Test_Strategy.trackme', self)
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB, [], self)
def test_settrackingstrategy_argcheck_fails(self):
with patch('os.path.exists', new=Mock(return_value=True)):
self.lcs.setTrackingStrategy(4812)
assertEqualsImproved(self.lcs.trackingStrategy, 4812, self)
self.assertTrue(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB, [4812], self)
def test_settrackingstrategy_lfn(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setTrackingStrategy('LFN:/my/track/strat.txt')
assertEqualsImproved(self.lcs.trackingStrategy,
'LFN:/my/track/strat.txt', self)
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.inputSB, ['LFN:/my/track/strat.txt'],
self)
def test_setextraparams(self):
self.assertFalse(self.lcs.willBeCut)
self.assertFalse(self.lcs._errorDict)
self.lcs.setExtraParams('myTestPar')
self.lcs.willRunSLICPandora()
self.assertFalse(self.lcs._errorDict)
assertEqualsImproved(self.lcs.extraParams, 'myTestPar', self)
self.assertTrue(self.lcs.willBeCut)
def test_setextraparams_argcheck_fails(self):
self.assertFalse(self.lcs._errorDict)
self.lcs.setExtraParams([8914])
self.assertTrue(self.lcs._errorDict)
def test_userjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.lcs._userjobmodules(module_mock), self)
def test_prodjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.lcs._prodjobmodules(module_mock), self)
def test_userjobmodules_fails(self):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith(self.lcs._userjobmodules(None),
'userjobmodules failed', self)
def test_prodjobmodules_fails(self):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith(self.lcs._prodjobmodules(None),
'prodjobmodules failed', self)
def test_checkwfconsistency(self):
assertDiracSucceeds(self.lcs._checkWorkflowConsistency(), self)
def test_resolveparameters(self):
step_mock = Mock()
assertDiracSucceeds(self.lcs._resolveLinkedStepParameters(step_mock()),
self)
def test_resolveparameters_setlink(self):
step_mock = Mock()
self.lcs._linkedidx = 1
self.lcs._jobsteps = [None, Mock()]
assertDiracSucceeds(self.lcs._resolveLinkedStepParameters(step_mock()),
self)
def test_checkconsistency(self):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ False, True, False ])) as exists_mock, \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())), \
patch.object(inspect.getmodule(LCSIM), 'checkXMLValidity', new=Mock(return_value=S_OK())) as xml_mock:
assertDiracFailsWith(self.lcs._checkConsistency('myTestJob'),
'you have to pass an existing .zip file',
self)
assertMockCalls(
exists_mock,
['myTestSteerFile.txt', 'myTestSteerFile.txt', 'myTestStrat'],
self)
xml_mock.assert_called_once_with('myTestSteerFile.txt')
def test_checkconsistency_noversion(self):
assertDiracFailsWith(self.lcs._checkConsistency(), 'no version found',
self)
def test_checkconsistency_basic(self):
self.lcs.energy = 2489
self.lcs.numberOfEvents = 8245
self.lcs.version = 'v1.2'
self.lcs.steeringFile = ''
self.lcs.detectorModel = ''
self.lcs._jobtype = 'User'
assertDiracSucceeds(self.lcs._checkConsistency(), self)
def test_checkconsistency_steering_Exists_fails(self):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_ERROR('Exists_test_err'))):
assertDiracFailsWith(self.lcs._checkConsistency('myTestJob'),
'exists_test_err', self)
def test_checkconsistency_invalidxml(self):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ True, True ])), \
patch.object(inspect.getmodule(LCSIM), 'checkXMLValidity', new=Mock(return_value=S_ERROR('xmlcheck_failed_testme'))):
assertDiracFailsWith(
self.lcs._checkConsistency('myTestJob'),
'supplied steering file cannot be'
' read by xml parser: xmlcheck_failed_testme', self)
def test_checkconsistency_tracking_Exists_fails(self):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ True, False, False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_ERROR('xmlcheck_failed_testme'))):
assertDiracFailsWith(self.lcs._checkConsistency('myTestJob'),
'xmlcheck_failed_testme', self)
def test_checkconsistency_othercase(self):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = ''
self.lcs.trackingStrategy = 'lfn:/myTestStrat'
self.lcs.detectorModel = 'correctDetector.zip'
self.lcs._jobtype = 'notUser'
self.lcs._listofoutput = []
app1 = Mock()
app1.appame = 'myTestApp'
app2 = Mock()
app2.appname = 'marlin'
self.lcs._inputapp = [app1, app2]
self.lcs.outputFile = ''
self.lcs.willBeCut = False
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.lcs._checkConsistency('myTestJob'), self)
expected_output_list = [{
"outputFile": "@{outputREC}",
"outputPath": "@{outputPathREC}",
"outputDataSE": "@{OutputSE}"
}, {
"outputFile": "@{outputDST}",
"outputPath": "@{outputPathDST}",
"outputDataSE": "@{OutputSE}"
}]
assertEqualsImproved(self.lcs._listofoutput, expected_output_list,
self)
prodparam_dict = self.lcs.prodparameters
assertEqualsImproved((prodparam_dict['detectorType'],
prodparam_dict['lcsim_steeringfile'],
prodparam_dict['lcsim_trackingstrategy']),
('SID', '', 'lfn:/myTestStrat'), self)
class LCSIMTestCase( unittest.TestCase ):
""" Base class for the LCSIM test cases
"""
def setUp(self):
"""set up the objects"""
self.lcs = LCSIM()
def test_setoutputrecfile( self ):
self.assertFalse( self.lcs._errorDict )
self.lcs.setOutputRecFile( 'myTestRECfile.rec', 'test/path/rec' )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.outputRecFile, 'myTestRECfile.rec', self )
assertEqualsImproved( self.lcs.outputRecPath, 'test/path/rec', self )
def test_setoutputrecfile_argcheck_fails( self ):
self.assertFalse( self.lcs._errorDict )
self.lcs.setOutputRecFile( 123 )
self.assertTrue( self.lcs._errorDict )
def test_setoutputdstfile( self ):
self.assertFalse( self.lcs._errorDict )
self.lcs.setOutputDstFile( 'myTestDSTfile.dst', 'test/path/dst' )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.outputDstFile, 'myTestDSTfile.dst', self )
assertEqualsImproved( self.lcs.outputDstPath, 'test/path/dst', self )
def test_setoutputdstfile_argcheck_fails( self ):
self.assertFalse( self.lcs._errorDict )
self.lcs.setOutputDstFile( 123 )
self.assertTrue( self.lcs._errorDict )
def test_setaliasproperties( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setAliasProperties( 'myAlias.Properties.Test' )
assertEqualsImproved( self.lcs.aliasProperties, 'myAlias.Properties.Test', self )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [], self )
def test_setaliasproperties_argcheck_fails( self ):
with patch('os.path.exists', new=Mock(return_value=True)):
self.lcs.setAliasProperties( 8934 )
assertEqualsImproved( self.lcs.aliasProperties, 8934, self )
self.assertTrue( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [ 8934 ], self )
def test_setaliasproperties_lfn( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setAliasProperties( 'LFN:/myAlias.Properties.Test' )
assertEqualsImproved( self.lcs.aliasProperties, 'LFN:/myAlias.Properties.Test', self )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [ 'LFN:/myAlias.Properties.Test' ], self )
def test_setdetectormodel( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setDetectorModel( 'Test_Detectorv103.clic' )
assertEqualsImproved( self.lcs.detectorModel, 'Test_Detectorv103.clic', self )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [], self )
def test_setdetectormodel_argcheck_fails( self ):
with patch('os.path.exists', new=Mock(return_value=True)):
self.lcs.setDetectorModel( 2489 )
assertEqualsImproved( self.lcs.detectorModel, 2489, self )
self.assertTrue( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [ 2489 ], self )
def test_setdetectormodel_lfn( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setDetectorModel( 'LFN:/dir/other_dir/DetectorTestmeModel.ilc' )
assertEqualsImproved( self.lcs.detectorModel, 'LFN:/dir/other_dir/DetectorTestmeModel.ilc', self )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [ 'LFN:/dir/other_dir/DetectorTestmeModel.ilc' ], self )
def test_settrackingstrategy( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setTrackingStrategy( 'Test_Strategy.trackme' )
assertEqualsImproved( self.lcs.trackingStrategy, 'Test_Strategy.trackme', self )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [], self )
def test_settrackingstrategy_argcheck_fails( self ):
with patch('os.path.exists', new=Mock(return_value=True)):
self.lcs.setTrackingStrategy( 4812 )
assertEqualsImproved( self.lcs.trackingStrategy, 4812, self )
self.assertTrue( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [ 4812 ], self )
def test_settrackingstrategy_lfn( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.lcs.setTrackingStrategy( 'LFN:/my/track/strat.txt' )
assertEqualsImproved( self.lcs.trackingStrategy, 'LFN:/my/track/strat.txt', self )
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.inputSB, [ 'LFN:/my/track/strat.txt' ], self )
def test_setextraparams( self ):
self.assertFalse( self.lcs.willBeCut )
self.assertFalse( self.lcs._errorDict )
self.lcs.setExtraParams( 'myTestPar' )
self.lcs.willRunSLICPandora()
self.assertFalse( self.lcs._errorDict )
assertEqualsImproved( self.lcs.extraParams, 'myTestPar', self )
self.assertTrue( self.lcs.willBeCut )
def test_setextraparams_argcheck_fails( self ):
self.assertFalse( self.lcs._errorDict )
self.lcs.setExtraParams( [ 8914 ] )
self.assertTrue( self.lcs._errorDict )
def test_userjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.lcs._userjobmodules( module_mock ), self )
def test_prodjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.lcs._prodjobmodules( module_mock ), self )
def test_userjobmodules_fails( self ):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith( self.lcs._userjobmodules( None ),
'userjobmodules failed', self )
def test_prodjobmodules_fails( self ):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith( self.lcs._prodjobmodules( None ),
'prodjobmodules failed', self )
def test_checkwfconsistency( self ):
assertDiracSucceeds( self.lcs._checkWorkflowConsistency(), self )
def test_resolveparameters( self ):
step_mock = Mock()
assertDiracSucceeds( self.lcs._resolveLinkedStepParameters( step_mock() ), self )
def test_resolveparameters_setlink( self ):
step_mock = Mock()
self.lcs._linkedidx = 1
self.lcs._jobsteps = [ None, Mock() ]
assertDiracSucceeds( self.lcs._resolveLinkedStepParameters( step_mock() ), self )
def test_checkconsistency( self ):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ False, True, False ])) as exists_mock, \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())), \
patch.object(inspect.getmodule(LCSIM), 'checkXMLValidity', new=Mock(return_value=S_OK())) as xml_mock:
assertDiracFailsWith( self.lcs._checkConsistency( 'myTestJob' ),
'you have to pass an existing .zip file', self )
assertMockCalls( exists_mock, [ 'myTestSteerFile.txt', 'myTestSteerFile.txt', 'myTestStrat' ], self )
xml_mock.assert_called_once_with( 'myTestSteerFile.txt' )
def test_checkconsistency_noversion( self ):
assertDiracFailsWith( self.lcs._checkConsistency(), 'no version found', self )
def test_checkconsistency_basic( self ):
self.lcs.energy = 2489
self.lcs.numberOfEvents = 8245
self.lcs.version = 'v1.2'
self.lcs.steeringFile = ''
self.lcs.detectorModel = ''
self.lcs._jobtype = 'User'
assertDiracSucceeds( self.lcs._checkConsistency(), self )
def test_checkconsistency_steering_Exists_fails( self ):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_ERROR('Exists_test_err'))):
assertDiracFailsWith( self.lcs._checkConsistency( 'myTestJob' ), 'exists_test_err', self )
def test_checkconsistency_invalidxml( self ):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ True, True ])), \
patch.object(inspect.getmodule(LCSIM), 'checkXMLValidity', new=Mock(return_value=S_ERROR('xmlcheck_failed_testme'))):
assertDiracFailsWith( self.lcs._checkConsistency( 'myTestJob' ), 'supplied steering file cannot be'
' read by xml parser: xmlcheck_failed_testme', self )
def test_checkconsistency_tracking_Exists_fails( self ):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = 'myTestSteerFile.txt'
self.lcs.trackingStrategy = 'myTestStrat'
self.lcs.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ True, False, False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_ERROR('xmlcheck_failed_testme'))):
assertDiracFailsWith( self.lcs._checkConsistency( 'myTestJob' ), 'xmlcheck_failed_testme', self )
def test_checkconsistency_othercase( self ):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = ''
self.lcs.trackingStrategy = 'lfn:/myTestStrat'
self.lcs.detectorModel = 'correctDetector.zip'
self.lcs._jobtype = 'notUser'
self.lcs._listofoutput = []
app1 = Mock()
app1.appame = 'myTestApp'
app2 = Mock()
app2.appname = 'marlin'
self.lcs._inputapp = [ app1, app2 ]
self.lcs.outputFile = ''
self.lcs.willBeCut = False
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())):
assertDiracSucceeds( self.lcs._checkConsistency( 'myTestJob' ), self )
expected_output_list = [ { "outputFile" : "@{outputREC}", "outputPath" : "@{outputPathREC}",
"outputDataSE" : "@{OutputSE}" }, { "outputFile" : "@{outputDST}",
"outputPath" : "@{outputPathDST}",
"outputDataSE" : "@{OutputSE}" } ]
assertEqualsImproved( self.lcs._listofoutput, expected_output_list, self )
prodparam_dict = self.lcs.prodparameters
assertEqualsImproved( ( prodparam_dict['detectorType'], prodparam_dict['lcsim_steeringfile'],
prodparam_dict['lcsim_trackingstrategy'] ), ( 'SID' , '', 'lfn:/myTestStrat' ),
self )