def getMarlin( self ):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def getMarlin(self):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def create_job(inputData, saveName, outputDir, dontPromptMe):
slcioFile = saveName + '.slcio'
rootFile = saveName + '.root'
if check_file_existence(outputDir, slcioFile, dontPromptMe):
remove_file(outputDir, slcioFile, dontPromptMe)
if check_file_existence(outputDir, rootFile, dontPromptMe):
remove_file(outputDir, rootFile, dontPromptMe)
dIlc = DiracILC()
job = UserJob()
job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml'])
if SAVE_SLCIO:
job.setOutputData([slcioFile, rootFile],
OutputPath=outputDir,
OutputSE=STORAGE_SE)
else:
job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE)
job.setJobGroup('myMarlinRun1')
job.setName('MyMarlinJob1')
# job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk'])
job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz'])
marl = Marlin()
marl.setVersion('ILCSoft-2017-11-15_gcc62')
marl.setInputFile(inputData)
job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData)))
# marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio'])
marl.setSteeringFile('marlin/FullChain.xml')
# marl.setSteeringFile('marlin/FullChainNewDetModel.xml')
marl.setOutputFile(slcioFile)
gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear'
if not os.path.isfile(gearFile):
print('Error: gear file', gearFile,
'does not exist! Abort submission.')
return
marl.setGearFile(gearFile)
marl.setExtraCLIArguments(
"--MyNtupleMaker.OutputFileName={rootOutfile}".format(
rootOutfile=rootFile))
# marl.setNumberOfEvents(1000)
job.append(marl)
if dontPromptMe:
job.dontPromptMe()
job.submit(dIlc)
return False
def defMarlin(outputFile, rootFile, **kwargs):
'''Defines Marlin job'''
##Jet Settings
jetAlgo = kwargs.pop('jetAlgo', 'ValenciaPlugin 1.5 1 1')
jetCluster = kwargs.pop('jetCluster', 'ExclusiveNJets 2')
jetRecomb = kwargs.pop('jetRecomb', 'E_scheme')
##Top Tagger Settings
deltaR = kwargs.pop('deltaR', 0.01)
deltaP = kwargs.pop('deltaP', 0.01)
cos_theta_W_max = kwargs.pop('cos_theta_W_max', 1.0)
##Sixfermion sample
sixFermionSample = kwargs.pop('sixFermionSample', True)
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
ma = Marlin()
ma.setVersion('ILCSoft-2017-05-30_gcc62')
ma.setSteeringFile(
"/home/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/topasymmetry_template_new.xml"
)
ma.setGearFile("/home/lstroem/clicdp/gear/clic_ild_cdr.gear")
ma.setOutputFile(outputFile, rootFile)
ma.setExtraCLIArguments(
"--MyTopTaggerProcessorJ2.algorithm=\"{algorithm}\"\
--MyTopTaggerProcessorJ2.clusteringMode=\"{clusteringMode}\"\
--MyTopTaggerProcessorJ2.recombinationScheme={recombinationScheme}\
--MyTopTaggerProcessorJ2.deltaR={deltaR:.2f}\
--MyTopTaggerProcessorJ2.deltaP={deltaP:.2f}\
--MyTopTaggerProcessorJ2.cos_theta_W_max={cos_theta_W_max:.2f}\
--MyTopTaggerProcessorJ2wTrimming.algorithm=\"{algorithm}\"\
--MyTopTaggerProcessorJ2wTrimming.clusteringMode=\"{clusteringMode}\"\
--MyTopTaggerProcessorJ2wTrimming.recombinationScheme={recombinationScheme}\
--MyTopTaggerProcessorJ2wTrimming.deltaR={deltaR:.2f}\
--MyTopTaggerProcessorJ2wTrimming.deltaP={deltaP:.2f}\
--MyTopTaggerProcessorJ2wTrimming.cos_theta_W_max={cos_theta_W_max:.2f}\
--MyTopAsymmetryStoreProcessor.useSixFermionSample={sixFermionSample}\
--MyTopAsymmetryStoreProcessor.outFileName={outfile}"
.format(algorithm=jetAlgo,
clusteringMode=jetCluster,
recombinationScheme=jetRecomb,
deltaR=deltaR,
deltaP=deltaP,
cos_theta_W_max=cos_theta_W_max,
sixFermionSample=sixFermionSample,
outfile=rootFile))
return ma
def getJob():
""" produce a job: it's always the same, so we don't need to put it in the main
"""
j = UserJob()
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile("clic_ild_cdr.gear")
result = j.append(ma)
if not result['OK']:
gLogger.error(result["Message"])
dexit(1)
j.setCPUTime(10000)
j.setOutputSandbox("*.log")
return j
def getJob():
""" produce a job: it's always the same, so we don't need to put it in the main
"""
j = UserJob()
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile("clic_ild_cdr.gear")
result = j.append(ma)
if not result['OK']:
gLogger.error(result["Message"])
dexit(1)
j.setCPUTime(10000)
j.setOutputSandbox("*.log")
return j
def getJob(jobid, jobpara):
iser=jobid+1
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_ovl_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
return j
ov = OverlayInput()
ov.setDetectorType("ILD")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
res = j.append(ov)
if not res["OK"]:
print res["Message"]
dexit(1)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res["OK"]:
print res["Message"]
dexit(1)
# print appplication's attributes.
ma.listAttributes()
j.setName("test")
j.setOutputSandbox("*.log")
res = dirac.checkparams(j)
if not res["OK"]:
print res["Message"]
def getJob(dirac, jobid, jobpara):
iser=jobid+100
outdir = "/ilc/user/a/amiyamoto/myprod2/test/"
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
dstlfn = outdir+"dst/"+outdst
reclfn = outdir+"rec/"+outrec
outsrm = "CERN-SRM"
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
# Upload files to different directories
upload_script="upload%i.sh"%iser
upload = GenericApplication()
# Create a script to upload files.
shfile = open(upload_script,"w")
shfile.write("#!/bin/bash\n")
shfile.write("/bin/ls -l \n")
shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n")
shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n")
shfile.close()
os.chmod(upload_script,0755)
upload.setScript(upload_script)
res = j.append(upload)
if not res['OK'] :
print res['Message']
exit(1)
# j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setInputSandbox([ setting_file, upload_script ] )
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
res = j.submit(dirac)
if not res["OK"] :
print "Failed submit job, jobid=%s" %jobid
print res
os.remove(upload_script)
return j
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)
ma = Marlin()
ma.setDebug()
# ma.setLogLevel("verbose")
# ma.setILDConfig("v01-16-p05_500")
ma.setVersion("v01-16-02")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile(gearfile)
ma.setInputFile([simfile, pandoraLikelihoodData, bg_aver])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh"])
j.dontPromptMe()
j.submit(d)
outdst = "toto-3.dst.slcio" #% i
outrec = "toto-3.rec.slcio" #% i
d= DiracILC(True,"repo.rep")
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)
j.setInputSandbox(setting_file)
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile("GearOutput.xml")
ma.setInputFile(simfile)
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
# j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.dontPromptMe()
j.submit(d)
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
"0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True,
setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent / nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector,
fileNumber, args.outputPath,
softVersions)
slicOutput = outputFiles['slicOutput']
prePandoraOutput = outputFiles['prePandoraOutput']
pandoraOutput = outputFiles['pandoraOutput']
vertexingOutput = outputFiles['vertexingOutput']
lcsimRecOutput = outputFiles['lcsimRecOutput']
lcsimDstOutput = outputFiles['lcsimDstOutput']
flavortagOutput = outputFiles['flavortagOutput']
diracOutput = outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(
'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
) # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy(
'steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites([
'LCG.IN2P3-CC.fr',
'LCG.RAL-LCG2.uk',
'LCG.DESY-HH.de',
'LCG.DESYZN.de',
'LCG.KEK.jp',
'OSG.PNNL.us',
])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
d = DiracILC() # Provides job checking utilities
j = UserJob() # You are running a user job
m = Marlin() # Get an application instance
m.setVersion("v0116") # Define the version to use
m.setSteeringFile("clic_ild_cdr_steering.xml") #What the app should do
m.setInputFile("LFN:/ilc/prod/clic/3tev/ee_h_bb/ILD/DST/00000375/000/\
ee_h_bb_dst_375_999.slcio") # Add some input
m.setGearFile("clic_ilc_cdr.gear") # Application specific field
res = j.append(m) # Add the application to the job
if not res['OK']:
print res['Message'] # Catch any error
j.submit(d) # Submit the job
#Not shown here: metadata queries, chaining of applications
overlay.setBXOverlay(BXOverlay)
overlay.setGGToHadInt(GGToHadInt250)
overlay.setBkgEvtType("aa_lowpt")
else:
print "Overlay ILD: No overlay parameters defined for this energy"
##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion(MarlinVer) ##PUT HERE YOUR MARLIN VERSION
if ild_rec_ov:
if energy in [250.0, 350.0, 500.0, 1000.0]:
if UseDD4hepGeometry:
mao.setSteeringFile("bbudsc_3evt_stdreco_dd4hep.xml")
mao.setGearFile("GearOutput.xml")
mao.setDetectorModel(detectorModel)
else:
mao.setSteeringFile("bbudsc_3evt_stdreco.xml")
mao.setGearFile("GearOutput.xml")
else:
print "Marlin: No reconstruction suitable for this energy"
##Reconstruction ILD w/o overlay
ma = Marlin()
ma.setDebug()
ma.setVersion(MarlinVer)
ma.setEnergy(energy)
if ild_rec:
if energy in [250.0, 350.0, 500.0, 1000.0]:
overlay.setDetectorModel("CLIC_ILD_CDR")
elif energy == 1400.:
overlay.setBXOverlay(60)
overlay.setGGToHadInt(1.3)##When running at 1.4TeV
overlay.setDetectorModel("CLIC_ILD_CDR")
else:
print "Overlay ILD: No overlay parameters defined for this energy"
##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion('v0111Prod')
if ild_rec_ov:
if energy==500.:
mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml")
mao.setGearFile('clic_ild_cdr500.gear')
elif energy==350.:
mao.setSteeringFile("clic_ild_cdr500_steering_overlay_350.0.xml")
mao.setGearFile('clic_ild_cdr500.gear')
elif energy==3000.0:
mao.setSteeringFile("clic_ild_cdr_steering_overlay_3000.0.xml")
mao.setGearFile('clic_ild_cdr.gear')
elif energy==1400.0:
mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml")
mao.setGearFile('clic_ild_cdr.gear')
else:
print "Marlin: No reconstruction suitable for this energy"
##Reconstruction w/o overlay
ma = Marlin()
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
print args.chain[0]
# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]
check_events_arguments(args.events, args.split)
detector = args.detector
alias_properties(detector)
outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
inputSandbox, outputSandbox = setup_sandboxes(args.macFile)
dirac = DiracILC(True, repoName)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(outputBase)
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ---> ", fileNumber
################## SLIC ##################################################
if 1 in args.chain:
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
slic.setDetectorModel(detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Overlay ###############################################
if 2 in args.chain:
'''
#Add the gghad background overlay.
gghad = OverlayInput()
#gghad.setProdID(1767)
gghad.setEnergy(500.0)
gghad.setBXOverlay('args.bunches')
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setMachine('ilc_dbd')
gghad.setDetectorModel('sidloi3')
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
#Add the pair background overlay.
pairs = OverlayInput()
pairs.setProdID(2)
pairs.setEnergy(500.0)
pairs.setBXOverlay('args.bunches')
pairs.setGGToHadInt(1.)
pairs.setNbSigEvtsPerJob(nOutputEvents)
pairs.setMachine('ilc_dbd')
pairs.setDetectorModel('sidloi3')
pairs.setBkgEvtType('eepairs')
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
gghad = OverlayInput()
gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
gghad.setBXOverlay(int(args.bunches))
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
pairs = OverlayInput()
pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
pairs.setBXOverlay(int(args.bunches))
pairs.setGGToHadInt(1.)
pairs.setBkgEvtType('eepairs')
pairs.setNbSigEvtsPerJob(nOutputEvents)
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
################## lcsim (digitization and tracking) #####################
if 3 in args.chain:
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
if 1 in args.chain:
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
lcsim.setAliasProperties('steeringFiles/alias.properties')
lcsim.setDetectorModel(detector+".zip")
#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
if 4 in args.chain:
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
if 5 in args.chain:
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/' + detector + '.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
lcsimDst.setAliasProperties('steeringFiles/alias.properties')
lcsimDst.setDetectorModel(detector+".zip")
lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if 6 in args.chain:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/' + detector + '.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
outputLevel = max(args.chain)
if outputLevel == 1:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 3:
#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 4:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 5:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 6:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')
job.setOutputSandbox(outputSandbox)
job.setInputData(args.Input)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
class MarlinTestCase( unittest.TestCase ):
""" Base class for the Marlin test cases
"""
def setUp( self ):
"""set up the objects"""
self.mar = Marlin( {} )
def test_setgear( self ):
self.mar.setGearFile( 'lfn:/my/gear/file.txt' )
self.assertFalse( self.mar._errorDict )
self.assertIn( 'lfn:/my/gear/file.txt', self.mar.inputSB )
def test_setKeepRecFile(self):
"""Tests for behaviour with KeepRecFile True/False."""
self.assertTrue(self.mar.keepRecFile)
self.mar.setKeepRecFile(False)
self.assertFalse(self.mar._errorDict)
self.mar.setKeepRecFile(True)
self.assertTrue(self.mar.keepRecFile)
self.assertFalse(self.mar._errorDict)
self.mar.setKeepRecFile(123)
self.assertIn('val = 123', str(self.mar._errorDict))
def test_setoutputrec( self ):
self.mar.setOutputRecFile( 'my/file.outputrec', 'mytestPath' )
assertEqualsImproved( self.mar.outputRecPath, 'mytestPath', self )
self.assertFalse( self.mar._errorDict )
def test_setoutputdst( self ):
self.mar.setOutputDstFile( 'my/file.outputdst', 'mytestPath' )
assertEqualsImproved( self.mar.outputDstPath, 'mytestPath', self )
self.assertFalse( self.mar._errorDict )
def test_setproclist( self ):
self.mar.setProcessorsToUse( [ 'proc1', 'proc2' ] )
self.assertFalse( self.mar._errorDict )
def test_setexcludeproclist( self ):
self.mar.setProcessorsToExclude( [ 'proc1', 'proc2' ] )
self.assertFalse( self.mar._errorDict )
def test_userjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.mar._userjobmodules( module_mock ), self )
def test_prodjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )
def test_prodjobmodules_outputpath( self ):
module_mock = Mock()
self.mar.outputPath = 'aef'
assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )
self.assertIn( { 'OutputFile' : '@{OutputFile}', 'outputPath' : '@{OutputPath}',
'outputDataSE' : '@{OutputSE}'}, self.mar._listofoutput )
def test_userjobmodules_fails( self ):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith( self.mar._userjobmodules( None ),
'userjobmodules failed', self )
def test_prodjobmodules_fails( self ):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith( self.mar._prodjobmodules( None ),
'prodjobmodules failed', self )
def test_checkconsistency( self ):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = None
self.mar._jobtype = 'notUser'
self.mar.outputFile = None
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
assertDiracSucceeds( self.mar._checkConsistency(), self )
self.assertIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
self.assertIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
self.assertIn( keyword, self.mar.prodparameters )
assertEqualsImproved( self.mar.gearFile, None, self )
def test_checkconsistency_noversion( self ):
self.mar.version = None
assertDiracFailsWith( self.mar._checkConsistency(), 'version not set', self )
def test_checkconsistency_invalidxml( self ):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
assertDiracFailsWith( self.mar._checkConsistency(), 'supplied steering file cannot be read with xml', self)
def test_checkconsistency_othercase( self ):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'notUser'
self.mar.outputFile = 'myoutput.test'
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds( self.mar._checkConsistency(), self )
self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
self.assertIn( keyword, self.mar.prodparameters )
assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )
def test_checkconsistency_lastcase( self ):
self.mar.version = '13'
self.mar.steeringFile = None
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'User'
self.mar.outputFile = 'myoutput.test'
assertDiracSucceeds( self.mar._checkConsistency(), self )
self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
self.assertNotIn( keyword, self.mar.prodparameters )
assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )
def test_resolvelinkedparams(self):
"""Test _resolveLinkedStepParameters with something happening."""
step_mock = Mock()
input_mock = Mock()
input_mock.getType.return_value = {'abc': False}
self.mar._linkedidx = 3
self.mar._jobsteps = [None, None, None, input_mock]
assertDiracSucceeds(self.mar._resolveLinkedStepParameters(step_mock), self)
step_mock.setLink.assert_called_once_with('InputFile', {'abc': False}, 'OutputFile')
def test_resolvelinkedparams_noinputstep(self):
"""Call _resolveLinkedStep function, which does nothing."""
self.mar._linkedidx = None
self.mar._inputappstep = []
assertDiracSucceeds(self.mar._resolveLinkedStepParameters(None), self)
overlay.setBXOverlay(300)
overlay.setGGToHadInt(0.3)##When running at 500geV
elif energy == 1000.:
overlay.setBXOverlay(60)
overlay.setGGToHadInt(1.3)##When running at 1tev
else:
print "Overlay ILD: No overlay parameters defined for this energy"
##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion('v0111Prod') ##PUT HERE YOUR MARLIN VERSION
if ild_rec_ov:
if energy==500.:
mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml") #STEERINGFILE for 500gev
mao.setGearFile('clic_ild_cdr500.gear') #GEAR FILE for 500gev
elif energy==1000.0:
mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml") #STEERINGFILE for 1tev
mao.setGearFile('clic_ild_cdr.gear') #GEAR FILE for 1tev
else:
print "Marlin: No reconstruction suitable for this energy"
##Reconstruction ILD w/o overlay
ma = Marlin()
ma.setDebug()
ma.setVersion('v0111Prod') ##PUT HERE YOUR MARLIN VERSION
if ild_rec:
if energy in [500.]:
ma.setSteeringFile("clic_ild_cdr500_steering.xml")##PUT HERE YOUR MARLIN steering files
ma.setGearFile('clic_ild_cdr500.gear')
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions)
slicOutput=outputFiles['slicOutput']
prePandoraOutput=outputFiles['prePandoraOutput']
pandoraOutput=outputFiles['pandoraOutput']
vertexingOutput=outputFiles['vertexingOutput']
lcsimRecOutput=outputFiles['lcsimRecOutput']
lcsimDstOutput=outputFiles['lcsimDstOutput']
flavortagOutput=outputFiles['flavortagOutput']
diracOutput=outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us',])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
if detectormodel in ILDDetectorModels:
overlay.setMachine("ilc_dbd")
overlay.setBackgroundType("aa_lowpt")
overlay.setBXOverlay(1)
overlay.setDetectorModel(detectormodel)
if energy == 250:
overlay.setGGToHadInt(0.3)
##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion(SOFTWAREVERSION)
if ild_rec_ov:
if energy == 500.:
mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml")
mao.setGearFile('clic_ild_cdr500.gear')
elif energy == 420.:
mao.setSteeringFile("clic_ild_cdr500_steering_overlay_420.0.xml")
mao.setGearFile('clic_ild_cdr500.gear')
elif energy == 350.:
mao.setSteeringFile("clic_ild_cdr500_steering_overlay_350.0.xml")
mao.setGearFile('clic_ild_cdr500.gear')
elif energy == 3000.0:
mao.setSteeringFile("clic_ild_cdr_steering_overlay_3000.0.xml")
mao.setGearFile('clic_ild_cdr.gear')
elif energy == 1400.0:
mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml")
mao.setGearFile('clic_ild_cdr.gear')
else:
print "Marlin: No reconstruction suitable for this energy"
overlay = [True, False]
for ov in overlay:
d = DiracILC(True, "repo_overlay_%s.rep" % ov)
for lfn in lfns:
j = UserJob()
steeringf = "clic_ild_cdr_steering.xml"
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print res['Message']
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:" + lfn)
ma.setNbEvts(10)
ma.setEnergy(500.)
ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov)
ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit()
j.setCPUTime(86400)
j.setOutputData("myrec_overlay_%s.slcio" % ov, "some/path")
j.setName("SomeName")
j.setJobGroup("SomeGroup")
class MarlinTestCase(unittest.TestCase):
""" Base class for the Marlin test cases
"""
def setUp(self):
"""set up the objects"""
self.mar = Marlin({})
def test_setgear(self):
self.mar.setGearFile('lfn:/my/gear/file.txt')
self.assertFalse(self.mar._errorDict)
self.assertIn('lfn:/my/gear/file.txt', self.mar.inputSB)
def test_setoutputrec(self):
self.mar.setOutputRecFile('my/file.outputrec', 'mytestPath')
assertEqualsImproved(self.mar.outputRecPath, 'mytestPath', self)
self.assertFalse(self.mar._errorDict)
def test_setoutputdst(self):
self.mar.setOutputDstFile('my/file.outputdst', 'mytestPath')
assertEqualsImproved(self.mar.outputDstPath, 'mytestPath', self)
self.assertFalse(self.mar._errorDict)
def test_setproclist(self):
self.mar.setProcessorsToUse(['proc1', 'proc2'])
self.assertFalse(self.mar._errorDict)
def test_setexcludeproclist(self):
self.mar.setProcessorsToExclude(['proc1', 'proc2'])
self.assertFalse(self.mar._errorDict)
def test_userjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.mar._userjobmodules(module_mock), self)
def test_prodjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)
def test_prodjobmodules_outputpath(self):
module_mock = Mock()
self.mar.outputPath = 'aef'
assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)
self.assertIn(
{
'OutputFile': '@{OutputFile}',
'outputPath': '@{OutputPath}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
def test_userjobmodules_fails(self):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith(self.mar._userjobmodules(None),
'userjobmodules failed', self)
def test_prodjobmodules_fails(self):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith(self.mar._prodjobmodules(None),
'prodjobmodules failed', self)
def test_checkconsistency(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = None
self.mar._jobtype = 'notUser'
self.mar.outputFile = None
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, None, self)
def test_checkconsistency_noversion(self):
self.mar.version = None
assertDiracFailsWith(self.mar._checkConsistency(), 'version not set',
self)
def test_checkconsistency_invalidxml(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
assertDiracFailsWith(
self.mar._checkConsistency(),
'supplied steering file cannot be read with xml', self)
def test_checkconsistency_othercase(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'notUser'
self.mar.outputFile = 'myoutput.test'
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertNotIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertNotIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)
def test_checkconsistency_lastcase(self):
self.mar.version = '13'
self.mar.steeringFile = None
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'User'
self.mar.outputFile = 'myoutput.test'
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertNotIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertNotIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertNotIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)