from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
dirac = DiracILC( True , "CLICdet_repo.cfg" )
job = UserJob( )
job.setName( "example" )
job.setInputData(["/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep"])
job.setCLICConfig("ILCSoft-2017-12-21")
job.setOutputSandbox( ["*.root","*.slcio","*.log"] )
ddsim = DDSim()
ddsim.setSteeringFile("CLICPerformance/examples/clic_steer.py")
ddsim.setVersion("ILCSoft-2017-12-21_gcc62")
ddsim.setDetectorModel("CLIC_o3_v14")
ddsim.setInputFile("/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep")
ddsim.setOutputFile("ddsim_output.slcio")
ddsim.setStartFrom(0)
ddsim.setNumberOfEvents(10)
res1 = job.append(ddsim)
if not res1['OK']:
print res1['Message']
sys.exit(2)
# -------------------- Comment if gg_had overlay not wanted -----------------------------------------------------#
over = OverlayInput()
over.setBXOverlay(30)
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print res['Message']
lfns = res['Value']
#print "Found %s files" % len(lfns)
filelist=[]
for lfn in lfns:
filelist.append(lfn)
#print filelist
#filelist2=filelist[0]
#check if big radius in fastjetanalyzer indeed 0.7 or 1.0
jobGroup = "HZAnalyzer_190417_ee_qqqq_m_qqqq_2TeV_13696_VLC7PFOs"
dirac = DiracILC(True,jobGroup+".rep")
job = UserJob()
#job.setExecutionEnv({'ROOT_INCLUDE_PATH':'./'})
job.setJobGroup(jobGroup)
job.setOutputSandbox ( [ "*.log","*.out","*.py"] )
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk'])
#pay attention that the Zuds200 here is NOT changed
job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190606/HZAnalyzerlib.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] )
job.setBannedSites(['LCG.INP3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk','LCG.Oxford.uk'])
job.setSplitInputData(filelist, numberOfFilesPerJob=50)
ma = Marlin()
ma.setVersion('ILCSoft-2019-04-17_gcc62')
#ma.setInputFile("LFN:/ilc/user/w/webermat/ddsimstdheptautau/ILC18-10-11_gcc62_CLIC_o3_v14/tautau200/ddsim_ILC181011_gcc62_tautau_200_CLIC_o3_v14_0%s.slcio"%(str(input_ind)))
ma.setSteeringFile("/eos/user/w/weberma2/steeringFiles/testHZAnalyzer.xml")
ma.setDetectorModel("CLIC_o3_v14")
HZrootfilename2="HZStudy_ee_qqqq_m_qqqq_2TeV_13696_polm80_3TeV_wO_CLIC_o3_v14_DR7.root"
RunStatRootfilename2="ee_qqqq_m_qqqq_2TeV_13696_RunEventStatisticsHistogram.root"
mokkaSteeringTemplate = ""
mokkaSteeringTemplate = getMokkaSteeringFileTemplate(baseXmlFile, calibConfigFile)
mokkaSteeringTemplate = setGearFile(mokkaSteeringTemplate, gearFile)
mokkaSteeringTemplate = setOutputFile(mokkaSteeringTemplate, outputFile)
mokkaSteeringTemplate = setStartNumber(mokkaSteeringTemplate, startEvent)
with open("MokkaSteering.steer", "w") as steeringFile:
steeringFile.write(mokkaSteeringTemplate)
MokkaApplication = Mokka()
MokkaApplication.setVersion(getMokkaVersion(calibConfigFile))
MokkaApplication.setSteeringFile("MokkaSteering.steer")
MokkaApplication.setNumberOfEvents(eventsPerJob)
MokkaApplication.setStartFrom(startEvent)
job = UserJob()
job.setJobGroup("HighEnergyPhotons")
job.setOutputSandbox(["*.log", "*.gear", "*.mac", "*.steer", "*.xml"])
job.setOutputData(outputFile, OutputPath=outputPath)
job.setInputData(hepevtFile)
job.setName("HighEnergyPhotons_Detector_Model_" + str(detectorModelNumber))
job.setBannedSites(["LCG.IN2P3-CC.fr", "LCG.IN2P3-IRES.fr", "LCG.KEK.jp"])
res = job.append(MokkaApplication)
if not res["OK"]:
print res["Message"]
exit()
job.dontPromptMe()
job.submit(diracInstance)
os.system("rm *.cfg")
# ma = Marlin()
# ma.setVersion('ILCSoft-01-17-07')
# ma.setSteeringFile('MarlinSteering.steer')
# ma.setGearFile(gearFileLocal)
# ma.setInputFile('lfn:' + slcioFile)
# outputFiles = []
# outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Default.root')
#outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '.slcio') # Not saving the output collections to speed up processing.
# if eventType == 'Z_uds':
# outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Muon.root')
# outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhoton.root')
# outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhotonNK0L.root')
# outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPFA.root')
job = UserJob()
job.setJobGroup('TestingHadd')
#job.setInputSandbox(['hadd.C']) # Local files
# job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
job.setInputData(arguements[1:])
job.setOutputData('HaddTest.root',OutputPath='/Testing') # On grid
job.setName('TestingHadd')
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp'])
job.dontPromptMe()
job.setCPUTime(1000)
res = job.append(genericApplication)
if not res['OK']:
print res['Message']
exit()
def SubmitJob(jobInfo):
#########################
# Unpack job information
#########################
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
reconstructionVariant = jobInfo['reconstructionVariant']
slcioFile = jobInfo['slcioFile']
analysisTag = jobInfo['analysisTag']
jobDescription = jobInfo['jobDescription']
idx = jobInfo['idx']
numberOfFiles = jobInfo['numberOfFiles']
gearFileLocal = jobInfo['gearFileLocal']
diracInstance = jobInfo['diracInstance']
slcioFormat = jobInfo['slcioFormat']
pandoraPFOsToUse = jobInfo['pandoraPFOsToUse']
jetClusteringMode = jobInfo['jetClusteringMode']
nJetsToCluster = jobInfo['nJetsToCluster']
jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm']
jetClusteringRadius = jobInfo['jetClusteringRadius']
jetAlgorithmConfigString = pandoraPFOsToUse + '_' + jetClusteringAlgorithm + '_' + str(
format(jetClusteringRadius, '.2f')).replace('.', 'p')
flavourTaggingWeights = ''
rootFileSuffix = ''
if pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.50':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p50.tar.gz'
rootFileSuffix = 'SPFOs_kt_0p50.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'SPFOs_kt_0p70.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.90':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p90.tar.gz'
rootFileSuffix = 'SPFOs_kt_0p90.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '1.00':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_1p00.tar.gz'
rootFileSuffix = 'SPFOs_kt_1p00.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '1.10':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_1p10.tar.gz'
rootFileSuffix = 'SPFOs_kt_1p10.root'
elif pandoraPFOsToUse == 'TightSelectedPandoraPFANewPFOs' and (int)(
energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_TightPFOs_kt_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'TPFOs_kt_0p70.root'
elif pandoraPFOsToUse == 'LooseSelectedPandoraPFANewPFOs' and (int)(
energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_LoosePFOs_kt_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'LPFOs_kt_0p70.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
int
)(energy) == 1400 and jetClusteringAlgorithm == 'cambridge_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_cambridge_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'SPFOs_cam_0p70.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
int
)(energy) == 1400 and jetClusteringAlgorithm == 'ee_kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.00':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_ee_kt_algorithm_2jets_0p00.tar.gz'
rootFileSuffix = 'SPFOs_ee_kt_0p70.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.50':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p50.tar.gz'
rootFileSuffix = 'SPFOs_kt_0p50.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'SPFOs_kt_0p70.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.90':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p90.tar.gz'
rootFileSuffix = 'SPFOs_kt_0p90.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '1.00':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_1p00.tar.gz'
rootFileSuffix = 'SPFOs_kt_1p00.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '1.10':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_1p10.tar.gz'
rootFileSuffix = 'SPFOs_kt_1p10.root'
elif pandoraPFOsToUse == 'TightSelectedPandoraPFANewPFOs' and (int)(
energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_TightPFOs_kt_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'TPFOs_kt_0p70.root'
elif pandoraPFOsToUse == 'LooseSelectedPandoraPFANewPFOs' and (int)(
energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_LoosePFOs_kt_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'LPFOs_kt_0p70.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
int
)(energy) == 3000 and jetClusteringAlgorithm == 'cambridge_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.70':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_cambridge_algorithm_2jets_0p70.tar.gz'
rootFileSuffix = 'SPFOs_cam_0p70.root'
elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
int
)(energy) == 3000 and jetClusteringAlgorithm == 'ee_kt_algorithm' and (
int)(nJetsToCluster) == 2 and str(
format(jetClusteringRadius, '.2f')) == '0.00':
flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_ee_kt_algorithm_2jets_0p00.tar.gz'
rootFileSuffix = 'SPFOs_ee_kt_0p70.root'
slcioFileNoPath = os.path.basename(slcioFile)
inputSandbox = [
'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/MarlinAnalysisProcessor.tar.gz',
'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz',
'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz',
flavourTaggingWeights
]
#########################
# Get info from file name
#########################
matchObj = re.match(slcioFormat, os.path.basename(slcioFile), re.M | re.I)
generatorSerialNumber = 0
numberOfEventsInFile = 0
startEventNumber = 0 # In generator level, not reconstruction. Start event for all reconstruction is 0.
if matchObj:
generatorSerialNumber = matchObj.group(1)
numberOfEventsInFile = matchObj.group(2)
startEventNumber = matchObj.group(3)
else:
print 'Wrong slcio format. Please check.'
return
#########################
# Modify Template
#########################
steeringTemplate = GetTemplate(jobInfo)
outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str(
energy) + 'GeV/AnalysisTag' + str(analysisTag)
rootFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
energy) + 'GeV_GenN_' + str(generatorSerialNumber) + '_' + str(
numberOfEventsInFile) + '_' + str(startEventNumber) + '_Tag' + str(
analysisTag) + rootFileSuffix
outputFiles = []
outputFiles.append(rootFileName)
steeringTemplate = re.sub('InputSlcioFile', slcioFileNoPath,
steeringTemplate)
steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate)
steeringTemplate = re.sub('MaximumNumberOfEventsToRecord', '-1',
steeringTemplate)
steeringTemplate = re.sub('AnalysisProcessorRootFile', rootFileName,
steeringTemplate)
#########################
# Check output doesn't exist already
#########################
print 'Checking ' + eventType + ' ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.'
skipJob = False
for outputFile in outputFiles:
lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
if doesFileExist(lfn):
skipJob = True
if skipJob:
return
print 'Submitting ' + eventType + ' ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.'
#########################
# Write Template File
#########################
marlinSteeringFilename = 'MarlinSteering_' + str(
idx + 1) + '_' + eventType + '_' + str(
energy) + '_' + jetAlgorithmConfigString + '.steer'
with open(marlinSteeringFilename, 'w') as SteeringFile:
SteeringFile.write(steeringTemplate)
#########################
# Setup Marlin Application
#########################
ma = Marlin()
ma.setVersion('v01-16-02')
ma.setSteeringFile(marlinSteeringFilename)
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + slcioFile)
ma.setProcessorsToUse([
'libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so',
'libAnalysisProcessor.so', 'libMarlinReco.so'
])
#########################
# Submit Job
#########################
jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
numberOfEventsInFile) + '_' + str(startEventNumber)
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(inputSandbox) # Local files
job.setOutputSandbox(['*.log', '*.gear', '*.mac', '*.steer', '*.xml'])
job.setOutputData(outputFiles, OutputPath=outputPath) # On grid
job.setName(jobDetailedName)
job.setBannedSites([
'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us',
'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il',
'LCG.Weizmann.il', 'OSG.BNL.us', 'LCG.GRIF.fr',
'LCG.UKI-SOUTHGRID-RALPP.uk', 'LCG.RAL-LCG2.uk'
])
job.setCPUTime(21600) # 6 hour, should be excessive
job.dontPromptMe()
res = job.append(ma)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
os.system('rm ' + marlinSteeringFilename)
print 'Job submitted to grid.'
import sys, os, subprocess
from string import Template
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
#beware end index now one lower than previously, pay attention also start from 0 and not 1
for ind in range(0, 200, 1):
jobGroup = "pim1500_CLIC_o3_v14_2018-11-01_CLIC_gcc62"
dirac = DiracILC(True, jobGroup + ".rep")
job = UserJob()
job.setJobGroup(jobGroup)
job.setOutputSandbox(["*.log", "*.out", "*.py"])
job.setBannedSites([
'LCG.IN2P3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
'LCG.QMUL.uk'
])
job = UserJob()
job.setJobGroup(jobGroup)
job.setOutputSandbox(["*.log", "*.out", "*.py"])
job.setInputSandbox([
"LFN:/ilc/user/w/webermat/181011/PandoraSettings/PandoraSettingsDefault.xml",
"LFN:/ilc/user/w/webermat/181011/PandoraSettings/PandoraLikelihoodData12EBin.xml"
])
#job.setInputSandbox( ["LFN:/ilc/user/w/webermat/171221/PandoraSettingsDefault_171221_SWC_CLIC_n_K0L_12bins_Ph_gcc62.xml","LFN:/ilc/user/w/webermat/171221/PandoraLikelihoodData12EBin_Zuds500_CLIC_o3_v14_171221_gcc62.xml"] )
job.setBannedSites([
'LCG.INP3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
meta['ProdID'] = '12957'
meta['Datatype'] = 'SIM'
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print res['Message']
lfns = res['Value']
print "Found %s files" % len(lfns)
filelist = []
for lfn in lfns:
filelist.append("LFN:" + lfn)
#print filelist
dirac = DiracILC(True, jobGroup + ".rep")
job = UserJob()
job.setJobGroup(jobGroup)
job.setOutputSandbox(["*.log", "*.out", "*.py", "*.xml"])
job.setBannedSites([
'OSG.UCon.us', 'LCG.Cracow.pl', 'OSG.MIT.us', 'LCG.Glasgow.uk',
'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk', 'LCG.QMUL.uk'
])
#default settings and default likelihood file
job.setInputSandbox([
"LFN:/ilc/user/w/webermat/190222/PandoraSettings/PandoraLikelihoodData12EBin.xml",
"LFN:/ilc/user/w/webermat/190222/PandoraSettings/PandoraSettingsDefault.xml",
"LFN:/ilc/user/w/webermat/190222/libClicPerformance.tar.gz"
])
job.setBannedSites([
'LCG.INP3-CC.fr', 'OSG.UCon.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
def SubmitMokkaJob(jobInfo):
#########################
# Unpack job information
#########################
jobDescription = jobInfo['jobDescription']
eventsPerFile = jobInfo['eventsPerFile']
eventsPerJob = jobInfo['eventsPerJob']
stdhepFormat = jobInfo['stdhepFormat']
stdhepFile = jobInfo['stdhepFile']
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
mokkaSteeringFile = jobInfo['mokkaSteeringFile']
diracInstance = jobInfo['diracInstance']
idx = jobInfo['idx']
base = open(mokkaSteeringFile, 'r')
mokkaSteeringTemplateContent = base.read()
base.close()
matchObj = re.match(
stdhepFormat, os.path.basename(stdhepFile),
re.M | re.I) # "whizard_ee_nunuww_nunuqqqq_1400GeV.100.stdhep"
generatorSerialNumber = ''
if matchObj:
jobSet = int(matchObj.group(1))
runNumber = int(matchObj.group(2))
generatorSerialNumber = str((1000 * jobSet) + runNumber)
else:
print 'Wrong stdhep format. Please check.'
return
for startEvent in xrange(0, eventsPerFile, eventsPerJob):
print 'Checking ' + eventType + ', ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Generator serial number ' + str(
generatorSerialNumber) + '. Start event number ' + str(
startEvent) + '.'
description = eventType + '_' + str(
energy) + 'GeV_GeneratorSerialNumber_' + str(generatorSerialNumber)
outputFile = 'MokkaSim_Detector_Model_' + detectorModel + '_' + description + '_' + str(
eventsPerJob) + '_' + str(startEvent) + '.slcio'
outputPath = '/' + jobDescription + '/MokkaJobs/Detector_Model_' + detectorModel + '/' + eventType + '/' + str(
energy) + 'GeV'
#########################
# Check if output exists
#########################
lfn = '/ilc/user/s/sgreen/' + outputPath + '/' + outputFile
if DoesFileExist(lfn):
print 'Output file already exists, skipping job.'
continue
#########################
# Edit Mokka Template
#########################
mokkaSteeringContent = mokkaSteeringTemplateContent
mokkaSteeringContent = re.sub('LcioOutputFile', outputFile,
mokkaSteeringContent)
mokkaSteeringContent = re.sub('StartEventNumber', str(startEvent),
mokkaSteeringContent)
mokkaSteeringFilename = 'MokkaSteering_' + eventType + '_' + str(
idx) + '_' + str(startEvent) + '.steer'
with open(mokkaSteeringFilename, 'w') as steeringFile:
steeringFile.write(mokkaSteeringContent)
#########################
# Mokka Application
#########################
MokkaApplication = Mokka()
MokkaApplication.setVersion(
'0706P08'
) # Version info for offical CLIC productions as of 8-6-16
MokkaApplication.setSteeringFile(mokkaSteeringFilename)
MokkaApplication.setNumberOfEvents(eventsPerJob)
MokkaApplication.setStartFrom(startEvent)
MokkaApplication.setDbSlice(
'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/MokkaJobs/CLICMokkaDatabase/CLICMokkaDB.sql'
)
#########################
# User Job Settings
#########################
job = UserJob()
job.setJobGroup(jobDescription)
job.setOutputSandbox(["*.log", "*.gear", "*.mac", "*.steer", "*.xml"])
job.setOutputData(outputFile, OutputPath=outputPath)
job.setInputData(stdhepFile)
jobDetailedName = jobDescription + '_' + detectorModel + '_' + eventType + '_' + str(
energy) + 'GeV_GeneratorSerialNumber_' + str(
generatorSerialNumber) + '_' + str(eventsPerJob) + '_' + str(
startEvent)
job.setName(jobDetailedName)
job.setBannedSites([
'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp',
'OSG.PNNL.us', 'OSG.CIT.us', 'LCG.LAPP.fr',
'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il', 'LCG.Weizmann.il',
'OSG.BNL.us', 'OSG.MIT.us'
])
# job.setCPUTime(43200) # Mokka jobs take a long time so set no limit.
res = job.append(MokkaApplication)
if not res['OK']:
print res['Message']
exit()
print 'Submitting ' + eventType + ', ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Generator serial number ' + str(
generatorSerialNumber) + '. Start event number ' + str(
startEvent) + '.'
job.dontPromptMe()
job.submit(diracInstance)
os.system('rm ' + mokkaSteeringFilename)
print 'Job submitted to grid.'
def SubmitMokkaJob(jobInfo):
#########################
# Unpack job information
#########################
jobDescription = jobInfo['jobDescription']
eventsPerFile = jobInfo['eventsPerFile']
eventsPerJob = jobInfo['eventsPerJob']
stdhepFormat = jobInfo['stdhepFormat']
stdhepFile = jobInfo['stdhepFile']
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
mokkaSteeringFile = jobInfo['mokkaSteeringFile']
diracInstance = jobInfo['diracInstance']
idx = jobInfo['idx']
base = open(mokkaSteeringFile,'r')
mokkaSteeringTemplateContent = base.read()
base.close()
matchObj = re.match(stdhepFormat, os.path.basename(stdhepFile), re.M|re.I) # "whizard_ee_nunuww_nunuqqqq_1400GeV.100.stdhep"
generatorSerialNumber = ''
if matchObj:
jobSet = int(matchObj.group(1))
runNumber = int(matchObj.group(2))
generatorSerialNumber = str((1000*jobSet)+runNumber)
else:
print 'Wrong stdhep format. Please check.'
return
for startEvent in xrange(0, eventsPerFile, eventsPerJob):
print 'Checking ' + eventType + ', ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Generator serial number ' + str(generatorSerialNumber) + '. Start event number ' + str(startEvent) + '.'
description = eventType + '_' + str(energy) + 'GeV_GeneratorSerialNumber_' + str(generatorSerialNumber)
outputFile = 'MokkaSim_Detector_Model_' + detectorModel + '_' + description + '_' + str(eventsPerJob) + '_' + str(startEvent) + '.slcio'
outputPath = '/' + jobDescription + '/MokkaJobs/Detector_Model_' + detectorModel + '/' + eventType + '/' + str(energy) + 'GeV'
#########################
# Check if output exists
#########################
lfn = '/ilc/user/s/sgreen/' + outputPath + '/' + outputFile
if DoesFileExist(lfn):
print 'Output file already exists, skipping job.'
continue
#########################
# Edit Mokka Template
#########################
mokkaSteeringContent = mokkaSteeringTemplateContent
mokkaSteeringContent = re.sub('LcioOutputFile',outputFile,mokkaSteeringContent)
mokkaSteeringContent = re.sub('StartEventNumber',str(startEvent),mokkaSteeringContent)
mokkaSteeringFilename = 'MokkaSteering_' + eventType + '_' + str(idx) + '_' + str(startEvent) + '.steer'
with open(mokkaSteeringFilename ,'w') as steeringFile:
steeringFile.write(mokkaSteeringContent)
#########################
# Mokka Application
#########################
MokkaApplication = Mokka()
MokkaApplication.setVersion('0706P08') # Version info for offical CLIC productions as of 8-6-16
MokkaApplication.setSteeringFile(mokkaSteeringFilename)
MokkaApplication.setNumberOfEvents(eventsPerJob)
MokkaApplication.setStartFrom(startEvent)
MokkaApplication.setDbSlice('LFN:/ilc/user/s/sgreen/PhysicsAnalysis/MokkaJobs/CLICMokkaDatabase/CLICMokkaDB.sql')
#########################
# User Job Settings
#########################
job = UserJob()
job.setJobGroup(jobDescription)
job.setOutputSandbox( ["*.log", "*.gear", "*.mac", "*.steer", "*.xml" ] )
job.setOutputData(outputFile, OutputPath=outputPath)
job.setInputData(stdhepFile)
jobDetailedName = jobDescription + '_' + detectorModel + '_' + eventType + '_' + str(energy) + 'GeV_GeneratorSerialNumber_' + str(generatorSerialNumber) + '_' + str(eventsPerJob) + '_' + str(startEvent)
job.setName(jobDetailedName)
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr','LCG.UKI-LT2-IC-HEP.uk','LCG.Tau.il','LCG.Weizmann.il','OSG.BNL.us', 'OSG.MIT.us'])
# job.setCPUTime(43200) # Mokka jobs take a long time so set no limit.
res = job.append(MokkaApplication)
if not res['OK']:
print res['Message']
exit()
print 'Submitting ' + eventType + ', ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Generator serial number ' + str(generatorSerialNumber) + '. Start event number ' + str(startEvent) + '.'
job.dontPromptMe()
job.submit(diracInstance)
os.system('rm ' + mokkaSteeringFilename)
print 'Job submitted to grid.'
ma = Marlin()
ma.setVersion('ILCSoft-01-17-07')
ma.setSteeringFile('MarlinSteering.steer')
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + slcioFile)
outputFiles = []
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Default.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '.slcio')
if eventType == 'Z_uds':
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Muon.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhoton.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhotonNK0L.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPFA.root')
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(pandoraSettingsFilesLocal.values()) # Local files
job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
job.setOutputData(outputFiles,OutputPath='/' + jobDescription + '/MarlinJobs/Detector_Model_' + str(mokkaJobNumber) + '/Reco_Stage_' + str(recoStageNumber) + '/' + eventType + '/' + energy + 'GeV') # On grid
job.setName(jobDescription + '_Detector_Model_' + str(mokkaJobNumber) + '_Reco_' + str(recoStageNumber))
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp'])
job.dontPromptMe()
res = job.append(ma)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
os.system('rm *.cfg')
'AnalysePerformance_PandoraSettings' + pandoraSettings + '_DetectorModel_' + str(detModel) + '_Reco_Stage_' + str(recoVar) + '_' + eventType + '_' + str(energy) + 'GeV.txt'
]
outputFiles = arguements[1:]
outputPath = '/' + jobDescription + '/AnalysePerformance/Detector_Model_' + str(detModel) + '_Run2/Reco_Stage_' + str(recoVar) + '/' + eventType + '/' + str(energy) + 'GeV'
lfn = '/ilc/user/s/sgreen/' + outputPath + '/' + arguements[1]
if doesFileExist(lfn):
continue
genericApplication = GenericApplication()
genericApplication.setScript('AnalysePerformance')
genericApplication.setArguments(' '.join(arguements))
genericApplication.setDependency({'ROOT':'5.34'})
job = UserJob()
job.setJobGroup(JobIdentificationString)
job.setInputSandbox(['LFN:/ilc/user/s/sgreen/AnalysePerformanceTarBall/lib.tar.gz', 'runfile.txt'])
job.setInputData(rootFilesToProcess)
job.setOutputData(outputFiles,OutputPath=outputPath)
job.setName(JobIdentificationString)
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr'])
job.setDestination('LCG.CERN.ch')
job.dontPromptMe()
#job.setCPUTime(1000)
res = job.append(genericApplication)
if not res['OK']:
print res['Message']
exit()
+ "_DetectorModel_"
+ str(detModel)
+ "_ReconstructionVariant_"
+ str(recoVar)
+ "_"
+ eventType
+ ".root",
]
outputFiles = arguements[3:]
genericApplication = GenericApplication()
genericApplication.setScript("SingleParticleResolution")
genericApplication.setArguments(" ".join(arguements))
genericApplication.setDependency({"Marlin": "ILCSoft-01-17-07"})
job = UserJob()
job.setJobGroup(JobIdentificationString)
job.setInputSandbox(["LFN:/ilc/user/s/sgreen/EnergyResolutionTarBall/lib.tar.gz", "RunFile.txt"])
job.setInputData(allRootFilesToUse)
outputPath = (
"/"
+ jobDescription
+ "/EnergyResolution/Detector_Model_"
+ str(detModel)
+ "/Reco_Stage_"
+ str(recoVar)
+ "/"
+ eventType
)
job.setOutputData(outputFiles, OutputPath=outputPath)
job.setName(JobIdentificationString)
def SubmitJob(jobInfo):
#########################
# Unpack job information
#########################
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
reconstructionVariant = jobInfo['reconstructionVariant']
slcioFile = jobInfo['slcioFile']
pandoraSettingsFileLocal = jobInfo['pandoraSettingsFileLocal']
slcioFormat = jobInfo['slcioFormat']
steeringTemplateContent = jobInfo['steeringTemplateContent']
jobDescription = jobInfo['jobDescription']
idx = jobInfo['idx']
gearFileLocal = jobInfo['gearFileLocal']
diracInstance = jobInfo['diracInstance']
ggHadBackground = jobInfo['ggHadBackground']
eventsPerFile = jobInfo['eventsPerFile']
slcioFileNoPath = os.path.basename(slcioFile)
inputSandbox = []
inputSandbox.append(pandoraSettingsFileLocal)
#########################
# Get info from file name
#########################
matchObj = re.match(
slcioFormat, os.path.basename(slcioFile), re.M | re.I
) # MokkaSim_Detector_Model_clic_ild_cdr_ee_nunuww_nunuqqqq_1400GeV_GeneratorSerialNumber_100_100_0.slcio
generatorSerialNumber = 0
numberOfEventsInFile = 0
startEventNumber = 0 # In generator level, not reconstruction. Start event for all reconstruction is 0.
if matchObj:
generatorSerialNumber = matchObj.group(1)
numberOfEventsInFile = matchObj.group(2)
startEventNumber = matchObj.group(3)
else:
print 'Wrong stdhep format. Please check.'
return
#########################
# Modify Template
#########################
steeringTemplate = steeringTemplateContent
outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str(
energy) + 'GeV'
recFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
numberOfEventsInFile) + '_' + str(startEventNumber) + '_REC.slcio'
dstFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
numberOfEventsInFile) + '_' + str(startEventNumber) + '_DST.slcio'
outputFiles = []
outputFiles.append(recFileName)
outputFiles.append(dstFileName)
steeringTemplate = re.sub('OutputRecFile', recFileName, steeringTemplate)
steeringTemplate = re.sub('OutputDstFile', dstFileName, steeringTemplate)
steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate)
steeringTemplate = re.sub('InputSlcioFile', slcioFileNoPath,
steeringTemplate)
steeringTemplate = re.sub('PandoraSettingsFile', pandoraSettingsFileLocal,
steeringTemplate)
#########################
# Check output doesn't exist already
#########################
print 'Checking ' + eventType + ' ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.'
skipJob = False
for outputFile in outputFiles:
lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
if doesFileExist(lfn):
skipJob = True
if skipJob:
return
print 'Submitting ' + eventType + ' ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.'
#########################
# Write Template File
#########################
marlinSteeringFilename = 'MarlinSteering_' + str(
idx + 1) + '_' + eventType + '.steer'
with open(marlinSteeringFilename, "w") as SteeringFile:
SteeringFile.write(steeringTemplate)
#########################
# Setup Marlin Application
#########################
ma = Marlin()
ma.setVersion('v0111Prod')
ma.setSteeringFile(marlinSteeringFilename)
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + slcioFile)
#########################
# Setup Overlay Information
#########################
overlay = OverlayInput()
if ggHadBackground:
overlay.setMachine('clic_cdr')
overlay.setBXOverlay(60)
overlay.setDetectorModel('CLIC_ILD_CDR')
overlay.setBackgroundType('gghad')
overlay.setNbSigEvtsPerJob((int)(eventsPerFile))
if (int)(energy) == 1400:
overlay.setEnergy(1400.0)
overlay.setGGToHadInt(1.3) # When running at 1.4TeV
elif (int)(energy) == 3000:
overlay.setEnergy(3000.0)
overlay.setGGToHadInt(3.2) # When running at 3TeV
else:
print 'Unknown number of GGHadToInts to use for this energy. Exiting job subission.'
return
#########################
# Submit Job
#########################
jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
numberOfEventsInFile) + '_' + str(startEventNumber)
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(inputSandbox) # Local files
job.setOutputSandbox(['*.log', '*.gear', '*.mac', '*.steer', '*.xml'])
job.setOutputData(outputFiles, OutputPath=outputPath) # On grid
job.setName(jobDetailedName)
job.setBannedSites([
'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us',
'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il',
'LCG.Weizmann.il', 'OSG.BNL.us'
])
job.setCPUTime(21600) # 6 hour, should be excessive
job.dontPromptMe()
if ggHadBackground:
res = job.append(overlay)
res = job.append(ma)
if not res['OK']:
print res['Message']
return
job.submit(diracInstance)
# Tidy Up
os.system('rm ' + marlinSteeringFilename)
print 'Job submitted to grid.'
inputFile = 'LFN:' + lfnFile
inputFile = inputFile.replace( 'LFN:', '' )
# base name for all application output files
outputFile = "output"
slicOutput='slic.slcio'
lcsimOutput='prePandora.slcio'
slicPandoraOutput='slicPandora.slcio'
# Storage path is based on your home directory on the grid. Eg. /ilc/user/u/username
storagePath = "output/test"
# We will only ever run a user job
job = UserJob()
# Setting up SLIC application
slic = SLIC()
slic.setVersion( slicVer )
slic.setSteeringFile( macroFile )
slic.setNumberOfEvents( nEvts )
slic.setStartFrom( 0 )
slic.setDetectorModel( detector )
slic.setOutputFile( slicOutput )
result = job.append( slic )
if not result['OK']:
# job.append will return an error message if your application object is not properly initialised
print result['Message']
sys.exit(2)
def SubmitCLICJob(jobInfo):
#########################
# Unpack job information
#########################
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
reconstructionVariant = jobInfo['reconstructionVariant']
clicFile = jobInfo['clicFile']
analysisTag = jobInfo['analysisTag']
jobDescription = jobInfo['jobDescription']
steeringTemplateContent = jobInfo['steeringTemplateContent']
prodID = jobInfo['prodID']
idx = jobInfo['idx']
numberOfFiles = jobInfo['numberOfFiles']
gearFileLocal = jobInfo['gearFileLocal']
diracInstance = jobInfo['diracInstance']
marlinVersion = jobInfo['marlinVersion']
pandoraPFOsToUse = jobInfo['pandoraPFOsToUse']
shortPandoraPFOsToUse = jobInfo['shortPandoraPFOsToUse']
jetClusteringMode = jobInfo['jetClusteringMode']
nJetsToCluster = jobInfo['nJetsToCluster']
jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm']
jetClusteringRadius = jobInfo['jetClusteringRadius']
jetAlgorithmConfigString = jobInfo['jetAlgorithmConfigString']
print 'Checking CLIC sample ' + eventType + ' ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. Slcio file ' + clicFile + '.'
clicFileNoPath = os.path.basename(clicFile)
inputSandbox = [
'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz',
'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz'
]
#########################
# Modify Template
#########################
steeringTemplate = steeringTemplateContent
outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '_ProdID_' + str(
prodID) + '/' + str(energy) + 'GeV/' + jetAlgorithmConfigString
rootFileName = 'ProdID_' + str(prodID) + '_' + eventType + '_' + str(
energy) + 'GeV_Analysis_' + str(analysisTag) + '_Number_' + str(
idx + 1) + '_Of_' + str(numberOfFiles) + '.root'
outputFiles = []
outputFiles.append(rootFileName)
steeringTemplate = re.sub('MakeNtupleRootFileName', rootFileName,
steeringTemplate)
steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate)
steeringTemplate = re.sub('InputSlcioFile', clicFileNoPath,
steeringTemplate)
steeringTemplate = re.sub('PandoraPFOsToUse', pandoraPFOsToUse,
steeringTemplate)
steeringTemplate = re.sub('JetClusteringMode', jetClusteringMode,
steeringTemplate)
steeringTemplate = re.sub('NJetsToCluster', str(nJetsToCluster),
steeringTemplate)
steeringTemplate = re.sub('JetClusteringAlgorithm', jetClusteringAlgorithm,
steeringTemplate)
steeringTemplate = re.sub('JetClusteringRadius',
str(format(jetClusteringRadius, '.2f')),
steeringTemplate)
# steeringTemplate = re.sub('JetAlgorithmConfig',jetAlgorithmConfigString,steeringTemplate)
#########################
# Write Template File
#########################
marlinSteeringFilename = 'MarlinSteering_' + str(
idx + 1) + '_' + eventType + '_' + jetAlgorithmConfigString + '.steer'
with open(marlinSteeringFilename, 'w') as SteeringFile:
SteeringFile.write(steeringTemplate)
#########################
# Check output doesn't exist already
#########################
skipJob = False
for outputFile in outputFiles:
lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
if doesFileExist(lfn):
skipJob = True
if skipJob:
return
print 'Submitting ' + eventType + ' ' + str(
energy
) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. CLIC file ' + clicFile + '.'
#########################
# Setup Marlin Application
#########################
ma = Marlin()
ma.setVersion(marlinVersion)
ma.setSteeringFile(marlinSteeringFilename)
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + clicFile)
ma.setProcessorsToUse(
['libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so'])
#########################
# Submit Job
#########################
jobDetailedName = jobDescription + '_' + eventType + '_' + str(
energy) + 'GeV_Tag' + str(analysisTag) + '_ProdID_' + str(
prodID) + '_Number_' + str(idx + 1) + '_Of_' + str(numberOfFiles)
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(inputSandbox) # Local files
job.setOutputSandbox(['*.log', '*.steer', '*.xml'])
job.setOutputData(outputFiles, OutputPath=outputPath) # On grid
job.setName(jobDetailedName)
job.setBannedSites([
'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us',
'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il',
'LCG.Weizmann.il', 'OSG.BNL.us'
])
job.setCPUTime(21600) # 6 hour, should be excessive
job.dontPromptMe()
# job.setDestination('LCG.CERN.ch')
res = job.append(ma)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
# Tidy Up
os.system('rm ' + marlinSteeringFilename)
print 'Job submitted to grid.'
lfn = '/ilc/user/s/sgreen/' + outputPath + '/' + outputFile
if not doesFileExist(lfn):
skipJob = True
if skipJob:
continue
print 'Submitting ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + str(detModel) + '. Reconstruction stage ' + str(recoVar) + '. Slcio file ' + slcioFile + '.'
ma = Marlin()
ma.setVersion('ILCSoft-01-17-07')
ma.setSteeringFile('MarlinSteering.steer')
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + slcioFile)
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(pandoraSettingsFilesLocal.values()) # Local files
job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
job.setOutputData(outputFiles,OutputPath=outputPath) # On grid
job.setName(jobDescription + '_Detector_Model_' + str(detModel) + '_Reco_' + str(recoVar))
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr'])
job.dontPromptMe()
res = job.append(ma)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
os.system('rm *.cfg')
inputFile = 'LFN:' + lfnFile
inputFile = inputFile.replace('LFN:', '')
# base name for all application output files
outputFile = "output"
slicOutput = 'slic.slcio'
lcsimOutput = 'prePandora.slcio'
slicPandoraOutput = 'slicPandora.slcio'
# Storage path is based on your home directory on the grid. Eg. /ilc/user/u/username
storagePath = "output/test"
# We will only ever run a user job
job = UserJob()
# Setting up SLIC application
slic = SLIC()
slic.setVersion(slicVer)
slic.setSteeringFile(macroFile)
slic.setNumberOfEvents(nEvts)
slic.setStartFrom(0)
slic.setDetectorModel(detector)
slic.setOutputFile(slicOutput)
result = job.append(slic)
if not result['OK']:
# job.append will return an error message if your application object is not properly initialised
print result['Message']
sys.exit(2)
ma = Marlin()
ma.setVersion('ILCSoft-01-17-07')
ma.setSteeringFile('MarlinSteering.steer')
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + slcioFile)
outputFiles = []
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Default.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '.slcio')
if eventType == 'Z_uds':
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Muon.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhoton.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhotonNK0L.root')
outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPFA.root')
job = UserJob()
job.setJobGroup('ReviewJER')
job.setInputSandbox(pandoraSettingsFilesLocal.values()) # Local files
job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
job.setOutputData(outputFiles,OutputPath='/ReviewJER/MarlinJobs/Detector_Model_' + str(gridJobNumber) + '/Reco_Stage_' + str(recoStageNumber) + '/' + eventType + '/' + energy + 'GeV') # On grid
job.setName('ReviewJER_Detector' + str(gridJobNumber) + '_Reco_' + str(recoStageNumber))
job.dontPromptMe()
res = job.append(ma)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
os.system('rm *.cfg')
# Tidy Up
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
dirac = DiracILC(True, "CLICdet_repo.cfg")
job = UserJob()
job.setName("example")
job.setInputData([
"/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep"
])
job.setCLICConfig("ILCSoft-2017-12-21")
job.setOutputSandbox(["*.root", "*.slcio", "*.log"])
ddsim = DDSim()
ddsim.setSteeringFile("CLICPerformance/examples/clic_steer.py")
ddsim.setVersion("ILCSoft-2017-12-21_gcc62")
ddsim.setDetectorModel("CLIC_o3_v14")
ddsim.setInputFile(
"/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep"
)
ddsim.setOutputFile("ddsim_output.slcio")
ddsim.setStartFrom(0)
ddsim.setNumberOfEvents(10)
res1 = job.append(ddsim)
if not res1['OK']:
print res1['Message']
sys.exit(2)
# -------------------- Comment if gg_had overlay not wanted -----------------------------------------------------#
#########################
# Setup Marlin Application
#########################
ma = Marlin()
ma.setVersion('ILCSoft-01-17-08_gcc48')
ma.setSteeringFile('MarlinSteering.steer')
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + slcioFile)
ma.setProcessorsToUse(['libSelectionProcessor.so','libMarlinFastJet.so'])
#########################
# Submit Job
#########################
jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber)
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(inputSandbox) # Local files
job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
job.setOutputData(outputFiles,OutputPath=outputPath) # On grid
job.setName(jobDetailedName)
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr'])
job.dontPromptMe()
res = job.append(ma)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
#sys.exit()
mokkaSteeringTemplate = ''
mokkaSteeringTemplate = getMokkaSteeringFileTemplate(baseXmlFile,detModelConfigFile)
mokkaSteeringTemplate = setGearFile(mokkaSteeringTemplate,gearFile)
mokkaSteeringTemplate = setOutputFile(mokkaSteeringTemplate,outputFile)
mokkaSteeringTemplate = setStartNumber(mokkaSteeringTemplate,startEvent)
with open("MokkaSteering.steer" ,"w") as steeringFile:
steeringFile.write(mokkaSteeringTemplate)
MokkaApplication = Mokka()
MokkaApplication.setVersion(getMokkaVersion(detModelConfigFile))
MokkaApplication.setSteeringFile('MokkaSteering.steer')
MokkaApplication.setNumberOfEvents(eventsPerJob)
MokkaApplication.setStartFrom(startEvent)
job = UserJob()
job.setJobGroup(jobName)
job.setOutputSandbox( ["*.log", "*.gear", "*.mac", "*.steer", "*.xml" ] )
job.setOutputData(outputFile, OutputPath=outputPath)
job.setInputData(hepevtFile)
job.setName(jobName + '_Detector_Model_' + str(detectorModelNumber))
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us'])
res = job.append(MokkaApplication)
if not res['OK']:
print res['Message']
exit()
job.dontPromptMe()
job.submit(diracInstance)
os.system('rm *.cfg')
##Example to submit Marlin job: MarlinExample.py
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
jobGroup = "ph10_sim_CLIC_o3_v05_2016_07_04"
dirac = DiracILC(True, jobGroup + ".rep")
job = UserJob()
#all input files to be send here
job.setInputSandbox([
"LFN:/ilc/user/w/webermat/160620/photonReReco/lib.tar.gz",
"LFN:/ilc/user/w/webermat/160502/photonReReco/PandoraSettingsFast.xml",
"LFN:/ilc/user/w/webermat/160502/photonReReco/PandoraLikelihoodData9EBin_CLIC_ILD.xml"
])
job.setOutputSandbox([
"*.log", "*.out",
"hnunu_rec_6265_photonrereco_3_0_TeV_ntuple160620_233_25.root"
])
job.setOutputData(
["hnunu_rec_6265_photonrereco_3_0_TeV_ntuple160620_233_25.root"],
"160620_" + jobGroup, "CERN-SRM")
job.setInputData([
#job.setParametricInputData([
"/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_233.slcio",
"/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_234.slcio",
"/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_235.slcio",
"/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_236.slcio",
"/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_237.slcio",
"/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_238.slcio",
"/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_239.slcio",
import sys, os, subprocess
from string import Template
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
jobGroup = "sim_K0L2_190220"
dirac = DiracILC(True, jobGroup + ".rep")
job = UserJob()
job.setJobGroup(jobGroup)
job.setOutputSandbox(["*.log", "*.out", "*.py"])
job.setBannedSites([
'LCG.IK0L2P3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
'LCG.QMUL.uk'
])
#pay attention that the Zuds200 here is NOT changed
job.setBannedSites([
'LCG.INP3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
'LCG.QMUL.uk', 'LCG.Oxford.uk'
])
job.setSplitFilesAcrossJobs(
"LFN:/ilc/user/w/webermat/slcioFiles/mcparticlesK0L2_uniformCosTheta_70000.slcio",
eventsPerFile=70000,
eventsPerJob=500)
ddsim = DDSim()
ddsim.setVersion("ILCSoft-2019-02-20_gcc62")
def SubmitCLICJob(jobInfo):
#########################
# Unpack job information
#########################
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
reconstructionVariant = jobInfo['reconstructionVariant']
clicFile = jobInfo['clicFile']
analysisTag = jobInfo['analysisTag']
jobDescription = jobInfo['jobDescription']
steeringTemplateContent = jobInfo['steeringTemplateContent']
prodID = jobInfo['prodID']
idx = jobInfo['idx']
numberOfFiles = jobInfo['numberOfFiles']
gearFileLocal = jobInfo['gearFileLocal']
diracInstance = jobInfo['diracInstance']
marlinVersion = jobInfo['marlinVersion']
pandoraPFOsToUse = jobInfo['pandoraPFOsToUse']
shortPandoraPFOsToUse = jobInfo['shortPandoraPFOsToUse']
jetClusteringMode = jobInfo['jetClusteringMode']
nJetsToCluster = jobInfo['nJetsToCluster']
jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm']
jetClusteringRadius = jobInfo['jetClusteringRadius']
jetAlgorithmConfigString = jobInfo['jetAlgorithmConfigString']
print 'Checking CLIC sample ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. Slcio file ' + clicFile + '.'
clicFileNoPath = os.path.basename(clicFile)
inputSandbox = ['LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz', 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz']
#########################
# Modify Template
#########################
steeringTemplate = steeringTemplateContent
outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '_ProdID_' + str(prodID) + '/' + str(energy) + 'GeV/' + jetAlgorithmConfigString
rootFileName = 'ProdID_' + str(prodID) + '_' + eventType + '_' + str(energy) + 'GeV_Analysis_' + str(analysisTag) + '_Number_' + str(idx+1) + '_Of_' + str(numberOfFiles) + '.root'
outputFiles = []
outputFiles.append(rootFileName)
steeringTemplate = re.sub('MakeNtupleRootFileName',rootFileName,steeringTemplate)
steeringTemplate = re.sub('GearFile',gearFileLocal,steeringTemplate)
steeringTemplate = re.sub('InputSlcioFile',clicFileNoPath,steeringTemplate)
steeringTemplate = re.sub('PandoraPFOsToUse', pandoraPFOsToUse,steeringTemplate)
steeringTemplate = re.sub('JetClusteringMode', jetClusteringMode,steeringTemplate)
steeringTemplate = re.sub('NJetsToCluster', str(nJetsToCluster),steeringTemplate)
steeringTemplate = re.sub('JetClusteringAlgorithm',jetClusteringAlgorithm,steeringTemplate)
steeringTemplate = re.sub('JetClusteringRadius', str(format(jetClusteringRadius,'.2f')),steeringTemplate)
# steeringTemplate = re.sub('JetAlgorithmConfig',jetAlgorithmConfigString,steeringTemplate)
#########################
# Write Template File
#########################
marlinSteeringFilename = 'MarlinSteering_' + str(idx+1) + '_' + eventType + '_' + jetAlgorithmConfigString + '.steer'
with open(marlinSteeringFilename ,'w') as SteeringFile:
SteeringFile.write(steeringTemplate)
#########################
# Check output doesn't exist already
#########################
skipJob = False
for outputFile in outputFiles:
lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
if doesFileExist(lfn):
skipJob = True
if skipJob:
return
print 'Submitting ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. CLIC file ' + clicFile + '.'
#########################
# Setup Marlin Application
#########################
ma = Marlin()
ma.setVersion(marlinVersion)
ma.setSteeringFile(marlinSteeringFilename)
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + clicFile)
ma.setProcessorsToUse(['libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so'])
#########################
# Submit Job
#########################
jobDetailedName = jobDescription + '_' + eventType + '_' + str(energy) + 'GeV_Tag' + str(analysisTag) + '_ProdID_' + str(prodID) + '_Number_' + str(idx+1) + '_Of_' + str(numberOfFiles)
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(inputSandbox) # Local files
job.setOutputSandbox(['*.log','*.steer','*.xml'])
job.setOutputData(outputFiles,OutputPath=outputPath) # On grid
job.setName(jobDetailedName)
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr','LCG.UKI-LT2-IC-HEP.uk','LCG.Tau.il','LCG.Weizmann.il','OSG.BNL.us'])
job.setCPUTime(21600) # 6 hour, should be excessive
job.dontPromptMe()
# job.setDestination('LCG.CERN.ch')
res = job.append(ma)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
# Tidy Up
os.system('rm ' + marlinSteeringFilename)
print 'Job submitted to grid.'
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
#0 to 10 for all, but 500, there it is 0 to 41
#for input_ind in range(9495,9496,1):
for input_ind in range(0, 10, 1):
for ind in range(
0, 20, 1
): #for low stat 91-200 1 job, for 380,500,750,1000,2000 10 jobs, for 1500 and 3000 do 20 jobs per file, 1-401 for bbar
#for 91-200 split 1000, for 500 slit 250, for 380,750,1000,2000 split 100, same for 1500 and 3000 split125
#jobGroup = "bb_180518_gcc62_SWC_CLIC_DR200"
jobGroup = "Zuds3000_180518_gcc62_SWC_CLIC_DR200"
dirac = DiracILC(True, jobGroup + ".rep")
job = UserJob()
job.setJobGroup(jobGroup)
job.setOutputSandbox(["*.log", "*.out", "*.py"])
job.setBannedSites([
'LCG.IN2P3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
'LCG.QMUL.uk'
])
#pay attention that the Zuds 500 here is NOT changed
job.setInputSandbox([
"LFN:/ilc/user/w/webermat/171109/PandoraSettingsDefault_171109_CLIC_o3_v13_12EBin.xml",
"LFN:/ilc/user/w/webermat/171109/PandoraLikelihoodData12EBin_Zuds500_CLIC_o3_v13_171109.xml"
])
job.setBannedSites([
'LCG.INP3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print res['Message']
lfns = res['Value']
#print "Found %s files" % len(lfns)
filelist=[]
for lfn in lfns:
filelist.append(lfn)
#print filelist
#filelist2=filelist[0]
#check if big radius in fastjetanalyzer indeed 0.7 or 1.0
jobGroup = "HHZAnalyzer_19-09-04_hhz_14344_VLC10PFOs_NJet3_polp80"
dirac = DiracILC(True,jobGroup+".rep")
job = UserJob()
#job.setExecutionEnv({'ROOT_INCLUDE_PATH':'./'})
job.setJobGroup(jobGroup)
job.setOutputSandbox ( [ "*.log","*.out","*.py"] )
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk'])
#pay attention that the Zuds200 here is NOT changed
#job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190904/libClicPerformanceNJet2.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] )
job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190904/libClicPerformance.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] )
job.setBannedSites(['LCG.INP3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk','LCG.Oxford.uk'])
job.setSplitInputData(filelist, numberOfFilesPerJob=50)
ma = Marlin()
ma.setVersion('ILCSoft-2019-09-04_gcc62')
#ma.setInputFile("LFN:/ilc/user/w/webermat/ddsimstdheptautau/ILC18-10-11_gcc62_CLIC_o3_v14/tautau200/ddsim_ILC181011_gcc62_tautau_200_CLIC_o3_v14_0%s.slcio"%(str(input_ind)))
ma.setSteeringFile("/eos/user/w/weberma2/steeringFiles/testHHZAnalyzer.xml")
ma.setDetectorModel("CLIC_o3_v14")
HHZrootfilename2="HHZStudy_hhz_14344_polp80_3TeV_wO_CLIC_o3_v14.root"
storagePath = detector + '/' + process + '/' + jobTitle
outputData = []
if slicPandoraVer and lcsimVer:
outputData.append(outputFile + '_REC.slcio')
outputData.append(outputFile + '_DST.slcio')
else:
outputData.append(
outputFile +
'.slcio') # default reco only creates one output file
startEvt = job * nEvts
job = UserJob()
if slicVer:
if nEvts < 0:
print 'ERROR: need to set number of events per job for SLIC. Use -n <nEvts> to set number of events.'
sys.exit(2)
slic = SLIC()
slic.setVersion(slicVer)
slic.setEnergy(energy)
slic.setSteeringFile(macroFile)
slic.setNumberOfEvents(nEvts)
slic.setStartFrom(startEvt)
slic.setDetectorModel(detector)
slic.setOutputFile(slicOutput)
result = job.append(slic)
if not result['OK']:
storagePath = detector+'/'+process+'/'+jobTitle
outputData = []
if slicPandoraVer and lcsimVer:
outputData.append( outputFile+'_REC.slcio' )
outputData.append( outputFile+'_DST.slcio' )
else:
outputData.append( outputFile+'.slcio' ) # default reco only creates one output file
startEvt = job*nEvts
job = UserJob()
if slicVer:
if nEvts < 0:
print 'ERROR: need to set number of events per job for SLIC. Use -n <nEvts> to set number of events.'
sys.exit(2)
slic = SLIC()
slic.setVersion( slicVer )
slic.setEnergy( energy )
slic.setSteeringFile( macroFile )
slic.setNumberOfEvents( nEvts )
slic.setStartFrom( startEvt )
slic.setDetectorModel( detector )
slic.setOutputFile( slicOutput )
result = job.append( slic )
if not result['OK']:
def SubmitJob(jobInfo):
#########################
# Unpack job information
#########################
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
reconstructionVariant = jobInfo['reconstructionVariant']
slcioFile = jobInfo['slcioFile']
pandoraSettingsFileLocal = jobInfo['pandoraSettingsFileLocal']
slcioFormat = jobInfo['slcioFormat']
steeringTemplateContent = jobInfo['steeringTemplateContent']
jobDescription = jobInfo['jobDescription']
idx = jobInfo['idx']
gearFileLocal = jobInfo['gearFileLocal']
diracInstance = jobInfo['diracInstance']
ggHadBackground = jobInfo['ggHadBackground']
eventsPerFile = jobInfo['eventsPerFile']
slcioFileNoPath = os.path.basename(slcioFile)
inputSandbox = []
inputSandbox.append(pandoraSettingsFileLocal)
#########################
# Get info from file name
#########################
matchObj = re.match(slcioFormat, os.path.basename(slcioFile), re.M|re.I) # MokkaSim_Detector_Model_clic_ild_cdr_ee_nunuww_nunuqqqq_1400GeV_GeneratorSerialNumber_100_100_0.slcio
generatorSerialNumber = 0
numberOfEventsInFile = 0
startEventNumber = 0 # In generator level, not reconstruction. Start event for all reconstruction is 0.
if matchObj:
generatorSerialNumber = matchObj.group(1)
numberOfEventsInFile = matchObj.group(2)
startEventNumber = matchObj.group(3)
else:
print 'Wrong stdhep format. Please check.'
return
#########################
# Modify Template
#########################
steeringTemplate = steeringTemplateContent
outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str(energy) + 'GeV'
recFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber) + '_REC.slcio'
dstFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber) + '_DST.slcio'
outputFiles = []
outputFiles.append(recFileName)
outputFiles.append(dstFileName)
steeringTemplate = re.sub('OutputRecFile',recFileName,steeringTemplate)
steeringTemplate = re.sub('OutputDstFile',dstFileName,steeringTemplate)
steeringTemplate = re.sub('GearFile',gearFileLocal,steeringTemplate)
steeringTemplate = re.sub('InputSlcioFile',slcioFileNoPath,steeringTemplate)
steeringTemplate = re.sub('PandoraSettingsFile',pandoraSettingsFileLocal,steeringTemplate)
#########################
# Check output doesn't exist already
#########################
print 'Checking ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.'
skipJob = False
for outputFile in outputFiles:
lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
if doesFileExist(lfn):
skipJob = True
if skipJob:
return
print 'Submitting ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.'
#########################
# Write Template File
#########################
marlinSteeringFilename = 'MarlinSteering_' + str(idx+1) + '_' + eventType + '.steer'
with open(marlinSteeringFilename ,"w") as SteeringFile:
SteeringFile.write(steeringTemplate)
#########################
# Setup Marlin Application
#########################
ma = Marlin()
ma.setVersion('v0111Prod')
ma.setSteeringFile(marlinSteeringFilename)
ma.setGearFile(gearFileLocal)
ma.setInputFile('lfn:' + slcioFile)
#########################
# Setup Overlay Information
#########################
overlay = OverlayInput()
if ggHadBackground:
overlay.setMachine('clic_cdr')
overlay.setBXOverlay(60)
overlay.setDetectorModel('CLIC_ILD_CDR')
overlay.setBackgroundType('gghad')
overlay.setNbSigEvtsPerJob((int)(eventsPerFile))
if (int)(energy) == 1400:
overlay.setEnergy(1400.0)
overlay.setGGToHadInt(1.3) # When running at 1.4TeV
elif (int)(energy) == 3000:
overlay.setEnergy(3000.0)
overlay.setGGToHadInt(3.2) # When running at 3TeV
else:
print 'Unknown number of GGHadToInts to use for this energy. Exiting job subission.'
return
#########################
# Submit Job
#########################
jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber)
job = UserJob()
job.setJobGroup(jobDescription)
job.setInputSandbox(inputSandbox) # Local files
job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
job.setOutputData(outputFiles,OutputPath=outputPath) # On grid
job.setName(jobDetailedName)
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr','LCG.UKI-LT2-IC-HEP.uk','LCG.Tau.il','LCG.Weizmann.il','OSG.BNL.us'])
job.setCPUTime(21600) # 6 hour, should be excessive
job.dontPromptMe()
if ggHadBackground:
res = job.append(overlay)
res = job.append(ma)
if not res['OK']:
print res['Message']
return
job.submit(diracInstance)
# Tidy Up
os.system('rm ' + marlinSteeringFilename)
print 'Job submitted to grid.'
runFile.write('===End_Entry===' + '\n')
runFile.close()
arguements = [
str(recoVar),
'RunFile.txt',
'EnergyResolution_PandoraSettings' + pandoraSettings + '_DetectorModel_' + str(detModel) + '_Reco_Stage_' + str(recoVar) + '_' + eventType + '.root'
]
outputFiles = arguements[2:]
genericApplication = GenericApplication()
genericApplication.setScript('SingleParticleResolution.exe')
genericApplication.setArguments(' '.join(arguements))
genericApplication.setDependency({'Marlin':'ILCSoft-01-17-07'})
job = UserJob()
job.setJobGroup(JobIdentificationString)
job.setInputSandbox(['LFN:/ilc/user/s/sgreen/EnergyResolutionTarBall/lib.tar.gz', 'RunFile.txt'])
job.setInputData(allRootFilesToUse)
job.setOutputData(outputFiles,OutputPath='/OptimisationStudies/EnergyResolution/Detector_Model_' + str(detModel) + '/Reco_Stage_' + str(recoVar) + '/' + eventType)
job.setName(JobIdentificationString)
job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.CIT.us'])
job.dontPromptMe()
res = job.append(genericApplication)
if not res['OK']:
print res['Message']
exit()
job.submit(diracInstance)
# Tidy Up
"CERN":{"dest":"LCG.CERN.ch","name":"testcern"}}
site="KEK"
opt=""
if (argc > 1) :
site = argvs[1]
if (argc > 2 ) :
opt = argvs[2]
jobname=param[site]["name"]
dest=param[site]["dest"]
print "Test getfile to "+site+"( jobname="+jobname+" dest="+dest+" opt="+opt+")"
job = UserJob()
job.setCPUTime(500) # time in sec
job.setExecutable('/bin/ls -l ')
job.setExecutable('/bin/bash test.sh')
job.setName("test1")
job.setInputSandbox(["test.sh"])
job.setOutputSandbox(["*.log","*.dat"])
# job.setDestination(dest)
job.dontPromptMe()
dirac = DiracILC(True,jobname+".rep")
jobID = job.submit(dirac)
print jobname+" submitted. JobID=",jobID
