コード例 #1
0
##Example to submit Marlin job: MarlinExample.py
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
jobGroup = "ph10_sim_CLIC_o3_v05_2016_07_04"
dirac = DiracILC(True, jobGroup + ".rep")
job = UserJob()
#all input files to be send here
job.setInputSandbox([
    "LFN:/ilc/user/w/webermat/160620/photonReReco/lib.tar.gz",
    "LFN:/ilc/user/w/webermat/160502/photonReReco/PandoraSettingsFast.xml",
    "LFN:/ilc/user/w/webermat/160502/photonReReco/PandoraLikelihoodData9EBin_CLIC_ILD.xml"
])
job.setOutputSandbox([
    "*.log", "*.out",
    "hnunu_rec_6265_photonrereco_3_0_TeV_ntuple160620_233_25.root"
])
job.setOutputData(
    ["hnunu_rec_6265_photonrereco_3_0_TeV_ntuple160620_233_25.root"],
    "160620_" + jobGroup, "CERN-SRM")
job.setInputData([
    #job.setParametricInputData([
    "/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_233.slcio",
    "/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_234.slcio",
    "/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_235.slcio",
    "/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_236.slcio",
    "/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_237.slcio",
    "/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_238.slcio",
    "/ilc/prod/clic/3tev/h_nunu/ILD/REC/00006265/000/h_nunu_rec_6265_239.slcio",
コード例 #2
0
def SubmitCLICJob(jobInfo):
    #########################
    # Unpack job information
    #########################
    eventType = jobInfo['eventType']
    energy = jobInfo['energy']
    detectorModel = jobInfo['detectorModel']
    reconstructionVariant = jobInfo['reconstructionVariant']
    clicFile = jobInfo['clicFile']
    analysisTag = jobInfo['analysisTag']
    jobDescription = jobInfo['jobDescription']
    steeringTemplateContent = jobInfo['steeringTemplateContent']
    prodID = jobInfo['prodID']
    idx = jobInfo['idx']
    numberOfFiles = jobInfo['numberOfFiles']
    gearFileLocal = jobInfo['gearFileLocal']
    diracInstance =  jobInfo['diracInstance']
    marlinVersion = jobInfo['marlinVersion']
    pandoraPFOsToUse = jobInfo['pandoraPFOsToUse']
    shortPandoraPFOsToUse = jobInfo['shortPandoraPFOsToUse']
    jetClusteringMode = jobInfo['jetClusteringMode']
    nJetsToCluster = jobInfo['nJetsToCluster']
    jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm']
    jetClusteringRadius = jobInfo['jetClusteringRadius']
    jetAlgorithmConfigString = jobInfo['jetAlgorithmConfigString']

    print 'Checking CLIC sample ' + eventType + ' ' + str(energy) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  Slcio file ' + clicFile + '.'
    clicFileNoPath = os.path.basename(clicFile)
    inputSandbox = ['LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz', 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz']

    #########################
    # Modify Template
    #########################
    steeringTemplate = steeringTemplateContent

    outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '_ProdID_' + str(prodID) + '/' + str(energy) + 'GeV/' + jetAlgorithmConfigString
    rootFileName = 'ProdID_' + str(prodID) + '_' + eventType + '_' + str(energy) + 'GeV_Analysis_' + str(analysisTag) + '_Number_' + str(idx+1) + '_Of_' + str(numberOfFiles) + '.root'
    outputFiles = []
    outputFiles.append(rootFileName)

    steeringTemplate = re.sub('MakeNtupleRootFileName',rootFileName,steeringTemplate)
    steeringTemplate = re.sub('GearFile',gearFileLocal,steeringTemplate)
    steeringTemplate = re.sub('InputSlcioFile',clicFileNoPath,steeringTemplate)
    steeringTemplate = re.sub('PandoraPFOsToUse', pandoraPFOsToUse,steeringTemplate)
    steeringTemplate = re.sub('JetClusteringMode', jetClusteringMode,steeringTemplate)
    steeringTemplate = re.sub('NJetsToCluster', str(nJetsToCluster),steeringTemplate)
    steeringTemplate = re.sub('JetClusteringAlgorithm',jetClusteringAlgorithm,steeringTemplate)
    steeringTemplate = re.sub('JetClusteringRadius',  str(format(jetClusteringRadius,'.2f')),steeringTemplate)
#    steeringTemplate = re.sub('JetAlgorithmConfig',jetAlgorithmConfigString,steeringTemplate)

    #########################
    # Write Template File
    #########################
    marlinSteeringFilename = 'MarlinSteering_' + str(idx+1) + '_' + eventType + '_' + jetAlgorithmConfigString + '.steer'

    with open(marlinSteeringFilename ,'w') as SteeringFile:
        SteeringFile.write(steeringTemplate)

    #########################
    # Check output doesn't exist already
    #########################
    skipJob = False
    for outputFile in outputFiles:
        lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
        if doesFileExist(lfn):
            skipJob = True

    if skipJob:
        return

    print 'Submitting ' + eventType + ' ' + str(energy) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  CLIC file ' + clicFile + '.'

    #########################
    # Setup Marlin Application
    #########################
    ma = Marlin()
    ma.setVersion(marlinVersion)
    ma.setSteeringFile(marlinSteeringFilename)
    ma.setGearFile(gearFileLocal)
    ma.setInputFile('lfn:' + clicFile)
    ma.setProcessorsToUse(['libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so'])

    #########################
    # Submit Job
    #########################
    jobDetailedName = jobDescription + '_' + eventType + '_' + str(energy) + 'GeV_Tag' + str(analysisTag) + '_ProdID_' + str(prodID) + '_Number_' + str(idx+1) + '_Of_' + str(numberOfFiles)

    job = UserJob()
    job.setJobGroup(jobDescription)
    job.setInputSandbox(inputSandbox) # Local files
    job.setOutputSandbox(['*.log','*.steer','*.xml'])
    job.setOutputData(outputFiles,OutputPath=outputPath) # On grid
    job.setName(jobDetailedName)
    job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr','LCG.UKI-LT2-IC-HEP.uk','LCG.Tau.il','LCG.Weizmann.il','OSG.BNL.us'])
    job.setCPUTime(21600) # 6 hour, should be excessive
    job.dontPromptMe()
#    job.setDestination('LCG.CERN.ch')

    res = job.append(ma)

    if not res['OK']:
        print res['Message']
        exit()
    job.submit(diracInstance)

    # Tidy Up
    os.system('rm ' + marlinSteeringFilename)
    print 'Job submitted to grid.'
コード例 #3
0
        #########################
        ma = Marlin()
        ma.setVersion('ILCSoft-01-17-08_gcc48')
        ma.setSteeringFile('MarlinSteering.steer')
        ma.setGearFile(gearFileLocal)
        ma.setInputFile('lfn:' + slcioFile)
        ma.setProcessorsToUse(['libSelectionProcessor.so','libMarlinFastJet.so'])

        #########################
        # Submit Job
        #########################
        jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' +  str(numberOfEventsInFile) + '_' + str(startEventNumber)

        job = UserJob()
        job.setJobGroup(jobDescription)
        job.setInputSandbox(inputSandbox) # Local files
        job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
        job.setOutputData(outputFiles,OutputPath=outputPath) # On grid
        job.setName(jobDetailedName)
        job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr'])
        job.dontPromptMe()

        res = job.append(ma)

        if not res['OK']:
            print res['Message']
            exit()
        job.submit(diracInstance)
        #sys.exit()

# Tidy Up
コード例 #4
0
for ind in range(0, 200, 1):
    jobGroup = "pim1500_CLIC_o3_v14_2018-11-01_CLIC_gcc62"
    dirac = DiracILC(True, jobGroup + ".rep")
    job = UserJob()
    job.setJobGroup(jobGroup)
    job.setOutputSandbox(["*.log", "*.out", "*.py"])
    job.setBannedSites([
        'LCG.IN2P3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
        'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
        'LCG.QMUL.uk'
    ])
    job = UserJob()
    job.setJobGroup(jobGroup)
    job.setOutputSandbox(["*.log", "*.out", "*.py"])
    job.setInputSandbox([
        "LFN:/ilc/user/w/webermat/181011/PandoraSettings/PandoraSettingsDefault.xml",
        "LFN:/ilc/user/w/webermat/181011/PandoraSettings/PandoraLikelihoodData12EBin.xml"
    ])
    #job.setInputSandbox( ["LFN:/ilc/user/w/webermat/171221/PandoraSettingsDefault_171221_SWC_CLIC_n_K0L_12bins_Ph_gcc62.xml","LFN:/ilc/user/w/webermat/171221/PandoraLikelihoodData12EBin_Zuds500_CLIC_o3_v14_171221_gcc62.xml"] )
    job.setBannedSites([
        'LCG.INP3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
        'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
        'LCG.QMUL.uk', 'LCG.Oxford.uk'
    ])
    ma = Marlin()
    ma.setVersion('ILCSoft-2018-11-01_gcc62')
    ma.setInputFile(
        "LFN:/ilc/user/w/webermat/ddsim/ILC18-10-11_gcc62_CLIC_o3_v14/pim1500_split100evts/ddsim_ILC18-10-11_gcc62_CLIC_o3_v14_pim1500_0%s.slcio"
        % ((str)(ind)))

    #ma.setSteeringFile("/afs/cern.ch/work/w/weberma2/steeringFiles/testCalibPerformance_171221_CLIC_o3_v14_PhLikelihood12Ebin_TauFinder_SWC_CLIC_n_K0L.xml")
    ma.setSteeringFile(
コード例 #5
0
def SubmitJob(jobInfo):
    #########################
    # Unpack job information
    #########################
    eventType = jobInfo['eventType']
    energy = jobInfo['energy']
    detectorModel = jobInfo['detectorModel']
    reconstructionVariant = jobInfo['reconstructionVariant']
    slcioFile = jobInfo['slcioFile']
    pandoraSettingsFileLocal = jobInfo['pandoraSettingsFileLocal']
    slcioFormat = jobInfo['slcioFormat']
    steeringTemplateContent = jobInfo['steeringTemplateContent']
    jobDescription = jobInfo['jobDescription']
    idx = jobInfo['idx']
    gearFileLocal = jobInfo['gearFileLocal']
    diracInstance = jobInfo['diracInstance']
    ggHadBackground = jobInfo['ggHadBackground']
    eventsPerFile = jobInfo['eventsPerFile']

    slcioFileNoPath = os.path.basename(slcioFile)
    inputSandbox = []
    inputSandbox.append(pandoraSettingsFileLocal)

    #########################
    # Get info from file name
    #########################
    matchObj = re.match(
        slcioFormat, os.path.basename(slcioFile), re.M | re.I
    )  # MokkaSim_Detector_Model_clic_ild_cdr_ee_nunuww_nunuqqqq_1400GeV_GeneratorSerialNumber_100_100_0.slcio
    generatorSerialNumber = 0
    numberOfEventsInFile = 0
    startEventNumber = 0  # In generator level, not reconstruction.  Start event for all reconstruction is 0.
    if matchObj:
        generatorSerialNumber = matchObj.group(1)
        numberOfEventsInFile = matchObj.group(2)
        startEventNumber = matchObj.group(3)
    else:
        print 'Wrong stdhep format.  Please check.'
        return

    #########################
    # Modify Template
    #########################
    steeringTemplate = steeringTemplateContent
    outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str(
        energy) + 'GeV'
    recFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
        energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
            numberOfEventsInFile) + '_' + str(startEventNumber) + '_REC.slcio'
    dstFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
        energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
            numberOfEventsInFile) + '_' + str(startEventNumber) + '_DST.slcio'

    outputFiles = []
    outputFiles.append(recFileName)
    outputFiles.append(dstFileName)

    steeringTemplate = re.sub('OutputRecFile', recFileName, steeringTemplate)
    steeringTemplate = re.sub('OutputDstFile', dstFileName, steeringTemplate)
    steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate)
    steeringTemplate = re.sub('InputSlcioFile', slcioFileNoPath,
                              steeringTemplate)
    steeringTemplate = re.sub('PandoraSettingsFile', pandoraSettingsFileLocal,
                              steeringTemplate)

    #########################
    # Check output doesn't exist already
    #########################
    print 'Checking ' + eventType + ' ' + str(
        energy
    ) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  SLCIO file ' + slcioFile + '.'
    skipJob = False
    for outputFile in outputFiles:
        lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
        if doesFileExist(lfn):
            skipJob = True
    if skipJob:
        return
    print 'Submitting ' + eventType + ' ' + str(
        energy
    ) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  SLCIO file ' + slcioFile + '.'

    #########################
    # Write Template File
    #########################
    marlinSteeringFilename = 'MarlinSteering_' + str(
        idx + 1) + '_' + eventType + '.steer'
    with open(marlinSteeringFilename, "w") as SteeringFile:
        SteeringFile.write(steeringTemplate)

    #########################
    # Setup Marlin Application
    #########################
    ma = Marlin()
    ma.setVersion('v0111Prod')
    ma.setSteeringFile(marlinSteeringFilename)
    ma.setGearFile(gearFileLocal)
    ma.setInputFile('lfn:' + slcioFile)

    #########################
    # Setup Overlay Information
    #########################
    overlay = OverlayInput()
    if ggHadBackground:
        overlay.setMachine('clic_cdr')
        overlay.setBXOverlay(60)
        overlay.setDetectorModel('CLIC_ILD_CDR')
        overlay.setBackgroundType('gghad')

        overlay.setNbSigEvtsPerJob((int)(eventsPerFile))
        if (int)(energy) == 1400:
            overlay.setEnergy(1400.0)
            overlay.setGGToHadInt(1.3)  # When running at 1.4TeV
        elif (int)(energy) == 3000:
            overlay.setEnergy(3000.0)
            overlay.setGGToHadInt(3.2)  # When running at 3TeV
        else:
            print 'Unknown number of GGHadToInts to use for this energy.  Exiting job subission.'
            return

    #########################
    # Submit Job
    #########################
    jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
        energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
            numberOfEventsInFile) + '_' + str(startEventNumber)

    job = UserJob()
    job.setJobGroup(jobDescription)
    job.setInputSandbox(inputSandbox)  # Local files
    job.setOutputSandbox(['*.log', '*.gear', '*.mac', '*.steer', '*.xml'])
    job.setOutputData(outputFiles, OutputPath=outputPath)  # On grid
    job.setName(jobDetailedName)
    job.setBannedSites([
        'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us',
        'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il',
        'LCG.Weizmann.il', 'OSG.BNL.us'
    ])
    job.setCPUTime(21600)  # 6 hour, should be excessive
    job.dontPromptMe()

    if ggHadBackground:
        res = job.append(overlay)

    res = job.append(ma)

    if not res['OK']:
        print res['Message']
        return
    job.submit(diracInstance)

    # Tidy Up
    os.system('rm ' + marlinSteeringFilename)
    print 'Job submitted to grid.'
コード例 #6
0
def SubmitJob(jobInfo):
    #########################
    # Unpack job information
    #########################
    eventType = jobInfo['eventType']
    energy = jobInfo['energy']
    detectorModel = jobInfo['detectorModel']
    reconstructionVariant = jobInfo['reconstructionVariant']
    slcioFile = jobInfo['slcioFile']
    analysisTag = jobInfo['analysisTag']
    jobDescription = jobInfo['jobDescription']
    idx = jobInfo['idx']
    numberOfFiles = jobInfo['numberOfFiles']
    gearFileLocal = jobInfo['gearFileLocal']
    diracInstance = jobInfo['diracInstance']
    slcioFormat = jobInfo['slcioFormat']

    pandoraPFOsToUse = jobInfo['pandoraPFOsToUse']
    jetClusteringMode = jobInfo['jetClusteringMode']
    nJetsToCluster = jobInfo['nJetsToCluster']
    jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm']
    jetClusteringRadius = jobInfo['jetClusteringRadius']
    jetAlgorithmConfigString = pandoraPFOsToUse + '_' + jetClusteringAlgorithm + '_' + str(
        format(jetClusteringRadius, '.2f')).replace('.', 'p')

    flavourTaggingWeights = ''
    rootFileSuffix = ''

    if pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.50':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p50.tar.gz'
        rootFileSuffix = 'SPFOs_kt_0p50.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'SPFOs_kt_0p70.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.90':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p90.tar.gz'
        rootFileSuffix = 'SPFOs_kt_0p90.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '1.00':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_1p00.tar.gz'
        rootFileSuffix = 'SPFOs_kt_1p00.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '1.10':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_1p10.tar.gz'
        rootFileSuffix = 'SPFOs_kt_1p10.root'

    elif pandoraPFOsToUse == 'TightSelectedPandoraPFANewPFOs' and (int)(
            energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_TightPFOs_kt_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'TPFOs_kt_0p70.root'

    elif pandoraPFOsToUse == 'LooseSelectedPandoraPFANewPFOs' and (int)(
            energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_LoosePFOs_kt_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'LPFOs_kt_0p70.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
            int
    )(energy) == 1400 and jetClusteringAlgorithm == 'cambridge_algorithm' and (
            int)(nJetsToCluster) == 2 and str(
                format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_cambridge_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'SPFOs_cam_0p70.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
            int
    )(energy) == 1400 and jetClusteringAlgorithm == 'ee_kt_algorithm' and (
            int)(nJetsToCluster) == 2 and str(
                format(jetClusteringRadius, '.2f')) == '0.00':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_ee_kt_algorithm_2jets_0p00.tar.gz'
        rootFileSuffix = 'SPFOs_ee_kt_0p70.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.50':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p50.tar.gz'
        rootFileSuffix = 'SPFOs_kt_0p50.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'SPFOs_kt_0p70.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.90':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p90.tar.gz'
        rootFileSuffix = 'SPFOs_kt_0p90.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '1.00':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_1p00.tar.gz'
        rootFileSuffix = 'SPFOs_kt_1p00.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)(
            energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '1.10':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_1p10.tar.gz'
        rootFileSuffix = 'SPFOs_kt_1p10.root'

    elif pandoraPFOsToUse == 'TightSelectedPandoraPFANewPFOs' and (int)(
            energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_TightPFOs_kt_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'TPFOs_kt_0p70.root'

    elif pandoraPFOsToUse == 'LooseSelectedPandoraPFANewPFOs' and (int)(
            energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and (
                int)(nJetsToCluster) == 2 and str(
                    format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_LoosePFOs_kt_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'LPFOs_kt_0p70.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
            int
    )(energy) == 3000 and jetClusteringAlgorithm == 'cambridge_algorithm' and (
            int)(nJetsToCluster) == 2 and str(
                format(jetClusteringRadius, '.2f')) == '0.70':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_cambridge_algorithm_2jets_0p70.tar.gz'
        rootFileSuffix = 'SPFOs_cam_0p70.root'

    elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (
            int
    )(energy) == 3000 and jetClusteringAlgorithm == 'ee_kt_algorithm' and (
            int)(nJetsToCluster) == 2 and str(
                format(jetClusteringRadius, '.2f')) == '0.00':
        flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_ee_kt_algorithm_2jets_0p00.tar.gz'
        rootFileSuffix = 'SPFOs_ee_kt_0p70.root'

    slcioFileNoPath = os.path.basename(slcioFile)
    inputSandbox = [
        'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/MarlinAnalysisProcessor.tar.gz',
        'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz',
        'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz',
        flavourTaggingWeights
    ]

    #########################
    # Get info from file name
    #########################
    matchObj = re.match(slcioFormat, os.path.basename(slcioFile), re.M | re.I)
    generatorSerialNumber = 0
    numberOfEventsInFile = 0
    startEventNumber = 0  # In generator level, not reconstruction.  Start event for all reconstruction is 0.

    if matchObj:
        generatorSerialNumber = matchObj.group(1)
        numberOfEventsInFile = matchObj.group(2)
        startEventNumber = matchObj.group(3)
    else:
        print 'Wrong slcio format.  Please check.'
        return

    #########################
    # Modify Template
    #########################
    steeringTemplate = GetTemplate(jobInfo)

    outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str(
        energy) + 'GeV/AnalysisTag' + str(analysisTag)
    rootFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
        energy) + 'GeV_GenN_' + str(generatorSerialNumber) + '_' + str(
            numberOfEventsInFile) + '_' + str(startEventNumber) + '_Tag' + str(
                analysisTag) + rootFileSuffix

    outputFiles = []
    outputFiles.append(rootFileName)

    steeringTemplate = re.sub('InputSlcioFile', slcioFileNoPath,
                              steeringTemplate)
    steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate)
    steeringTemplate = re.sub('MaximumNumberOfEventsToRecord', '-1',
                              steeringTemplate)
    steeringTemplate = re.sub('AnalysisProcessorRootFile', rootFileName,
                              steeringTemplate)

    #########################
    # Check output doesn't exist already
    #########################
    print 'Checking ' + eventType + ' ' + str(
        energy
    ) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  SLCIO file ' + slcioFile + '.'
    skipJob = False
    for outputFile in outputFiles:
        lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
        if doesFileExist(lfn):
            skipJob = True
    if skipJob:
        return
    print 'Submitting ' + eventType + ' ' + str(
        energy
    ) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  SLCIO file ' + slcioFile + '.'

    #########################
    # Write Template File
    #########################
    marlinSteeringFilename = 'MarlinSteering_' + str(
        idx + 1) + '_' + eventType + '_' + str(
            energy) + '_' + jetAlgorithmConfigString + '.steer'
    with open(marlinSteeringFilename, 'w') as SteeringFile:
        SteeringFile.write(steeringTemplate)

    #########################
    # Setup Marlin Application
    #########################
    ma = Marlin()
    ma.setVersion('v01-16-02')
    ma.setSteeringFile(marlinSteeringFilename)
    ma.setGearFile(gearFileLocal)
    ma.setInputFile('lfn:' + slcioFile)
    ma.setProcessorsToUse([
        'libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so',
        'libAnalysisProcessor.so', 'libMarlinReco.so'
    ])

    #########################
    # Submit Job
    #########################
    jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(
        energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(
            numberOfEventsInFile) + '_' + str(startEventNumber)
    job = UserJob()
    job.setJobGroup(jobDescription)
    job.setInputSandbox(inputSandbox)  # Local files
    job.setOutputSandbox(['*.log', '*.gear', '*.mac', '*.steer', '*.xml'])
    job.setOutputData(outputFiles, OutputPath=outputPath)  # On grid
    job.setName(jobDetailedName)
    job.setBannedSites([
        'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us',
        'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il',
        'LCG.Weizmann.il', 'OSG.BNL.us', 'LCG.GRIF.fr',
        'LCG.UKI-SOUTHGRID-RALPP.uk', 'LCG.RAL-LCG2.uk'
    ])
    job.setCPUTime(21600)  # 6 hour, should be excessive
    job.dontPromptMe()

    res = job.append(ma)
    if not res['OK']:
        print res['Message']
        exit()

    job.submit(diracInstance)
    os.system('rm ' + marlinSteeringFilename)
    print 'Job submitted to grid.'
コード例 #7
0
def SubmitJob(jobInfo):
    #########################
    # Unpack job information
    #########################
    eventType = jobInfo['eventType']
    energy = jobInfo['energy']
    detectorModel = jobInfo['detectorModel']
    reconstructionVariant = jobInfo['reconstructionVariant']
    slcioFile = jobInfo['slcioFile']
    pandoraSettingsFileLocal = jobInfo['pandoraSettingsFileLocal']
    slcioFormat = jobInfo['slcioFormat']
    steeringTemplateContent = jobInfo['steeringTemplateContent']
    jobDescription = jobInfo['jobDescription']
    idx = jobInfo['idx']
    gearFileLocal = jobInfo['gearFileLocal']
    diracInstance =  jobInfo['diracInstance']
    ggHadBackground = jobInfo['ggHadBackground']
    eventsPerFile = jobInfo['eventsPerFile']

    slcioFileNoPath = os.path.basename(slcioFile)
    inputSandbox = []
    inputSandbox.append(pandoraSettingsFileLocal)

    #########################
    # Get info from file name
    #########################
    matchObj = re.match(slcioFormat, os.path.basename(slcioFile), re.M|re.I) # MokkaSim_Detector_Model_clic_ild_cdr_ee_nunuww_nunuqqqq_1400GeV_GeneratorSerialNumber_100_100_0.slcio
    generatorSerialNumber = 0
    numberOfEventsInFile = 0
    startEventNumber = 0 # In generator level, not reconstruction.  Start event for all reconstruction is 0.
    if matchObj:
        generatorSerialNumber = matchObj.group(1)
        numberOfEventsInFile = matchObj.group(2)
        startEventNumber = matchObj.group(3)
    else:
        print 'Wrong stdhep format.  Please check.'
        return 

    #########################
    # Modify Template
    #########################
    steeringTemplate = steeringTemplateContent
    outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str(energy) + 'GeV'
    recFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber) + '_REC.slcio'
    dstFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber) + '_DST.slcio'

    outputFiles = []
    outputFiles.append(recFileName)
    outputFiles.append(dstFileName)

    steeringTemplate = re.sub('OutputRecFile',recFileName,steeringTemplate)
    steeringTemplate = re.sub('OutputDstFile',dstFileName,steeringTemplate)
    steeringTemplate = re.sub('GearFile',gearFileLocal,steeringTemplate)
    steeringTemplate = re.sub('InputSlcioFile',slcioFileNoPath,steeringTemplate)
    steeringTemplate = re.sub('PandoraSettingsFile',pandoraSettingsFileLocal,steeringTemplate)

    #########################
    # Check output doesn't exist already
    #########################
    print 'Checking ' + eventType + ' ' + str(energy) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  SLCIO file ' + slcioFile + '.'
    skipJob = False
    for outputFile in outputFiles:
        lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
        if doesFileExist(lfn):
            skipJob = True
    if skipJob:
        return
    print 'Submitting ' + eventType + ' ' + str(energy) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  SLCIO file ' + slcioFile + '.'

    #########################
    # Write Template File
    #########################
    marlinSteeringFilename = 'MarlinSteering_' + str(idx+1) + '_' + eventType + '.steer'
    with open(marlinSteeringFilename ,"w") as SteeringFile:
        SteeringFile.write(steeringTemplate)

    #########################
    # Setup Marlin Application
    #########################
    ma = Marlin()
    ma.setVersion('v0111Prod')
    ma.setSteeringFile(marlinSteeringFilename)
    ma.setGearFile(gearFileLocal)
    ma.setInputFile('lfn:' + slcioFile)

    #########################
    # Setup Overlay Information
    #########################
    overlay = OverlayInput()
    if ggHadBackground:
        overlay.setMachine('clic_cdr')
        overlay.setBXOverlay(60)
        overlay.setDetectorModel('CLIC_ILD_CDR')
        overlay.setBackgroundType('gghad')

        overlay.setNbSigEvtsPerJob((int)(eventsPerFile))
        if (int)(energy) == 1400:
            overlay.setEnergy(1400.0)
            overlay.setGGToHadInt(1.3) # When running at 1.4TeV
        elif (int)(energy) == 3000:
            overlay.setEnergy(3000.0)
            overlay.setGGToHadInt(3.2) # When running at 3TeV
        else:
            print 'Unknown number of GGHadToInts to use for this energy.  Exiting job subission.'
            return 

    #########################
    # Submit Job
    #########################
    jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' +  str(numberOfEventsInFile) + '_' + str(startEventNumber)

    job = UserJob()
    job.setJobGroup(jobDescription)
    job.setInputSandbox(inputSandbox) # Local files
    job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
    job.setOutputData(outputFiles,OutputPath=outputPath) # On grid
    job.setName(jobDetailedName)
    job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr','LCG.UKI-LT2-IC-HEP.uk','LCG.Tau.il','LCG.Weizmann.il','OSG.BNL.us'])
    job.setCPUTime(21600) # 6 hour, should be excessive
    job.dontPromptMe()

    if ggHadBackground:
        res = job.append(overlay)

    res = job.append(ma)

    if not res['OK']:
        print res['Message']
        return
    job.submit(diracInstance)

    # Tidy Up
    os.system('rm ' + marlinSteeringFilename)
    print 'Job submitted to grid.'
コード例 #8
0
filelist=[]
for lfn in lfns:
   filelist.append(lfn)
#print filelist
#filelist2=filelist[0]

#check if big radius in fastjetanalyzer indeed 0.7 or 1.0
jobGroup = "HZAnalyzer_190417_ee_qqqq_m_qqqq_2TeV_13696_VLC7PFOs"
dirac = DiracILC(True,jobGroup+".rep")
job = UserJob()
#job.setExecutionEnv({'ROOT_INCLUDE_PATH':'./'})
job.setJobGroup(jobGroup)
job.setOutputSandbox ( [ "*.log","*.out","*.py"] )
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk'])
#pay attention that the Zuds200 here is NOT changed
job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190606/HZAnalyzerlib.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] ) 
job.setBannedSites(['LCG.INP3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk','LCG.Oxford.uk'])
job.setSplitInputData(filelist, numberOfFilesPerJob=50)
ma = Marlin()
ma.setVersion('ILCSoft-2019-04-17_gcc62')
#ma.setInputFile("LFN:/ilc/user/w/webermat/ddsimstdheptautau/ILC18-10-11_gcc62_CLIC_o3_v14/tautau200/ddsim_ILC181011_gcc62_tautau_200_CLIC_o3_v14_0%s.slcio"%(str(input_ind)))
ma.setSteeringFile("/eos/user/w/weberma2/steeringFiles/testHZAnalyzer.xml")
ma.setDetectorModel("CLIC_o3_v14")
HZrootfilename2="HZStudy_ee_qqqq_m_qqqq_2TeV_13696_polm80_3TeV_wO_CLIC_o3_v14_DR7.root"
RunStatRootfilename2="ee_qqqq_m_qqqq_2TeV_13696_RunEventStatisticsHistogram.root"
#TrackPtMin default -1, saveMEInfo=true and Rmax=0.7
ma.setExtraCLIArguments("--MyHZAnalyzer.saveMEInfo=true --MyHZAnalyzer.OutputRootFileName={HZrootfilename} --MyRunEventStatisticsHistogram.OutputRootFileName={RunStatRootfilename}".format(HZrootfilename=HZrootfilename2,RunStatRootfilename=RunStatRootfilename2))
#ma.setExtraCLIArguments("--MyHZAnalyzer.saveMEInfo=false --MyHZAnalyzer.Rmax=0.7 --MyHZAnalyzer.TrackPtMin=-1 --MyHZAnalyzer.OutputRootFileName={HZrootfilename}".format(HZrootfilename=HZrootfilename2))
#lcoutputreco ="ddsmp_ILC181101_gcc62_tautau200_0%s.slcio"%(str(input_ind))
#ma.setOutputFile(lcoutputreco)
res=job.append(ma)
コード例 #9
0
 ):  #for low stat 91-200 1 job, for 380,500,750,1000,2000 10 jobs, for 1500 and 3000 do 20 jobs per file, 1-401 for bbar
     #for 91-200 split 1000, for 500 slit 250, for 380,750,1000,2000 split 100,  same for 1500 and 3000 split125
     #jobGroup = "bb_180518_gcc62_SWC_CLIC_DR200"
     jobGroup = "Zuds3000_180518_gcc62_SWC_CLIC_DR200"
     dirac = DiracILC(True, jobGroup + ".rep")
     job = UserJob()
     job.setJobGroup(jobGroup)
     job.setOutputSandbox(["*.log", "*.out", "*.py"])
     job.setBannedSites([
         'LCG.IN2P3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
         'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
         'LCG.QMUL.uk'
     ])
     #pay attention that the Zuds 500 here is NOT changed
     job.setInputSandbox([
         "LFN:/ilc/user/w/webermat/171109/PandoraSettingsDefault_171109_CLIC_o3_v13_12EBin.xml",
         "LFN:/ilc/user/w/webermat/171109/PandoraLikelihoodData12EBin_Zuds500_CLIC_o3_v13_171109.xml"
     ])
     job.setBannedSites([
         'LCG.INP3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
         'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
         'LCG.QMUL.uk', 'LCG.Oxford.uk'
     ])
     ma = Marlin()
     ma.setVersion('ILCSoft-2018-05-18_gcc62')
     #ma.setInputFile("LFN:/ilc/prod/clic/3tev/bb/CLIC_o3_v14/SIM/00009495/000/bb_sim_%s_%s.slcio"%(str(input_ind),(str)(ind)))
     ma.setInputFile(
         "LFN:/ilc/user/w/webermat/ddsimstdhep/ILC18-03-02_CLIC_o3_v14/Zuds3000_split125/ddsim_ILC18-03-02_Zuds3000_CLIC_o3_v14_0%s_%s.slcio"
         % (str(input_ind), (str)(ind)))
     ma.setSteeringFile(
         "/afs/cern.ch/work/w/weberma2/steeringFiles/testCalibPerformance_180514_CLIC_o3_v14_PhLikelihood12Ebin_TauFinder_SWC_CLIC.xml"
     )
コード例 #10
0
            lcsimFinalize.setSteeringFile(xmlPostPandora)
            lcsimFinalize.setAliasProperties(aliasFile)
            lcsimFinalize.setDetectorModel(detector + '.zip')
            lcsimFinalize.setOutputRecFile(outputFile + '_REC.slcio')
            lcsimFinalize.setOutputDstFile(outputFile + '_DST.slcio')
            result = job.append(lcsimFinalize)
            if not result['OK']:
                print result['Message']
                sys.exit(2)
        elif marlinVer or slicPandoraVer:
            defaultOutputFile = outputFile + '.slcio'
            if not defaultOutputFile in outputData:
                outputData.append(defaultOutputFile)

        job.setOutputSandbox(outputSandbox)
        job.setInputSandbox(inputSandbox)
        if inputSlcios:
            job.setInputData(inputSlcios)
        elif inputFile:
            job.setInputData([inputFile])
        if storeOutput == True:
            job.setOutputData(outputData, storagePath, storageElement)
        job.setCPUTime(cpuLimit)
        job.setSystemConfig(systemConfig)
        job.setName(detector + "_" + process + "_" + jobTitle)
        job.setJobGroup(detector + "_" + process + "_" + jobTitle)
        if destination:
            job.setDestination(destination)
        else:
            job.setBannedSites(bannedSites)
コード例 #11
0
def SubmitCLICJob(jobInfo):
    #########################
    # Unpack job information
    #########################
    eventType = jobInfo['eventType']
    energy = jobInfo['energy']
    detectorModel = jobInfo['detectorModel']
    reconstructionVariant = jobInfo['reconstructionVariant']
    clicFile = jobInfo['clicFile']
    analysisTag = jobInfo['analysisTag']
    jobDescription = jobInfo['jobDescription']
    steeringTemplateContent = jobInfo['steeringTemplateContent']
    prodID = jobInfo['prodID']
    idx = jobInfo['idx']
    numberOfFiles = jobInfo['numberOfFiles']
    gearFileLocal = jobInfo['gearFileLocal']
    diracInstance = jobInfo['diracInstance']
    marlinVersion = jobInfo['marlinVersion']
    pandoraPFOsToUse = jobInfo['pandoraPFOsToUse']
    shortPandoraPFOsToUse = jobInfo['shortPandoraPFOsToUse']
    jetClusteringMode = jobInfo['jetClusteringMode']
    nJetsToCluster = jobInfo['nJetsToCluster']
    jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm']
    jetClusteringRadius = jobInfo['jetClusteringRadius']
    jetAlgorithmConfigString = jobInfo['jetAlgorithmConfigString']

    print 'Checking CLIC sample ' + eventType + ' ' + str(
        energy
    ) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  Slcio file ' + clicFile + '.'
    clicFileNoPath = os.path.basename(clicFile)
    inputSandbox = [
        'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz',
        'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz'
    ]

    #########################
    # Modify Template
    #########################
    steeringTemplate = steeringTemplateContent

    outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '_ProdID_' + str(
        prodID) + '/' + str(energy) + 'GeV/' + jetAlgorithmConfigString
    rootFileName = 'ProdID_' + str(prodID) + '_' + eventType + '_' + str(
        energy) + 'GeV_Analysis_' + str(analysisTag) + '_Number_' + str(
            idx + 1) + '_Of_' + str(numberOfFiles) + '.root'
    outputFiles = []
    outputFiles.append(rootFileName)

    steeringTemplate = re.sub('MakeNtupleRootFileName', rootFileName,
                              steeringTemplate)
    steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate)
    steeringTemplate = re.sub('InputSlcioFile', clicFileNoPath,
                              steeringTemplate)
    steeringTemplate = re.sub('PandoraPFOsToUse', pandoraPFOsToUse,
                              steeringTemplate)
    steeringTemplate = re.sub('JetClusteringMode', jetClusteringMode,
                              steeringTemplate)
    steeringTemplate = re.sub('NJetsToCluster', str(nJetsToCluster),
                              steeringTemplate)
    steeringTemplate = re.sub('JetClusteringAlgorithm', jetClusteringAlgorithm,
                              steeringTemplate)
    steeringTemplate = re.sub('JetClusteringRadius',
                              str(format(jetClusteringRadius, '.2f')),
                              steeringTemplate)
    #    steeringTemplate = re.sub('JetAlgorithmConfig',jetAlgorithmConfigString,steeringTemplate)

    #########################
    # Write Template File
    #########################
    marlinSteeringFilename = 'MarlinSteering_' + str(
        idx + 1) + '_' + eventType + '_' + jetAlgorithmConfigString + '.steer'

    with open(marlinSteeringFilename, 'w') as SteeringFile:
        SteeringFile.write(steeringTemplate)

    #########################
    # Check output doesn't exist already
    #########################
    skipJob = False
    for outputFile in outputFiles:
        lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile
        if doesFileExist(lfn):
            skipJob = True

    if skipJob:
        return

    print 'Submitting ' + eventType + ' ' + str(
        energy
    ) + 'GeV jobs.  Detector model ' + detectorModel + '.  Reconstruction stage ' + reconstructionVariant + '.  CLIC file ' + clicFile + '.'

    #########################
    # Setup Marlin Application
    #########################
    ma = Marlin()
    ma.setVersion(marlinVersion)
    ma.setSteeringFile(marlinSteeringFilename)
    ma.setGearFile(gearFileLocal)
    ma.setInputFile('lfn:' + clicFile)
    ma.setProcessorsToUse(
        ['libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so'])

    #########################
    # Submit Job
    #########################
    jobDetailedName = jobDescription + '_' + eventType + '_' + str(
        energy) + 'GeV_Tag' + str(analysisTag) + '_ProdID_' + str(
            prodID) + '_Number_' + str(idx + 1) + '_Of_' + str(numberOfFiles)

    job = UserJob()
    job.setJobGroup(jobDescription)
    job.setInputSandbox(inputSandbox)  # Local files
    job.setOutputSandbox(['*.log', '*.steer', '*.xml'])
    job.setOutputData(outputFiles, OutputPath=outputPath)  # On grid
    job.setName(jobDetailedName)
    job.setBannedSites([
        'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us',
        'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il',
        'LCG.Weizmann.il', 'OSG.BNL.us'
    ])
    job.setCPUTime(21600)  # 6 hour, should be excessive
    job.dontPromptMe()
    #    job.setDestination('LCG.CERN.ch')

    res = job.append(ma)

    if not res['OK']:
        print res['Message']
        exit()
    job.submit(diracInstance)

    # Tidy Up
    os.system('rm ' + marlinSteeringFilename)
    print 'Job submitted to grid.'
コード例 #12
0
for lfn in lfns:
    filelist.append("LFN:" + lfn)
#print filelist

dirac = DiracILC(True, jobGroup + ".rep")
job = UserJob()
job.setJobGroup(jobGroup)
job.setOutputSandbox(["*.log", "*.out", "*.py", "*.xml"])
job.setBannedSites([
    'OSG.UCon.us', 'LCG.Cracow.pl', 'OSG.MIT.us', 'LCG.Glasgow.uk',
    'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk', 'LCG.QMUL.uk'
])
#default settings and default likelihood file
job.setInputSandbox([
    "LFN:/ilc/user/w/webermat/190222/PandoraSettings/PandoraLikelihoodData12EBin.xml",
    "LFN:/ilc/user/w/webermat/190222/PandoraSettings/PandoraSettingsDefault.xml",
    "LFN:/ilc/user/w/webermat/190222/libClicPerformance.tar.gz"
])
job.setBannedSites([
    'LCG.INP3-CC.fr', 'OSG.UCon.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
    'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk',
    'LCG.QMUL.uk', 'LCG.Oxford.uk'
])
job.setSplitInputData(filelist, numberOfFilesPerJob=30)
ma = Marlin()
ma.setVersion('ILCSoft-2018-11-01_gcc62')
#ma.setSteeringFile("LFN:/ilc/user/w/webermat/190222/clicReconstruction20181101JetAnalyzer.xml")
ma.setSteeringFile(
    "/afs/cern.ch/user/w/weberma2/work/steeringFiles/clicReconstruction20181101JetAnalyzer.xml"
)
#ma.setSteeringFile("/home/weberma2/clicReconstruction20181101JetAnalyzer.xml")
        + "_ReconstructionVariant_"
        + str(recoVar)
        + "_"
        + eventType
        + ".root",
    ]
    outputFiles = arguements[3:]

    genericApplication = GenericApplication()
    genericApplication.setScript("SingleParticleResolution")
    genericApplication.setArguments(" ".join(arguements))
    genericApplication.setDependency({"Marlin": "ILCSoft-01-17-07"})

    job = UserJob()
    job.setJobGroup(JobIdentificationString)
    job.setInputSandbox(["LFN:/ilc/user/s/sgreen/EnergyResolutionTarBall/lib.tar.gz", "RunFile.txt"])
    job.setInputData(allRootFilesToUse)
    outputPath = (
        "/"
        + jobDescription
        + "/EnergyResolution/Detector_Model_"
        + str(detModel)
        + "/Reco_Stage_"
        + str(recoVar)
        + "/"
        + eventType
    )
    job.setOutputData(outputFiles, OutputPath=outputPath)
    job.setName(JobIdentificationString)
    job.setBannedSites(
        ["LCG.IN2P3-CC.fr", "LCG.IN2P3-IRES.fr", "LCG.KEK.jp", "OSG.PNNL.us", "OSG.CIT.us", "LCG.LAPP.fr"]
コード例 #14
0
for lfn in lfns:
   filelist.append(lfn)
#print filelist
#filelist2=filelist[0]

#check if big radius in fastjetanalyzer indeed 0.7 or 1.0
jobGroup = "HHZAnalyzer_19-09-04_hhz_14344_VLC10PFOs_NJet3_polp80"
dirac = DiracILC(True,jobGroup+".rep")
job = UserJob()
#job.setExecutionEnv({'ROOT_INCLUDE_PATH':'./'})
job.setJobGroup(jobGroup)
job.setOutputSandbox ( [ "*.log","*.out","*.py"] )
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk'])
#pay attention that the Zuds200 here is NOT changed
#job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190904/libClicPerformanceNJet2.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] ) 
job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190904/libClicPerformance.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] ) 
job.setBannedSites(['LCG.INP3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk','LCG.Oxford.uk'])
job.setSplitInputData(filelist, numberOfFilesPerJob=50)
ma = Marlin()
ma.setVersion('ILCSoft-2019-09-04_gcc62')
#ma.setInputFile("LFN:/ilc/user/w/webermat/ddsimstdheptautau/ILC18-10-11_gcc62_CLIC_o3_v14/tautau200/ddsim_ILC181011_gcc62_tautau_200_CLIC_o3_v14_0%s.slcio"%(str(input_ind)))
ma.setSteeringFile("/eos/user/w/weberma2/steeringFiles/testHHZAnalyzer.xml")
ma.setDetectorModel("CLIC_o3_v14")
HHZrootfilename2="HHZStudy_hhz_14344_polp80_3TeV_wO_CLIC_o3_v14.root"
#RunStatRootfilename2="hhz _14344_RunEventStatisticsHistogram.root"
#TrackPtMin default -1, saveMEInfo=true and Rmax=0.7
ma.setExtraCLIArguments("--MyHHZAnalyzer.saveMEInfo=true --MyHHZAnalyzer.OutputRootFileName={HHZrootfilename}".format(HHZrootfilename=HHZrootfilename2))
res=job.append(ma)
if not res['OK']:
   print res['Message']
   exit()
        arguements = [
                       'runfile.txt',
                       'AnalysePerformance_PandoraSettings' + pandoraSettings + '_DetectorModel_' + str(detModel) + '_Reco_Stage_' + str(recoVar) + '_' + eventType + '_' + str(energy) + 'GeV.root',
                       'AnalysePerformance_PandoraSettings' + pandoraSettings + '_DetectorModel_' + str(detModel) + '_Reco_Stage_' + str(recoVar) + '_' + eventType + '_' + str(energy) + 'GeV.txt'
                     ]

        outputFiles = arguements[1:]

        genericApplication = GenericApplication()
        genericApplication.setScript('AnalysePerformance')
        genericApplication.setArguments(' '.join(arguements))
        genericApplication.setDependency({'ROOT':'5.34'})

        job = UserJob()
        job.setJobGroup(JobIdentificationString)
        job.setInputSandbox(['LFN:/ilc/user/s/sgreen/AnalysePerformanceTarBall/lib.tar.gz', 'runfile.txt']) 
        job.setInputData(rootFilesToProcess)
        job.setOutputData(outputFiles,OutputPath='/OptimisationStudies/AnalysePerformance/Detector_Model_' + str(detModel) + '/Reco_Stage_' + str(recoVar) + '/' + eventType + '/' + str(energy) + 'GeV')

        job.setName(JobIdentificationString)
        job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.CIT.us'])
        job.dontPromptMe()
        #job.setCPUTime(1000)
        res = job.append(genericApplication)

        if not res['OK']:
            print res['Message']
            exit()
        job.submit(diracInstance)
        os.system('rm *.cfg')
        os.system('rm runfile.txt')
コード例 #16
0
     lcsimFinalize.setSteeringFile( xmlPostPandora )
     lcsimFinalize.setAliasProperties( aliasFile )
     lcsimFinalize.setDetectorModel( detector + '.zip' )
     lcsimFinalize.setOutputRecFile( outputFile+'_REC.slcio' )
     lcsimFinalize.setOutputDstFile( outputFile+'_DST.slcio' )
     result = job.append( lcsimFinalize )
     if not result['OK']:
         print result['Message']
         sys.exit(2)
 elif marlinVer or slicPandoraVer:
     defaultOutputFile = outputFile + '.slcio'
     if not defaultOutputFile in outputData:
         outputData.append( defaultOutputFile )
 
 job.setOutputSandbox ( outputSandbox )
 job.setInputSandbox ( inputSandbox )
 if inputSlcios:
     job.setInputData( inputSlcios )
 elif inputFile:
     job.setInputData( [ inputFile ] )
 if storeOutput == True:
     job.setOutputData ( outputData, storagePath, storageElement )
 job.setCPUTime( cpuLimit )
 job.setSystemConfig ( systemConfig )
 job.setName ( detector+"_"+process+"_"+jobTitle )
 job.setJobGroup( detector+"_"+process+"_"+jobTitle )
 if destination:
     job.setDestination( destination )
 else:
     job.setBannedSites( bannedSites )
 
コード例 #17
0
            ma.setSteeringFile('MarlinSteering.steer')
            ma.setGearFile(gearFileLocal)
            ma.setInputFile('lfn:' + slcioFile)

            outputFiles = []
            outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Default.root')
            outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '.slcio')
            if eventType == 'Z_uds':
                outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Muon.root')
                outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhoton.root')
                outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhotonNK0L.root')
                outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPFA.root')

            job = UserJob()
            job.setJobGroup('ReviewJER')
            job.setInputSandbox(pandoraSettingsFilesLocal.values()) # Local files
            job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml'])
            job.setOutputData(outputFiles,OutputPath='/ReviewJER/MarlinJobs/Detector_Model_' + str(gridJobNumber) + '/Reco_Stage_' + str(recoStageNumber) + '/' + eventType + '/' + energy + 'GeV') # On grid
            job.setName('ReviewJER_Detector' + str(gridJobNumber) + '_Reco_' + str(recoStageNumber))
            job.dontPromptMe()
            res = job.append(ma)

            if not res['OK']:
                print res['Message']
                exit()
            job.submit(diracInstance)
            os.system('rm *.cfg')

# Tidy Up
os.system('rm MarlinSteering.steer')
os.system('rm ' + gearFileLocal)
    arguements = [
                   str(recoVar),
                   'RunFile.txt',
                   'EnergyResolution_PandoraSettings' + pandoraSettings + '_DetectorModel_' + str(detModel) + '_Reco_Stage_' + str(recoVar) + '_' + eventType + '.root'
                 ]
    outputFiles = arguements[2:]

    genericApplication = GenericApplication()
    genericApplication.setScript('SingleParticleResolution.exe')
    genericApplication.setArguments(' '.join(arguements))
    genericApplication.setDependency({'Marlin':'ILCSoft-01-17-07'})

    job = UserJob()
    job.setJobGroup(JobIdentificationString)
    job.setInputSandbox(['LFN:/ilc/user/s/sgreen/EnergyResolutionTarBall/lib.tar.gz', 'RunFile.txt']) 
    job.setInputData(allRootFilesToUse)
    job.setOutputData(outputFiles,OutputPath='/OptimisationStudies/EnergyResolution/Detector_Model_' + str(detModel) + '/Reco_Stage_' + str(recoVar) + '/' + eventType)
    job.setName(JobIdentificationString)
    job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.CIT.us'])
    job.dontPromptMe()
    res = job.append(genericApplication)

    if not res['OK']:
        print res['Message']
        exit()
    job.submit(diracInstance)

    # Tidy Up
    os.system('rm *.cfg')
    os.system('rm RunFile.txt')
コード例 #19
0
ファイル: testjob.py プロジェクト: akiyamiyamoto/ildprod
opt=""
if (argc > 1) :
  site = argvs[1]

if (argc > 2 ) :
  opt = argvs[2]

jobname=param[site]["name"]
dest=param[site]["dest"]

print "Test getfile to "+site+"( jobname="+jobname+" dest="+dest+" opt="+opt+")"

job = UserJob()
job.setCPUTime(500)  # time in sec
job.setExecutable('/bin/ls -l ')
job.setExecutable('/bin/bash test.sh')

job.setName("test1")

job.setInputSandbox(["test.sh"])
job.setOutputSandbox(["*.log","*.dat"])
# job.setDestination(dest)
job.dontPromptMe()

dirac = DiracILC(True,jobname+".rep")
jobID = job.submit(dirac)
print jobname+" submitted. JobID=",jobID