Exemple #1
0
def create_job(inputData, saveName, outputDir, dontPromptMe):

    slcioFile = saveName + '.slcio'
    rootFile = saveName + '.root'

    if check_file_existence(outputDir, slcioFile, dontPromptMe):
        remove_file(outputDir, slcioFile, dontPromptMe)
    if check_file_existence(outputDir, rootFile, dontPromptMe):
        remove_file(outputDir, rootFile, dontPromptMe)

    dIlc = DiracILC()

    job = UserJob()
    job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml'])
    if SAVE_SLCIO:
        job.setOutputData([slcioFile, rootFile],
                          OutputPath=outputDir,
                          OutputSE=STORAGE_SE)
    else:
        job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE)
    job.setJobGroup('myMarlinRun1')
    job.setName('MyMarlinJob1')
    # job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk'])
    job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz'])

    marl = Marlin()
    marl.setVersion('ILCSoft-2017-11-15_gcc62')

    marl.setInputFile(inputData)
    job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData)))
    # marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio'])
    marl.setSteeringFile('marlin/FullChain.xml')
    # marl.setSteeringFile('marlin/FullChainNewDetModel.xml')
    marl.setOutputFile(slcioFile)
    gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear'
    if not os.path.isfile(gearFile):
        print('Error: gear file', gearFile,
              'does not exist! Abort submission.')
        return
    marl.setGearFile(gearFile)
    marl.setExtraCLIArguments(
        "--MyNtupleMaker.OutputFileName={rootOutfile}".format(
            rootOutfile=rootFile))
    # marl.setNumberOfEvents(1000)

    job.append(marl)
    if dontPromptMe:
        job.dontPromptMe()
    job.submit(dIlc)

    return False
Exemple #2
0
def defMarlin(outputFile, rootFile, **kwargs):
    '''Defines Marlin job'''

    ##Jet Settings
    jetAlgo = kwargs.pop('jetAlgo', 'ValenciaPlugin 1.5 1 1')
    jetCluster = kwargs.pop('jetCluster', 'ExclusiveNJets 2')
    jetRecomb = kwargs.pop('jetRecomb', 'E_scheme')

    ##Top Tagger Settings
    deltaR = kwargs.pop('deltaR', 0.01)
    deltaP = kwargs.pop('deltaP', 0.01)
    cos_theta_W_max = kwargs.pop('cos_theta_W_max', 1.0)

    ##Sixfermion sample
    sixFermionSample = kwargs.pop('sixFermionSample', True)

    from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
    ma = Marlin()
    ma.setVersion('ILCSoft-2017-05-30_gcc62')
    ma.setSteeringFile(
        "/home/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/topasymmetry_template_new.xml"
    )
    ma.setGearFile("/home/lstroem/clicdp/gear/clic_ild_cdr.gear")
    ma.setOutputFile(outputFile, rootFile)
    ma.setExtraCLIArguments(
        "--MyTopTaggerProcessorJ2.algorithm=\"{algorithm}\"\
                           --MyTopTaggerProcessorJ2.clusteringMode=\"{clusteringMode}\"\
                           --MyTopTaggerProcessorJ2.recombinationScheme={recombinationScheme}\
                           --MyTopTaggerProcessorJ2.deltaR={deltaR:.2f}\
                           --MyTopTaggerProcessorJ2.deltaP={deltaP:.2f}\
                           --MyTopTaggerProcessorJ2.cos_theta_W_max={cos_theta_W_max:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.algorithm=\"{algorithm}\"\
                           --MyTopTaggerProcessorJ2wTrimming.clusteringMode=\"{clusteringMode}\"\
                           --MyTopTaggerProcessorJ2wTrimming.recombinationScheme={recombinationScheme}\
                           --MyTopTaggerProcessorJ2wTrimming.deltaR={deltaR:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.deltaP={deltaP:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.cos_theta_W_max={cos_theta_W_max:.2f}\
                           --MyTopAsymmetryStoreProcessor.useSixFermionSample={sixFermionSample}\
                           --MyTopAsymmetryStoreProcessor.outFileName={outfile}"
        .format(algorithm=jetAlgo,
                clusteringMode=jetCluster,
                recombinationScheme=jetRecomb,
                deltaR=deltaR,
                deltaP=deltaP,
                cos_theta_W_max=cos_theta_W_max,
                sixFermionSample=sixFermionSample,
                outfile=rootFile))

    return ma
  def createMarlinApplication(self, energy, cliReco, over):
    """Create Marlin application with or without overlay."""
    from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
    marlin = Marlin()
    marlin.setDebug()
    marlin.setVersion( self.softwareVersion )
    marlin.setDetectorModel( self.detectorModel )
    marlin.detectortype = self.detectorModel
    marlin.setKeepRecFile(False)

    if over:
      self.addOverlayOptionsToMarlin( energy )

    self.cliReco = ' '.join([self.cliRecoOption, self.cliReco, cliReco]).strip()
    marlin.setExtraCLIArguments(self.cliReco)
    self.cliReco = ''

    marlin.setSteeringFile(self.marlinSteeringFile)

    self._setApplicationOptions("Marlin", marlin)

    return marlin
Exemple #4
0
  def createMarlinApplication( self, energy ):
    """ create Marlin Application without overlay """
    from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
    marlin = Marlin()
    marlin.setDebug()
    marlin.setVersion( self.softwareVersion )
    marlin.setDetectorModel( self.detectorModel )
    marlin.detectortype = self.detectorModel

    if self._flags.over:
      self.addOverlayOptionsToMarlin( energy )

    marlin.setExtraCLIArguments( self.cliReco )

    steeringFile = {
      350. : "clicReconstruction.xml",
      380. : "clicReconstruction.xml",
      420. : "clicReconstruction.xml",
      1400.: "clicReconstruction.xml",
      3000.: "clicReconstruction.xml",
    }.get( energy, 'clicReconstruction.xml' )

    marlin.setSteeringFile( steeringFile )
    return marlin
print jobname

#lcoutputSIM  = simOutfile + "_SIM_%s.slcio"%(indx) 
lcoutputSIM  = simOutfile + "_%s.SIM.slcio"%(indx) 
lcinputREC = "/ilc/user/k/kurca/MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" + ireq +"/sim/" + lcoutputSIM
print lcinputREC

# Marlin
lcoutputDST  = recOutfile + "_%s.DST.slcio"%(indx) 
lcoutputREC  = recOutfile + "_%s.REC.slcio"%(indx) 

ma = Marlin()
ma.setVersion(ILCSoftVer)
ma.setDetectorModel(detectorModel)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments(" --constant.DetectorModel=%s "%(detectorModel) )
ma.setLogFile("marlin.log")
#ma.getInputFromApp(sim)
ma.setInputFile([lcoutputSIM])
ma.setEnergy(energy)
ma.setNumberOfEvents(evtsPerRun)
ma.setOutputDstFile(lcoutputDST)
ma.setOutputRecFile(lcoutputREC)

RECoutput.append(lcoutputDST)
RECoutput.append(lcoutputREC)

job = UserJob()
job.setName(jobname)
job.setJobGroup(jobGrName)
job.setILDConfig(ILDConfigVer)
Exemple #6
0
        "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/clic_ild_cdr.gear",
        "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/lib.tar.gz"
    ])
    job.setInputData(
        "LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
        i)
    job.setOutputSandbox(
        ["*.log", "*.sh", "*.py", "*.out", "*.xml", "*.steer "])
    job.setJobGroup("myRoot")
    job.setName("root_aa_%d" % i)
    marl = Marlin()
    marl.setVersion('ILCSoft-2016-09-27_gcc48')
    marl.setInputFile([
        "LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
        i
    ])
    # marl.setNumberOfEvents(10)
    marl.setSteeringFile("steering.xml")
    marl.setGearFile("clic_ild_cdr.gear")
    marl.setExtraCLIArguments("--MyProcessor.RootFilename=%s" % lcoutput)
    res = job.append(marl)
    if not res['OK']:
        print res['Message']
        quit()  #do something, like quit
    job.setOutputData(lcoutput, "hgamgam/Marlin/rootFiles/aa/workingVersion",
                      "CERN-SRM")
    print lcoutput
    job.dontPromptMe()
    print job.submit(dIlc)
    time.sleep(0.5)
Exemple #7
0
def subOverlay():

    # Decide parameters for a job
    outputSE = "KEK-SRM"

    isLocal = _clip.isLocal
    nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
    nbevts = 0  # To analize all input events
    outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix
    outputDir = _clip.outputDir
    inputFile = _clip.inputFile
    if inputFile == "":
        gLogger.error("Input file for ddsim does not given.")
        exit(-1)

    recfile = outputFilePrefix + ".rec.slcio"
    dstfile = outputFilePrefix + ".dst.slcio"
    detector_model = "ILD_l5_o1_v02"
    key = detector_model.split('_')
    sim_detectorModel = "_".join([key[0], key[1], key[3]])

    # Create DIRAC objects for job submission

    dIlc = DiracILC()

    job = UserJob()
    job.setJobGroup("myoverlayjob")
    job.setName("myoverlay")
    job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
    job.setILDConfig("v02-00-02")

    # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
    # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"])  # job submission destination
    # job.setBannedSites([])         # a list of sites not to submit job
    # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )

    # Create Overlay application
    ovldata = [{
        "ProcessorName": "BgOverlayWW",
        "evttype": "aa_lowpt_WW",
        "ProdID": 10237,
        "expBG": 0.211,
        "subdir": "000"
    }, {
        "ProcessorName": "BgOverlayWB",
        "evttype": "aa_lowpt_WB",
        "ProdID": 10241,
        "expBG": 0.24605,
        "subdir": "000"
    }, {
        "ProcessorName": "BgOverlayBW",
        "evttype": "aa_lowpt_BW",
        "ProdID": 10239,
        "expBG": 0.243873,
        "subdir": "000"
    }, {
        "ProcessorName": "BgOverlayBB",
        "evttype": "aa_lowpt_BB",
        "ProdID": 10235,
        "expBG": 0.35063,
        "subdir": "000"
    }, {
        "ProcessorName": "PairBgOverlay",
        "evttype": "seeablepairs",
        "ProdID": 10233,
        "expBG": 1.0,
        "subdir": "100"
    }]

    BXOverlay = 1
    NbSigEvtsPerJob = 100
    numberOfSignalEvents = NbSigEvtsPerJob
    basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws"
    energy = "500"

    for ovl in ovldata:
        print "### OverlayInput ... " + ovl["ProcessorName"]
        ovlapp = OverlayInput()
        ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \
       ( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] )
        print "    OverlayPath ... " + ovlpath
        ovlapp.setMachine("ilc_dbd")
        # ovlapp.setEnergy(energy)
        # ovlapp.setDetectorModel(sim_detectorModel)
        ovlapp.setProcessorName(ovl["ProcessorName"])
        ovlapp.setBkgEvtType(ovl["evttype"])
        ovlapp.setPathToFiles(ovlpath)
        ovlapp.setGGToHadInt(ovl["expBG"])
        ovlapp.setBXOverlay(BXOverlay)
        ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob)
        ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents)
        res = job.append(ovlapp)
        if not res['OK']:
            print res['Message']
            exit(1)

    # Create Marlin application
    marlin = Marlin()
    marlin.setVersion("ILCSoft-02-00-02_gcc49")
    marlin.setDetectorModel(detector_model)
    marlin.setSteeringFile("MarlinStdReco.xml")
    marlin.setInputFile(inputFile)
    marlin.setNumberOfEvents(nbevts)
    marlin.setOutputDstFile(dstfile)
    marlin.setOutputRecFile(recfile)
    extraCLIArguments = " --constant.DetectorModel=%s " % detector_model
    extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str(
        energy)
    extraCLIArguments += " --global.Verbosity=MESSAGE "
    marlin.setExtraCLIArguments(extraCLIArguments)

    job.append(marlin)

    if outputDir != "":
        job.setOutputData([dstfile, recfile],
                          OutputPath=outputDir,
                          OutputSE=outputSE)

    if isLocal:
        job.submit(dIlc, mode="local")
    else:
        job.submit(dIlc)
def main(argv):
    # Input arguments
    ildconfig_version   = "$ILDCONFIGVER"
    ilcsoft_version     = "$ILCSOFTVER"

    evts_per_run    = $EVTSPERRUN
    detector_model  = "$DETECTOR"
    sim_input       = "$SIMINPUT"
    process_name    = "$PROCESS"

    index           = $IND

    sim_input = diracpath_from_pnfspath( sim_input )
    sim_detector_model = detector_model_wo_option( detector_model )

    job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model
    dirac = DiracILC(True,job_group+".rep")

    # outputs to be saved onto grid SE
    RECoutput = []

    # DDSim

    evtStart   = (index-1)*evts_per_run
    evtEnd     = index*evts_per_run - 1
    RandSeed = random.randrange(11623, 99999)

    lcinputSIM  = "LFN:" + sim_input
    lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd))

    sim = DDSim()
    sim.setVersion(ilcsoft_version)

    sim.setDetectorModel(sim_detector_model)
    sim.setInputFile(lcinputSIM)
    sim.setSteeringFile("ddsim_steer.py")
    sim.setNumberOfEvents(evts_per_run)
    sim.setRandomSeed(RandSeed)
    sim.setEnergy(1000)
    sim.setStartFrom(evtStart)
    sim.setOutputFile(lcoutputSIM)

    # Marlin
    lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd))

    ma = Marlin()
    ma.setVersion(ilcsoft_version)
    ma.setDetectorModel(detector_model)
    ma.setSteeringFile("MarlinStdReco.xml")
    ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) )
    ma.setLogFile("marlin.log")
    ma.getInputFromApp(sim)
    ma.setEnergy(1000)
    ma.setOutputDstFile(lcoutputDST)

    RECoutput.append(lcoutputDST)

    # ILCDirac user job
    job = UserJob()
    job.setName("user_sim_reco")

    job.setJobGroup(job_group)

    job.setILDConfig(ildconfig_version)
    job.setCPUTime(86400)

    tmp_file_name = process_name + "_sim_reco_job_tmp.py"
    job.setInputSandbox([tmp_file_name])
    job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])

    job.dontPromptMe()
    job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])

    # run simulation job
    simres = job.append(sim)
    if not simres['OK']:
            print simres['Not ok appending ddsim to job']
            quit()


    # run Malrin reco jobs
    mares = job.append(ma)
    if not mares['OK']:
            print mares['Not ok appending Marlin to job']
            quit()

    job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM")
    print RECoutput

    submit_output = job.submit(dirac)
    print submit_output