Exemple #1
0
 def getMarlin( self ):
   """ Define a marlin step
   """
   from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
   marlin = Marlin()
 #  marlin.setVersion("v0111Prod")
   marlin.setVersion(self.marlinVersion)
   marlin.setSteeringFile(self.marlinSteeringFile)
   marlin.setGearFile(self.gearFile)
   marlin.setOutputDstFile("testmarlinDST.slcio")
   marlin.setOutputRecFile("testmarlinREC.slcio")
   marlin.setNumberOfEvents(1)
   return marlin
 def getMarlin(self):
     """ Define a marlin step
 """
     from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
     marlin = Marlin()
     #  marlin.setVersion("v0111Prod")
     marlin.setVersion(self.marlinVersion)
     marlin.setSteeringFile(self.marlinSteeringFile)
     marlin.setGearFile(self.gearFile)
     marlin.setOutputDstFile("testmarlinDST.slcio")
     marlin.setOutputRecFile("testmarlinREC.slcio")
     marlin.setNumberOfEvents(1)
     return marlin
Exemple #3
0
def create_job(inputData, saveName, outputDir, dontPromptMe):

    slcioFile = saveName + '.slcio'
    rootFile = saveName + '.root'

    if check_file_existence(outputDir, slcioFile, dontPromptMe):
        remove_file(outputDir, slcioFile, dontPromptMe)
    if check_file_existence(outputDir, rootFile, dontPromptMe):
        remove_file(outputDir, rootFile, dontPromptMe)

    dIlc = DiracILC()

    job = UserJob()
    job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml'])
    if SAVE_SLCIO:
        job.setOutputData([slcioFile, rootFile],
                          OutputPath=outputDir,
                          OutputSE=STORAGE_SE)
    else:
        job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE)
    job.setJobGroup('myMarlinRun1')
    job.setName('MyMarlinJob1')
    # job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk'])
    job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz'])

    marl = Marlin()
    marl.setVersion('ILCSoft-2017-11-15_gcc62')

    marl.setInputFile(inputData)
    job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData)))
    # marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio'])
    marl.setSteeringFile('marlin/FullChain.xml')
    # marl.setSteeringFile('marlin/FullChainNewDetModel.xml')
    marl.setOutputFile(slcioFile)
    gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear'
    if not os.path.isfile(gearFile):
        print('Error: gear file', gearFile,
              'does not exist! Abort submission.')
        return
    marl.setGearFile(gearFile)
    marl.setExtraCLIArguments(
        "--MyNtupleMaker.OutputFileName={rootOutfile}".format(
            rootOutfile=rootFile))
    # marl.setNumberOfEvents(1000)

    job.append(marl)
    if dontPromptMe:
        job.dontPromptMe()
    job.submit(dIlc)

    return False
Exemple #4
0
def defMarlin(outputFile, rootFile, **kwargs):
    '''Defines Marlin job'''

    ##Jet Settings
    jetAlgo = kwargs.pop('jetAlgo', 'ValenciaPlugin 1.5 1 1')
    jetCluster = kwargs.pop('jetCluster', 'ExclusiveNJets 2')
    jetRecomb = kwargs.pop('jetRecomb', 'E_scheme')

    ##Top Tagger Settings
    deltaR = kwargs.pop('deltaR', 0.01)
    deltaP = kwargs.pop('deltaP', 0.01)
    cos_theta_W_max = kwargs.pop('cos_theta_W_max', 1.0)

    ##Sixfermion sample
    sixFermionSample = kwargs.pop('sixFermionSample', True)

    from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
    ma = Marlin()
    ma.setVersion('ILCSoft-2017-05-30_gcc62')
    ma.setSteeringFile(
        "/home/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/topasymmetry_template_new.xml"
    )
    ma.setGearFile("/home/lstroem/clicdp/gear/clic_ild_cdr.gear")
    ma.setOutputFile(outputFile, rootFile)
    ma.setExtraCLIArguments(
        "--MyTopTaggerProcessorJ2.algorithm=\"{algorithm}\"\
                           --MyTopTaggerProcessorJ2.clusteringMode=\"{clusteringMode}\"\
                           --MyTopTaggerProcessorJ2.recombinationScheme={recombinationScheme}\
                           --MyTopTaggerProcessorJ2.deltaR={deltaR:.2f}\
                           --MyTopTaggerProcessorJ2.deltaP={deltaP:.2f}\
                           --MyTopTaggerProcessorJ2.cos_theta_W_max={cos_theta_W_max:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.algorithm=\"{algorithm}\"\
                           --MyTopTaggerProcessorJ2wTrimming.clusteringMode=\"{clusteringMode}\"\
                           --MyTopTaggerProcessorJ2wTrimming.recombinationScheme={recombinationScheme}\
                           --MyTopTaggerProcessorJ2wTrimming.deltaR={deltaR:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.deltaP={deltaP:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.cos_theta_W_max={cos_theta_W_max:.2f}\
                           --MyTopAsymmetryStoreProcessor.useSixFermionSample={sixFermionSample}\
                           --MyTopAsymmetryStoreProcessor.outFileName={outfile}"
        .format(algorithm=jetAlgo,
                clusteringMode=jetCluster,
                recombinationScheme=jetRecomb,
                deltaR=deltaR,
                deltaP=deltaP,
                cos_theta_W_max=cos_theta_W_max,
                sixFermionSample=sixFermionSample,
                outfile=rootFile))

    return ma
Exemple #5
0
def getJob():
    """ produce a job: it's always the same, so we don't need to put it in the main
  """
    j = UserJob()
    ma = Marlin()
    ma.setVersion("v0111Prod")
    ma.setSteeringFile("clic_ild_cdr_steering.xml")
    ma.setGearFile("clic_ild_cdr.gear")
    result = j.append(ma)
    if not result['OK']:
        gLogger.error(result["Message"])
        dexit(1)
    j.setCPUTime(10000)
    j.setOutputSandbox("*.log")
    return j
def getJob():
  """ produce a job: it's always the same, so we don't need to put it in the main
  """
  j = UserJob()
  ma = Marlin()
  ma.setVersion("v0111Prod")
  ma.setSteeringFile("clic_ild_cdr_steering.xml")
  ma.setGearFile("clic_ild_cdr.gear")
  result = j.append(ma)
  if not result['OK']:
    gLogger.error(result["Message"])
    dexit(1)
  j.setCPUTime(10000)
  j.setOutputSandbox("*.log")
  return j
def getJob(jobid, jobpara):
  iser=jobid+1

  outdst = "toto-ovl-%5.5i.dst.slcio"%iser
  outrec = "toto-ovl-%5.5i.rec.slcio"%iser

###In case one wants a loop: comment the folowing.
#for i in range(2):
  j = UserJob()
  j.setJobGroup("Tutorial")
  j.setName("MarlinOverlayParametric%i"%iser)
  j.setInputSandbox(jobpara["setting_file"])

## Define the overlay
  ov = OverlayInput()
  ov.setMachine("ilc_dbd")
  ov.setEnergy(energy)
  ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
  ov.setBXOverlay(int(jobpara["BXOverlay"]))
  ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
  ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
  ov.setDetectorModel("ILD_o1_v05")
  res = j.append(ov)
  if not res['OK']:
    print res['Message']
    exit(1)

## Define Marlin job
  ma = Marlin()
  ma.setDebug()
  ma.setVersion("ILCSoft-01-17-09")
  ma.setSteeringFile("marlin_ovl_stdreco.xml")
  ma.setGearFile("GearOutput.xml")
#   ma.setInputFile(simfile)
  ma.setInputFile(simlists[jobid])
  ma.setOutputDstFile(outdst)
  ma.setOutputRecFile(outrec)
  res = j.append(ma)
  if not res['OK']:
    print res['Message']
    exit(1)
  
  j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
  j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
  j.setCPUTime(10000)
  j.dontPromptMe()
  return j
Exemple #8
0
ov = OverlayInput()
ov.setDetectorType("ILD")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
res = j.append(ov)
if not res["OK"]:
    print res["Message"]
    dexit(1)


ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res["OK"]:
    print res["Message"]
    dexit(1)
# print appplication's attributes.
ma.listAttributes()

j.setName("test")
j.setOutputSandbox("*.log")

res = dirac.checkparams(j)
if not res["OK"]:
    print res["Message"]
def getJob(dirac, jobid, jobpara):
  iser=jobid+100

  outdir = "/ilc/user/a/amiyamoto/myprod2/test/"
  outdst = "toto-ovl-%5.5i.dst.slcio"%iser
  outrec = "toto-ovl-%5.5i.rec.slcio"%iser
  dstlfn = outdir+"dst/"+outdst
  reclfn = outdir+"rec/"+outrec
  outsrm = "CERN-SRM"

###In case one wants a loop: comment the folowing.
#for i in range(2):
  j = UserJob()
  j.setJobGroup("Tutorial")
  j.setName("MarlinOverlayParametric%i"%iser)
  j.setInputSandbox(jobpara["setting_file"])

## Define the overlay
  ov = OverlayInput()
  ov.setMachine("ilc_dbd")
  ov.setEnergy(energy)
  ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
  ov.setBXOverlay(int(jobpara["BXOverlay"]))
  ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
  ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
  ov.setDetectorModel("ILD_o1_v05")
  res = j.append(ov)
  if not res['OK']:
    print res['Message']
    exit(1)

## Define Marlin job
  ma = Marlin()
  ma.setDebug()
  ma.setVersion("ILCSoft-01-17-09")
  ma.setSteeringFile("marlin_stdreco.xml")
  ma.setGearFile("GearOutput.xml")
#   ma.setInputFile(simfile)
  ma.setInputFile(simlists[jobid])
  ma.setOutputDstFile(outdst)
  ma.setOutputRecFile(outrec)
  res = j.append(ma)
  if not res['OK']:
    print res['Message']
    exit(1)

# Upload files to different directories
  upload_script="upload%i.sh"%iser
  upload = GenericApplication()
# Create a script to upload files.
  shfile = open(upload_script,"w")
  shfile.write("#!/bin/bash\n")
  shfile.write("/bin/ls -l \n")
  shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n")
  shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n")
  shfile.close()
  os.chmod(upload_script,0755)
  upload.setScript(upload_script)

  res = j.append(upload)
  if not res['OK'] :
    print res['Message']
    exit(1)

#   j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
  j.setInputSandbox([ setting_file, upload_script ] )
  j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
  j.setCPUTime(10000)
  j.dontPromptMe()

  res = j.submit(dirac)
  if not res["OK"] :
    print "Failed submit job, jobid=%s" %jobid
    print res


  os.remove(upload_script)

  return j
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)



ma = Marlin()
ma.setDebug()
# ma.setLogLevel("verbose")
# ma.setILDConfig("v01-16-p05_500") 

ma.setVersion("v01-16-02")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile(gearfile)
ma.setInputFile([simfile, pandoraLikelihoodData, bg_aver])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)


res = j.append(ma)
if not res['OK']:
    print res['Message']
    exit(1)
  
j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh"])
j.dontPromptMe()
j.submit(d)
outdst = "toto-3.dst.slcio" #% i
outrec = "toto-3.rec.slcio" #% i

d= DiracILC(True,"repo.rep")

###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)
j.setInputSandbox(setting_file)

ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile("GearOutput.xml")
ma.setInputFile(simfile)
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)

res = j.append(ma)
if not res['OK']:
    print res['Message']
    exit(1)
  
# j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.dontPromptMe()
j.submit(d)
Exemple #12
0
def main():
    # Take the input arguments from the argument parser, and check they exist...
    args = parse_args()
    if not args:
        print 'Invalid Arguments'
        sys.exit(1)

#### Software Versions ####
    softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
                    "0116"]  # Working (recommended)
    # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
    # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working

    # Check the --runs and --split arguments to make sure they are compatible, if not exit...
    if not check_events_arguments(args.events, args.split):
        sys.exit(1)

    # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
    lfn_check, lfn = check_input_LFN(args.stdhepInput)
    if not lfn_check:
        sys.exit(1)

    # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
    dirac = DiracILC(True,
                     setup_repository_name(args.stdhepInput, args.detector))

    # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
    # into the input sandbox
    inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)

    # Prepares values for the job loop...
    if args.split < 0:
        nInputEvents = int(args.events)
        nOutputEvents = int(args.events)
    if args.split > 0:
        nInputEvents = int(args.events)
        nOutputEvents = int(args.split)

    # Loop that runs through the required number of jobs to be executed...
    for startEvent in range(0, nInputEvents, nOutputEvents):

        ################## Job Initialise ########################################
        job = UserJob()
        job.setName(path.basename(args.stdhepInput))
        job.setJobGroup('JobGroup')
        job.setInputSandbox(inputSandbox)
        fileNumber = startEvent / nOutputEvents
        print "Job ", fileNumber

        outputFiles = setup_output_dict(args.stdhepInput, args.detector,
                                        fileNumber, args.outputPath,
                                        softVersions)
        slicOutput = outputFiles['slicOutput']
        prePandoraOutput = outputFiles['prePandoraOutput']
        pandoraOutput = outputFiles['pandoraOutput']
        vertexingOutput = outputFiles['vertexingOutput']
        lcsimRecOutput = outputFiles['lcsimRecOutput']
        lcsimDstOutput = outputFiles['lcsimDstOutput']
        flavortagOutput = outputFiles['flavortagOutput']
        diracOutput = outputFiles['diracOutput']

        ################## SLIC ##################################################
        slic = SLIC()
        slic.setVersion(softVersions[0])
        slic.setSteeringFile(args.macFile)
        # slic.setInputFile(lfn)
        slic.setOutputFile(slicOutput)
        slic.setDetectorModel(args.detector)
        slic.setNumberOfEvents(nOutputEvents)
        slic.setStartFrom(startEvent)
        #print slic.listAttributes()
        result = job.append(slic)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## lcsim (digitization and tracking) #####################
        lcsim = LCSIM()
        lcsim.setVersion(softVersions[1])
        lcsim.setSteeringFile(
            'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
        )  # Another version is included in /steeringFiles
        lcsim.getInputFromApp(slic)
        lcsim.setTrackingStrategy(
            'steeringFiles/sidloi3_trackingStrategies_default.xml')
        # lcsim.setAliasProperties('alias.properties')
        lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
        lcsim.setOutputFile(prePandoraOutput)
        lcsim.setNumberOfEvents(nOutputEvents)
        #print lcsim.listAttributes()
        result = job.append(lcsim)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## slicPandora ###########################################
        slicPandora = SLICPandora()
        slicPandora.setVersion(softVersions[2])
        slicPandora.setDetectorModel(args.detector)
        slicPandora.getInputFromApp(lcsim)
        slicPandora.setOutputFile(pandoraOutput)
        slicPandora.setPandoraSettings('pandoraSettings.xml')
        slicPandora.setNumberOfEvents(nOutputEvents)
        #print slicPandora.listAttributes()
        result = job.append(slicPandora)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## Marlin, LCFIPlus Vertexing ############################
        vertexing = Marlin()
        vertexing.setVersion(softVersions[3])
        vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
        vertexing.setGearFile('steeringFiles/sidloi3.gear')
        vertexing.getInputFromApp(slicPandora)
        vertexing.setOutputFile(vertexingOutput)
        vertexing.setNumberOfEvents(nOutputEvents)
        #print vertexing.listAttributes()
        result = job.append(vertexing)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## lcsim (DST production) ################################
        lcsimDst = LCSIM()
        lcsimDst.setVersion(softVersions[1])
        lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
        lcsimDst.getInputFromApp(vertexing)
        lcsimDst.setNumberOfEvents(nOutputEvents)
        # lcsimDst.setAliasProperties('alias.properties')
        lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
        lcsimDst.setOutputRecFile(lcsimRecOutput)
        lcsimDst.setOutputDstFile(lcsimDstOutput)
        #print lcsimDst.listAttributes()
        result = job.append(lcsimDst)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## Marlin, LCFIPlus flavortag ############################
        if args.flavortag:
            flavortag = Marlin()
            flavortag.setVersion(softVersions[3])
            flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
            flavortag.setGearFile('steeringFiles/sidloi3.gear')
            flavortag.setInputFile(lcsimDstOutput)
            flavortag.setOutputFile(flavortagOutput)
            flavortag.setNumberOfEvents(nOutputEvents)
            #print flavortag.listAttributes()
            result = job.append(flavortag)
            if not result['OK']:
                print result['Message']
                sys.exit(2)


################## Job Finalise ##########################################

# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
        job.setBannedSites([
            'LCG.IN2P3-CC.fr',
            'LCG.RAL-LCG2.uk',
            'LCG.DESY-HH.de',
            'LCG.DESYZN.de',
            'LCG.KEK.jp',
            'OSG.PNNL.us',
        ])

        job.setCPUTime(50000)
        job.setPlatform('x86_64-slc5-gcc43-opt')

        # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
        # directory on the grid.
        if args.flavortag:
            job.setOutputData(flavortagOutput, diracOutput, args.SE)

        else:
            job.setOutputData(lcsimDstOutput, diracOutput, args.SE)

        job.setOutputSandbox(outputSandbox)
        job.setInputData(lfn)

        if args.dontPromptMe:
            job.dontPromptMe()
        # Submits Job!!!
        job.submit()

    return 0
Exemple #13
0
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
d = DiracILC()                            # Provides job checking utilities
j = UserJob()                             # You are running a user job
m = Marlin()                              # Get an application instance
m.setVersion("v0116")                     # Define the version to use
m.setSteeringFile("clic_ild_cdr_steering.xml") #What the app should do
m.setInputFile("LFN:/ilc/prod/clic/3tev/ee_h_bb/ILD/DST/00000375/000/\
ee_h_bb_dst_375_999.slcio")               # Add some input
m.setGearFile("clic_ilc_cdr.gear")        # Application specific field
res = j.append(m)                         # Add the application to the job
if not res['OK']:
  print res['Message']                    # Catch any error
j.submit(d)                               # Submit the job
#Not shown here: metadata queries, chaining of applications
Exemple #14
0
  overlay.setBXOverlay(BXOverlay)
  overlay.setGGToHadInt(GGToHadInt250)
  overlay.setBkgEvtType("aa_lowpt")

else:
  print "Overlay ILD: No overlay parameters defined for this energy"

##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion(MarlinVer) ##PUT HERE YOUR MARLIN VERSION
if ild_rec_ov:
  if energy in [250.0, 350.0, 500.0, 1000.0]:
    if UseDD4hepGeometry:
      mao.setSteeringFile("bbudsc_3evt_stdreco_dd4hep.xml")
      mao.setGearFile("GearOutput.xml")
      mao.setDetectorModel(detectorModel)
    else:
      mao.setSteeringFile("bbudsc_3evt_stdreco.xml")
      mao.setGearFile("GearOutput.xml")
  else:
    print "Marlin: No reconstruction suitable for this energy"


##Reconstruction ILD w/o overlay
ma = Marlin()
ma.setDebug()
ma.setVersion(MarlinVer)
ma.setEnergy(energy)
if ild_rec:
  if energy in [250.0, 350.0, 500.0, 1000.0]:
   overlay.setDetectorModel("CLIC_ILD_CDR")
 elif energy == 1400.:
   overlay.setBXOverlay(60)
   overlay.setGGToHadInt(1.3)##When running at 1.4TeV
   overlay.setDetectorModel("CLIC_ILD_CDR")
 else:
   print "Overlay ILD: No overlay parameters defined for this energy"  
 
 ##Reconstruction ILD with overlay
 mao = Marlin()
 mao.setDebug()
 mao.setVersion('v0111Prod')
 if ild_rec_ov:
   if energy==500.:
     mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml")
     mao.setGearFile('clic_ild_cdr500.gear')
   elif energy==350.:
     mao.setSteeringFile("clic_ild_cdr500_steering_overlay_350.0.xml")
     mao.setGearFile('clic_ild_cdr500.gear')
   elif energy==3000.0:
     mao.setSteeringFile("clic_ild_cdr_steering_overlay_3000.0.xml")
     mao.setGearFile('clic_ild_cdr.gear')
   elif energy==1400.0:
     mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml")
     mao.setGearFile('clic_ild_cdr.gear')
   else:
     print "Marlin: No reconstruction suitable for this energy"
 
 
 ##Reconstruction w/o overlay
 ma = Marlin()
def main():
	# Take the input arguments from the argument parser, and check they exist...
	args = parse_args()
	if not args:
		print 'Invalid Arguments'
		sys.exit(1)

	print args.chain[0]

	# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
	softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
	# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working 
	# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
	# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]

	check_events_arguments(args.events, args.split)
	detector = args.detector
	alias_properties(detector)
	outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
	inputSandbox, outputSandbox = setup_sandboxes(args.macFile)

	dirac = DiracILC(True, repoName)

	# Prepares values for the job loop...
	if args.split < 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.events)
	if args.split > 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.split)

	# Loop that runs through the required number of jobs to be executed...
	for startEvent in range(0, nInputEvents, nOutputEvents):

################## Job Initialise ########################################		
		job = UserJob()
		job.setName(outputBase)
		job.setJobGroup('JobGroup')
		job.setInputSandbox(inputSandbox)
		fileNumber = startEvent/nOutputEvents
		print "Job ---> ", fileNumber

################## SLIC ##################################################
		if 1 in args.chain:
			slic = SLIC()
			slic.setVersion(softVersions[0])
			slic.setSteeringFile(args.macFile)
			# slic.setInputFile(lfn)
			slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
			slic.setDetectorModel(detector)
			slic.setNumberOfEvents(nOutputEvents)
			slic.setStartFrom(startEvent)
			#print slic.listAttributes()
			result = job.append(slic)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Overlay ###############################################
		if 2 in args.chain:
			'''
			#Add the gghad background overlay.
			gghad = OverlayInput()
			#gghad.setProdID(1767)
			gghad.setEnergy(500.0)
			gghad.setBXOverlay('args.bunches')
			gghad.setGGToHadInt( 4.1 )
			gghad.setNbSigEvtsPerJob(nOutputEvents)
			gghad.setMachine('ilc_dbd')
			gghad.setDetectorModel('sidloi3')
			gghad.setBkgEvtType('aa_lowpt')
			result = job.append( gghad )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			
			#Add the pair background overlay.
			pairs = OverlayInput()
			pairs.setProdID(2)
			pairs.setEnergy(500.0)
			pairs.setBXOverlay('args.bunches')
			pairs.setGGToHadInt(1.)
			pairs.setNbSigEvtsPerJob(nOutputEvents)
			pairs.setMachine('ilc_dbd')
			pairs.setDetectorModel('sidloi3')
			pairs.setBkgEvtType('eepairs')
			result = job.append( pairs )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			'''
			gghad = OverlayInput()
			gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
			gghad.setBXOverlay(int(args.bunches))
			gghad.setGGToHadInt( 4.1 )
			gghad.setNbSigEvtsPerJob(nOutputEvents)
			gghad.setBkgEvtType('aa_lowpt')		
			result = job.append( gghad )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			'''
			pairs = OverlayInput()
			pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
			pairs.setBXOverlay(int(args.bunches))
			pairs.setGGToHadInt(1.)
			pairs.setBkgEvtType('eepairs')
			pairs.setNbSigEvtsPerJob(nOutputEvents)
			result = job.append( pairs )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			'''
			
################## lcsim (digitization and tracking) #####################
		if 3 in args.chain:
			lcsim = LCSIM()
			lcsim.setVersion(softVersions[1])
			lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
			if 1 in args.chain:
				lcsim.getInputFromApp(slic)
			lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
			lcsim.setAliasProperties('steeringFiles/alias.properties')
			lcsim.setDetectorModel(detector+".zip")
			#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
			lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
			lcsim.setNumberOfEvents(nOutputEvents)
			#print lcsim.listAttributes()
			result = job.append(lcsim)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## slicPandora ###########################################
		if 4 in args.chain:
			slicPandora = SLICPandora()
			slicPandora.setVersion(softVersions[2])
			slicPandora.setDetectorModel(detector)
			slicPandora.getInputFromApp(lcsim)
			slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
			slicPandora.setPandoraSettings('pandoraSettings.xml')
			slicPandora.setNumberOfEvents(nOutputEvents)
			#print slicPandora.listAttributes()
			result = job.append(slicPandora)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Marlin, LCFIPlus Vertexing ############################
		if 5 in args.chain:
			vertexing = Marlin()
			vertexing.setVersion(softVersions[3])
			vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
			vertexing.setGearFile('steeringFiles/' + detector + '.gear')
			vertexing.getInputFromApp(slicPandora)
			vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
			vertexing.setNumberOfEvents(nOutputEvents)
			#print vertexing.listAttributes()
			result = job.append(vertexing)
			if not result['OK']:
				print result['Message']
				sys.exit(2)
################## lcsim (DST production) ################################
			lcsimDst = LCSIM()
			lcsimDst.setVersion(softVersions[1])
			lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
			lcsimDst.getInputFromApp(vertexing)
			lcsimDst.setNumberOfEvents(nOutputEvents)
			lcsimDst.setAliasProperties('steeringFiles/alias.properties')
			lcsimDst.setDetectorModel(detector+".zip")
			lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
			lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
			#print lcsimDst.listAttributes()
			result = job.append(lcsimDst)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Marlin, LCFIPlus flavortag ############################
		if 6 in args.chain:
			flavortag = Marlin()
			flavortag.setVersion(softVersions[3])
			flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
			flavortag.setGearFile('steeringFiles/' + detector + '.gear')
			flavortag.setInputFile(lcsimDstOutput)
			flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
			flavortag.setNumberOfEvents(nOutputEvents)
			#print flavortag.listAttributes()
			result = job.append(flavortag)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Job Finalise ##########################################

		# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
		# This list is likely to change.
		job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
							'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
							'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])

		job.setCPUTime(50000)
		job.setPlatform('x86_64-slc5-gcc43-opt')

		# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
		# directory on the grid.
		outputLevel = max(args.chain)
		if outputLevel == 1:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 3:
			#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 4:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 5:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 6:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')

		job.setOutputSandbox(outputSandbox)
		job.setInputData(args.Input)

		if args.dontPromptMe:
			job.dontPromptMe()
		# Submits Job!!!
		job.submit()

	return 0;
Exemple #17
0
class MarlinTestCase( unittest.TestCase ):
  """ Base class for the Marlin test cases
  """
  def setUp( self ):
    """set up the objects"""
    self.mar = Marlin( {} )

  def test_setgear( self ):
    self.mar.setGearFile( 'lfn:/my/gear/file.txt' )
    self.assertFalse( self.mar._errorDict )
    self.assertIn( 'lfn:/my/gear/file.txt', self.mar.inputSB )

  def test_setKeepRecFile(self):
    """Tests for behaviour with KeepRecFile True/False."""
    self.assertTrue(self.mar.keepRecFile)
    self.mar.setKeepRecFile(False)
    self.assertFalse(self.mar._errorDict)
    self.mar.setKeepRecFile(True)
    self.assertTrue(self.mar.keepRecFile)
    self.assertFalse(self.mar._errorDict)
    self.mar.setKeepRecFile(123)
    self.assertIn('val = 123', str(self.mar._errorDict))

  def test_setoutputrec( self ):
    self.mar.setOutputRecFile( 'my/file.outputrec', 'mytestPath' )
    assertEqualsImproved( self.mar.outputRecPath, 'mytestPath', self )
    self.assertFalse( self.mar._errorDict )

  def test_setoutputdst( self ):
    self.mar.setOutputDstFile( 'my/file.outputdst', 'mytestPath' )
    assertEqualsImproved( self.mar.outputDstPath, 'mytestPath', self )
    self.assertFalse( self.mar._errorDict )

  def test_setproclist( self ):
    self.mar.setProcessorsToUse( [ 'proc1', 'proc2' ] )
    self.assertFalse( self.mar._errorDict )

  def test_setexcludeproclist( self ):
    self.mar.setProcessorsToExclude( [ 'proc1', 'proc2' ] )
    self.assertFalse( self.mar._errorDict )

  def test_userjobmodules( self ):
    module_mock = Mock()
    assertDiracSucceeds( self.mar._userjobmodules( module_mock ), self )

  def test_prodjobmodules( self ):
    module_mock = Mock()
    assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )

  def test_prodjobmodules_outputpath( self ):
    module_mock = Mock()
    self.mar.outputPath = 'aef'
    assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )
    self.assertIn( { 'OutputFile' : '@{OutputFile}', 'outputPath' : '@{OutputPath}',
                     'outputDataSE' : '@{OutputSE}'}, self.mar._listofoutput )

  def test_userjobmodules_fails( self ):
    with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
         patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
      assertDiracFailsWith( self.mar._userjobmodules( None ),
                            'userjobmodules failed', self )

  def test_prodjobmodules_fails( self ):
    with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
         patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
      assertDiracFailsWith( self.mar._prodjobmodules( None ),
                            'prodjobmodules failed', self )

  def test_checkconsistency( self ):
    self.mar.version = '13'
    self.mar.steeringFile = '/mysteer/file.stdhep'
    self.mar.gearFile = None
    self.mar._jobtype = 'notUser'
    self.mar.outputFile = None
    with patch('os.path.exists', new=Mock(return_value=True)), \
         patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
      assertDiracSucceeds( self.mar._checkConsistency(), self )
      self.assertIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
                       'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      self.assertIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
                       'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
        self.assertIn( keyword, self.mar.prodparameters )
      assertEqualsImproved( self.mar.gearFile, None, self )

  def test_checkconsistency_noversion( self ):
    self.mar.version = None
    assertDiracFailsWith( self.mar._checkConsistency(), 'version not set', self )

  def test_checkconsistency_invalidxml( self ):
    self.mar.version = '13'
    self.mar.steeringFile = '/mysteer/file.stdhep'
    with patch('os.path.exists', new=Mock(return_value=True)), \
         patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
      assertDiracFailsWith( self.mar._checkConsistency(), 'supplied steering file cannot be read with xml', self)

  def test_checkconsistency_othercase( self ):
    self.mar.version = '13'
    self.mar.steeringFile = '/mysteer/file.stdhep'
    self.mar.gearFile = 'myGearOutput.mock'
    self.mar._jobtype = 'notUser'
    self.mar.outputFile = 'myoutput.test'
    with patch('os.path.exists', new=Mock(return_value=False)):
      assertDiracSucceeds( self.mar._checkConsistency(), self )
      self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
                          'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
                          'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
        self.assertIn( keyword, self.mar.prodparameters )
      assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )

  def test_checkconsistency_lastcase( self ):
    self.mar.version = '13'
    self.mar.steeringFile = None
    self.mar.gearFile = 'myGearOutput.mock'
    self.mar._jobtype = 'User'
    self.mar.outputFile = 'myoutput.test'
    assertDiracSucceeds( self.mar._checkConsistency(), self )
    self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
                        'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
    self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
                        'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
    for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
      self.assertNotIn( keyword, self.mar.prodparameters )
    assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )

  def test_resolvelinkedparams(self):
    """Test _resolveLinkedStepParameters with something happening."""
    step_mock = Mock()
    input_mock = Mock()
    input_mock.getType.return_value = {'abc': False}
    self.mar._linkedidx = 3
    self.mar._jobsteps = [None, None, None, input_mock]
    assertDiracSucceeds(self.mar._resolveLinkedStepParameters(step_mock), self)
    step_mock.setLink.assert_called_once_with('InputFile', {'abc': False}, 'OutputFile')

  def test_resolvelinkedparams_noinputstep(self):
    """Call _resolveLinkedStep function, which does nothing."""
    self.mar._linkedidx = None
    self.mar._inputappstep = []
    assertDiracSucceeds(self.mar._resolveLinkedStepParameters(None), self)
Exemple #18
0
  overlay.setBXOverlay(300)
  overlay.setGGToHadInt(0.3)##When running at 500geV
elif energy == 1000.:
  overlay.setBXOverlay(60)
  overlay.setGGToHadInt(1.3)##When running at 1tev
else:
  print "Overlay ILD: No overlay parameters defined for this energy"  

##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion('v0111Prod') ##PUT HERE YOUR MARLIN VERSION
if ild_rec_ov:
  if energy==500.:
    mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml") #STEERINGFILE for 500gev
    mao.setGearFile('clic_ild_cdr500.gear') #GEAR FILE for 500gev
  elif energy==1000.0:
    mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml") #STEERINGFILE for 1tev
    mao.setGearFile('clic_ild_cdr.gear') #GEAR FILE for 1tev
  else:
    print "Marlin: No reconstruction suitable for this energy"


##Reconstruction ILD w/o overlay
ma = Marlin()
ma.setDebug()
ma.setVersion('v0111Prod') ##PUT HERE YOUR MARLIN VERSION
if ild_rec:
  if energy in [500.]:
    ma.setSteeringFile("clic_ild_cdr500_steering.xml")##PUT HERE YOUR MARLIN steering files
    ma.setGearFile('clic_ild_cdr500.gear')
def main():
	# Take the input arguments from the argument parser, and check they exist...
	args = parse_args()
	if not args:
		print 'Invalid Arguments'
		sys.exit(1)

#### Software Versions ####
	softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
	# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working 
	# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working

	# Check the --runs and --split arguments to make sure they are compatible, if not exit... 
	if not check_events_arguments(args.events, args.split):
		sys.exit(1)

	# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
	lfn_check, lfn = check_input_LFN(args.stdhepInput)
	if not lfn_check:
		sys.exit(1)

	# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
	dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector))

	# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
	# into the input sandbox
	inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)

	# Prepares values for the job loop...
	if args.split < 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.events)
	if args.split > 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.split)

	# Loop that runs through the required number of jobs to be executed...
	for startEvent in range(0, nInputEvents, nOutputEvents):

################## Job Initialise ########################################		
		job = UserJob()
		job.setName(path.basename(args.stdhepInput))
		job.setJobGroup('JobGroup')
		job.setInputSandbox(inputSandbox)
		fileNumber = startEvent/nOutputEvents
		print "Job ", fileNumber

		outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions)
		slicOutput=outputFiles['slicOutput']
		prePandoraOutput=outputFiles['prePandoraOutput']
		pandoraOutput=outputFiles['pandoraOutput']
		vertexingOutput=outputFiles['vertexingOutput']
		lcsimRecOutput=outputFiles['lcsimRecOutput']
		lcsimDstOutput=outputFiles['lcsimDstOutput']
		flavortagOutput=outputFiles['flavortagOutput']
		diracOutput=outputFiles['diracOutput']

################## SLIC ##################################################
		slic = SLIC()
		slic.setVersion(softVersions[0])
		slic.setSteeringFile(args.macFile)
		# slic.setInputFile(lfn)
		slic.setOutputFile(slicOutput)
		slic.setDetectorModel(args.detector)
		slic.setNumberOfEvents(nOutputEvents)
		slic.setStartFrom(startEvent)
		#print slic.listAttributes()
		result = job.append(slic)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## lcsim (digitization and tracking) #####################
		lcsim = LCSIM()
		lcsim.setVersion(softVersions[1])
		lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles
		lcsim.getInputFromApp(slic)
		lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
		# lcsim.setAliasProperties('alias.properties')
		lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
		lcsim.setOutputFile(prePandoraOutput)
		lcsim.setNumberOfEvents(nOutputEvents)
		#print lcsim.listAttributes()
		result = job.append(lcsim)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## slicPandora ###########################################
		slicPandora = SLICPandora()
		slicPandora.setVersion(softVersions[2])
		slicPandora.setDetectorModel(args.detector)
		slicPandora.getInputFromApp(lcsim)
		slicPandora.setOutputFile(pandoraOutput)
		slicPandora.setPandoraSettings('pandoraSettings.xml')
		slicPandora.setNumberOfEvents(nOutputEvents)
		#print slicPandora.listAttributes()
		result = job.append(slicPandora)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## Marlin, LCFIPlus Vertexing ############################
		vertexing = Marlin()
		vertexing.setVersion(softVersions[3])
		vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
		vertexing.setGearFile('steeringFiles/sidloi3.gear')
		vertexing.getInputFromApp(slicPandora)
		vertexing.setOutputFile(vertexingOutput)
		vertexing.setNumberOfEvents(nOutputEvents)
		#print vertexing.listAttributes()
		result = job.append(vertexing)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## lcsim (DST production) ################################
		lcsimDst = LCSIM()
		lcsimDst.setVersion(softVersions[1])
		lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
		lcsimDst.getInputFromApp(vertexing)
		lcsimDst.setNumberOfEvents(nOutputEvents)
		# lcsimDst.setAliasProperties('alias.properties')
		lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
		lcsimDst.setOutputRecFile(lcsimRecOutput)
		lcsimDst.setOutputDstFile(lcsimDstOutput)
		#print lcsimDst.listAttributes()
		result = job.append(lcsimDst)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## Marlin, LCFIPlus flavortag ############################
		if args.flavortag:
			flavortag = Marlin()
			flavortag.setVersion(softVersions[3])
			flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
			flavortag.setGearFile('steeringFiles/sidloi3.gear')
			flavortag.setInputFile(lcsimDstOutput)
			flavortag.setOutputFile(flavortagOutput)
			flavortag.setNumberOfEvents(nOutputEvents)
			#print flavortag.listAttributes()
			result = job.append(flavortag)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Job Finalise ##########################################

		# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
		# This list is likely to change.
		job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
							'OSG.PNNL.us',])

		job.setCPUTime(50000)
		job.setPlatform('x86_64-slc5-gcc43-opt')

		# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
		# directory on the grid.
		if args.flavortag:
			job.setOutputData(flavortagOutput, diracOutput, args.SE)

		else: 
			job.setOutputData(lcsimDstOutput, diracOutput, args.SE)

		job.setOutputSandbox(outputSandbox)
		job.setInputData(lfn)

		if args.dontPromptMe:
			job.dontPromptMe()
		# Submits Job!!!
		job.submit()

	return 0;
Exemple #20
0
    if detectormodel in ILDDetectorModels:
        overlay.setMachine("ilc_dbd")
        overlay.setBackgroundType("aa_lowpt")
        overlay.setBXOverlay(1)
        overlay.setDetectorModel(detectormodel)
        if energy == 250:
            overlay.setGGToHadInt(0.3)

    ##Reconstruction ILD with overlay
    mao = Marlin()
    mao.setDebug()
    mao.setVersion(SOFTWAREVERSION)
    if ild_rec_ov:
        if energy == 500.:
            mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml")
            mao.setGearFile('clic_ild_cdr500.gear')
        elif energy == 420.:
            mao.setSteeringFile("clic_ild_cdr500_steering_overlay_420.0.xml")
            mao.setGearFile('clic_ild_cdr500.gear')
        elif energy == 350.:
            mao.setSteeringFile("clic_ild_cdr500_steering_overlay_350.0.xml")
            mao.setGearFile('clic_ild_cdr500.gear')
        elif energy == 3000.0:
            mao.setSteeringFile("clic_ild_cdr_steering_overlay_3000.0.xml")
            mao.setGearFile('clic_ild_cdr.gear')
        elif energy == 1400.0:
            mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml")
            mao.setGearFile('clic_ild_cdr.gear')
        else:
            print "Marlin: No reconstruction suitable for this energy"
overlay = [True, False]

for ov in overlay:
    d = DiracILC(True, "repo_overlay_%s.rep" % ov)
    for lfn in lfns:
        j = UserJob()
        steeringf = "clic_ild_cdr_steering.xml"
        if ov:
            steeringf = "clic_ild_cdr_steering_overlay.xml"
            res = j.append(ovi)
            if not res['OK']:
                print res['Message']
                continue
        ma = Marlin()
        ma.setVersion("v0111Prod")
        ma.setGearFile("clic_ild_cdr.gear")
        ma.setSteeringFile(steeringf)
        ma.setInputFile("LFN:" + lfn)
        ma.setNbEvts(10)
        ma.setEnergy(500.)
        ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov)
        ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov)
        res = j.append(ma)
        if not res['OK']:
            print res['Message']
            exit()

        j.setCPUTime(86400)
        j.setOutputData("myrec_overlay_%s.slcio" % ov, "some/path")
        j.setName("SomeName")
        j.setJobGroup("SomeGroup")
Exemple #22
0
class MarlinTestCase(unittest.TestCase):
    """ Base class for the Marlin test cases
  """
    def setUp(self):
        """set up the objects"""
        self.mar = Marlin({})

    def test_setgear(self):
        self.mar.setGearFile('lfn:/my/gear/file.txt')
        self.assertFalse(self.mar._errorDict)
        self.assertIn('lfn:/my/gear/file.txt', self.mar.inputSB)

    def test_setoutputrec(self):
        self.mar.setOutputRecFile('my/file.outputrec', 'mytestPath')
        assertEqualsImproved(self.mar.outputRecPath, 'mytestPath', self)
        self.assertFalse(self.mar._errorDict)

    def test_setoutputdst(self):
        self.mar.setOutputDstFile('my/file.outputdst', 'mytestPath')
        assertEqualsImproved(self.mar.outputDstPath, 'mytestPath', self)
        self.assertFalse(self.mar._errorDict)

    def test_setproclist(self):
        self.mar.setProcessorsToUse(['proc1', 'proc2'])
        self.assertFalse(self.mar._errorDict)

    def test_setexcludeproclist(self):
        self.mar.setProcessorsToExclude(['proc1', 'proc2'])
        self.assertFalse(self.mar._errorDict)

    def test_userjobmodules(self):
        module_mock = Mock()
        assertDiracSucceeds(self.mar._userjobmodules(module_mock), self)

    def test_prodjobmodules(self):
        module_mock = Mock()
        assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)

    def test_prodjobmodules_outputpath(self):
        module_mock = Mock()
        self.mar.outputPath = 'aef'
        assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)
        self.assertIn(
            {
                'OutputFile': '@{OutputFile}',
                'outputPath': '@{OutputPath}',
                'outputDataSE': '@{OutputSE}'
            }, self.mar._listofoutput)

    def test_userjobmodules_fails(self):
        with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
             patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
            assertDiracFailsWith(self.mar._userjobmodules(None),
                                 'userjobmodules failed', self)

    def test_prodjobmodules_fails(self):
        with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
             patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
            assertDiracFailsWith(self.mar._prodjobmodules(None),
                                 'prodjobmodules failed', self)

    def test_checkconsistency(self):
        self.mar.version = '13'
        self.mar.steeringFile = '/mysteer/file.stdhep'
        self.mar.gearFile = None
        self.mar._jobtype = 'notUser'
        self.mar.outputFile = None
        with patch('os.path.exists', new=Mock(return_value=True)), \
             patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
            assertDiracSucceeds(self.mar._checkConsistency(), self)
            self.assertIn(
                {
                    'outputFile': '@{outputREC}',
                    'outputPath': '@{outputPathREC}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            self.assertIn(
                {
                    'outputFile': '@{outputDST}',
                    'outputPath': '@{outputPathDST}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            for keyword in [
                    'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
            ]:
                self.assertIn(keyword, self.mar.prodparameters)
            assertEqualsImproved(self.mar.gearFile, None, self)

    def test_checkconsistency_noversion(self):
        self.mar.version = None
        assertDiracFailsWith(self.mar._checkConsistency(), 'version not set',
                             self)

    def test_checkconsistency_invalidxml(self):
        self.mar.version = '13'
        self.mar.steeringFile = '/mysteer/file.stdhep'
        with patch('os.path.exists', new=Mock(return_value=True)), \
             patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
            assertDiracFailsWith(
                self.mar._checkConsistency(),
                'supplied steering file cannot be read with xml', self)

    def test_checkconsistency_othercase(self):
        self.mar.version = '13'
        self.mar.steeringFile = '/mysteer/file.stdhep'
        self.mar.gearFile = 'myGearOutput.mock'
        self.mar._jobtype = 'notUser'
        self.mar.outputFile = 'myoutput.test'
        with patch('os.path.exists', new=Mock(return_value=False)):
            assertDiracSucceeds(self.mar._checkConsistency(), self)
            self.assertNotIn(
                {
                    'outputFile': '@{outputREC}',
                    'outputPath': '@{outputPathREC}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            self.assertNotIn(
                {
                    'outputFile': '@{outputDST}',
                    'outputPath': '@{outputPathDST}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            for keyword in [
                    'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
            ]:
                self.assertIn(keyword, self.mar.prodparameters)
            assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)

    def test_checkconsistency_lastcase(self):
        self.mar.version = '13'
        self.mar.steeringFile = None
        self.mar.gearFile = 'myGearOutput.mock'
        self.mar._jobtype = 'User'
        self.mar.outputFile = 'myoutput.test'
        assertDiracSucceeds(self.mar._checkConsistency(), self)
        self.assertNotIn(
            {
                'outputFile': '@{outputREC}',
                'outputPath': '@{outputPathREC}',
                'outputDataSE': '@{OutputSE}'
            }, self.mar._listofoutput)
        self.assertNotIn(
            {
                'outputFile': '@{outputDST}',
                'outputPath': '@{outputPathDST}',
                'outputDataSE': '@{OutputSE}'
            }, self.mar._listofoutput)
        for keyword in [
                'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
        ]:
            self.assertNotIn(keyword, self.mar.prodparameters)
        assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)