예제 #1
0
def getJob():
    """ produce a job: it's always the same, so we don't need to put it in the main
  """
    j = UserJob()
    ma = Marlin()
    ma.setVersion("v0111Prod")
    ma.setSteeringFile("clic_ild_cdr_steering.xml")
    ma.setGearFile("clic_ild_cdr.gear")
    result = j.append(ma)
    if not result['OK']:
        gLogger.error(result["Message"])
        dexit(1)
    j.setCPUTime(10000)
    j.setOutputSandbox("*.log")
    return j
def getJob():
  """ produce a job: it's always the same, so we don't need to put it in the main
  """
  j = UserJob()
  ma = Marlin()
  ma.setVersion("v0111Prod")
  ma.setSteeringFile("clic_ild_cdr_steering.xml")
  ma.setGearFile("clic_ild_cdr.gear")
  result = j.append(ma)
  if not result['OK']:
    gLogger.error(result["Message"])
    dexit(1)
  j.setCPUTime(10000)
  j.setOutputSandbox("*.log")
  return j
예제 #3
0
    ddsim.setDetectorModel(detectorModel)
    ddsim.setOutputFile(outputFile)
    ddsim.setSteeringFile("/afs/cern.ch/work/v/viazlo/gridSubmission/FCCee_o5/files/clic_steer.py")
    ddsim.setNumberOfEvents(nEvts)
    res = job.append(ddsim)

    if not res['OK']:
        print res['Message']
        sys.exit(2)

    #####################################################################  


    # #####################################################################  
    #marlin
    ma = Marlin()
    ma.setVersion(marlinVersion)
    ma.setNbEvts(nEvts)
    ma.setDetectorModel(detectorModel)
    ma.getInputFromApp(ddsim)
    ma.setProcessorsToUse([])
    ma.setSteeringFile(nameSteeringMarlin)
    res = job.append(ma) 

    if not res['OK']:
        print res['Message']
        sys.exit(2)

    
    #####################################################################  
    #submit          
예제 #4
0
  overlay.setGGToHadInt(GGToHadInt1000)
  overlay.setBkgEvtType("aa_lowpt")
elif energy == 350.:
  overlay.setBXOverlay(BXOverlay)
  overlay.setGGToHadInt(GGToHadInt350)
  overlay.setBkgEvtType("aa_lowpt")
elif energy == 250.:
  overlay.setBXOverlay(BXOverlay)
  overlay.setGGToHadInt(GGToHadInt250)
  overlay.setBkgEvtType("aa_lowpt")

else:
  print "Overlay ILD: No overlay parameters defined for this energy"

##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion(MarlinVer) ##PUT HERE YOUR MARLIN VERSION
if ild_rec_ov:
  if energy in [250.0, 350.0, 500.0, 1000.0]:
    if UseDD4hepGeometry:
      mao.setSteeringFile("bbudsc_3evt_stdreco_dd4hep.xml")
      mao.setGearFile("GearOutput.xml")
      mao.setDetectorModel(detectorModel)
    else:
      mao.setSteeringFile("bbudsc_3evt_stdreco.xml")
      mao.setGearFile("GearOutput.xml")
  else:
    print "Marlin: No reconstruction suitable for this energy"

예제 #5
0
   overlay.setBXOverlay(300)
   overlay.setGGToHadInt(0.0464)##When running at 350geV
   overlay.setDetectorModel("CLIC_ILD_CDR500")
 elif energy == 3000.:
   overlay.setBXOverlay(60)
   overlay.setGGToHadInt(3.2)##When running at 3TeV
   overlay.setDetectorModel("CLIC_ILD_CDR")
 elif energy == 1400.:
   overlay.setBXOverlay(60)
   overlay.setGGToHadInt(1.3)##When running at 1.4TeV
   overlay.setDetectorModel("CLIC_ILD_CDR")
 else:
   print "Overlay ILD: No overlay parameters defined for this energy"  
 
 ##Reconstruction ILD with overlay
 mao = Marlin()
 mao.setDebug()
 mao.setVersion('v0111Prod')
 if ild_rec_ov:
   if energy==500.:
     mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml")
     mao.setGearFile('clic_ild_cdr500.gear')
   elif energy==350.:
     mao.setSteeringFile("clic_ild_cdr500_steering_overlay_350.0.xml")
     mao.setGearFile('clic_ild_cdr500.gear')
   elif energy==3000.0:
     mao.setSteeringFile("clic_ild_cdr_steering_overlay_3000.0.xml")
     mao.setGearFile('clic_ild_cdr.gear')
   elif energy==1400.0:
     mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml")
     mao.setGearFile('clic_ild_cdr.gear')
예제 #6
0
class MarlinTestCase(unittest.TestCase):
    """ Base class for the Marlin test cases
  """
    def setUp(self):
        """set up the objects"""
        self.mar = Marlin({})

    def test_setgear(self):
        self.mar.setGearFile('lfn:/my/gear/file.txt')
        self.assertFalse(self.mar._errorDict)
        self.assertIn('lfn:/my/gear/file.txt', self.mar.inputSB)

    def test_setoutputrec(self):
        self.mar.setOutputRecFile('my/file.outputrec', 'mytestPath')
        assertEqualsImproved(self.mar.outputRecPath, 'mytestPath', self)
        self.assertFalse(self.mar._errorDict)

    def test_setoutputdst(self):
        self.mar.setOutputDstFile('my/file.outputdst', 'mytestPath')
        assertEqualsImproved(self.mar.outputDstPath, 'mytestPath', self)
        self.assertFalse(self.mar._errorDict)

    def test_setproclist(self):
        self.mar.setProcessorsToUse(['proc1', 'proc2'])
        self.assertFalse(self.mar._errorDict)

    def test_setexcludeproclist(self):
        self.mar.setProcessorsToExclude(['proc1', 'proc2'])
        self.assertFalse(self.mar._errorDict)

    def test_userjobmodules(self):
        module_mock = Mock()
        assertDiracSucceeds(self.mar._userjobmodules(module_mock), self)

    def test_prodjobmodules(self):
        module_mock = Mock()
        assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)

    def test_prodjobmodules_outputpath(self):
        module_mock = Mock()
        self.mar.outputPath = 'aef'
        assertDiracSucceeds(self.mar._prodjobmodules(module_mock), self)
        self.assertIn(
            {
                'OutputFile': '@{OutputFile}',
                'outputPath': '@{OutputPath}',
                'outputDataSE': '@{OutputSE}'
            }, self.mar._listofoutput)

    def test_userjobmodules_fails(self):
        with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
             patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
            assertDiracFailsWith(self.mar._userjobmodules(None),
                                 'userjobmodules failed', self)

    def test_prodjobmodules_fails(self):
        with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
             patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
            assertDiracFailsWith(self.mar._prodjobmodules(None),
                                 'prodjobmodules failed', self)

    def test_checkconsistency(self):
        self.mar.version = '13'
        self.mar.steeringFile = '/mysteer/file.stdhep'
        self.mar.gearFile = None
        self.mar._jobtype = 'notUser'
        self.mar.outputFile = None
        with patch('os.path.exists', new=Mock(return_value=True)), \
             patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
            assertDiracSucceeds(self.mar._checkConsistency(), self)
            self.assertIn(
                {
                    'outputFile': '@{outputREC}',
                    'outputPath': '@{outputPathREC}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            self.assertIn(
                {
                    'outputFile': '@{outputDST}',
                    'outputPath': '@{outputPathDST}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            for keyword in [
                    'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
            ]:
                self.assertIn(keyword, self.mar.prodparameters)
            assertEqualsImproved(self.mar.gearFile, None, self)

    def test_checkconsistency_noversion(self):
        self.mar.version = None
        assertDiracFailsWith(self.mar._checkConsistency(), 'version not set',
                             self)

    def test_checkconsistency_invalidxml(self):
        self.mar.version = '13'
        self.mar.steeringFile = '/mysteer/file.stdhep'
        with patch('os.path.exists', new=Mock(return_value=True)), \
             patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
            assertDiracFailsWith(
                self.mar._checkConsistency(),
                'supplied steering file cannot be read with xml', self)

    def test_checkconsistency_othercase(self):
        self.mar.version = '13'
        self.mar.steeringFile = '/mysteer/file.stdhep'
        self.mar.gearFile = 'myGearOutput.mock'
        self.mar._jobtype = 'notUser'
        self.mar.outputFile = 'myoutput.test'
        with patch('os.path.exists', new=Mock(return_value=False)):
            assertDiracSucceeds(self.mar._checkConsistency(), self)
            self.assertNotIn(
                {
                    'outputFile': '@{outputREC}',
                    'outputPath': '@{outputPathREC}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            self.assertNotIn(
                {
                    'outputFile': '@{outputDST}',
                    'outputPath': '@{outputPathDST}',
                    'outputDataSE': '@{OutputSE}'
                }, self.mar._listofoutput)
            for keyword in [
                    'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
            ]:
                self.assertIn(keyword, self.mar.prodparameters)
            assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)

    def test_checkconsistency_lastcase(self):
        self.mar.version = '13'
        self.mar.steeringFile = None
        self.mar.gearFile = 'myGearOutput.mock'
        self.mar._jobtype = 'User'
        self.mar.outputFile = 'myoutput.test'
        assertDiracSucceeds(self.mar._checkConsistency(), self)
        self.assertNotIn(
            {
                'outputFile': '@{outputREC}',
                'outputPath': '@{outputPathREC}',
                'outputDataSE': '@{OutputSE}'
            }, self.mar._listofoutput)
        self.assertNotIn(
            {
                'outputFile': '@{outputDST}',
                'outputPath': '@{outputPathDST}',
                'outputDataSE': '@{OutputSE}'
            }, self.mar._listofoutput)
        for keyword in [
                'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
        ]:
            self.assertNotIn(keyword, self.mar.prodparameters)
        assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)
예제 #7
0
for i in range(n_jobs):
    j = UserJob()

    mo = Mokka()
    mo.setEnergy(3000)
    mo.setVersion("0706P08")
    mo.setSteeringFile("clic_ild_cdr.steer")
    mo.setMacFile("particlegun_electron.mac")
    mo.setOutputFile("MyFile.slcio")
    mo.setNbEvts(n_evts_per_job)
    res = j.append(mo)
    if not res['OK']:
        print res['Message']
        break
    ma = Marlin()
    ma.setVersion("v0111Prod")
    ma.setSteeringFile("clic_ild_cdr_steering.xml")
    ma.getInputFromApp(mo)
    ma.setOutputDstFile("mydst_no_ov_%s.slcio" % i)
    res = j.append(ma)
    if not res['OK']:
        print res['Message']
        break
    ov = OverlayInput()
    ov.setBXOverlay(60)
    ov.setGGToHadInt(3.2)
    ov.setNbSigEvtsPerJob(n_evts_per_job)
    ov.setBkgEvtType("gghad")
    ov.setDetectorModel("CLIC_ILD_CDR")
예제 #8
0
#RandSeed = random.randrange(11623, 99999)

#  "_%n"  .... gets the corresponding split number !!!!
jobname = "r" + idin + "_" + str(indx)
print jobname

#lcoutputSIM  = simOutfile + "_SIM_%s.slcio"%(indx) 
lcoutputSIM  = simOutfile + "_%s.SIM.slcio"%(indx) 
lcinputREC = "/ilc/user/k/kurca/MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" + ireq +"/sim/" + lcoutputSIM
print lcinputREC

# Marlin
lcoutputDST  = recOutfile + "_%s.DST.slcio"%(indx) 
lcoutputREC  = recOutfile + "_%s.REC.slcio"%(indx) 

ma = Marlin()
ma.setVersion(ILCSoftVer)
ma.setDetectorModel(detectorModel)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments(" --constant.DetectorModel=%s "%(detectorModel) )
ma.setLogFile("marlin.log")
#ma.getInputFromApp(sim)
ma.setInputFile([lcoutputSIM])
ma.setEnergy(energy)
ma.setNumberOfEvents(evtsPerRun)
ma.setOutputDstFile(lcoutputDST)
ma.setOutputRecFile(lcoutputREC)

RECoutput.append(lcoutputDST)
RECoutput.append(lcoutputREC)
예제 #9
0
def getJob(dirac, jobid, jobpara):
  iser=jobid+100

  outdir = "/ilc/user/a/amiyamoto/myprod2/test/"
  outdst = "toto-ovl-%5.5i.dst.slcio"%iser
  outrec = "toto-ovl-%5.5i.rec.slcio"%iser
  dstlfn = outdir+"dst/"+outdst
  reclfn = outdir+"rec/"+outrec
  outsrm = "CERN-SRM"

###In case one wants a loop: comment the folowing.
#for i in range(2):
  j = UserJob()
  j.setJobGroup("Tutorial")
  j.setName("MarlinOverlayParametric%i"%iser)
  j.setInputSandbox(jobpara["setting_file"])

## Define the overlay
  ov = OverlayInput()
  ov.setMachine("ilc_dbd")
  ov.setEnergy(energy)
  ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
  ov.setBXOverlay(int(jobpara["BXOverlay"]))
  ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
  ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
  ov.setDetectorModel("ILD_o1_v05")
  res = j.append(ov)
  if not res['OK']:
    print res['Message']
    exit(1)

## Define Marlin job
  ma = Marlin()
  ma.setDebug()
  ma.setVersion("ILCSoft-01-17-09")
  ma.setSteeringFile("marlin_stdreco.xml")
  ma.setGearFile("GearOutput.xml")
#   ma.setInputFile(simfile)
  ma.setInputFile(simlists[jobid])
  ma.setOutputDstFile(outdst)
  ma.setOutputRecFile(outrec)
  res = j.append(ma)
  if not res['OK']:
    print res['Message']
    exit(1)

# Upload files to different directories
  upload_script="upload%i.sh"%iser
  upload = GenericApplication()
# Create a script to upload files.
  shfile = open(upload_script,"w")
  shfile.write("#!/bin/bash\n")
  shfile.write("/bin/ls -l \n")
  shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n")
  shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n")
  shfile.close()
  os.chmod(upload_script,0755)
  upload.setScript(upload_script)

  res = j.append(upload)
  if not res['OK'] :
    print res['Message']
    exit(1)

#   j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
  j.setInputSandbox([ setting_file, upload_script ] )
  j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
  j.setCPUTime(10000)
  j.dontPromptMe()

  res = j.submit(dirac)
  if not res["OK"] :
    print "Failed submit job, jobid=%s" %jobid
    print res


  os.remove(upload_script)

  return j
예제 #10
0
def main():
	# Take the input arguments from the argument parser, and check they exist...
	args = parse_args()
	if not args:
		print 'Invalid Arguments'
		sys.exit(1)

#### Software Versions ####
	softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
	# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working 
	# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working

	# Check the --runs and --split arguments to make sure they are compatible, if not exit... 
	if not check_events_arguments(args.events, args.split):
		sys.exit(1)

	# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
	lfn_check, lfn = check_input_LFN(args.stdhepInput)
	if not lfn_check:
		sys.exit(1)

	# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
	dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector))

	# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
	# into the input sandbox
	inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)

	# Prepares values for the job loop...
	if args.split < 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.events)
	if args.split > 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.split)

	# Loop that runs through the required number of jobs to be executed...
	for startEvent in range(0, nInputEvents, nOutputEvents):

################## Job Initialise ########################################		
		job = UserJob()
		job.setName(path.basename(args.stdhepInput))
		job.setJobGroup('JobGroup')
		job.setInputSandbox(inputSandbox)
		fileNumber = startEvent/nOutputEvents
		print "Job ", fileNumber

		outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions)
		slicOutput=outputFiles['slicOutput']
		prePandoraOutput=outputFiles['prePandoraOutput']
		pandoraOutput=outputFiles['pandoraOutput']
		vertexingOutput=outputFiles['vertexingOutput']
		lcsimRecOutput=outputFiles['lcsimRecOutput']
		lcsimDstOutput=outputFiles['lcsimDstOutput']
		flavortagOutput=outputFiles['flavortagOutput']
		diracOutput=outputFiles['diracOutput']

################## SLIC ##################################################
		slic = SLIC()
		slic.setVersion(softVersions[0])
		slic.setSteeringFile(args.macFile)
		# slic.setInputFile(lfn)
		slic.setOutputFile(slicOutput)
		slic.setDetectorModel(args.detector)
		slic.setNumberOfEvents(nOutputEvents)
		slic.setStartFrom(startEvent)
		#print slic.listAttributes()
		result = job.append(slic)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## lcsim (digitization and tracking) #####################
		lcsim = LCSIM()
		lcsim.setVersion(softVersions[1])
		lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles
		lcsim.getInputFromApp(slic)
		lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
		# lcsim.setAliasProperties('alias.properties')
		lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
		lcsim.setOutputFile(prePandoraOutput)
		lcsim.setNumberOfEvents(nOutputEvents)
		#print lcsim.listAttributes()
		result = job.append(lcsim)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## slicPandora ###########################################
		slicPandora = SLICPandora()
		slicPandora.setVersion(softVersions[2])
		slicPandora.setDetectorModel(args.detector)
		slicPandora.getInputFromApp(lcsim)
		slicPandora.setOutputFile(pandoraOutput)
		slicPandora.setPandoraSettings('pandoraSettings.xml')
		slicPandora.setNumberOfEvents(nOutputEvents)
		#print slicPandora.listAttributes()
		result = job.append(slicPandora)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## Marlin, LCFIPlus Vertexing ############################
		vertexing = Marlin()
		vertexing.setVersion(softVersions[3])
		vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
		vertexing.setGearFile('steeringFiles/sidloi3.gear')
		vertexing.getInputFromApp(slicPandora)
		vertexing.setOutputFile(vertexingOutput)
		vertexing.setNumberOfEvents(nOutputEvents)
		#print vertexing.listAttributes()
		result = job.append(vertexing)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## lcsim (DST production) ################################
		lcsimDst = LCSIM()
		lcsimDst.setVersion(softVersions[1])
		lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
		lcsimDst.getInputFromApp(vertexing)
		lcsimDst.setNumberOfEvents(nOutputEvents)
		# lcsimDst.setAliasProperties('alias.properties')
		lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
		lcsimDst.setOutputRecFile(lcsimRecOutput)
		lcsimDst.setOutputDstFile(lcsimDstOutput)
		#print lcsimDst.listAttributes()
		result = job.append(lcsimDst)
		if not result['OK']:
			print result['Message']
			sys.exit(2)

################## Marlin, LCFIPlus flavortag ############################
		if args.flavortag:
			flavortag = Marlin()
			flavortag.setVersion(softVersions[3])
			flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
			flavortag.setGearFile('steeringFiles/sidloi3.gear')
			flavortag.setInputFile(lcsimDstOutput)
			flavortag.setOutputFile(flavortagOutput)
			flavortag.setNumberOfEvents(nOutputEvents)
			#print flavortag.listAttributes()
			result = job.append(flavortag)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Job Finalise ##########################################

		# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
		# This list is likely to change.
		job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
							'OSG.PNNL.us',])

		job.setCPUTime(50000)
		job.setPlatform('x86_64-slc5-gcc43-opt')

		# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
		# directory on the grid.
		if args.flavortag:
			job.setOutputData(flavortagOutput, diracOutput, args.SE)

		else: 
			job.setOutputData(lcsimDstOutput, diracOutput, args.SE)

		job.setOutputSandbox(outputSandbox)
		job.setInputData(lfn)

		if args.dontPromptMe:
			job.dontPromptMe()
		# Submits Job!!!
		job.submit()

	return 0;
예제 #11
0
def getJob(jobid, jobpara):
  iser=jobid+1

  outdst = "toto-ovl-%5.5i.dst.slcio"%iser
  outrec = "toto-ovl-%5.5i.rec.slcio"%iser

###In case one wants a loop: comment the folowing.
#for i in range(2):
  j = UserJob()
  j.setJobGroup("Tutorial")
  j.setName("MarlinOverlayParametric%i"%iser)
  j.setInputSandbox(jobpara["setting_file"])

## Define the overlay
  ov = OverlayInput()
  ov.setMachine("ilc_dbd")
  ov.setEnergy(energy)
  ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
  ov.setBXOverlay(int(jobpara["BXOverlay"]))
  ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
  ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
  ov.setDetectorModel("ILD_o1_v05")
  res = j.append(ov)
  if not res['OK']:
    print res['Message']
    exit(1)

## Define Marlin job
  ma = Marlin()
  ma.setDebug()
  ma.setVersion("ILCSoft-01-17-09")
  ma.setSteeringFile("marlin_ovl_stdreco.xml")
  ma.setGearFile("GearOutput.xml")
#   ma.setInputFile(simfile)
  ma.setInputFile(simlists[jobid])
  ma.setOutputDstFile(outdst)
  ma.setOutputRecFile(outrec)
  res = j.append(ma)
  if not res['OK']:
    print res['Message']
    exit(1)
  
  j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
  j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
  j.setCPUTime(10000)
  j.dontPromptMe()
  return j
예제 #12
0
def main():
    # Take the input arguments from the argument parser, and check they exist...
    args = parse_args()
    if not args:
        print 'Invalid Arguments'
        sys.exit(1)

#### Software Versions ####
    softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
                    "0116"]  # Working (recommended)
    # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
    # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working

    # Check the --runs and --split arguments to make sure they are compatible, if not exit...
    if not check_events_arguments(args.events, args.split):
        sys.exit(1)

    # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
    lfn_check, lfn = check_input_LFN(args.stdhepInput)
    if not lfn_check:
        sys.exit(1)

    # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
    dirac = DiracILC(True,
                     setup_repository_name(args.stdhepInput, args.detector))

    # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
    # into the input sandbox
    inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)

    # Prepares values for the job loop...
    if args.split < 0:
        nInputEvents = int(args.events)
        nOutputEvents = int(args.events)
    if args.split > 0:
        nInputEvents = int(args.events)
        nOutputEvents = int(args.split)

    # Loop that runs through the required number of jobs to be executed...
    for startEvent in range(0, nInputEvents, nOutputEvents):

        ################## Job Initialise ########################################
        job = UserJob()
        job.setName(path.basename(args.stdhepInput))
        job.setJobGroup('JobGroup')
        job.setInputSandbox(inputSandbox)
        fileNumber = startEvent / nOutputEvents
        print "Job ", fileNumber

        outputFiles = setup_output_dict(args.stdhepInput, args.detector,
                                        fileNumber, args.outputPath,
                                        softVersions)
        slicOutput = outputFiles['slicOutput']
        prePandoraOutput = outputFiles['prePandoraOutput']
        pandoraOutput = outputFiles['pandoraOutput']
        vertexingOutput = outputFiles['vertexingOutput']
        lcsimRecOutput = outputFiles['lcsimRecOutput']
        lcsimDstOutput = outputFiles['lcsimDstOutput']
        flavortagOutput = outputFiles['flavortagOutput']
        diracOutput = outputFiles['diracOutput']

        ################## SLIC ##################################################
        slic = SLIC()
        slic.setVersion(softVersions[0])
        slic.setSteeringFile(args.macFile)
        # slic.setInputFile(lfn)
        slic.setOutputFile(slicOutput)
        slic.setDetectorModel(args.detector)
        slic.setNumberOfEvents(nOutputEvents)
        slic.setStartFrom(startEvent)
        #print slic.listAttributes()
        result = job.append(slic)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## lcsim (digitization and tracking) #####################
        lcsim = LCSIM()
        lcsim.setVersion(softVersions[1])
        lcsim.setSteeringFile(
            'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
        )  # Another version is included in /steeringFiles
        lcsim.getInputFromApp(slic)
        lcsim.setTrackingStrategy(
            'steeringFiles/sidloi3_trackingStrategies_default.xml')
        # lcsim.setAliasProperties('alias.properties')
        lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
        lcsim.setOutputFile(prePandoraOutput)
        lcsim.setNumberOfEvents(nOutputEvents)
        #print lcsim.listAttributes()
        result = job.append(lcsim)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## slicPandora ###########################################
        slicPandora = SLICPandora()
        slicPandora.setVersion(softVersions[2])
        slicPandora.setDetectorModel(args.detector)
        slicPandora.getInputFromApp(lcsim)
        slicPandora.setOutputFile(pandoraOutput)
        slicPandora.setPandoraSettings('pandoraSettings.xml')
        slicPandora.setNumberOfEvents(nOutputEvents)
        #print slicPandora.listAttributes()
        result = job.append(slicPandora)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## Marlin, LCFIPlus Vertexing ############################
        vertexing = Marlin()
        vertexing.setVersion(softVersions[3])
        vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
        vertexing.setGearFile('steeringFiles/sidloi3.gear')
        vertexing.getInputFromApp(slicPandora)
        vertexing.setOutputFile(vertexingOutput)
        vertexing.setNumberOfEvents(nOutputEvents)
        #print vertexing.listAttributes()
        result = job.append(vertexing)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## lcsim (DST production) ################################
        lcsimDst = LCSIM()
        lcsimDst.setVersion(softVersions[1])
        lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
        lcsimDst.getInputFromApp(vertexing)
        lcsimDst.setNumberOfEvents(nOutputEvents)
        # lcsimDst.setAliasProperties('alias.properties')
        lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
        lcsimDst.setOutputRecFile(lcsimRecOutput)
        lcsimDst.setOutputDstFile(lcsimDstOutput)
        #print lcsimDst.listAttributes()
        result = job.append(lcsimDst)
        if not result['OK']:
            print result['Message']
            sys.exit(2)

################## Marlin, LCFIPlus flavortag ############################
        if args.flavortag:
            flavortag = Marlin()
            flavortag.setVersion(softVersions[3])
            flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
            flavortag.setGearFile('steeringFiles/sidloi3.gear')
            flavortag.setInputFile(lcsimDstOutput)
            flavortag.setOutputFile(flavortagOutput)
            flavortag.setNumberOfEvents(nOutputEvents)
            #print flavortag.listAttributes()
            result = job.append(flavortag)
            if not result['OK']:
                print result['Message']
                sys.exit(2)


################## Job Finalise ##########################################

# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
        job.setBannedSites([
            'LCG.IN2P3-CC.fr',
            'LCG.RAL-LCG2.uk',
            'LCG.DESY-HH.de',
            'LCG.DESYZN.de',
            'LCG.KEK.jp',
            'OSG.PNNL.us',
        ])

        job.setCPUTime(50000)
        job.setPlatform('x86_64-slc5-gcc43-opt')

        # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
        # directory on the grid.
        if args.flavortag:
            job.setOutputData(flavortagOutput, diracOutput, args.SE)

        else:
            job.setOutputData(lcsimDstOutput, diracOutput, args.SE)

        job.setOutputSandbox(outputSandbox)
        job.setInputData(lfn)

        if args.dontPromptMe:
            job.dontPromptMe()
        # Submits Job!!!
        job.submit()

    return 0
def main():
	# Take the input arguments from the argument parser, and check they exist...
	args = parse_args()
	if not args:
		print 'Invalid Arguments'
		sys.exit(1)

	print args.chain[0]

	# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
	softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
	# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working 
	# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
	# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]

	check_events_arguments(args.events, args.split)
	detector = args.detector
	alias_properties(detector)
	outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
	inputSandbox, outputSandbox = setup_sandboxes(args.macFile)

	dirac = DiracILC(True, repoName)

	# Prepares values for the job loop...
	if args.split < 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.events)
	if args.split > 0:
		nInputEvents = int(args.events)
		nOutputEvents = int(args.split)

	# Loop that runs through the required number of jobs to be executed...
	for startEvent in range(0, nInputEvents, nOutputEvents):

################## Job Initialise ########################################		
		job = UserJob()
		job.setName(outputBase)
		job.setJobGroup('JobGroup')
		job.setInputSandbox(inputSandbox)
		fileNumber = startEvent/nOutputEvents
		print "Job ---> ", fileNumber

################## SLIC ##################################################
		if 1 in args.chain:
			slic = SLIC()
			slic.setVersion(softVersions[0])
			slic.setSteeringFile(args.macFile)
			# slic.setInputFile(lfn)
			slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
			slic.setDetectorModel(detector)
			slic.setNumberOfEvents(nOutputEvents)
			slic.setStartFrom(startEvent)
			#print slic.listAttributes()
			result = job.append(slic)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Overlay ###############################################
		if 2 in args.chain:
			'''
			#Add the gghad background overlay.
			gghad = OverlayInput()
			#gghad.setProdID(1767)
			gghad.setEnergy(500.0)
			gghad.setBXOverlay('args.bunches')
			gghad.setGGToHadInt( 4.1 )
			gghad.setNbSigEvtsPerJob(nOutputEvents)
			gghad.setMachine('ilc_dbd')
			gghad.setDetectorModel('sidloi3')
			gghad.setBkgEvtType('aa_lowpt')
			result = job.append( gghad )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			
			#Add the pair background overlay.
			pairs = OverlayInput()
			pairs.setProdID(2)
			pairs.setEnergy(500.0)
			pairs.setBXOverlay('args.bunches')
			pairs.setGGToHadInt(1.)
			pairs.setNbSigEvtsPerJob(nOutputEvents)
			pairs.setMachine('ilc_dbd')
			pairs.setDetectorModel('sidloi3')
			pairs.setBkgEvtType('eepairs')
			result = job.append( pairs )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			'''
			gghad = OverlayInput()
			gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
			gghad.setBXOverlay(int(args.bunches))
			gghad.setGGToHadInt( 4.1 )
			gghad.setNbSigEvtsPerJob(nOutputEvents)
			gghad.setBkgEvtType('aa_lowpt')		
			result = job.append( gghad )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			'''
			pairs = OverlayInput()
			pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
			pairs.setBXOverlay(int(args.bunches))
			pairs.setGGToHadInt(1.)
			pairs.setBkgEvtType('eepairs')
			pairs.setNbSigEvtsPerJob(nOutputEvents)
			result = job.append( pairs )
			if not result['OK']:
				print result['Message']
				sys.exit(2)
			'''
			
################## lcsim (digitization and tracking) #####################
		if 3 in args.chain:
			lcsim = LCSIM()
			lcsim.setVersion(softVersions[1])
			lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
			if 1 in args.chain:
				lcsim.getInputFromApp(slic)
			lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
			lcsim.setAliasProperties('steeringFiles/alias.properties')
			lcsim.setDetectorModel(detector+".zip")
			#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
			lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
			lcsim.setNumberOfEvents(nOutputEvents)
			#print lcsim.listAttributes()
			result = job.append(lcsim)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## slicPandora ###########################################
		if 4 in args.chain:
			slicPandora = SLICPandora()
			slicPandora.setVersion(softVersions[2])
			slicPandora.setDetectorModel(detector)
			slicPandora.getInputFromApp(lcsim)
			slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
			slicPandora.setPandoraSettings('pandoraSettings.xml')
			slicPandora.setNumberOfEvents(nOutputEvents)
			#print slicPandora.listAttributes()
			result = job.append(slicPandora)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Marlin, LCFIPlus Vertexing ############################
		if 5 in args.chain:
			vertexing = Marlin()
			vertexing.setVersion(softVersions[3])
			vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
			vertexing.setGearFile('steeringFiles/' + detector + '.gear')
			vertexing.getInputFromApp(slicPandora)
			vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
			vertexing.setNumberOfEvents(nOutputEvents)
			#print vertexing.listAttributes()
			result = job.append(vertexing)
			if not result['OK']:
				print result['Message']
				sys.exit(2)
################## lcsim (DST production) ################################
			lcsimDst = LCSIM()
			lcsimDst.setVersion(softVersions[1])
			lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
			lcsimDst.getInputFromApp(vertexing)
			lcsimDst.setNumberOfEvents(nOutputEvents)
			lcsimDst.setAliasProperties('steeringFiles/alias.properties')
			lcsimDst.setDetectorModel(detector+".zip")
			lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
			lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
			#print lcsimDst.listAttributes()
			result = job.append(lcsimDst)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Marlin, LCFIPlus flavortag ############################
		if 6 in args.chain:
			flavortag = Marlin()
			flavortag.setVersion(softVersions[3])
			flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
			flavortag.setGearFile('steeringFiles/' + detector + '.gear')
			flavortag.setInputFile(lcsimDstOutput)
			flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
			flavortag.setNumberOfEvents(nOutputEvents)
			#print flavortag.listAttributes()
			result = job.append(flavortag)
			if not result['OK']:
				print result['Message']
				sys.exit(2)

################## Job Finalise ##########################################

		# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
		# This list is likely to change.
		job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
							'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
							'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])

		job.setCPUTime(50000)
		job.setPlatform('x86_64-slc5-gcc43-opt')

		# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
		# directory on the grid.
		outputLevel = max(args.chain)
		if outputLevel == 1:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 3:
			#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 4:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 5:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
		if outputLevel == 6:
			job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')

		job.setOutputSandbox(outputSandbox)
		job.setInputData(args.Input)

		if args.dontPromptMe:
			job.dontPromptMe()
		# Submits Job!!!
		job.submit()

	return 0;
예제 #14
0
def defMarlin(outputFile, rootFile, **kwargs):
    '''Defines Marlin job'''

    ##Jet Settings
    jetAlgo = kwargs.pop('jetAlgo', 'ValenciaPlugin 1.5 1 1')
    jetCluster = kwargs.pop('jetCluster', 'ExclusiveNJets 2')
    jetRecomb = kwargs.pop('jetRecomb', 'E_scheme')

    ##Top Tagger Settings
    deltaR = kwargs.pop('deltaR', 0.01)
    deltaP = kwargs.pop('deltaP', 0.01)
    cos_theta_W_max = kwargs.pop('cos_theta_W_max', 1.0)

    ##Sixfermion sample
    sixFermionSample = kwargs.pop('sixFermionSample', True)

    from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
    ma = Marlin()
    ma.setVersion('ILCSoft-2017-05-30_gcc62')
    ma.setSteeringFile(
        "/home/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/topasymmetry_template_new.xml"
    )
    ma.setGearFile("/home/lstroem/clicdp/gear/clic_ild_cdr.gear")
    ma.setOutputFile(outputFile, rootFile)
    ma.setExtraCLIArguments(
        "--MyTopTaggerProcessorJ2.algorithm=\"{algorithm}\"\
                           --MyTopTaggerProcessorJ2.clusteringMode=\"{clusteringMode}\"\
                           --MyTopTaggerProcessorJ2.recombinationScheme={recombinationScheme}\
                           --MyTopTaggerProcessorJ2.deltaR={deltaR:.2f}\
                           --MyTopTaggerProcessorJ2.deltaP={deltaP:.2f}\
                           --MyTopTaggerProcessorJ2.cos_theta_W_max={cos_theta_W_max:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.algorithm=\"{algorithm}\"\
                           --MyTopTaggerProcessorJ2wTrimming.clusteringMode=\"{clusteringMode}\"\
                           --MyTopTaggerProcessorJ2wTrimming.recombinationScheme={recombinationScheme}\
                           --MyTopTaggerProcessorJ2wTrimming.deltaR={deltaR:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.deltaP={deltaP:.2f}\
                           --MyTopTaggerProcessorJ2wTrimming.cos_theta_W_max={cos_theta_W_max:.2f}\
                           --MyTopAsymmetryStoreProcessor.useSixFermionSample={sixFermionSample}\
                           --MyTopAsymmetryStoreProcessor.outFileName={outfile}"
        .format(algorithm=jetAlgo,
                clusteringMode=jetCluster,
                recombinationScheme=jetRecomb,
                deltaR=deltaR,
                deltaP=deltaP,
                cos_theta_W_max=cos_theta_W_max,
                sixFermionSample=sixFermionSample,
                outfile=rootFile))

    return ma
예제 #15
0
  def createMarlinApplication( self, energy ):
    """ create Marlin Application without overlay """
    from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
    marlin = Marlin()
    marlin.setDebug()
    marlin.setVersion( self.softwareVersion )
    marlin.setDetectorModel( self.detectorModel )
    marlin.detectortype = self.detectorModel

    if self._flags.over:
      self.addOverlayOptionsToMarlin( energy )

    marlin.setExtraCLIArguments( self.cliReco )

    steeringFile = {
      350. : "clicReconstruction.xml",
      380. : "clicReconstruction.xml",
      420. : "clicReconstruction.xml",
      1400.: "clicReconstruction.xml",
      3000.: "clicReconstruction.xml",
    }.get( energy, 'clicReconstruction.xml' )

    marlin.setSteeringFile( steeringFile )
    return marlin
예제 #16
0
simfile="LFN:/ilc/prod/ilc/mc-dbd/ild/sim/500-TDR_ws/6f_ttbar/ILD_o1_v05/v01-14-01-p00/sv01-14-01-p00.mILD_o1_v05.E500-TDR_ws.I37623.P6f_bbcyyc.eR.pL-00001.slcio"
gearfile=basedir+"GearOutput.xml"
outdst = "toto.dst.slcio" #% i
outrec = "toto.rec.slcio" #% i

d= DiracILC(True,"repo.rep")

###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)



ma = Marlin()
ma.setDebug()
# ma.setLogLevel("verbose")
# ma.setILDConfig("v01-16-p05_500") 

ma.setVersion("v01-16-02")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile(gearfile)
ma.setInputFile([simfile, pandoraLikelihoodData, bg_aver])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)


res = j.append(ma)
if not res['OK']:
    print res['Message']
예제 #17
0
ovi.setDetectorModel("CLIC_ILD_CDR")

overlay = [True,False]

for ov in overlay:
  d = DiracILC(True,"repo_overlay_%s.rep"%ov)
  for lfn in lfns:
    j = UserJob()
    steeringf = "clic_ild_cdr_steering.xml"
    if ov:
      steeringf = "clic_ild_cdr_steering_overlay.xml"
      res = j.append(ovi)
      if not res['OK']:
        print(res['Message'])
        continue
    ma = Marlin() 
    ma.setVersion("v0111Prod")
    ma.setGearFile("clic_ild_cdr.gear")
    ma.setSteeringFile(steeringf)
    ma.setInputFile("LFN:"+lfn)
    ma.setNbEvts(10)
    ma.setEnergy(500.)
    ma.setOutputRecFile("myrec_overlay_%s.slcio"%ov)
    ma.setOutputDstFile("mydst_overlay_%s.slcio"%ov)
    res =j.append(ma)
    if not res['OK']:
      print(res['Message'])
      exit()

    j.setCPUTime(86400)
    j.setOutputData("myrec_overlay_%s.slcio"%ov,"some/path")
예제 #18
0
class MarlinTestCase( unittest.TestCase ):
  """ Base class for the Marlin test cases
  """
  def setUp( self ):
    """set up the objects"""
    self.mar = Marlin( {} )

  def test_setgear( self ):
    self.mar.setGearFile( 'lfn:/my/gear/file.txt' )
    self.assertFalse( self.mar._errorDict )
    self.assertIn( 'lfn:/my/gear/file.txt', self.mar.inputSB )

  def test_setKeepRecFile(self):
    """Tests for behaviour with KeepRecFile True/False."""
    self.assertTrue(self.mar.keepRecFile)
    self.mar.setKeepRecFile(False)
    self.assertFalse(self.mar._errorDict)
    self.mar.setKeepRecFile(True)
    self.assertTrue(self.mar.keepRecFile)
    self.assertFalse(self.mar._errorDict)
    self.mar.setKeepRecFile(123)
    self.assertIn('val = 123', str(self.mar._errorDict))

  def test_setoutputrec( self ):
    self.mar.setOutputRecFile( 'my/file.outputrec', 'mytestPath' )
    assertEqualsImproved( self.mar.outputRecPath, 'mytestPath', self )
    self.assertFalse( self.mar._errorDict )

  def test_setoutputdst( self ):
    self.mar.setOutputDstFile( 'my/file.outputdst', 'mytestPath' )
    assertEqualsImproved( self.mar.outputDstPath, 'mytestPath', self )
    self.assertFalse( self.mar._errorDict )

  def test_setproclist( self ):
    self.mar.setProcessorsToUse( [ 'proc1', 'proc2' ] )
    self.assertFalse( self.mar._errorDict )

  def test_setexcludeproclist( self ):
    self.mar.setProcessorsToExclude( [ 'proc1', 'proc2' ] )
    self.assertFalse( self.mar._errorDict )

  def test_userjobmodules( self ):
    module_mock = Mock()
    assertDiracSucceeds( self.mar._userjobmodules( module_mock ), self )

  def test_prodjobmodules( self ):
    module_mock = Mock()
    assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )

  def test_prodjobmodules_outputpath( self ):
    module_mock = Mock()
    self.mar.outputPath = 'aef'
    assertDiracSucceeds( self.mar._prodjobmodules( module_mock ), self )
    self.assertIn( { 'OutputFile' : '@{OutputFile}', 'outputPath' : '@{OutputPath}',
                     'outputDataSE' : '@{OutputSE}'}, self.mar._listofoutput )

  def test_userjobmodules_fails( self ):
    with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
         patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
      assertDiracFailsWith( self.mar._userjobmodules( None ),
                            'userjobmodules failed', self )

  def test_prodjobmodules_fails( self ):
    with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
         patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
      assertDiracFailsWith( self.mar._prodjobmodules( None ),
                            'prodjobmodules failed', self )

  def test_checkconsistency( self ):
    self.mar.version = '13'
    self.mar.steeringFile = '/mysteer/file.stdhep'
    self.mar.gearFile = None
    self.mar._jobtype = 'notUser'
    self.mar.outputFile = None
    with patch('os.path.exists', new=Mock(return_value=True)), \
         patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
      assertDiracSucceeds( self.mar._checkConsistency(), self )
      self.assertIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
                       'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      self.assertIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
                       'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
        self.assertIn( keyword, self.mar.prodparameters )
      assertEqualsImproved( self.mar.gearFile, None, self )

  def test_checkconsistency_noversion( self ):
    self.mar.version = None
    assertDiracFailsWith( self.mar._checkConsistency(), 'version not set', self )

  def test_checkconsistency_invalidxml( self ):
    self.mar.version = '13'
    self.mar.steeringFile = '/mysteer/file.stdhep'
    with patch('os.path.exists', new=Mock(return_value=True)), \
         patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_ERROR('mytesterrxml'))):
      assertDiracFailsWith( self.mar._checkConsistency(), 'supplied steering file cannot be read with xml', self)

  def test_checkconsistency_othercase( self ):
    self.mar.version = '13'
    self.mar.steeringFile = '/mysteer/file.stdhep'
    self.mar.gearFile = 'myGearOutput.mock'
    self.mar._jobtype = 'notUser'
    self.mar.outputFile = 'myoutput.test'
    with patch('os.path.exists', new=Mock(return_value=False)):
      assertDiracSucceeds( self.mar._checkConsistency(), self )
      self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
                          'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
                          'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
      for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
        self.assertIn( keyword, self.mar.prodparameters )
      assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )

  def test_checkconsistency_lastcase( self ):
    self.mar.version = '13'
    self.mar.steeringFile = None
    self.mar.gearFile = 'myGearOutput.mock'
    self.mar._jobtype = 'User'
    self.mar.outputFile = 'myoutput.test'
    assertDiracSucceeds( self.mar._checkConsistency(), self )
    self.assertNotIn( { 'outputFile' : '@{outputREC}', 'outputPath' : '@{outputPathREC}',
                        'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
    self.assertNotIn( { 'outputFile' : '@{outputDST}', 'outputPath' : '@{outputPathDST}',
                        'outputDataSE' : '@{OutputSE}' }, self.mar._listofoutput )
    for keyword in [ 'detectorType', 'marlin_gearfile', 'marlin_steeringfile' ]:
      self.assertNotIn( keyword, self.mar.prodparameters )
    assertEqualsImproved( self.mar.gearFile, 'myGearOutput.mock', self )

  def test_resolvelinkedparams(self):
    """Test _resolveLinkedStepParameters with something happening."""
    step_mock = Mock()
    input_mock = Mock()
    input_mock.getType.return_value = {'abc': False}
    self.mar._linkedidx = 3
    self.mar._jobsteps = [None, None, None, input_mock]
    assertDiracSucceeds(self.mar._resolveLinkedStepParameters(step_mock), self)
    step_mock.setLink.assert_called_once_with('InputFile', {'abc': False}, 'OutputFile')

  def test_resolvelinkedparams_noinputstep(self):
    """Call _resolveLinkedStep function, which does nothing."""
    self.mar._linkedidx = None
    self.mar._inputappstep = []
    assertDiracSucceeds(self.mar._resolveLinkedStepParameters(None), self)
예제 #19
0
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
d= DiracILC(True,"repo.rep")

###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("example")#%i)

ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.setInputFile("LFN:/ilc/prod/clic/3tev/gghad/ILD/SIM/00000187/000/gghad_sim_187_97.slcio")
outdst = "toto.dst.slcio" #% i
outrec = "toto.rec.slcio" #% i
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)

res = j.append(ma)
if not res['OK']:
    print res['Message']
    exit(1)
  
j.setOutputData([outdst,outrec],"some/path","KEK-SRM")
j.setOutputSandbox("*.log")
j.dontPromptMe()
예제 #20
0
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(n_events_per_job)
ov.setBXOverlay(BXOverlay)
ov.setGGToHadInt(GGToHadInt500)
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
    print res['Message']
    exit(1)

ma = Marlin()
ma.setDebug()
# ma.setLogLevel("verbose")
# ma.setILDConfig("v01-16-p05_500") 

ma.setVersion("v01-16-02")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile(gearfile)
ma.setInputFile(simfile)
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)


res = j.append(ma)
if not res['OK']:
    print res['Message']
예제 #21
0
for i in range(n_jobs):
  j = UserJob()

  mo = Mokka()
  mo.setEnergy(3000)
  mo.setVersion("0706P08")
  mo.setSteeringFile("clic_ild_cdr.steer")
  mo.setMacFile("particlegun_electron.mac")
  mo.setOutputFile("MyFile.slcio")
  mo.setNbEvts(n_evts_per_job)
  res = j.append(mo)
  if not res['OK']:
    print res['Message']
    break
  ma = Marlin()
  ma.setVersion("v0111Prod")
  ma.setSteeringFile("clic_ild_cdr_steering.xml")
  ma.getInputFromApp(mo)
  ma.setOutputDstFile("mydst_no_ov_%s.slcio"%i)
  res = j.append(ma)
  if not res['OK']:
    print res['Message']
    break
  ov  = OverlayInput()
  ov.setBXOverlay(60)
  ov.setGGToHadInt(3.2)
  ov.setNbSigEvtsPerJob(n_evts_per_job)
  ov.setDetectorType("ILD")

  res = j.append(ov)
예제 #22
0
    print res["Message"]
    dexit(1)


ov = OverlayInput()
ov.setDetectorType("ILD")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
res = j.append(ov)
if not res["OK"]:
    print res["Message"]
    dexit(1)


ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res["OK"]:
    print res["Message"]
    dexit(1)
# print appplication's attributes.
ma.listAttributes()

j.setName("test")
j.setOutputSandbox("*.log")
예제 #23
0
            overlay.setBXOverlay(60)
            overlay.setGGToHadInt(1.3)  ##When running at 1.4TeV
            overlay.setDetectorModel("CLIC_ILD_CDR")
        else:
            print "Overlay CLIC_ILD: No overlay parameters defined for this energy"

    if detectormodel in ILDDetectorModels:
        overlay.setMachine("ilc_dbd")
        overlay.setBackgroundType("aa_lowpt")
        overlay.setBXOverlay(1)
        overlay.setDetectorModel(detectormodel)
        if energy == 250:
            overlay.setGGToHadInt(0.3)

    ##Reconstruction ILD with overlay
    mao = Marlin()
    mao.setDebug()
    mao.setVersion(SOFTWAREVERSION)
    if ild_rec_ov:
        if energy == 500.:
            mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml")
            mao.setGearFile('clic_ild_cdr500.gear')
        elif energy == 420.:
            mao.setSteeringFile("clic_ild_cdr500_steering_overlay_420.0.xml")
            mao.setGearFile('clic_ild_cdr500.gear')
        elif energy == 350.:
            mao.setSteeringFile("clic_ild_cdr500_steering_overlay_350.0.xml")
            mao.setGearFile('clic_ild_cdr500.gear')
        elif energy == 3000.0:
            mao.setSteeringFile("clic_ild_cdr_steering_overlay_3000.0.xml")
            mao.setGearFile('clic_ild_cdr.gear')
예제 #24
0
 def setUp(self):
     """set up the objects"""
     self.mar = Marlin({})
예제 #25
0
def subOverlay():

    # Decide parameters for a job
    outputSE = "KEK-SRM"

    isLocal = _clip.isLocal
    nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
    nbevts = 0  # To analize all input events
    outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix
    outputDir = _clip.outputDir
    inputFile = _clip.inputFile
    if inputFile == "":
        gLogger.error("Input file for ddsim does not given.")
        exit(-1)

    recfile = outputFilePrefix + ".rec.slcio"
    dstfile = outputFilePrefix + ".dst.slcio"
    detector_model = "ILD_l5_o1_v02"
    key = detector_model.split('_')
    sim_detectorModel = "_".join([key[0], key[1], key[3]])

    # Create DIRAC objects for job submission

    dIlc = DiracILC()

    job = UserJob()
    job.setJobGroup("myoverlayjob")
    job.setName("myoverlay")
    job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
    job.setILDConfig("v02-00-02")

    # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
    # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"])  # job submission destination
    # job.setBannedSites([])         # a list of sites not to submit job
    # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )

    # Create Overlay application
    ovldata = [{
        "ProcessorName": "BgOverlayWW",
        "evttype": "aa_lowpt_WW",
        "ProdID": 10237,
        "expBG": 0.211,
        "subdir": "000"
    }, {
        "ProcessorName": "BgOverlayWB",
        "evttype": "aa_lowpt_WB",
        "ProdID": 10241,
        "expBG": 0.24605,
        "subdir": "000"
    }, {
        "ProcessorName": "BgOverlayBW",
        "evttype": "aa_lowpt_BW",
        "ProdID": 10239,
        "expBG": 0.243873,
        "subdir": "000"
    }, {
        "ProcessorName": "BgOverlayBB",
        "evttype": "aa_lowpt_BB",
        "ProdID": 10235,
        "expBG": 0.35063,
        "subdir": "000"
    }, {
        "ProcessorName": "PairBgOverlay",
        "evttype": "seeablepairs",
        "ProdID": 10233,
        "expBG": 1.0,
        "subdir": "100"
    }]

    BXOverlay = 1
    NbSigEvtsPerJob = 100
    numberOfSignalEvents = NbSigEvtsPerJob
    basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws"
    energy = "500"

    for ovl in ovldata:
        print "### OverlayInput ... " + ovl["ProcessorName"]
        ovlapp = OverlayInput()
        ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \
       ( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] )
        print "    OverlayPath ... " + ovlpath
        ovlapp.setMachine("ilc_dbd")
        # ovlapp.setEnergy(energy)
        # ovlapp.setDetectorModel(sim_detectorModel)
        ovlapp.setProcessorName(ovl["ProcessorName"])
        ovlapp.setBkgEvtType(ovl["evttype"])
        ovlapp.setPathToFiles(ovlpath)
        ovlapp.setGGToHadInt(ovl["expBG"])
        ovlapp.setBXOverlay(BXOverlay)
        ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob)
        ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents)
        res = job.append(ovlapp)
        if not res['OK']:
            print res['Message']
            exit(1)

    # Create Marlin application
    marlin = Marlin()
    marlin.setVersion("ILCSoft-02-00-02_gcc49")
    marlin.setDetectorModel(detector_model)
    marlin.setSteeringFile("MarlinStdReco.xml")
    marlin.setInputFile(inputFile)
    marlin.setNumberOfEvents(nbevts)
    marlin.setOutputDstFile(dstfile)
    marlin.setOutputRecFile(recfile)
    extraCLIArguments = " --constant.DetectorModel=%s " % detector_model
    extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str(
        energy)
    extraCLIArguments += " --global.Verbosity=MESSAGE "
    marlin.setExtraCLIArguments(extraCLIArguments)

    job.append(marlin)

    if outputDir != "":
        job.setOutputData([dstfile, recfile],
                          OutputPath=outputDir,
                          OutputSE=outputSE)

    if isLocal:
        job.submit(dIlc, mode="local")
    else:
        job.submit(dIlc)
예제 #26
0
  overlay.setGGToHadInt(GGToHadInt1000)
  overlay.setBkgEvtType("aa_lowpt")
elif energy == 350.:
  overlay.setBXOverlay(BXOverlay)
  overlay.setGGToHadInt(GGToHadInt350)
  overlay.setBkgEvtType("aa_lowpt")
elif energy == 250.:
  overlay.setBXOverlay(BXOverlay)
  overlay.setGGToHadInt(GGToHadInt250)
  overlay.setBkgEvtType("aa_lowpt")

else:
  print("Overlay ILD: No overlay parameters defined for this energy")

##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion(MarlinVer) ##PUT HERE YOUR MARLIN VERSION
if ild_rec_ov:
  if energy in [250.0, 350.0, 500.0, 1000.0]:
    mao.setSteeringFile("bbudsc_3evt_stdreco.xml")
    mao.setGearFile("GearOutput.xml")
  else:
    print("Marlin: No reconstruction suitable for this energy")


##Reconstruction ILD w/o overlay
ma = Marlin()
ma.setDebug()
ma.setVersion(MarlinVer)
ma.setEnergy(energy)
  def createMarlinApplication(self, energy, cliReco, over):
    """Create Marlin application with or without overlay."""
    from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
    marlin = Marlin()
    marlin.setDebug()
    marlin.setVersion( self.softwareVersion )
    marlin.setDetectorModel( self.detectorModel )
    marlin.detectortype = self.detectorModel
    marlin.setKeepRecFile(False)

    if over:
      self.addOverlayOptionsToMarlin( energy )

    self.cliReco = ' '.join([self.cliRecoOption, self.cliReco, cliReco]).strip()
    marlin.setExtraCLIArguments(self.cliReco)
    self.cliReco = ''

    marlin.setSteeringFile(self.marlinSteeringFile)

    self._setApplicationOptions("Marlin", marlin)

    return marlin
예제 #28
0
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
d = DiracILC()                            # Provides job checking utilities
j = UserJob()                             # You are running a user job
m = Marlin()                              # Get an application instance
m.setVersion("v0116")                     # Define the version to use
m.setSteeringFile("clic_ild_cdr_steering.xml") #What the app should do
m.setInputFile("LFN:/ilc/prod/clic/3tev/ee_h_bb/ILD/DST/00000375/000/\
ee_h_bb_dst_375_999.slcio")               # Add some input
m.setGearFile("clic_ilc_cdr.gear")        # Application specific field
res = j.append(m)                         # Add the application to the job
if not res['OK']:
  print res['Message']                    # Catch any error
j.submit(d)                               # Submit the job
#Not shown here: metadata queries, chaining of applications
예제 #29
0
 def getMarlin(self):
     """ Define a marlin step
 """
     from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
     marlin = Marlin()
     #  marlin.setVersion("v0111Prod")
     marlin.setVersion(self.marlinVersion)
     marlin.setSteeringFile(self.marlinSteeringFile)
     marlin.setGearFile(self.gearFile)
     marlin.setOutputDstFile("testmarlinDST.slcio")
     marlin.setOutputRecFile("testmarlinREC.slcio")
     marlin.setNumberOfEvents(1)
     return marlin
예제 #30
0
 def getMarlin( self ):
   """ Define a marlin step
   """
   from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
   marlin = Marlin()
 #  marlin.setVersion("v0111Prod")
   marlin.setVersion(self.marlinVersion)
   marlin.setSteeringFile(self.marlinSteeringFile)
   marlin.setGearFile(self.gearFile)
   marlin.setOutputDstFile("testmarlinDST.slcio")
   marlin.setOutputRecFile("testmarlinREC.slcio")
   marlin.setNumberOfEvents(1)
   return marlin
예제 #31
0
wh.setFullParameterDict(pdict)

##Simulation
mo = Mokka()
mo.setVersion('0706P08')
mo.setSteeringFile("clic_ild_cdr.steer")


overlay = OverlayInput()
overlay.setBXOverlay(60)
overlay.setGGToHadInt(1.3)##When running at 1.4TeV
overlay.setDetectorModel("CLIC_ILD_CDR")
overlay.setBkgEvtType("gghad")

##Reconstruction
ma = Marlin()
ma.setVersion('v0111Prod')
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile("clic_ild_cdr.gear")

##########################################
##Define the generation production.
pwh = ProductionJob()
pwh.setOutputSE("CERN-SRM")
pwh.setProdType("MCGeneration")
pwh.setWorkflowName(process+"_"+str(energy))
pwh.setProdGroup(process+"_"+str(energy))
res = pwh.append(wh)
if not res['OK']:
    print res['Message']
    exit(1)
예제 #32
0
simfile="LFN:/ilc/prod/ilc/mc-dbd/ild/sim/500-TDR_ws/6f_ttbar/ILD_o1_v05/v01-14-01-p00/sv01-14-01-p00.mILD_o1_v05.E500-TDR_ws.I37623.P6f_bbcyyc.eR.pL-00001.slcio"
setting_file="LFN:/ilc/user/a/amiyamot/software/Settings/marlinSettings-v01-17-09_500.tar.gz"

outdst = "toto-3.dst.slcio" #% i
outrec = "toto-3.rec.slcio" #% i

d= DiracILC(True,"repo.rep")

###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)
j.setInputSandbox(setting_file)

ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile("GearOutput.xml")
ma.setInputFile(simfile)
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)

res = j.append(ma)
if not res['OK']:
    print res['Message']
    exit(1)
  
# j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
예제 #33
0
overlay.setMachine("ilc_dbd") #Don't touch, this is how the system knows what files to get
overlay.setEnergy(energy)#Don't touch, this is how the system knows what files to get
overlay.setDetectorModel(detectorModel)#Don't touch, this is how the system knows what files to get
if energy==500.: #here you chose the overlay parameters as this determines how many files you need
  #it does NOT affect the content of the marlin steering file whatsoever, you need to make sure the values 
  #there are correct. Only the file names are handled properly so that you don't need to care
  overlay.setBXOverlay(300)
  overlay.setGGToHadInt(0.3)##When running at 500geV
elif energy == 1000.:
  overlay.setBXOverlay(60)
  overlay.setGGToHadInt(1.3)##When running at 1tev
else:
  print "Overlay ILD: No overlay parameters defined for this energy"  

##Reconstruction ILD with overlay
mao = Marlin()
mao.setDebug()
mao.setVersion('v0111Prod') ##PUT HERE YOUR MARLIN VERSION
if ild_rec_ov:
  if energy==500.:
    mao.setSteeringFile("clic_ild_cdr500_steering_overlay.xml") #STEERINGFILE for 500gev
    mao.setGearFile('clic_ild_cdr500.gear') #GEAR FILE for 500gev
  elif energy==1000.0:
    mao.setSteeringFile("clic_ild_cdr_steering_overlay_1400.0.xml") #STEERINGFILE for 1tev
    mao.setGearFile('clic_ild_cdr.gear') #GEAR FILE for 1tev
  else:
    print "Marlin: No reconstruction suitable for this energy"


##Reconstruction ILD w/o overlay
ma = Marlin()
예제 #34
0
lcsim_prepandora.setVersion('CLIC_CDR')#This will change only once
lcsim_prepandora.setSteeringFile("clic_cdr_prePandora.lcsim")#The steering files should NOT CHANGE
lcsim_prepandora.setTrackingStrategy(tracking_strategy)
#lcsim_prepandora.setDetectorModel(detector_model)
lcsim_prepandora.setOutputFile("prePandora.slcio")#NEVER CHANGE THIS, this file is not stored in any case
lcsim_prepandora.setNumberOfEvents(n_events)
lcsim_prepandora.willRunSLICPandora()

slicpandora = SLICPandora()
slicpandora.setVersion('CLIC_CDR')#This will change only once
slicpandora.setDetectorModel(detector_model)
slicpandora.setPandoraSettings("PandoraSettingsSlic.xml")
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')#NEVER CHANGE THIS, this file is not stored in any case

lcfivertex = Marlin()
lcfivertex.setVersion()
lcfivertex.getInputFromApp(slicpandora)
lcfivertex.setSteeringFile()
lcfivertex.setOutputFile("lcfivertex.slcio")#NEVER CHANGE THIS, this file is not stored in any case

#Final step. Outpufile is resolved automatically 
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion('CLIC_CDR')#This will change only once
lcsim_postpandora.getInputFromApp(lcfivertex)
lcsim_postpandora.setSteeringFile("clic_cdr_postPandoraOverlay.lcsim")#This will change only once
lcsim_postpandora.setTrackingStrategy(tracking_strategy)
#lcsim_postpandora.setDetectorModel(detector_model)

## SID Reco w/o overlay
overlay_sid_gg = OverlayInput()
예제 #35
0
#lcsim_prepandora.setDetectorModel(detector_model)
lcsim_prepandora.setOutputFile(
    "prePandora.slcio"
)  #NEVER CHANGE THIS, this file is not stored in any case
lcsim_prepandora.setNumberOfEvents(n_events)
lcsim_prepandora.willRunSLICPandora()

slicpandora = SLICPandora()
slicpandora.setVersion('CLIC_CDR')  #This will change only once
slicpandora.setDetectorModel(detector_model)
slicpandora.setPandoraSettings("PandoraSettingsSlic.xml")
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile(
    'pandora.slcio')  #NEVER CHANGE THIS, this file is not stored in any case

lcfivertex = Marlin()
lcfivertex.setVersion()
lcfivertex.getInputFromApp(slicpandora)
lcfivertex.setSteeringFile()
lcfivertex.setOutputFile(
    "lcfivertex.slcio"
)  #NEVER CHANGE THIS, this file is not stored in any case

#Final step. Outpufile is resolved automatically
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion('CLIC_CDR')  #This will change only once
lcsim_postpandora.getInputFromApp(lcfivertex)
lcsim_postpandora.setSteeringFile(
    "clic_cdr_postPandoraOverlay.lcsim")  #This will change only once
lcsim_postpandora.setTrackingStrategy(tracking_strategy)
#lcsim_postpandora.setDetectorModel(detector_model)
예제 #36
0
def create_job(inputData, saveName, outputDir, dontPromptMe):

    slcioFile = saveName + '.slcio'
    rootFile = saveName + '.root'

    if check_file_existence(outputDir, slcioFile, dontPromptMe):
        remove_file(outputDir, slcioFile, dontPromptMe)
    if check_file_existence(outputDir, rootFile, dontPromptMe):
        remove_file(outputDir, rootFile, dontPromptMe)

    dIlc = DiracILC()

    job = UserJob()
    job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml'])
    if SAVE_SLCIO:
        job.setOutputData([slcioFile, rootFile],
                          OutputPath=outputDir,
                          OutputSE=STORAGE_SE)
    else:
        job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE)
    job.setJobGroup('myMarlinRun1')
    job.setName('MyMarlinJob1')
    # job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk'])
    job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz'])

    marl = Marlin()
    marl.setVersion('ILCSoft-2017-11-15_gcc62')

    marl.setInputFile(inputData)
    job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData)))
    # marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio'])
    marl.setSteeringFile('marlin/FullChain.xml')
    # marl.setSteeringFile('marlin/FullChainNewDetModel.xml')
    marl.setOutputFile(slcioFile)
    gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear'
    if not os.path.isfile(gearFile):
        print('Error: gear file', gearFile,
              'does not exist! Abort submission.')
        return
    marl.setGearFile(gearFile)
    marl.setExtraCLIArguments(
        "--MyNtupleMaker.OutputFileName={rootOutfile}".format(
            rootOutfile=rootFile))
    # marl.setNumberOfEvents(1000)

    job.append(marl)
    if dontPromptMe:
        job.dontPromptMe()
    job.submit(dIlc)

    return False
예제 #37
0
 job.setDestination('LCG.CERN.ch')
 job.setInputSandbox([
     "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraLikelihoodData9EBin_CLIC_ILD.xml",
     "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraSettingsFast.xml",
     "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/steering.xml",
     "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/clic_ild_cdr.gear",
     "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/lib.tar.gz"
 ])
 job.setInputData(
     "LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
     i)
 job.setOutputSandbox(
     ["*.log", "*.sh", "*.py", "*.out", "*.xml", "*.steer "])
 job.setJobGroup("myRoot")
 job.setName("root_aa_%d" % i)
 marl = Marlin()
 marl.setVersion('ILCSoft-2016-09-27_gcc48')
 marl.setInputFile([
     "LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
     i
 ])
 # marl.setNumberOfEvents(10)
 marl.setSteeringFile("steering.xml")
 marl.setGearFile("clic_ild_cdr.gear")
 marl.setExtraCLIArguments("--MyProcessor.RootFilename=%s" % lcoutput)
 res = job.append(marl)
 if not res['OK']:
     print res['Message']
     quit()  #do something, like quit
 job.setOutputData(lcoutput, "hgamgam/Marlin/rootFiles/aa/workingVersion",
                   "CERN-SRM")
예제 #38
0
 def setUp( self ):
   """set up the objects"""
   self.mar = Marlin( {} )
예제 #39
0
    dexit(1)


ov = OverlayInput()
ov.setDetectorModel("CLIC_ILD_CDR")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
ov.setBkgEvtType("gghad")
res = j.append(ov)
if not res['OK']:
    print res['Message']
    dexit(1)


ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res['OK']:
    print res['Message']
    dexit(1)
#print appplication's attributes.
ma.listAttributes()

j.setName("test")
j.setOutputSandbox("*.log")
예제 #40
0
ovi.setDetectorModel("CLIC_ILD_CDR")

overlay = [True, False]

for ov in overlay:
    d = DiracILC(True, "repo_overlay_%s.rep" % ov)
    for lfn in lfns:
        j = UserJob()
        steeringf = "clic_ild_cdr_steering.xml"
        if ov:
            steeringf = "clic_ild_cdr_steering_overlay.xml"
            res = j.append(ovi)
            if not res['OK']:
                print res['Message']
                continue
        ma = Marlin()
        ma.setVersion("v0111Prod")
        ma.setGearFile("clic_ild_cdr.gear")
        ma.setSteeringFile(steeringf)
        ma.setInputFile("LFN:" + lfn)
        ma.setNbEvts(10)
        ma.setEnergy(500.)
        ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov)
        ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov)
        res = j.append(ma)
        if not res['OK']:
            print res['Message']
            exit()

        j.setCPUTime(86400)
        j.setOutputData("myrec_overlay_%s.slcio" % ov, "some/path")
def main(argv):
    # Input arguments
    ildconfig_version   = "$ILDCONFIGVER"
    ilcsoft_version     = "$ILCSOFTVER"

    evts_per_run    = $EVTSPERRUN
    detector_model  = "$DETECTOR"
    sim_input       = "$SIMINPUT"
    process_name    = "$PROCESS"

    index           = $IND

    sim_input = diracpath_from_pnfspath( sim_input )
    sim_detector_model = detector_model_wo_option( detector_model )

    job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model
    dirac = DiracILC(True,job_group+".rep")

    # outputs to be saved onto grid SE
    RECoutput = []

    # DDSim

    evtStart   = (index-1)*evts_per_run
    evtEnd     = index*evts_per_run - 1
    RandSeed = random.randrange(11623, 99999)

    lcinputSIM  = "LFN:" + sim_input
    lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd))

    sim = DDSim()
    sim.setVersion(ilcsoft_version)

    sim.setDetectorModel(sim_detector_model)
    sim.setInputFile(lcinputSIM)
    sim.setSteeringFile("ddsim_steer.py")
    sim.setNumberOfEvents(evts_per_run)
    sim.setRandomSeed(RandSeed)
    sim.setEnergy(1000)
    sim.setStartFrom(evtStart)
    sim.setOutputFile(lcoutputSIM)

    # Marlin
    lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd))

    ma = Marlin()
    ma.setVersion(ilcsoft_version)
    ma.setDetectorModel(detector_model)
    ma.setSteeringFile("MarlinStdReco.xml")
    ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) )
    ma.setLogFile("marlin.log")
    ma.getInputFromApp(sim)
    ma.setEnergy(1000)
    ma.setOutputDstFile(lcoutputDST)

    RECoutput.append(lcoutputDST)

    # ILCDirac user job
    job = UserJob()
    job.setName("user_sim_reco")

    job.setJobGroup(job_group)

    job.setILDConfig(ildconfig_version)
    job.setCPUTime(86400)

    tmp_file_name = process_name + "_sim_reco_job_tmp.py"
    job.setInputSandbox([tmp_file_name])
    job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])

    job.dontPromptMe()
    job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])

    # run simulation job
    simres = job.append(sim)
    if not simres['OK']:
            print simres['Not ok appending ddsim to job']
            quit()


    # run Malrin reco jobs
    mares = job.append(ma)
    if not mares['OK']:
            print mares['Not ok appending Marlin to job']
            quit()

    job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM")
    print RECoutput

    submit_output = job.submit(dirac)
    print submit_output