# outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_Muon.root') # outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhoton.root') # outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPhotonNK0L.root') # outputFiles.append('MarlinReco_' + slcioFileNoPath[:-6] + '_PerfectPFA.root') job = UserJob() job.setJobGroup('TestingHadd') #job.setInputSandbox(['hadd.C']) # Local files # job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml']) job.setInputData(arguements[1:]) job.setOutputData('HaddTest.root',OutputPath='/Testing') # On grid job.setName('TestingHadd') job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp']) job.dontPromptMe() job.setCPUTime(1000) res = job.append(genericApplication) if not res['OK']: print res['Message'] exit() job.submit(diracInstance) os.system('rm *.cfg') # Tidy Up #os.system('rm MarlinSteering.steer') #os.system('rm ' + gearFileLocal) #for key, value in pandoraSettingsFilesLocal.iteritems(): # os.system('rm ' + value)
print result['Message'] sys.exit(2) elif marlinVer or slicPandoraVer: defaultOutputFile = outputFile + '.slcio' if not defaultOutputFile in outputData: outputData.append( defaultOutputFile ) job.setOutputSandbox ( outputSandbox ) job.setInputSandbox ( inputSandbox ) if inputSlcios: job.setInputData( inputSlcios ) elif inputFile: job.setInputData( [ inputFile ] ) if storeOutput == True: job.setOutputData ( outputData, storagePath, storageElement ) job.setCPUTime( cpuLimit ) job.setSystemConfig ( systemConfig ) job.setName ( detector+"_"+process+"_"+jobTitle ) job.setJobGroup( detector+"_"+process+"_"+jobTitle ) if destination: job.setDestination( destination ) else: job.setBannedSites( bannedSites ) if debug: print '' print 'Jobs to submit:' nTotal = len(lfnlist)*nJobs/mergeSlcioFiles if inputFile: print ' Number of input files:', len(lfnlist) if maxFiles > 0:
def SubmitJob(jobInfo): ######################### # Unpack job information ######################### eventType = jobInfo['eventType'] energy = jobInfo['energy'] detectorModel = jobInfo['detectorModel'] reconstructionVariant = jobInfo['reconstructionVariant'] slcioFile = jobInfo['slcioFile'] analysisTag = jobInfo['analysisTag'] jobDescription = jobInfo['jobDescription'] idx = jobInfo['idx'] numberOfFiles = jobInfo['numberOfFiles'] gearFileLocal = jobInfo['gearFileLocal'] diracInstance = jobInfo['diracInstance'] slcioFormat = jobInfo['slcioFormat'] pandoraPFOsToUse = jobInfo['pandoraPFOsToUse'] jetClusteringMode = jobInfo['jetClusteringMode'] nJetsToCluster = jobInfo['nJetsToCluster'] jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm'] jetClusteringRadius = jobInfo['jetClusteringRadius'] jetAlgorithmConfigString = pandoraPFOsToUse + '_' + jetClusteringAlgorithm + '_' + str( format(jetClusteringRadius, '.2f')).replace('.', 'p') flavourTaggingWeights = '' rootFileSuffix = '' if pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.50': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p50.tar.gz' rootFileSuffix = 'SPFOs_kt_0p50.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'SPFOs_kt_0p70.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.90': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_0p90.tar.gz' rootFileSuffix = 'SPFOs_kt_0p90.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '1.00': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_1p00.tar.gz' rootFileSuffix = 'SPFOs_kt_1p00.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '1.10': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_kt_algorithm_2jets_1p10.tar.gz' rootFileSuffix = 'SPFOs_kt_1p10.root' elif pandoraPFOsToUse == 'TightSelectedPandoraPFANewPFOs' and (int)( energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_TightPFOs_kt_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'TPFOs_kt_0p70.root' elif pandoraPFOsToUse == 'LooseSelectedPandoraPFANewPFOs' and (int)( energy) == 1400 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_LoosePFOs_kt_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'LPFOs_kt_0p70.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and ( int )(energy) == 1400 and jetClusteringAlgorithm == 'cambridge_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_cambridge_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'SPFOs_cam_0p70.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and ( int )(energy) == 1400 and jetClusteringAlgorithm == 'ee_kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.00': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_1400GeV_SelectedPFOs_ee_kt_algorithm_2jets_0p00.tar.gz' rootFileSuffix = 'SPFOs_ee_kt_0p70.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.50': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p50.tar.gz' rootFileSuffix = 'SPFOs_kt_0p50.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'SPFOs_kt_0p70.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.90': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_0p90.tar.gz' rootFileSuffix = 'SPFOs_kt_0p90.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '1.00': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_1p00.tar.gz' rootFileSuffix = 'SPFOs_kt_1p00.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and (int)( energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '1.10': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_kt_algorithm_2jets_1p10.tar.gz' rootFileSuffix = 'SPFOs_kt_1p10.root' elif pandoraPFOsToUse == 'TightSelectedPandoraPFANewPFOs' and (int)( energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_TightPFOs_kt_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'TPFOs_kt_0p70.root' elif pandoraPFOsToUse == 'LooseSelectedPandoraPFANewPFOs' and (int)( energy) == 3000 and jetClusteringAlgorithm == 'kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_LoosePFOs_kt_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'LPFOs_kt_0p70.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and ( int )(energy) == 3000 and jetClusteringAlgorithm == 'cambridge_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.70': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_cambridge_algorithm_2jets_0p70.tar.gz' rootFileSuffix = 'SPFOs_cam_0p70.root' elif pandoraPFOsToUse == 'SelectedPandoraPFANewPFOs' and ( int )(energy) == 3000 and jetClusteringAlgorithm == 'ee_kt_algorithm' and ( int)(nJetsToCluster) == 2 and str( format(jetClusteringRadius, '.2f')) == '0.00': flavourTaggingWeights = 'LFN:/ilc/user/s/sgreen/PhysicsAnalysis/LcfiWeights/lcfiweights_3000GeV_SelectedPFOs_ee_kt_algorithm_2jets_0p00.tar.gz' rootFileSuffix = 'SPFOs_ee_kt_0p70.root' slcioFileNoPath = os.path.basename(slcioFile) inputSandbox = [ 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/MarlinAnalysisProcessor.tar.gz', 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz', 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz', flavourTaggingWeights ] ######################### # Get info from file name ######################### matchObj = re.match(slcioFormat, os.path.basename(slcioFile), re.M | re.I) generatorSerialNumber = 0 numberOfEventsInFile = 0 startEventNumber = 0 # In generator level, not reconstruction. Start event for all reconstruction is 0. if matchObj: generatorSerialNumber = matchObj.group(1) numberOfEventsInFile = matchObj.group(2) startEventNumber = matchObj.group(3) else: print 'Wrong slcio format. Please check.' return ######################### # Modify Template ######################### steeringTemplate = GetTemplate(jobInfo) outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str( energy) + 'GeV/AnalysisTag' + str(analysisTag) rootFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str( energy) + 'GeV_GenN_' + str(generatorSerialNumber) + '_' + str( numberOfEventsInFile) + '_' + str(startEventNumber) + '_Tag' + str( analysisTag) + rootFileSuffix outputFiles = [] outputFiles.append(rootFileName) steeringTemplate = re.sub('InputSlcioFile', slcioFileNoPath, steeringTemplate) steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate) steeringTemplate = re.sub('MaximumNumberOfEventsToRecord', '-1', steeringTemplate) steeringTemplate = re.sub('AnalysisProcessorRootFile', rootFileName, steeringTemplate) ######################### # Check output doesn't exist already ######################### print 'Checking ' + eventType + ' ' + str( energy ) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.' skipJob = False for outputFile in outputFiles: lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile if doesFileExist(lfn): skipJob = True if skipJob: return print 'Submitting ' + eventType + ' ' + str( energy ) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.' ######################### # Write Template File ######################### marlinSteeringFilename = 'MarlinSteering_' + str( idx + 1) + '_' + eventType + '_' + str( energy) + '_' + jetAlgorithmConfigString + '.steer' with open(marlinSteeringFilename, 'w') as SteeringFile: SteeringFile.write(steeringTemplate) ######################### # Setup Marlin Application ######################### ma = Marlin() ma.setVersion('v01-16-02') ma.setSteeringFile(marlinSteeringFilename) ma.setGearFile(gearFileLocal) ma.setInputFile('lfn:' + slcioFile) ma.setProcessorsToUse([ 'libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so', 'libAnalysisProcessor.so', 'libMarlinReco.so' ]) ######################### # Submit Job ######################### jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str( energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str( numberOfEventsInFile) + '_' + str(startEventNumber) job = UserJob() job.setJobGroup(jobDescription) job.setInputSandbox(inputSandbox) # Local files job.setOutputSandbox(['*.log', '*.gear', '*.mac', '*.steer', '*.xml']) job.setOutputData(outputFiles, OutputPath=outputPath) # On grid job.setName(jobDetailedName) job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us', 'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il', 'LCG.Weizmann.il', 'OSG.BNL.us', 'LCG.GRIF.fr', 'LCG.UKI-SOUTHGRID-RALPP.uk', 'LCG.RAL-LCG2.uk' ]) job.setCPUTime(21600) # 6 hour, should be excessive job.dontPromptMe() res = job.append(ma) if not res['OK']: print res['Message'] exit() job.submit(diracInstance) os.system('rm ' + marlinSteeringFilename) print 'Job submitted to grid.'
def SubmitCLICJob(jobInfo): ######################### # Unpack job information ######################### eventType = jobInfo['eventType'] energy = jobInfo['energy'] detectorModel = jobInfo['detectorModel'] reconstructionVariant = jobInfo['reconstructionVariant'] clicFile = jobInfo['clicFile'] analysisTag = jobInfo['analysisTag'] jobDescription = jobInfo['jobDescription'] steeringTemplateContent = jobInfo['steeringTemplateContent'] prodID = jobInfo['prodID'] idx = jobInfo['idx'] numberOfFiles = jobInfo['numberOfFiles'] gearFileLocal = jobInfo['gearFileLocal'] diracInstance = jobInfo['diracInstance'] marlinVersion = jobInfo['marlinVersion'] pandoraPFOsToUse = jobInfo['pandoraPFOsToUse'] shortPandoraPFOsToUse = jobInfo['shortPandoraPFOsToUse'] jetClusteringMode = jobInfo['jetClusteringMode'] nJetsToCluster = jobInfo['nJetsToCluster'] jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm'] jetClusteringRadius = jobInfo['jetClusteringRadius'] jetAlgorithmConfigString = jobInfo['jetAlgorithmConfigString'] print 'Checking CLIC sample ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. Slcio file ' + clicFile + '.' clicFileNoPath = os.path.basename(clicFile) inputSandbox = ['LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz', 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz'] ######################### # Modify Template ######################### steeringTemplate = steeringTemplateContent outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '_ProdID_' + str(prodID) + '/' + str(energy) + 'GeV/' + jetAlgorithmConfigString rootFileName = 'ProdID_' + str(prodID) + '_' + eventType + '_' + str(energy) + 'GeV_Analysis_' + str(analysisTag) + '_Number_' + str(idx+1) + '_Of_' + str(numberOfFiles) + '.root' outputFiles = [] outputFiles.append(rootFileName) steeringTemplate = re.sub('MakeNtupleRootFileName',rootFileName,steeringTemplate) steeringTemplate = re.sub('GearFile',gearFileLocal,steeringTemplate) steeringTemplate = re.sub('InputSlcioFile',clicFileNoPath,steeringTemplate) steeringTemplate = re.sub('PandoraPFOsToUse', pandoraPFOsToUse,steeringTemplate) steeringTemplate = re.sub('JetClusteringMode', jetClusteringMode,steeringTemplate) steeringTemplate = re.sub('NJetsToCluster', str(nJetsToCluster),steeringTemplate) steeringTemplate = re.sub('JetClusteringAlgorithm',jetClusteringAlgorithm,steeringTemplate) steeringTemplate = re.sub('JetClusteringRadius', str(format(jetClusteringRadius,'.2f')),steeringTemplate) # steeringTemplate = re.sub('JetAlgorithmConfig',jetAlgorithmConfigString,steeringTemplate) ######################### # Write Template File ######################### marlinSteeringFilename = 'MarlinSteering_' + str(idx+1) + '_' + eventType + '_' + jetAlgorithmConfigString + '.steer' with open(marlinSteeringFilename ,'w') as SteeringFile: SteeringFile.write(steeringTemplate) ######################### # Check output doesn't exist already ######################### skipJob = False for outputFile in outputFiles: lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile if doesFileExist(lfn): skipJob = True if skipJob: return print 'Submitting ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. CLIC file ' + clicFile + '.' ######################### # Setup Marlin Application ######################### ma = Marlin() ma.setVersion(marlinVersion) ma.setSteeringFile(marlinSteeringFilename) ma.setGearFile(gearFileLocal) ma.setInputFile('lfn:' + clicFile) ma.setProcessorsToUse(['libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so']) ######################### # Submit Job ######################### jobDetailedName = jobDescription + '_' + eventType + '_' + str(energy) + 'GeV_Tag' + str(analysisTag) + '_ProdID_' + str(prodID) + '_Number_' + str(idx+1) + '_Of_' + str(numberOfFiles) job = UserJob() job.setJobGroup(jobDescription) job.setInputSandbox(inputSandbox) # Local files job.setOutputSandbox(['*.log','*.steer','*.xml']) job.setOutputData(outputFiles,OutputPath=outputPath) # On grid job.setName(jobDetailedName) job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr','LCG.UKI-LT2-IC-HEP.uk','LCG.Tau.il','LCG.Weizmann.il','OSG.BNL.us']) job.setCPUTime(21600) # 6 hour, should be excessive job.dontPromptMe() # job.setDestination('LCG.CERN.ch') res = job.append(ma) if not res['OK']: print res['Message'] exit() job.submit(diracInstance) # Tidy Up os.system('rm ' + marlinSteeringFilename) print 'Job submitted to grid.'
def SubmitJob(jobInfo): ######################### # Unpack job information ######################### eventType = jobInfo['eventType'] energy = jobInfo['energy'] detectorModel = jobInfo['detectorModel'] reconstructionVariant = jobInfo['reconstructionVariant'] slcioFile = jobInfo['slcioFile'] pandoraSettingsFileLocal = jobInfo['pandoraSettingsFileLocal'] slcioFormat = jobInfo['slcioFormat'] steeringTemplateContent = jobInfo['steeringTemplateContent'] jobDescription = jobInfo['jobDescription'] idx = jobInfo['idx'] gearFileLocal = jobInfo['gearFileLocal'] diracInstance = jobInfo['diracInstance'] ggHadBackground = jobInfo['ggHadBackground'] eventsPerFile = jobInfo['eventsPerFile'] slcioFileNoPath = os.path.basename(slcioFile) inputSandbox = [] inputSandbox.append(pandoraSettingsFileLocal) ######################### # Get info from file name ######################### matchObj = re.match(slcioFormat, os.path.basename(slcioFile), re.M|re.I) # MokkaSim_Detector_Model_clic_ild_cdr_ee_nunuww_nunuqqqq_1400GeV_GeneratorSerialNumber_100_100_0.slcio generatorSerialNumber = 0 numberOfEventsInFile = 0 startEventNumber = 0 # In generator level, not reconstruction. Start event for all reconstruction is 0. if matchObj: generatorSerialNumber = matchObj.group(1) numberOfEventsInFile = matchObj.group(2) startEventNumber = matchObj.group(3) else: print 'Wrong stdhep format. Please check.' return ######################### # Modify Template ######################### steeringTemplate = steeringTemplateContent outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str(energy) + 'GeV' recFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber) + '_REC.slcio' dstFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber) + '_DST.slcio' outputFiles = [] outputFiles.append(recFileName) outputFiles.append(dstFileName) steeringTemplate = re.sub('OutputRecFile',recFileName,steeringTemplate) steeringTemplate = re.sub('OutputDstFile',dstFileName,steeringTemplate) steeringTemplate = re.sub('GearFile',gearFileLocal,steeringTemplate) steeringTemplate = re.sub('InputSlcioFile',slcioFileNoPath,steeringTemplate) steeringTemplate = re.sub('PandoraSettingsFile',pandoraSettingsFileLocal,steeringTemplate) ######################### # Check output doesn't exist already ######################### print 'Checking ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.' skipJob = False for outputFile in outputFiles: lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile if doesFileExist(lfn): skipJob = True if skipJob: return print 'Submitting ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.' ######################### # Write Template File ######################### marlinSteeringFilename = 'MarlinSteering_' + str(idx+1) + '_' + eventType + '.steer' with open(marlinSteeringFilename ,"w") as SteeringFile: SteeringFile.write(steeringTemplate) ######################### # Setup Marlin Application ######################### ma = Marlin() ma.setVersion('v0111Prod') ma.setSteeringFile(marlinSteeringFilename) ma.setGearFile(gearFileLocal) ma.setInputFile('lfn:' + slcioFile) ######################### # Setup Overlay Information ######################### overlay = OverlayInput() if ggHadBackground: overlay.setMachine('clic_cdr') overlay.setBXOverlay(60) overlay.setDetectorModel('CLIC_ILD_CDR') overlay.setBackgroundType('gghad') overlay.setNbSigEvtsPerJob((int)(eventsPerFile)) if (int)(energy) == 1400: overlay.setEnergy(1400.0) overlay.setGGToHadInt(1.3) # When running at 1.4TeV elif (int)(energy) == 3000: overlay.setEnergy(3000.0) overlay.setGGToHadInt(3.2) # When running at 3TeV else: print 'Unknown number of GGHadToInts to use for this energy. Exiting job subission.' return ######################### # Submit Job ######################### jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str(numberOfEventsInFile) + '_' + str(startEventNumber) job = UserJob() job.setJobGroup(jobDescription) job.setInputSandbox(inputSandbox) # Local files job.setOutputSandbox(['*.log','*.gear','*.mac','*.steer','*.xml']) job.setOutputData(outputFiles,OutputPath=outputPath) # On grid job.setName(jobDetailedName) job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.IN2P3-IRES.fr','LCG.KEK.jp','OSG.PNNL.us','OSG.CIT.us','LCG.LAPP.fr','LCG.UKI-LT2-IC-HEP.uk','LCG.Tau.il','LCG.Weizmann.il','OSG.BNL.us']) job.setCPUTime(21600) # 6 hour, should be excessive job.dontPromptMe() if ggHadBackground: res = job.append(overlay) res = job.append(ma) if not res['OK']: print res['Message'] return job.submit(diracInstance) # Tidy Up os.system('rm ' + marlinSteeringFilename) print 'Job submitted to grid.'
print result['Message'] sys.exit(2) elif marlinVer or slicPandoraVer: defaultOutputFile = outputFile + '.slcio' if not defaultOutputFile in outputData: outputData.append(defaultOutputFile) job.setOutputSandbox(outputSandbox) job.setInputSandbox(inputSandbox) if inputSlcios: job.setInputData(inputSlcios) elif inputFile: job.setInputData([inputFile]) if storeOutput == True: job.setOutputData(outputData, storagePath, storageElement) job.setCPUTime(cpuLimit) job.setSystemConfig(systemConfig) job.setName(detector + "_" + process + "_" + jobTitle) job.setJobGroup(detector + "_" + process + "_" + jobTitle) if destination: job.setDestination(destination) else: job.setBannedSites(bannedSites) if debug: print '' print 'Jobs to submit:' nTotal = len(lfnlist) * nJobs / mergeSlcioFiles if inputFile: print ' Number of input files:', len(lfnlist) if maxFiles > 0:
def SubmitCLICJob(jobInfo): ######################### # Unpack job information ######################### eventType = jobInfo['eventType'] energy = jobInfo['energy'] detectorModel = jobInfo['detectorModel'] reconstructionVariant = jobInfo['reconstructionVariant'] clicFile = jobInfo['clicFile'] analysisTag = jobInfo['analysisTag'] jobDescription = jobInfo['jobDescription'] steeringTemplateContent = jobInfo['steeringTemplateContent'] prodID = jobInfo['prodID'] idx = jobInfo['idx'] numberOfFiles = jobInfo['numberOfFiles'] gearFileLocal = jobInfo['gearFileLocal'] diracInstance = jobInfo['diracInstance'] marlinVersion = jobInfo['marlinVersion'] pandoraPFOsToUse = jobInfo['pandoraPFOsToUse'] shortPandoraPFOsToUse = jobInfo['shortPandoraPFOsToUse'] jetClusteringMode = jobInfo['jetClusteringMode'] nJetsToCluster = jobInfo['nJetsToCluster'] jetClusteringAlgorithm = jobInfo['jetClusteringAlgorithm'] jetClusteringRadius = jobInfo['jetClusteringRadius'] jetAlgorithmConfigString = jobInfo['jetAlgorithmConfigString'] print 'Checking CLIC sample ' + eventType + ' ' + str( energy ) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. Slcio file ' + clicFile + '.' clicFileNoPath = os.path.basename(clicFile) inputSandbox = [ 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/JetsToPFOProcessor.tar.gz', 'LFN:/ilc/user/s/sgreen/AnalysisProcessorTarBall/vtxprob.tar.gz' ] ######################### # Modify Template ######################### steeringTemplate = steeringTemplateContent outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '_ProdID_' + str( prodID) + '/' + str(energy) + 'GeV/' + jetAlgorithmConfigString rootFileName = 'ProdID_' + str(prodID) + '_' + eventType + '_' + str( energy) + 'GeV_Analysis_' + str(analysisTag) + '_Number_' + str( idx + 1) + '_Of_' + str(numberOfFiles) + '.root' outputFiles = [] outputFiles.append(rootFileName) steeringTemplate = re.sub('MakeNtupleRootFileName', rootFileName, steeringTemplate) steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate) steeringTemplate = re.sub('InputSlcioFile', clicFileNoPath, steeringTemplate) steeringTemplate = re.sub('PandoraPFOsToUse', pandoraPFOsToUse, steeringTemplate) steeringTemplate = re.sub('JetClusteringMode', jetClusteringMode, steeringTemplate) steeringTemplate = re.sub('NJetsToCluster', str(nJetsToCluster), steeringTemplate) steeringTemplate = re.sub('JetClusteringAlgorithm', jetClusteringAlgorithm, steeringTemplate) steeringTemplate = re.sub('JetClusteringRadius', str(format(jetClusteringRadius, '.2f')), steeringTemplate) # steeringTemplate = re.sub('JetAlgorithmConfig',jetAlgorithmConfigString,steeringTemplate) ######################### # Write Template File ######################### marlinSteeringFilename = 'MarlinSteering_' + str( idx + 1) + '_' + eventType + '_' + jetAlgorithmConfigString + '.steer' with open(marlinSteeringFilename, 'w') as SteeringFile: SteeringFile.write(steeringTemplate) ######################### # Check output doesn't exist already ######################### skipJob = False for outputFile in outputFiles: lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile if doesFileExist(lfn): skipJob = True if skipJob: return print 'Submitting ' + eventType + ' ' + str( energy ) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. CLIC file ' + clicFile + '.' ######################### # Setup Marlin Application ######################### ma = Marlin() ma.setVersion(marlinVersion) ma.setSteeringFile(marlinSteeringFilename) ma.setGearFile(gearFileLocal) ma.setInputFile('lfn:' + clicFile) ma.setProcessorsToUse( ['libMarlinFastJet.so', 'libJetsToPFOs.so', 'libLCFIPlus.so']) ######################### # Submit Job ######################### jobDetailedName = jobDescription + '_' + eventType + '_' + str( energy) + 'GeV_Tag' + str(analysisTag) + '_ProdID_' + str( prodID) + '_Number_' + str(idx + 1) + '_Of_' + str(numberOfFiles) job = UserJob() job.setJobGroup(jobDescription) job.setInputSandbox(inputSandbox) # Local files job.setOutputSandbox(['*.log', '*.steer', '*.xml']) job.setOutputData(outputFiles, OutputPath=outputPath) # On grid job.setName(jobDetailedName) job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us', 'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il', 'LCG.Weizmann.il', 'OSG.BNL.us' ]) job.setCPUTime(21600) # 6 hour, should be excessive job.dontPromptMe() # job.setDestination('LCG.CERN.ch') res = job.append(ma) if not res['OK']: print res['Message'] exit() job.submit(diracInstance) # Tidy Up os.system('rm ' + marlinSteeringFilename) print 'Job submitted to grid.'
def SubmitJob(jobInfo): ######################### # Unpack job information ######################### eventType = jobInfo['eventType'] energy = jobInfo['energy'] detectorModel = jobInfo['detectorModel'] reconstructionVariant = jobInfo['reconstructionVariant'] slcioFile = jobInfo['slcioFile'] pandoraSettingsFileLocal = jobInfo['pandoraSettingsFileLocal'] slcioFormat = jobInfo['slcioFormat'] steeringTemplateContent = jobInfo['steeringTemplateContent'] jobDescription = jobInfo['jobDescription'] idx = jobInfo['idx'] gearFileLocal = jobInfo['gearFileLocal'] diracInstance = jobInfo['diracInstance'] ggHadBackground = jobInfo['ggHadBackground'] eventsPerFile = jobInfo['eventsPerFile'] slcioFileNoPath = os.path.basename(slcioFile) inputSandbox = [] inputSandbox.append(pandoraSettingsFileLocal) ######################### # Get info from file name ######################### matchObj = re.match( slcioFormat, os.path.basename(slcioFile), re.M | re.I ) # MokkaSim_Detector_Model_clic_ild_cdr_ee_nunuww_nunuqqqq_1400GeV_GeneratorSerialNumber_100_100_0.slcio generatorSerialNumber = 0 numberOfEventsInFile = 0 startEventNumber = 0 # In generator level, not reconstruction. Start event for all reconstruction is 0. if matchObj: generatorSerialNumber = matchObj.group(1) numberOfEventsInFile = matchObj.group(2) startEventNumber = matchObj.group(3) else: print 'Wrong stdhep format. Please check.' return ######################### # Modify Template ######################### steeringTemplate = steeringTemplateContent outputPath = '/' + jobDescription + '/MarlinJobs/Detector_Model_' + detectorModel + '/Reconstruction_Variant_' + reconstructionVariant + '/' + eventType + '/' + str( energy) + 'GeV' recFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str( energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str( numberOfEventsInFile) + '_' + str(startEventNumber) + '_REC.slcio' dstFileName = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str( energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str( numberOfEventsInFile) + '_' + str(startEventNumber) + '_DST.slcio' outputFiles = [] outputFiles.append(recFileName) outputFiles.append(dstFileName) steeringTemplate = re.sub('OutputRecFile', recFileName, steeringTemplate) steeringTemplate = re.sub('OutputDstFile', dstFileName, steeringTemplate) steeringTemplate = re.sub('GearFile', gearFileLocal, steeringTemplate) steeringTemplate = re.sub('InputSlcioFile', slcioFileNoPath, steeringTemplate) steeringTemplate = re.sub('PandoraSettingsFile', pandoraSettingsFileLocal, steeringTemplate) ######################### # Check output doesn't exist already ######################### print 'Checking ' + eventType + ' ' + str( energy ) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.' skipJob = False for outputFile in outputFiles: lfn = '/ilc/user/s/sgreen' + outputPath + '/' + outputFile if doesFileExist(lfn): skipJob = True if skipJob: return print 'Submitting ' + eventType + ' ' + str( energy ) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. SLCIO file ' + slcioFile + '.' ######################### # Write Template File ######################### marlinSteeringFilename = 'MarlinSteering_' + str( idx + 1) + '_' + eventType + '.steer' with open(marlinSteeringFilename, "w") as SteeringFile: SteeringFile.write(steeringTemplate) ######################### # Setup Marlin Application ######################### ma = Marlin() ma.setVersion('v0111Prod') ma.setSteeringFile(marlinSteeringFilename) ma.setGearFile(gearFileLocal) ma.setInputFile('lfn:' + slcioFile) ######################### # Setup Overlay Information ######################### overlay = OverlayInput() if ggHadBackground: overlay.setMachine('clic_cdr') overlay.setBXOverlay(60) overlay.setDetectorModel('CLIC_ILD_CDR') overlay.setBackgroundType('gghad') overlay.setNbSigEvtsPerJob((int)(eventsPerFile)) if (int)(energy) == 1400: overlay.setEnergy(1400.0) overlay.setGGToHadInt(1.3) # When running at 1.4TeV elif (int)(energy) == 3000: overlay.setEnergy(3000.0) overlay.setGGToHadInt(3.2) # When running at 3TeV else: print 'Unknown number of GGHadToInts to use for this energy. Exiting job subission.' return ######################### # Submit Job ######################### jobDetailedName = jobDescription + '_DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str( energy) + 'GeV_GenNumber_' + str(generatorSerialNumber) + '_' + str( numberOfEventsInFile) + '_' + str(startEventNumber) job = UserJob() job.setJobGroup(jobDescription) job.setInputSandbox(inputSandbox) # Local files job.setOutputSandbox(['*.log', '*.gear', '*.mac', '*.steer', '*.xml']) job.setOutputData(outputFiles, OutputPath=outputPath) # On grid job.setName(jobDetailedName) job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'LCG.IN2P3-IRES.fr', 'LCG.KEK.jp', 'OSG.PNNL.us', 'OSG.CIT.us', 'LCG.LAPP.fr', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.Tau.il', 'LCG.Weizmann.il', 'OSG.BNL.us' ]) job.setCPUTime(21600) # 6 hour, should be excessive job.dontPromptMe() if ggHadBackground: res = job.append(overlay) res = job.append(ma) if not res['OK']: print res['Message'] return job.submit(diracInstance) # Tidy Up os.system('rm ' + marlinSteeringFilename) print 'Job submitted to grid.'
site="KEK" opt="" if (argc > 1) : site = argvs[1] if (argc > 2 ) : opt = argvs[2] jobname=param[site]["name"] dest=param[site]["dest"] print "Test getfile to "+site+"( jobname="+jobname+" dest="+dest+" opt="+opt+")" job = UserJob() job.setCPUTime(500) # time in sec job.setExecutable('/bin/ls -l ') job.setExecutable('/bin/bash test.sh') job.setName("test1") job.setInputSandbox(["test.sh"]) job.setOutputSandbox(["*.log","*.dat"]) # job.setDestination(dest) job.dontPromptMe() dirac = DiracILC(True,jobname+".rep") jobID = job.submit(dirac) print jobname+" submitted. JobID=",jobID