def main(): cmsswBase=os.environ['CMSSW_BASE'] eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string') (opt, args) = parser.parse_args() Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate() #prepare output directory if opt.outDir is None: opt.outDir=opt.inDir Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate() dset_list=getEOSlslist(directory=opt.inDir,prepend='') for dset in dset_list: dsetname=dset.split('/')[-1] pub_list=getEOSlslist(directory=dset,prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname continue pub=pubDir.split('/crab_')[-1] localMerge='python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s'%(opt.inDir,opt.outDir,pub) cmd='bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (opt.queue,cmsswBase,localMerge) os.system(cmd) Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate()
def main(): cmsswBase = os.environ['CMSSW_BASE'] eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string') (opt, args) = parser.parse_args() Popen([eos_cmd, ' -b fuse mount', 'eos'], stdout=PIPE).communicate() #prepare output directory if opt.outDir is None: opt.outDir = opt.inDir Popen([eos_cmd, 'mkdir', '/eos/cms/' + opt.outDir], stdout=PIPE).communicate() dset_list = getEOSlslist(directory=opt.inDir, prepend='') for dset in dset_list: dsetname = dset.split('/')[-1] pub_list = getEOSlslist(directory=dset, prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname continue pub = pubDir.split('/crab_')[-1] localMerge = 'python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s' % ( opt.inDir, opt.outDir, pub) cmd = 'bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % ( opt.queue, cmsswBase, localMerge) os.system(cmd) Popen([eos_cmd, ' -b fuse umount', 'eos'], stdout=PIPE).communicate()
def main(): eos_cmd = '/afs/cern.ch/project/eos/installation/cms/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string') (opt, args) = parser.parse_args() #prepare output directory if opt.outDir is None: opt.outDir=opt.inDir cmsswBase=os.environ['CMSSW_BASE'] FARMDIR='%s/src/TopLJets2015/TopAnalysis/FARM'%cmsswBase os.system('mkdir -p %s'%FARMDIR) dset_list=getEOSlslist(directory=opt.inDir,prepend='') for dset in dset_list: dsetname=dset.split('/')[-1] pub_list=getEOSlslist(directory=dset,prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname continue pub=pubDir.split('/crab_')[-1] cfgfile='%s/mergejob_%s.sh'%(FARMDIR,pub) with open(cfgfile,'w') as cfg: cfg.write('WORKDIR=`pwd`\n') cfg.write('echo "Working directory is ${WORKDIR}"\n') cfg.write('cd %s\n'%cmsswBase) cfg.write('eval `scram r -sh`\n') cfg.write('cd ${WORKDIR}\n') cfg.write('echo "Preparing output directory"\n') cfg.write('%s mkdir %s\n'%(eos_cmd,opt.outDir)) cfg.write('echo "Running integrity checker"\n') cfg.write('python ${CMSSW_BASE}/src/TopLJets2015/TopAnalysis/scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --mount --only %s'%(opt.inDir,opt.outDir,pub)) os.system('chmod u+x %s'%cfgfile) cmd='bsub -q %s %s' % (opt.queue,cfgfile) os.system(cmd)
def createAnalysisTasks(opt): onlyList = opt.only.split("v") ## Local directory file_list = [] if os.path.isdir(opt.input): for file_path in os.listdir(opt.input): if file_path.endswith(".root"): file_list.append(os.path.join(opt.input, file_path)) elif opt.input.startswith("/store/"): file_list = getEOSlslist(opt.input) elif ".root" in opt.input: file_list.append(opt.input) # list of files to analyse tasklist = [] for filename in file_list: baseFileName = os.path.basename(filename) tag, ext = os.path.splitext(baseFileName) if len(onlyList) > 0: processThis = False for filtTag in onlyList: if filtTag in tag: processThis = True if not processThis: continue tasklist.append((filename, "%s/%s" % (opt.output, baseFileName))) # loop over tasks if opt.queue == "local": if opt.jobs > 1: print " Submitting jobs in %d threads" % opt.jobs import multiprocessing as MP pool = MP.Pool(opt.jobs) pool.map(runTopWidthAnalysisPacked, tasklist) else: for fileName, outFileName in tasklist: runTopWidthAnalysis(fileName, outFileName) else: cmsswBase = os.environ["CMSSW_BASE"] for fileName, _ in tasklist: localRun = "python %s/src/TopLJets2015/TopAnalysis/scripts/runTopWidthAnalysis.py -i %s -o %s -q local" % ( cmsswBase, fileName, opt.output, ) cmd = 'bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh "%s"' % ( opt.queue, cmsswBase, localRun, ) print cmd os.system(cmd)
def createAnalysisTasks(opt): onlyList = opt.only.split('v') ## Local directory file_list = [] if os.path.isdir(opt.input): for file_path in os.listdir(opt.input): if file_path.endswith('.root'): file_list.append(os.path.join(opt.input, file_path)) elif opt.input.startswith('/store/'): file_list = getEOSlslist(opt.input) elif '.root' in opt.input: file_list.append(opt.input) #list of files to analyse tasklist = [] for filename in file_list: baseFileName = os.path.basename(filename) tag, ext = os.path.splitext(baseFileName) if len(onlyList) > 0: processThis = False for filtTag in onlyList: if filtTag in tag: processThis = True if not processThis: continue tasklist.append( (filename, '%s/%s' % (opt.output, baseFileName), opt.filter)) #loop over tasks if opt.queue == 'local': if opt.jobs > 1: print ' Submitting jobs in %d threads' % opt.jobs import multiprocessing as MP pool = MP.Pool(opt.jobs) pool.map(runAnomalousTopProductionAnalysisPacked, tasklist) else: for fileName, outFileName, filterName in tasklist: runAnomalousTopProductionAnalysis(fileName, outFileName, filterName) else: cmsswBase = os.environ['CMSSW_BASE'] for fileName, _, filterName in tasklist: localRun = 'python %s/src/TopLJets2015/TopAnalysis/scripts/runAnomalousTopProductionAnalysis.py -i %s -o %s -q local' % ( cmsswBase, fileName, opt.output) if filterName: localRun += ' --filter %s' % filterName cmd = 'bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % ( opt.queue, cmsswBase, localRun) print cmd os.system(cmd)
def createAnalysisTasks(opt): onlyList=opt.only.split('v') ## Local directory file_list=[] if os.path.isdir(opt.input): for file_path in os.listdir(opt.input): if file_path.endswith('.root'): file_list.append(os.path.join(opt.input,file_path)) elif opt.input.startswith('/store/'): file_list = getEOSlslist(opt.input) elif '.root' in opt.input: file_list.append(opt.input) #list of files to analyse tasklist=[] for filename in file_list: baseFileName=os.path.basename(filename) tag,ext=os.path.splitext(baseFileName) if len(onlyList)>0: processThis=False for filtTag in onlyList: if filtTag in tag: processThis=True if not processThis : continue tasklist.append((filename,'%s/%s'%(opt.output,baseFileName),opt.filter)) #loop over tasks if opt.queue=='local': if opt.jobs>1: print ' Submitting jobs in %d threads' % opt.jobs import multiprocessing as MP pool = MP.Pool(opt.jobs) pool.map(runAnomalousTopProductionAnalysisPacked,tasklist) else: for fileName,outFileName,filterName in tasklist: runAnomalousTopProductionAnalysis(fileName,outFileName,filterName) else: cmsswBase=os.environ['CMSSW_BASE'] for fileName,_,filterName in tasklist: localRun='python %s/src/TopLJets2015/TopAnalysis/scripts/runAnomalousTopProductionAnalysis.py -i %s -o %s -q local'%(cmsswBase,fileName,opt.output) if filterName : localRun+=' --filter %s'%filterName cmd='bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (opt.queue,cmsswBase,localRun) print cmd os.system(cmd)
def main(): cmsswBase=os.environ['CMSSW_BASE'] eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string') parser.add_option( '--only', dest='only', help='only this tag', default=None, type='string') (opt, args) = parser.parse_args() Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate() #prepare output directory if opt.outDir is None: opt.outDir=opt.inDir Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate() dset_list=getEOSlslist(directory=opt.inDir,prepend='') for dset in dset_list: dsetname=dset.split('/')[-1] #if not 'pp_TuneCUETP8M1_Hydjet_Min_Bias' : continue pub_list=getEOSlslist(directory=dset,prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname continue pub=pubDir.split('/crab_')[-1] #if not 'V4' in pub : continue if opt.only: if pub not in opt.only: continue #if 'Data13TeV' in pub : continue localMerge='python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s'%(opt.inDir,opt.outDir,pub) ############### Submit to condor ############### target = '%s/src/TopLJets2015/TopAnalysis/%s' % (cmsswBase,pub) condorFile = open(target,'w') condorFile.write('universe = vanilla\n') #condorFile.write('executable = condor/cond_wrapper.sh\n') condorFile.write('executable = condor/cond_check.sh\n') condorFile.write('arguments = $(ClusterID) $(ProcId) %s %s %s\n' % (opt.inDir,opt.outDir,pub)) condorFile.write('output = condor/log/%s_EOS_$(ProcId).out\n' % pub) condorFile.write('error = condor/log/%s_EOS_$(ProcId).err\n' % pub) condorFile.write('log = condor/log/%s_EOS.log\n' % pub) condorFile.write('+JobFlavour = "workday"\n') condorFile.write('Should_Transfer_Files = NO\n') condorFile.write('queue') condorFile.close() os.system('condor_submit %s -batch-name %s' % (target,pub)) os.system('rm %s' % (pub)) #cmd='bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (opt.queue,cmsswBase,localMerge) #os.system(cmd) Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate()
def main(): eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-c', '--cleanup', dest='cleanup', help='removes original crab directory', default=False, action='store_true') parser.add_option('--nocheck', dest='nocheck', help='do not prompt user', default=False, action='store_true') parser.add_option('--only', dest='only', help='only this tag', default=None, type='string') (opt, args) = parser.parse_args() Popen([eos_cmd, ' -b fuse mount', 'eos'], stdout=PIPE).communicate() #prepare output directory if opt.outDir is None: opt.outDir = opt.inDir Popen([eos_cmd, 'mkdir', '/eos/cms/' + opt.outDir], stdout=PIPE).communicate() dset_list = getEOSlslist(directory=opt.inDir, prepend='') for dset in dset_list: dsetname = dset.split('/')[-1] pub_list = getEOSlslist(directory=dset, prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname continue #select if it doesn't match the required selection pub = pubDir.split('/crab_')[-1] if opt.only: if pub != opt.only: continue #check if it's an extension pubExt = None try: extSplit = pub.split('_ext') pubExt = 'ext' + extSplit[1] pub = extSplit[0] print 'Extension will be postfixed with ', pubExt except: print 'Core sample (no extension)' time_list = getEOSlslist(directory=pubDir, prepend='') if len(time_list) != 1: print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out' % dsetname continue time_stamp = time_list[0].split('/')[-1] out_list = [] count_list = getEOSlslist(directory=time_list[0], prepend='') for count in count_list: out_list += getEOSlslist(directory=count, prepend='') file_list = [x for x in out_list if '.root' in x] newDir = '%s/%s' % (opt.outDir, pub) print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % ( dsetname, pub, time_stamp, len(file_list)) if not opt.nocheck: choice = raw_input( 'Will move to %s current output directory. [y/n] ?' % newDir).lower() if not 'y' in choice: continue Popen([eos_cmd, 'mkdir', '/eos/cms/' + newDir], stdout=PIPE).communicate() moveIndividualFiles = True if len(file_list) > 0: #subgroupMerge = int( raw_input('This set has %d files. Merge into groups? (enter 0 if no merging)' % len(file_list)) ) subgroupMerge = 60 if 'Data' in dsetname else 20 if subgroupMerge > 0: moveIndividualFiles = False splitFunc = lambda A, n=subgroupMerge: [ A[i:i + n] for i in range(0, len(A), n) ] split_file_lists = splitFunc(file_list) for ilist in xrange(0, len(split_file_lists)): if pubExt: mergedFileName = '/tmp/MergedMiniEvents_%d_%s.root ' % ( ilist, pubExt) else: mergedFileName = '/tmp/MergedMiniEvents_%d.root ' % ilist toAdd = '%s ' % mergedFileName for f in split_file_lists[ilist]: toAdd += 'eos/cms/%s ' % f finalOutput = 'eos/cms/%s/%s' % ( newDir, mergedFileName.replace('/tmp/', '')) fIn = ROOT.TFile.Open(finalOutput) try: if fIn or not fIn.IsZombie(): print '%s already in EOS, skipping' % finalOutput fIn.Close() continue except: pass os.system('hadd -f %s' % toAdd) os.system('cp %s eos/cms/%s/' % (mergedFileName, newDir)) os.system('rm %s' % mergedFileName) #os.system('xrdcp -f %s root://eoscms//eos/cms/%s/MergedMiniEvents_%d.root' %(mergedFileName,newDir,ilist)) #if still needed copy individual files if moveIndividualFiles: for f in file_list: #os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir) ) newF = f if pubExt: newF = f.replace('.root', '_%s.root' % pubExt) os.system('cp %s eos/cms/%s/%s' % (f, newDir, newF)) if not opt.nocheck and opt.cleanup: choice = raw_input( 'Will remove output directory. [y/n] ?').lower() if 'y' in choice: Popen([eos_cmd, 'rm', '-r /eos/cms/' + dset], stdout=PIPE).communicate() print 'Crab outputs may now be found in %s' % newDir #Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate() print '-' * 50 print 'All done. In case errors were found check that the crab output structure is ' print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>' print '-' * 50
process.GlobalTag.globaltag = '74X_dataRun2_v2' if options.runOnData else '74X_mcRun2_asymptotic_v2' # Set the process options -- Display summary at the end, enable unscheduled execution process.options = cms.untracked.PSet(allowUnscheduled=cms.untracked.bool(True), # wantSummary = cms.untracked.bool(True) ) #Number of events to process process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(-1)) from TopLJets2015.TopAnalysis.storeTools import getEOSlslist process.source = cms.Source( "PoolSource", fileNames=cms.untracked.vstring( getEOSlslist( directory= '/store/mc/RunIISpring15MiniAODv2/TTJets_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/MINIAODSIM/74X_mcRun2_asymptotic_v2-v3/60000/' ))) if options.inputDir != '': print 'Will process files from', options.inputDir process.source.fileNames = cms.untracked.vstring( getEOSlslist(directory=options.inputDir)) #reduce verbosity process.load("FWCore.MessageService.MessageLogger_cfi") process.MessageLogger.cerr.threshold = '' process.MessageLogger.cerr.FwkReport.reportEvery = 1000 #Tfileservice process.TFileService = cms.Service("TFileService", fileName=cms.string(options.outFilename))
def main(): #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-c', '--cleanup', dest='cleanup', help='removes original crab directory', default=False, action='store_true') parser.add_option('--nocheck', dest='nocheck', help='do not prompt user', default=False, action='store_true') parser.add_option('--only', dest='only', help='only this tag', default=None, type='string') (opt, args) = parser.parse_args() baseEOS = '/eos/cms/' #prepare output directory if opt.outDir is None: opt.outDir = opt.inDir os.system('mkdir -p %s/%s' % (baseEOS, opt.outDir)) dset_list = getEOSlslist(directory=opt.inDir, prepend='') for dset in dset_list: dsetname = dset.split('/')[-1] pub_list = getEOSlslist(directory=dset, prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname continue #select if it doesn't match the required selection pub = pubDir.split('/crab_')[-1] if opt.only: if pub != opt.only: continue #check if it's an extension pubExt = None try: extSplit = pub.split('_ext') pubExt = 'ext%d' % (len(extSplit) - 1) pub = extSplit[0] print 'Extension will be postfixed with ', pubExt except: print 'Core sample (no extension)' time_list = getEOSlslist(directory=pubDir, prepend='') if len(time_list) != 1: print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out' % dsetname continue time_stamp = time_list[0].split('/')[-1] out_list = [] count_list = getEOSlslist(directory=time_list[0], prepend='') for count in count_list: if '/merge' in count and 'group/hintt' in count: continue out_list += getEOSlslist(directory=count, prepend='') file_list = [x for x in out_list if '.root' in x] newDir = '%s/%s' % (opt.outDir, pub) print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % ( dsetname, pub, time_stamp, len(file_list)) if not opt.nocheck: choice = raw_input( 'Will move to %s current output directory. [y/n] ?' % newDir).lower() if not 'y' in choice: continue os.system('mkdir -p %s/%s' % (baseEOS, newDir)) moveIndividualFiles = True if len(file_list) > 0: subgroupMerge = 10 if 'Data' in pub: subgroupMerge = 50 moveIndividualFiles = False splitFunc = lambda A, n=subgroupMerge: [ A[i:i + n] for i in range(0, len(A), n) ] split_file_lists = splitFunc(file_list) for ilist in xrange(0, len(split_file_lists)): if pubExt: mergedFileName = 'MergedMiniEvents_%d_%s.root ' % ( ilist, pubExt) else: mergedFileName = 'MergedMiniEvents_%d.root ' % ilist toAdd = '%s ' % mergedFileName for f in split_file_lists[ilist]: toAdd += '%s/%s ' % (baseEOS, f) finalOutput = '%s/%s/%s' % (baseEOS, newDir, mergedFileName) fIn = ROOT.TFile.Open(finalOutput) try: if fIn or not fIn.IsZombie(): print '%s already in EOS, skipping' % finalOutput fIn.Close() continue except: pass os.system('hadd -f -k %s' % toAdd) os.system( 'mv -v %s %s/%s/%s' % (mergedFileName, baseEOS, newDir, mergedFileName)) #if still needed copy individual files if moveIndividualFiles: for f in file_list: newF = f if pubExt: newF = f.replace('.root', '_%s.root' % pubExt) os.system('mv -v %s %s/%s/%s' % (f, baseEOS, newDir, newF)) if not opt.nocheck and opt.cleanup: choice = raw_input( 'Will remove output directory. [y/n] ?').lower() if 'y' in choice: os.system('rm -r %s/%s' % (baseEOS, dset)) print 'Crab outputs may now be found in %s' % newDir print '-' * 50 print 'All done. In case errors were found check that the crab output structure is ' print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>' print '-' * 50
def createAnalysisTasks(opt): onlyList=opt.only.split('v') ## Local directory file_list=[] if os.path.isdir(opt.input): for file_path in os.listdir(opt.input): if file_path.endswith('.root'): file_list.append(os.path.join(opt.input,file_path)) elif opt.input.startswith('/store/'): file_list = getEOSlslist(opt.input) elif '.root' in opt.input: file_list.append(opt.input) #list of files to analyse tasklist=[] for filename in file_list: baseFileName=os.path.basename(filename) tag,ext=os.path.splitext(baseFileName) if len(onlyList)>0: processThis=False for filtTag in onlyList: if filtTag in tag: processThis=True if not processThis : continue tasklist.append((filename,'%s/%s'%(opt.output,baseFileName))) #loop over tasks if opt.queue=='local': if opt.jobs>1: print ' Submitting jobs in %d threads' % opt.jobs import multiprocessing as MP pool = MP.Pool(opt.jobs) pool.map(runTopWidthAnalysisPacked,tasklist) else: for fileName,outFileName in tasklist: runTopWidthAnalysis(fileName,outFileName) else: cmsswBase=os.environ['CMSSW_BASE'] FarmDirectory = '%s/%s'%(cmsswBase,opt.farm) os.system('mkdir -p %s'%FarmDirectory) with open ('%s/condor.sub'%FarmDirectory,'w') as condor: condor.write('executable = {0}/$(jobName).sh\n'.format(FarmDirectory)) condor.write('output = {0}/output_$(jobName).out\n'.format(FarmDirectory)) condor.write('error = {0}/output_$(jobName).err\n'.format(FarmDirectory)) condor.write('+JobFlavour = workday\n') condor.write('RequestCpus = 4\n') for fileName,_ in tasklist: jobName='%s'%(os.path.splitext(os.path.basename(fileName))[0]) jobScript='%s/%s.sh'%(FarmDirectory,jobName) with open(jobScript,'w') as job: job.write('#!/bin/bash\n') job.write('WORKDIR=`pwd`\n') job.write('echo "Working directory is ${WORKDIR}"\n') job.write('cd %s\n'%cmsswBase) job.write('eval `scram r -sh`\n') job.write('cd ${WORKDIR}\n') job.write('python {0}/src/TopLJets2015/TopAnalysis/scripts/runTopWidthAnalysis.py -o {1} -q local -i {2}\n'.format(cmsswBase,opt.output,fileName)) job.write('echo "All done"\n') os.system('chmod u+x %s'%jobScript) condor.write('jobName=%s\n'%jobName) condor.write('queue 1\n') os.system('condor_submit %s/condor.sub'%FarmDirectory)
from Configuration.AlCa.GlobalTag import GlobalTag process.GlobalTag.globaltag = '74X_dataRun2_v2' if options.runOnData else '74X_mcRun2_asymptotic_v4' # Set the process options -- Display summary at the end, enable unscheduled execution process.options = cms.untracked.PSet( allowUnscheduled = cms.untracked.bool(True), # wantSummary = cms.untracked.bool(True) ) #Number of events to process process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1)) from TopLJets2015.TopAnalysis.storeTools import getEOSlslist process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring(getEOSlslist(directory='/store/mc/RunIISpring15MiniAODv2/TTJets_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/MINIAODSIM/74X_mcRun2_asymptotic_v2-v3/60000/') ) ) if options.inputDir!='': print 'Will process files from',options.inputDir process.source.fileNames=cms.untracked.vstring(getEOSlslist(directory=options.inputDir)) #reduce verbosity process.load("FWCore.MessageService.MessageLogger_cfi") process.MessageLogger.cerr.threshold = '' process.MessageLogger.cerr.FwkReport.reportEvery = 1000 #Tfileservice process.TFileService = cms.Service("TFileService", fileName = cms.string(options.outFilename))
def main(): #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='workday', type='string') (opt, args) = parser.parse_args() #prepare output directory if opt.outDir is None: opt.outDir = opt.inDir cmsswBase = os.environ['CMSSW_BASE'] FARMDIR = '%s/INTEGRITYFARM' % cmsswBase os.system('mkdir -p %s' % FARMDIR) dset_list = getEOSlslist(directory=opt.inDir, prepend='') with open('%s/checkInteg.sh' % FARMDIR, 'w') as shell: shell.write('#!/bin/bash\n') shell.write('WORKDIR=`pwd`\n') shell.write('echo "Working directory is ${WORKDIR}"\n') shell.write('cd %s\n' % cmsswBase) shell.write('eval `scram r -sh`\n') shell.write('cd ${WORKDIR}\n') shell.write('echo "CMSSW_BASE=${CMSSW_BASE}"\n') shell.write('echo "Preparing output directory"\n') shell.write('mkdir -p /eos/cms/%s\n' % opt.outDir) shell.write('echo "Running integrity checker"\n') shell.write( 'python ${CMSSW_BASE}/src/TopLJets2015/TopAnalysis/scripts/checkProductionIntegrity.py -i ${1} -o ${2} --nocheck --only ${3}\n' ) os.system('chmod u+x %s/checkInteg.sh' % FARMDIR) condor = open('%s/condor.sub' % FARMDIR, 'w') condor.write('executable = %s/checkInteg.sh\n' % FARMDIR) condor.write('output = %s/job_$(ProcId).out\n' % FARMDIR) condor.write('error = %s/job_$(ProcId).err\n' % FARMDIR) condor.write('log = %s/job_$(ProcId).log\n' % FARMDIR) condor.write('+JobFlavour ="%s"\n' % opt.queue) for dset in dset_list: dsetname = dset.split('/')[-1] pub_list = getEOSlslist(directory=dset, prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname continue pub = pubDir.split('/crab_')[-1] condor.write('arguments = %s %s %s\n' % (opt.inDir, opt.outDir, pub)) condor.write('queue 1\n') condor.close() os.system('condor_submit %s/condor.sub' % FARMDIR)
from Configuration.AlCa.GlobalTag import GlobalTag process.GlobalTag.globaltag = '74X_dataRun2_v2' if options.runOnData else '74X_mcRun2_asymptotic_v2' # Set the process options -- Display summary at the end, enable unscheduled execution process.options = cms.untracked.PSet( allowUnscheduled = cms.untracked.bool(True), # wantSummary = cms.untracked.bool(True) ) #Number of events to process process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1)) from TopLJets2015.TopAnalysis.storeTools import getEOSlslist process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring(getEOSlslist(directory='/store/mc/RunIISpring15MiniAODv2/TTJets_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/MINIAODSIM/74X_mcRun2_asymptotic_v2-v3/60000/') ) ) #reduce verbosity process.load("FWCore.MessageService.MessageLogger_cfi") process.MessageLogger.cerr.threshold = '' process.MessageLogger.cerr.FwkReport.reportEvery = 1000 #Tfileservice process.TFileService = cms.Service("TFileService", fileName = cms.string(options.outFilename)) # Set up electron ID (VID framework) from PhysicsTools.SelectorUtils.tools.vid_id_tools import * dataFormat = DataFormat.MiniAOD
def main(): cmsswBase = os.environ['CMSSW_BASE'] #eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string') parser.add_option('--only', dest='only', help='only this tag', default=None, type='string') (opt, args) = parser.parse_args() #Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate() #prepare output directory if opt.outDir is None: opt.outDir = opt.inDir #Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate() dset_list = getEOSlslist(directory=opt.inDir, prepend='') for dset in dset_list: dsetname = dset.split('/')[-1] #if not 'pp_TuneCUETP8M1_Hydjet_Min_Bias' : continue pub_list = getEOSlslist(directory=dset, prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname continue pub = pubDir.split('/crab_')[-1] #if not 'V4' in pub : continue if opt.only: if pub not in opt.only: continue #if 'Data13TeV' in pub : continue localMerge = 'python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s' % ( opt.inDir, opt.outDir, pub) ############### Submit to condor ############### target = '%s/src/TopLJets2015/TopAnalysis/%s' % (cmsswBase, pub) condorFile = open(target, 'w') condorFile.write('universe = vanilla\n') #condorFile.write('executable = condor/cond_wrapper.sh\n') condorFile.write('executable = condor/cond_check.sh\n') condorFile.write( 'arguments = $(ClusterID) $(ProcId) %s %s %s\n' % (opt.inDir, opt.outDir, pub)) condorFile.write( 'output = condor/log/%s_EOS_$(ProcId).out\n' % pub) condorFile.write( 'error = condor/log/%s_EOS_$(ProcId).err\n' % pub) condorFile.write( 'log = condor/log/%s_EOS.log\n' % pub) condorFile.write('+JobFlavour = "tomorrow"\n') condorFile.write('Should_Transfer_Files = NO\n') condorFile.write('queue') condorFile.close() os.system('condor_submit %s -batch-name %s' % (target, pub)) os.system('rm %s' % (pub))
def main(): eos_cmd = '/afs/cern.ch/project/eos/installation/0.2.41/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-c', '--cleanup', dest='cleanup', help='removes original crab directory', default =False, action='store_true') parser.add_option( '--nocheck', dest='nocheck', help='do not prompt user', default=False, action='store_true') (opt, args) = parser.parse_args() Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate() if opt.outDir is None: opt.outDir=opt.inDir dset_list=getEOSlslist(directory=opt.inDir,prepend='') for dset in dset_list: dsetname=dset.split('/')[-1] pub_list=getEOSlslist(directory=dset,prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname continue pub=pubDir.split('/crab_')[-1] time_list=getEOSlslist(directory=pubDir,prepend='') if len(time_list)!=1: print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out'%dsetname continue time_stamp=time_list[0].split('/')[-1] out_list=[] count_list=getEOSlslist(directory=time_list[0],prepend='') for count in count_list: out_list += getEOSlslist(directory=count,prepend='') file_list=[x for x in out_list if '.root' in x] newDir='%s/%s' % (opt.outDir,pub) print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % (dsetname,pub,time_stamp,len(file_list) ) if not opt.nocheck: choice = raw_input('Will move to %s current output directory. [y/n] ?' % newDir ).lower() if not 'y' in choice : continue Popen([eos_cmd, 'mkdir', '/eos/cms/'+newDir],stdout=PIPE).communicate() moveIndividualFiles=True if len(file_list)>0: subgroupMerge = int( raw_input('This set has %d files. Merge into groups? (enter 0 if no merging)' % len(file_list)) ) if subgroupMerge>0: moveIndividualFiles=False splitFunc = lambda A, n=subgroupMerge: [A[i:i+n] for i in range(0, len(A), n)] split_file_lists = splitFunc( file_list ) for ilist in xrange(0,len(split_file_lists)): mergedFileName='/tmp/MergedMiniEvents_%d.root '%ilist toAdd='%s ' % mergedFileName for f in split_file_lists[ilist]: toAdd += 'eos/cms/%s '%f os.system('hadd -f %s'%toAdd) os.system('xrdcp -f %s root://eoscms//eos/cms/%s/' %(mergedFileName,newDir)) #if still needed copy individual files if moveIndividualFiles: for f in file_list : os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir) ) if not opt.nocheck and opt.cleanup : choice = raw_input('Will remove output directory. [y/n] ?').lower() if 'y' in choice: Popen([eos_cmd, 'rm', '-r /eos/cms/'+dset],stdout=PIPE).communicate() print 'Crab outputs may now be found in %s' % newDir Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate() print '-'*50 print 'All done. In case errors were found check that the crab output structure is ' print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>' print '-'*50
def main(): eos_cmd = '/afs/cern.ch/project/eos/installation/0.2.41/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-c', '--cleanup', dest='cleanup', help='removes original crab directory', default=False, action='store_true') parser.add_option('--nocheck', dest='nocheck', help='do not prompt user', default=False, action='store_true') (opt, args) = parser.parse_args() Popen([eos_cmd, ' -b fuse mount', 'eos'], stdout=PIPE).communicate() if opt.outDir is None: opt.outDir = opt.inDir dset_list = getEOSlslist(directory=opt.inDir, prepend='') for dset in dset_list: dsetname = dset.split('/')[-1] pub_list = getEOSlslist(directory=dset, prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname continue pub = pubDir.split('/crab_')[-1] time_list = getEOSlslist(directory=pubDir, prepend='') if len(time_list) != 1: print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out' % dsetname continue time_stamp = time_list[0].split('/')[-1] out_list = [] count_list = getEOSlslist(directory=time_list[0], prepend='') for count in count_list: out_list += getEOSlslist(directory=count, prepend='') file_list = [x for x in out_list if '.root' in x] newDir = '%s/%s' % (opt.outDir, pub) print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % ( dsetname, pub, time_stamp, len(file_list)) if not opt.nocheck: choice = raw_input( 'Will move to %s current output directory. [y/n] ?' % newDir).lower() if not 'y' in choice: continue Popen([eos_cmd, 'mkdir', '/eos/cms/' + newDir], stdout=PIPE).communicate() moveIndividualFiles = True if len(file_list) > 0: subgroupMerge = int( raw_input( 'This set has %d files. Merge into groups? (enter 0 if no merging)' % len(file_list))) if subgroupMerge > 0: moveIndividualFiles = False splitFunc = lambda A, n=subgroupMerge: [ A[i:i + n] for i in range(0, len(A), n) ] split_file_lists = splitFunc(file_list) for ilist in xrange(0, len(split_file_lists)): mergedFileName = '/tmp/MergedMiniEvents_%d.root ' % ilist toAdd = '%s ' % mergedFileName for f in split_file_lists[ilist]: toAdd += 'eos/cms/%s ' % f os.system('hadd -f %s' % toAdd) os.system('xrdcp -f %s root://eoscms//eos/cms/%s/' % (mergedFileName, newDir)) #if still needed copy individual files if moveIndividualFiles: for f in file_list: os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir)) if not opt.nocheck and opt.cleanup: choice = raw_input( 'Will remove output directory. [y/n] ?').lower() if 'y' in choice: Popen([eos_cmd, 'rm', '-r /eos/cms/' + dset], stdout=PIPE).communicate() print 'Crab outputs may now be found in %s' % newDir Popen([eos_cmd, ' -b fuse umount', 'eos'], stdout=PIPE).communicate() print '-' * 50 print 'All done. In case errors were found check that the crab output structure is ' print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>' print '-' * 50
def main(): eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select' #configuration usage = 'usage: %prog [options]' parser = optparse.OptionParser(usage) parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string') parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string') parser.add_option('-c', '--cleanup', dest='cleanup', help='removes original crab directory', default =False, action='store_true') parser.add_option( '--nocheck', dest='nocheck', help='do not prompt user', default=False, action='store_true') parser.add_option( '--only', dest='only', help='only this tag', default=None, type='string') (opt, args) = parser.parse_args() Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate() #prepare output directory if opt.outDir is None: opt.outDir=opt.inDir Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate() dset_list=getEOSlslist(directory=opt.inDir,prepend='') for dset in dset_list: dsetname=dset.split('/')[-1] pub_list=getEOSlslist(directory=dset,prepend='') for pubDir in pub_list: if not 'crab' in pubDir: print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname continue #select if it doesn't match the required selection pub=pubDir.split('/crab_')[-1] if opt.only: if pub!=opt.only: continue #check if it's an extension pubExt=None try: extSplit=pub.split('_ext') pubExt='ext'+extSplit[1] pub=extSplit[0] print 'Extension will be postfixed with ',pubExt except: print 'Core sample (no extension)' time_list=getEOSlslist(directory=pubDir,prepend='') if len(time_list)!=1: print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out'%dsetname continue time_stamp=time_list[0].split('/')[-1] out_list=[] count_list=getEOSlslist(directory=time_list[0],prepend='') for count in count_list: out_list += getEOSlslist(directory=count,prepend='') file_list=[x for x in out_list if '.root' in x] newDir='%s/%s' % (opt.outDir,pub) print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % (dsetname,pub,time_stamp,len(file_list) ) if not opt.nocheck: choice = raw_input('Will move to %s current output directory. [y/n] ?' % newDir ).lower() if not 'y' in choice : continue Popen([eos_cmd, 'mkdir', '/eos/cms/'+newDir],stdout=PIPE).communicate() moveIndividualFiles=True if len(file_list)>0: #subgroupMerge = int( raw_input('This set has %d files. Merge into groups? (enter 0 if no merging)' % len(file_list)) ) subgroupMerge=60 if 'Data' in dsetname else 20 if subgroupMerge>0: moveIndividualFiles=False splitFunc = lambda A, n=subgroupMerge: [A[i:i+n] for i in range(0, len(A), n)] split_file_lists = splitFunc( file_list ) for ilist in xrange(0,len(split_file_lists)): if pubExt: mergedFileName='/tmp/MergedMiniEvents_%d_%s.root '%(ilist,pubExt) else: mergedFileName='/tmp/MergedMiniEvents_%d.root '%ilist toAdd='%s ' % mergedFileName for f in split_file_lists[ilist]: toAdd += 'eos/cms/%s '%f finalOutput='eos/cms/%s/%s'%(newDir,mergedFileName.replace('/tmp/','')) fIn=ROOT.TFile.Open(finalOutput) try: if fIn or not fIn.IsZombie(): print '%s already in EOS, skipping'%finalOutput fIn.Close() continue except: pass os.system('hadd -f %s'%toAdd) os.system('cp %s eos/cms/%s/'%(mergedFileName,newDir)) os.system('rm %s'%mergedFileName) #os.system('xrdcp -f %s root://eoscms//eos/cms/%s/MergedMiniEvents_%d.root' %(mergedFileName,newDir,ilist)) #if still needed copy individual files if moveIndividualFiles: for f in file_list : #os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir) ) newF=f if pubExt: newF=f.replace('.root','_%s.root'%pubExt) os.system('cp %s eos/cms/%s/%s' % (f, newDir,newF) ) if not opt.nocheck and opt.cleanup : choice = raw_input('Will remove output directory. [y/n] ?').lower() if 'y' in choice: Popen([eos_cmd, 'rm', '-r /eos/cms/'+dset],stdout=PIPE).communicate() print 'Crab outputs may now be found in %s' % newDir #Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate() print '-'*50 print 'All done. In case errors were found check that the crab output structure is ' print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>' print '-'*50