def main():
cmsswBase=os.environ['CMSSW_BASE']
eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string')
parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string')
parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string')
(opt, args) = parser.parse_args()
Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate()
#prepare output directory
if opt.outDir is None: opt.outDir=opt.inDir
Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate()
dset_list=getEOSlslist(directory=opt.inDir,prepend='')
for dset in dset_list:
dsetname=dset.split('/')[-1]
pub_list=getEOSlslist(directory=dset,prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname
continue
pub=pubDir.split('/crab_')[-1]
localMerge='python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s'%(opt.inDir,opt.outDir,pub)
cmd='bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (opt.queue,cmsswBase,localMerge)
os.system(cmd)
Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate()
def main():
cmsswBase = os.environ['CMSSW_BASE']
eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i',
'--inDir',
dest='inDir',
help='input directory with files',
default=None,
type='string')
parser.add_option('-o',
'--outDir',
dest='outDir',
help='output directory with files',
default=None,
type='string')
parser.add_option('-q',
'--queue',
dest='queue',
help='batch queue',
default='2nd',
type='string')
(opt, args) = parser.parse_args()
Popen([eos_cmd, ' -b fuse mount', 'eos'], stdout=PIPE).communicate()
#prepare output directory
if opt.outDir is None: opt.outDir = opt.inDir
Popen([eos_cmd, 'mkdir', '/eos/cms/' + opt.outDir],
stdout=PIPE).communicate()
dset_list = getEOSlslist(directory=opt.inDir, prepend='')
for dset in dset_list:
dsetname = dset.split('/')[-1]
pub_list = getEOSlslist(directory=dset, prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname
continue
pub = pubDir.split('/crab_')[-1]
localMerge = 'python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s' % (
opt.inDir, opt.outDir, pub)
cmd = 'bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (
opt.queue, cmsswBase, localMerge)
os.system(cmd)
Popen([eos_cmd, ' -b fuse umount', 'eos'], stdout=PIPE).communicate()
def main():
eos_cmd = '/afs/cern.ch/project/eos/installation/cms/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string')
parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string')
parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string')
(opt, args) = parser.parse_args()
#prepare output directory
if opt.outDir is None: opt.outDir=opt.inDir
cmsswBase=os.environ['CMSSW_BASE']
FARMDIR='%s/src/TopLJets2015/TopAnalysis/FARM'%cmsswBase
os.system('mkdir -p %s'%FARMDIR)
dset_list=getEOSlslist(directory=opt.inDir,prepend='')
for dset in dset_list:
dsetname=dset.split('/')[-1]
pub_list=getEOSlslist(directory=dset,prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname
continue
pub=pubDir.split('/crab_')[-1]
cfgfile='%s/mergejob_%s.sh'%(FARMDIR,pub)
with open(cfgfile,'w') as cfg:
cfg.write('WORKDIR=`pwd`\n')
cfg.write('echo "Working directory is ${WORKDIR}"\n')
cfg.write('cd %s\n'%cmsswBase)
cfg.write('eval `scram r -sh`\n')
cfg.write('cd ${WORKDIR}\n')
cfg.write('echo "Preparing output directory"\n')
cfg.write('%s mkdir %s\n'%(eos_cmd,opt.outDir))
cfg.write('echo "Running integrity checker"\n')
cfg.write('python ${CMSSW_BASE}/src/TopLJets2015/TopAnalysis/scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --mount --only %s'%(opt.inDir,opt.outDir,pub))
os.system('chmod u+x %s'%cfgfile)
cmd='bsub -q %s %s' % (opt.queue,cfgfile)
os.system(cmd)
def createAnalysisTasks(opt):
onlyList = opt.only.split("v")
## Local directory
file_list = []
if os.path.isdir(opt.input):
for file_path in os.listdir(opt.input):
if file_path.endswith(".root"):
file_list.append(os.path.join(opt.input, file_path))
elif opt.input.startswith("/store/"):
file_list = getEOSlslist(opt.input)
elif ".root" in opt.input:
file_list.append(opt.input)
# list of files to analyse
tasklist = []
for filename in file_list:
baseFileName = os.path.basename(filename)
tag, ext = os.path.splitext(baseFileName)
if len(onlyList) > 0:
processThis = False
for filtTag in onlyList:
if filtTag in tag:
processThis = True
if not processThis:
continue
tasklist.append((filename, "%s/%s" % (opt.output, baseFileName)))
# loop over tasks
if opt.queue == "local":
if opt.jobs > 1:
print " Submitting jobs in %d threads" % opt.jobs
import multiprocessing as MP
pool = MP.Pool(opt.jobs)
pool.map(runTopWidthAnalysisPacked, tasklist)
else:
for fileName, outFileName in tasklist:
runTopWidthAnalysis(fileName, outFileName)
else:
cmsswBase = os.environ["CMSSW_BASE"]
for fileName, _ in tasklist:
localRun = "python %s/src/TopLJets2015/TopAnalysis/scripts/runTopWidthAnalysis.py -i %s -o %s -q local" % (
cmsswBase,
fileName,
opt.output,
)
cmd = 'bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh "%s"' % (
opt.queue,
cmsswBase,
localRun,
)
print cmd
os.system(cmd)
def createAnalysisTasks(opt):
onlyList = opt.only.split('v')
## Local directory
file_list = []
if os.path.isdir(opt.input):
for file_path in os.listdir(opt.input):
if file_path.endswith('.root'):
file_list.append(os.path.join(opt.input, file_path))
elif opt.input.startswith('/store/'):
file_list = getEOSlslist(opt.input)
elif '.root' in opt.input:
file_list.append(opt.input)
#list of files to analyse
tasklist = []
for filename in file_list:
baseFileName = os.path.basename(filename)
tag, ext = os.path.splitext(baseFileName)
if len(onlyList) > 0:
processThis = False
for filtTag in onlyList:
if filtTag in tag:
processThis = True
if not processThis: continue
tasklist.append(
(filename, '%s/%s' % (opt.output, baseFileName), opt.filter))
#loop over tasks
if opt.queue == 'local':
if opt.jobs > 1:
print ' Submitting jobs in %d threads' % opt.jobs
import multiprocessing as MP
pool = MP.Pool(opt.jobs)
pool.map(runAnomalousTopProductionAnalysisPacked, tasklist)
else:
for fileName, outFileName, filterName in tasklist:
runAnomalousTopProductionAnalysis(fileName, outFileName,
filterName)
else:
cmsswBase = os.environ['CMSSW_BASE']
for fileName, _, filterName in tasklist:
localRun = 'python %s/src/TopLJets2015/TopAnalysis/scripts/runAnomalousTopProductionAnalysis.py -i %s -o %s -q local' % (
cmsswBase, fileName, opt.output)
if filterName: localRun += ' --filter %s' % filterName
cmd = 'bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (
opt.queue, cmsswBase, localRun)
print cmd
os.system(cmd)
def createAnalysisTasks(opt):
onlyList=opt.only.split('v')
## Local directory
file_list=[]
if os.path.isdir(opt.input):
for file_path in os.listdir(opt.input):
if file_path.endswith('.root'):
file_list.append(os.path.join(opt.input,file_path))
elif opt.input.startswith('/store/'):
file_list = getEOSlslist(opt.input)
elif '.root' in opt.input:
file_list.append(opt.input)
#list of files to analyse
tasklist=[]
for filename in file_list:
baseFileName=os.path.basename(filename)
tag,ext=os.path.splitext(baseFileName)
if len(onlyList)>0:
processThis=False
for filtTag in onlyList:
if filtTag in tag:
processThis=True
if not processThis : continue
tasklist.append((filename,'%s/%s'%(opt.output,baseFileName),opt.filter))
#loop over tasks
if opt.queue=='local':
if opt.jobs>1:
print ' Submitting jobs in %d threads' % opt.jobs
import multiprocessing as MP
pool = MP.Pool(opt.jobs)
pool.map(runAnomalousTopProductionAnalysisPacked,tasklist)
else:
for fileName,outFileName,filterName in tasklist:
runAnomalousTopProductionAnalysis(fileName,outFileName,filterName)
else:
cmsswBase=os.environ['CMSSW_BASE']
for fileName,_,filterName in tasklist:
localRun='python %s/src/TopLJets2015/TopAnalysis/scripts/runAnomalousTopProductionAnalysis.py -i %s -o %s -q local'%(cmsswBase,fileName,opt.output)
if filterName : localRun+=' --filter %s'%filterName
cmd='bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (opt.queue,cmsswBase,localRun)
print cmd
os.system(cmd)
def main():
cmsswBase=os.environ['CMSSW_BASE']
eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string')
parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string')
parser.add_option('-q', '--queue', dest='queue', help='batch queue', default='2nd', type='string')
parser.add_option( '--only', dest='only', help='only this tag', default=None, type='string')
(opt, args) = parser.parse_args()
Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate()
#prepare output directory
if opt.outDir is None: opt.outDir=opt.inDir
Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate()
dset_list=getEOSlslist(directory=opt.inDir,prepend='')
for dset in dset_list:
dsetname=dset.split('/')[-1]
#if not 'pp_TuneCUETP8M1_Hydjet_Min_Bias' : continue
pub_list=getEOSlslist(directory=dset,prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname
continue
pub=pubDir.split('/crab_')[-1]
#if not 'V4' in pub : continue
if opt.only:
if pub not in opt.only:
continue
#if 'Data13TeV' in pub : continue
localMerge='python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s'%(opt.inDir,opt.outDir,pub)
############### Submit to condor ###############
target = '%s/src/TopLJets2015/TopAnalysis/%s' % (cmsswBase,pub)
condorFile = open(target,'w')
condorFile.write('universe = vanilla\n')
#condorFile.write('executable = condor/cond_wrapper.sh\n')
condorFile.write('executable = condor/cond_check.sh\n')
condorFile.write('arguments = $(ClusterID) $(ProcId) %s %s %s\n' % (opt.inDir,opt.outDir,pub))
condorFile.write('output = condor/log/%s_EOS_$(ProcId).out\n' % pub)
condorFile.write('error = condor/log/%s_EOS_$(ProcId).err\n' % pub)
condorFile.write('log = condor/log/%s_EOS.log\n' % pub)
condorFile.write('+JobFlavour = "workday"\n')
condorFile.write('Should_Transfer_Files = NO\n')
condorFile.write('queue')
condorFile.close()
os.system('condor_submit %s -batch-name %s' % (target,pub))
os.system('rm %s' % (pub))
#cmd='bsub -q %s %s/src/TopLJets2015/TopAnalysis/scripts/wrapLocalAnalysisRun.sh \"%s\"' % (opt.queue,cmsswBase,localMerge)
#os.system(cmd)
Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate()
def main():
eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i',
'--inDir',
dest='inDir',
help='input directory with files',
default=None,
type='string')
parser.add_option('-o',
'--outDir',
dest='outDir',
help='output directory with files',
default=None,
type='string')
parser.add_option('-c',
'--cleanup',
dest='cleanup',
help='removes original crab directory',
default=False,
action='store_true')
parser.add_option('--nocheck',
dest='nocheck',
help='do not prompt user',
default=False,
action='store_true')
parser.add_option('--only',
dest='only',
help='only this tag',
default=None,
type='string')
(opt, args) = parser.parse_args()
Popen([eos_cmd, ' -b fuse mount', 'eos'], stdout=PIPE).communicate()
#prepare output directory
if opt.outDir is None: opt.outDir = opt.inDir
Popen([eos_cmd, 'mkdir', '/eos/cms/' + opt.outDir],
stdout=PIPE).communicate()
dset_list = getEOSlslist(directory=opt.inDir, prepend='')
for dset in dset_list:
dsetname = dset.split('/')[-1]
pub_list = getEOSlslist(directory=dset, prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname
continue
#select if it doesn't match the required selection
pub = pubDir.split('/crab_')[-1]
if opt.only:
if pub != opt.only:
continue
#check if it's an extension
pubExt = None
try:
extSplit = pub.split('_ext')
pubExt = 'ext' + extSplit[1]
pub = extSplit[0]
print 'Extension will be postfixed with ', pubExt
except:
print 'Core sample (no extension)'
time_list = getEOSlslist(directory=pubDir, prepend='')
if len(time_list) != 1:
print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out' % dsetname
continue
time_stamp = time_list[0].split('/')[-1]
out_list = []
count_list = getEOSlslist(directory=time_list[0], prepend='')
for count in count_list:
out_list += getEOSlslist(directory=count, prepend='')
file_list = [x for x in out_list if '.root' in x]
newDir = '%s/%s' % (opt.outDir, pub)
print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % (
dsetname, pub, time_stamp, len(file_list))
if not opt.nocheck:
choice = raw_input(
'Will move to %s current output directory. [y/n] ?' %
newDir).lower()
if not 'y' in choice: continue
Popen([eos_cmd, 'mkdir', '/eos/cms/' + newDir],
stdout=PIPE).communicate()
moveIndividualFiles = True
if len(file_list) > 0:
#subgroupMerge = int( raw_input('This set has %d files. Merge into groups? (enter 0 if no merging)' % len(file_list)) )
subgroupMerge = 60 if 'Data' in dsetname else 20
if subgroupMerge > 0:
moveIndividualFiles = False
splitFunc = lambda A, n=subgroupMerge: [
A[i:i + n] for i in range(0, len(A), n)
]
split_file_lists = splitFunc(file_list)
for ilist in xrange(0, len(split_file_lists)):
if pubExt:
mergedFileName = '/tmp/MergedMiniEvents_%d_%s.root ' % (
ilist, pubExt)
else:
mergedFileName = '/tmp/MergedMiniEvents_%d.root ' % ilist
toAdd = '%s ' % mergedFileName
for f in split_file_lists[ilist]:
toAdd += 'eos/cms/%s ' % f
finalOutput = 'eos/cms/%s/%s' % (
newDir, mergedFileName.replace('/tmp/', ''))
fIn = ROOT.TFile.Open(finalOutput)
try:
if fIn or not fIn.IsZombie():
print '%s already in EOS, skipping' % finalOutput
fIn.Close()
continue
except:
pass
os.system('hadd -f %s' % toAdd)
os.system('cp %s eos/cms/%s/' %
(mergedFileName, newDir))
os.system('rm %s' % mergedFileName)
#os.system('xrdcp -f %s root://eoscms//eos/cms/%s/MergedMiniEvents_%d.root' %(mergedFileName,newDir,ilist))
#if still needed copy individual files
if moveIndividualFiles:
for f in file_list:
#os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir) )
newF = f
if pubExt:
newF = f.replace('.root', '_%s.root' % pubExt)
os.system('cp %s eos/cms/%s/%s' % (f, newDir, newF))
if not opt.nocheck and opt.cleanup:
choice = raw_input(
'Will remove output directory. [y/n] ?').lower()
if 'y' in choice:
Popen([eos_cmd, 'rm', '-r /eos/cms/' + dset],
stdout=PIPE).communicate()
print 'Crab outputs may now be found in %s' % newDir
#Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate()
print '-' * 50
print 'All done. In case errors were found check that the crab output structure is '
print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>'
print '-' * 50
process.GlobalTag.globaltag = '74X_dataRun2_v2' if options.runOnData else '74X_mcRun2_asymptotic_v2'
# Set the process options -- Display summary at the end, enable unscheduled execution
process.options = cms.untracked.PSet(allowUnscheduled=cms.untracked.bool(True),
# wantSummary = cms.untracked.bool(True)
)
#Number of events to process
process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(-1))
from TopLJets2015.TopAnalysis.storeTools import getEOSlslist
process.source = cms.Source(
"PoolSource",
fileNames=cms.untracked.vstring(
getEOSlslist(
directory=
'/store/mc/RunIISpring15MiniAODv2/TTJets_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/MINIAODSIM/74X_mcRun2_asymptotic_v2-v3/60000/'
)))
if options.inputDir != '':
print 'Will process files from', options.inputDir
process.source.fileNames = cms.untracked.vstring(
getEOSlslist(directory=options.inputDir))
#reduce verbosity
process.load("FWCore.MessageService.MessageLogger_cfi")
process.MessageLogger.cerr.threshold = ''
process.MessageLogger.cerr.FwkReport.reportEvery = 1000
#Tfileservice
process.TFileService = cms.Service("TFileService",
fileName=cms.string(options.outFilename))
def main():
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i',
'--inDir',
dest='inDir',
help='input directory with files',
default=None,
type='string')
parser.add_option('-o',
'--outDir',
dest='outDir',
help='output directory with files',
default=None,
type='string')
parser.add_option('-c',
'--cleanup',
dest='cleanup',
help='removes original crab directory',
default=False,
action='store_true')
parser.add_option('--nocheck',
dest='nocheck',
help='do not prompt user',
default=False,
action='store_true')
parser.add_option('--only',
dest='only',
help='only this tag',
default=None,
type='string')
(opt, args) = parser.parse_args()
baseEOS = '/eos/cms/'
#prepare output directory
if opt.outDir is None: opt.outDir = opt.inDir
os.system('mkdir -p %s/%s' % (baseEOS, opt.outDir))
dset_list = getEOSlslist(directory=opt.inDir, prepend='')
for dset in dset_list:
dsetname = dset.split('/')[-1]
pub_list = getEOSlslist(directory=dset, prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname
continue
#select if it doesn't match the required selection
pub = pubDir.split('/crab_')[-1]
if opt.only:
if pub != opt.only:
continue
#check if it's an extension
pubExt = None
try:
extSplit = pub.split('_ext')
pubExt = 'ext%d' % (len(extSplit) - 1)
pub = extSplit[0]
print 'Extension will be postfixed with ', pubExt
except:
print 'Core sample (no extension)'
time_list = getEOSlslist(directory=pubDir, prepend='')
if len(time_list) != 1:
print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out' % dsetname
continue
time_stamp = time_list[0].split('/')[-1]
out_list = []
count_list = getEOSlslist(directory=time_list[0], prepend='')
for count in count_list:
if '/merge' in count and 'group/hintt' in count: continue
out_list += getEOSlslist(directory=count, prepend='')
file_list = [x for x in out_list if '.root' in x]
newDir = '%s/%s' % (opt.outDir, pub)
print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % (
dsetname, pub, time_stamp, len(file_list))
if not opt.nocheck:
choice = raw_input(
'Will move to %s current output directory. [y/n] ?' %
newDir).lower()
if not 'y' in choice: continue
os.system('mkdir -p %s/%s' % (baseEOS, newDir))
moveIndividualFiles = True
if len(file_list) > 0:
subgroupMerge = 10
if 'Data' in pub: subgroupMerge = 50
moveIndividualFiles = False
splitFunc = lambda A, n=subgroupMerge: [
A[i:i + n] for i in range(0, len(A), n)
]
split_file_lists = splitFunc(file_list)
for ilist in xrange(0, len(split_file_lists)):
if pubExt:
mergedFileName = 'MergedMiniEvents_%d_%s.root ' % (
ilist, pubExt)
else:
mergedFileName = 'MergedMiniEvents_%d.root ' % ilist
toAdd = '%s ' % mergedFileName
for f in split_file_lists[ilist]:
toAdd += '%s/%s ' % (baseEOS, f)
finalOutput = '%s/%s/%s' % (baseEOS, newDir,
mergedFileName)
fIn = ROOT.TFile.Open(finalOutput)
try:
if fIn or not fIn.IsZombie():
print '%s already in EOS, skipping' % finalOutput
fIn.Close()
continue
except:
pass
os.system('hadd -f -k %s' % toAdd)
os.system(
'mv -v %s %s/%s/%s' %
(mergedFileName, baseEOS, newDir, mergedFileName))
#if still needed copy individual files
if moveIndividualFiles:
for f in file_list:
newF = f
if pubExt:
newF = f.replace('.root', '_%s.root' % pubExt)
os.system('mv -v %s %s/%s/%s' % (f, baseEOS, newDir, newF))
if not opt.nocheck and opt.cleanup:
choice = raw_input(
'Will remove output directory. [y/n] ?').lower()
if 'y' in choice:
os.system('rm -r %s/%s' % (baseEOS, dset))
print 'Crab outputs may now be found in %s' % newDir
print '-' * 50
print 'All done. In case errors were found check that the crab output structure is '
print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>'
print '-' * 50
def createAnalysisTasks(opt):
onlyList=opt.only.split('v')
## Local directory
file_list=[]
if os.path.isdir(opt.input):
for file_path in os.listdir(opt.input):
if file_path.endswith('.root'):
file_list.append(os.path.join(opt.input,file_path))
elif opt.input.startswith('/store/'):
file_list = getEOSlslist(opt.input)
elif '.root' in opt.input:
file_list.append(opt.input)
#list of files to analyse
tasklist=[]
for filename in file_list:
baseFileName=os.path.basename(filename)
tag,ext=os.path.splitext(baseFileName)
if len(onlyList)>0:
processThis=False
for filtTag in onlyList:
if filtTag in tag:
processThis=True
if not processThis : continue
tasklist.append((filename,'%s/%s'%(opt.output,baseFileName)))
#loop over tasks
if opt.queue=='local':
if opt.jobs>1:
print ' Submitting jobs in %d threads' % opt.jobs
import multiprocessing as MP
pool = MP.Pool(opt.jobs)
pool.map(runTopWidthAnalysisPacked,tasklist)
else:
for fileName,outFileName in tasklist:
runTopWidthAnalysis(fileName,outFileName)
else:
cmsswBase=os.environ['CMSSW_BASE']
FarmDirectory = '%s/%s'%(cmsswBase,opt.farm)
os.system('mkdir -p %s'%FarmDirectory)
with open ('%s/condor.sub'%FarmDirectory,'w') as condor:
condor.write('executable = {0}/$(jobName).sh\n'.format(FarmDirectory))
condor.write('output = {0}/output_$(jobName).out\n'.format(FarmDirectory))
condor.write('error = {0}/output_$(jobName).err\n'.format(FarmDirectory))
condor.write('+JobFlavour = workday\n')
condor.write('RequestCpus = 4\n')
for fileName,_ in tasklist:
jobName='%s'%(os.path.splitext(os.path.basename(fileName))[0])
jobScript='%s/%s.sh'%(FarmDirectory,jobName)
with open(jobScript,'w') as job:
job.write('#!/bin/bash\n')
job.write('WORKDIR=`pwd`\n')
job.write('echo "Working directory is ${WORKDIR}"\n')
job.write('cd %s\n'%cmsswBase)
job.write('eval `scram r -sh`\n')
job.write('cd ${WORKDIR}\n')
job.write('python {0}/src/TopLJets2015/TopAnalysis/scripts/runTopWidthAnalysis.py -o {1} -q local -i {2}\n'.format(cmsswBase,opt.output,fileName))
job.write('echo "All done"\n')
os.system('chmod u+x %s'%jobScript)
condor.write('jobName=%s\n'%jobName)
condor.write('queue 1\n')
os.system('condor_submit %s/condor.sub'%FarmDirectory)
from Configuration.AlCa.GlobalTag import GlobalTag
process.GlobalTag.globaltag = '74X_dataRun2_v2' if options.runOnData else '74X_mcRun2_asymptotic_v4'
# Set the process options -- Display summary at the end, enable unscheduled execution
process.options = cms.untracked.PSet(
allowUnscheduled = cms.untracked.bool(True),
# wantSummary = cms.untracked.bool(True)
)
#Number of events to process
process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1))
from TopLJets2015.TopAnalysis.storeTools import getEOSlslist
process.source = cms.Source("PoolSource",
fileNames = cms.untracked.vstring(getEOSlslist(directory='/store/mc/RunIISpring15MiniAODv2/TTJets_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/MINIAODSIM/74X_mcRun2_asymptotic_v2-v3/60000/')
)
)
if options.inputDir!='':
print 'Will process files from',options.inputDir
process.source.fileNames=cms.untracked.vstring(getEOSlslist(directory=options.inputDir))
#reduce verbosity
process.load("FWCore.MessageService.MessageLogger_cfi")
process.MessageLogger.cerr.threshold = ''
process.MessageLogger.cerr.FwkReport.reportEvery = 1000
#Tfileservice
process.TFileService = cms.Service("TFileService",
fileName = cms.string(options.outFilename))
def main():
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i',
'--inDir',
dest='inDir',
help='input directory with files',
default=None,
type='string')
parser.add_option('-o',
'--outDir',
dest='outDir',
help='output directory with files',
default=None,
type='string')
parser.add_option('-q',
'--queue',
dest='queue',
help='batch queue',
default='workday',
type='string')
(opt, args) = parser.parse_args()
#prepare output directory
if opt.outDir is None: opt.outDir = opt.inDir
cmsswBase = os.environ['CMSSW_BASE']
FARMDIR = '%s/INTEGRITYFARM' % cmsswBase
os.system('mkdir -p %s' % FARMDIR)
dset_list = getEOSlslist(directory=opt.inDir, prepend='')
with open('%s/checkInteg.sh' % FARMDIR, 'w') as shell:
shell.write('#!/bin/bash\n')
shell.write('WORKDIR=`pwd`\n')
shell.write('echo "Working directory is ${WORKDIR}"\n')
shell.write('cd %s\n' % cmsswBase)
shell.write('eval `scram r -sh`\n')
shell.write('cd ${WORKDIR}\n')
shell.write('echo "CMSSW_BASE=${CMSSW_BASE}"\n')
shell.write('echo "Preparing output directory"\n')
shell.write('mkdir -p /eos/cms/%s\n' % opt.outDir)
shell.write('echo "Running integrity checker"\n')
shell.write(
'python ${CMSSW_BASE}/src/TopLJets2015/TopAnalysis/scripts/checkProductionIntegrity.py -i ${1} -o ${2} --nocheck --only ${3}\n'
)
os.system('chmod u+x %s/checkInteg.sh' % FARMDIR)
condor = open('%s/condor.sub' % FARMDIR, 'w')
condor.write('executable = %s/checkInteg.sh\n' % FARMDIR)
condor.write('output = %s/job_$(ProcId).out\n' % FARMDIR)
condor.write('error = %s/job_$(ProcId).err\n' % FARMDIR)
condor.write('log = %s/job_$(ProcId).log\n' % FARMDIR)
condor.write('+JobFlavour ="%s"\n' % opt.queue)
for dset in dset_list:
dsetname = dset.split('/')[-1]
pub_list = getEOSlslist(directory=dset, prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname
continue
pub = pubDir.split('/crab_')[-1]
condor.write('arguments = %s %s %s\n' %
(opt.inDir, opt.outDir, pub))
condor.write('queue 1\n')
condor.close()
os.system('condor_submit %s/condor.sub' % FARMDIR)
from Configuration.AlCa.GlobalTag import GlobalTag
process.GlobalTag.globaltag = '74X_dataRun2_v2' if options.runOnData else '74X_mcRun2_asymptotic_v2'
# Set the process options -- Display summary at the end, enable unscheduled execution
process.options = cms.untracked.PSet(
allowUnscheduled = cms.untracked.bool(True),
# wantSummary = cms.untracked.bool(True)
)
#Number of events to process
process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1))
from TopLJets2015.TopAnalysis.storeTools import getEOSlslist
process.source = cms.Source("PoolSource",
fileNames = cms.untracked.vstring(getEOSlslist(directory='/store/mc/RunIISpring15MiniAODv2/TTJets_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/MINIAODSIM/74X_mcRun2_asymptotic_v2-v3/60000/')
)
)
#reduce verbosity
process.load("FWCore.MessageService.MessageLogger_cfi")
process.MessageLogger.cerr.threshold = ''
process.MessageLogger.cerr.FwkReport.reportEvery = 1000
#Tfileservice
process.TFileService = cms.Service("TFileService",
fileName = cms.string(options.outFilename))
# Set up electron ID (VID framework)
from PhysicsTools.SelectorUtils.tools.vid_id_tools import *
dataFormat = DataFormat.MiniAOD
def main():
cmsswBase = os.environ['CMSSW_BASE']
#eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i',
'--inDir',
dest='inDir',
help='input directory with files',
default=None,
type='string')
parser.add_option('-o',
'--outDir',
dest='outDir',
help='output directory with files',
default=None,
type='string')
parser.add_option('-q',
'--queue',
dest='queue',
help='batch queue',
default='2nd',
type='string')
parser.add_option('--only',
dest='only',
help='only this tag',
default=None,
type='string')
(opt, args) = parser.parse_args()
#Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate()
#prepare output directory
if opt.outDir is None: opt.outDir = opt.inDir
#Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate()
dset_list = getEOSlslist(directory=opt.inDir, prepend='')
for dset in dset_list:
dsetname = dset.split('/')[-1]
#if not 'pp_TuneCUETP8M1_Hydjet_Min_Bias' : continue
pub_list = getEOSlslist(directory=dset, prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname
continue
pub = pubDir.split('/crab_')[-1]
#if not 'V4' in pub : continue
if opt.only:
if pub not in opt.only:
continue
#if 'Data13TeV' in pub : continue
localMerge = 'python scripts/checkProductionIntegrity.py -i %s -o %s --nocheck --only %s' % (
opt.inDir, opt.outDir, pub)
############### Submit to condor ###############
target = '%s/src/TopLJets2015/TopAnalysis/%s' % (cmsswBase, pub)
condorFile = open(target, 'w')
condorFile.write('universe = vanilla\n')
#condorFile.write('executable = condor/cond_wrapper.sh\n')
condorFile.write('executable = condor/cond_check.sh\n')
condorFile.write(
'arguments = $(ClusterID) $(ProcId) %s %s %s\n' %
(opt.inDir, opt.outDir, pub))
condorFile.write(
'output = condor/log/%s_EOS_$(ProcId).out\n' %
pub)
condorFile.write(
'error = condor/log/%s_EOS_$(ProcId).err\n' %
pub)
condorFile.write(
'log = condor/log/%s_EOS.log\n' % pub)
condorFile.write('+JobFlavour = "tomorrow"\n')
condorFile.write('Should_Transfer_Files = NO\n')
condorFile.write('queue')
condorFile.close()
os.system('condor_submit %s -batch-name %s' % (target, pub))
os.system('rm %s' % (pub))
def main():
eos_cmd = '/afs/cern.ch/project/eos/installation/0.2.41/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string')
parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string')
parser.add_option('-c', '--cleanup', dest='cleanup', help='removes original crab directory', default =False, action='store_true')
parser.add_option( '--nocheck', dest='nocheck', help='do not prompt user', default=False, action='store_true')
(opt, args) = parser.parse_args()
Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate()
if opt.outDir is None: opt.outDir=opt.inDir
dset_list=getEOSlslist(directory=opt.inDir,prepend='')
for dset in dset_list:
dsetname=dset.split('/')[-1]
pub_list=getEOSlslist(directory=dset,prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname
continue
pub=pubDir.split('/crab_')[-1]
time_list=getEOSlslist(directory=pubDir,prepend='')
if len(time_list)!=1:
print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out'%dsetname
continue
time_stamp=time_list[0].split('/')[-1]
out_list=[]
count_list=getEOSlslist(directory=time_list[0],prepend='')
for count in count_list: out_list += getEOSlslist(directory=count,prepend='')
file_list=[x for x in out_list if '.root' in x]
newDir='%s/%s' % (opt.outDir,pub)
print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % (dsetname,pub,time_stamp,len(file_list) )
if not opt.nocheck:
choice = raw_input('Will move to %s current output directory. [y/n] ?' % newDir ).lower()
if not 'y' in choice : continue
Popen([eos_cmd, 'mkdir', '/eos/cms/'+newDir],stdout=PIPE).communicate()
moveIndividualFiles=True
if len(file_list)>0:
subgroupMerge = int( raw_input('This set has %d files. Merge into groups? (enter 0 if no merging)' % len(file_list)) )
if subgroupMerge>0:
moveIndividualFiles=False
splitFunc = lambda A, n=subgroupMerge: [A[i:i+n] for i in range(0, len(A), n)]
split_file_lists = splitFunc( file_list )
for ilist in xrange(0,len(split_file_lists)):
mergedFileName='/tmp/MergedMiniEvents_%d.root '%ilist
toAdd='%s ' % mergedFileName
for f in split_file_lists[ilist]:
toAdd += 'eos/cms/%s '%f
os.system('hadd -f %s'%toAdd)
os.system('xrdcp -f %s root://eoscms//eos/cms/%s/' %(mergedFileName,newDir))
#if still needed copy individual files
if moveIndividualFiles:
for f in file_list : os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir) )
if not opt.nocheck and opt.cleanup :
choice = raw_input('Will remove output directory. [y/n] ?').lower()
if 'y' in choice:
Popen([eos_cmd, 'rm', '-r /eos/cms/'+dset],stdout=PIPE).communicate()
print 'Crab outputs may now be found in %s' % newDir
Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate()
print '-'*50
print 'All done. In case errors were found check that the crab output structure is '
print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>'
print '-'*50
def main():
eos_cmd = '/afs/cern.ch/project/eos/installation/0.2.41/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i',
'--inDir',
dest='inDir',
help='input directory with files',
default=None,
type='string')
parser.add_option('-o',
'--outDir',
dest='outDir',
help='output directory with files',
default=None,
type='string')
parser.add_option('-c',
'--cleanup',
dest='cleanup',
help='removes original crab directory',
default=False,
action='store_true')
parser.add_option('--nocheck',
dest='nocheck',
help='do not prompt user',
default=False,
action='store_true')
(opt, args) = parser.parse_args()
Popen([eos_cmd, ' -b fuse mount', 'eos'], stdout=PIPE).communicate()
if opt.outDir is None: opt.outDir = opt.inDir
dset_list = getEOSlslist(directory=opt.inDir, prepend='')
for dset in dset_list:
dsetname = dset.split('/')[-1]
pub_list = getEOSlslist(directory=dset, prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out' % dsetname
continue
pub = pubDir.split('/crab_')[-1]
time_list = getEOSlslist(directory=pubDir, prepend='')
if len(time_list) != 1:
print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out' % dsetname
continue
time_stamp = time_list[0].split('/')[-1]
out_list = []
count_list = getEOSlslist(directory=time_list[0], prepend='')
for count in count_list:
out_list += getEOSlslist(directory=count, prepend='')
file_list = [x for x in out_list if '.root' in x]
newDir = '%s/%s' % (opt.outDir, pub)
print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % (
dsetname, pub, time_stamp, len(file_list))
if not opt.nocheck:
choice = raw_input(
'Will move to %s current output directory. [y/n] ?' %
newDir).lower()
if not 'y' in choice: continue
Popen([eos_cmd, 'mkdir', '/eos/cms/' + newDir],
stdout=PIPE).communicate()
moveIndividualFiles = True
if len(file_list) > 0:
subgroupMerge = int(
raw_input(
'This set has %d files. Merge into groups? (enter 0 if no merging)'
% len(file_list)))
if subgroupMerge > 0:
moveIndividualFiles = False
splitFunc = lambda A, n=subgroupMerge: [
A[i:i + n] for i in range(0, len(A), n)
]
split_file_lists = splitFunc(file_list)
for ilist in xrange(0, len(split_file_lists)):
mergedFileName = '/tmp/MergedMiniEvents_%d.root ' % ilist
toAdd = '%s ' % mergedFileName
for f in split_file_lists[ilist]:
toAdd += 'eos/cms/%s ' % f
os.system('hadd -f %s' % toAdd)
os.system('xrdcp -f %s root://eoscms//eos/cms/%s/' %
(mergedFileName, newDir))
#if still needed copy individual files
if moveIndividualFiles:
for f in file_list:
os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir))
if not opt.nocheck and opt.cleanup:
choice = raw_input(
'Will remove output directory. [y/n] ?').lower()
if 'y' in choice:
Popen([eos_cmd, 'rm', '-r /eos/cms/' + dset],
stdout=PIPE).communicate()
print 'Crab outputs may now be found in %s' % newDir
Popen([eos_cmd, ' -b fuse umount', 'eos'], stdout=PIPE).communicate()
print '-' * 50
print 'All done. In case errors were found check that the crab output structure is '
print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>'
print '-' * 50
def main():
eos_cmd = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select'
#configuration
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('-i', '--inDir', dest='inDir', help='input directory with files', default=None, type='string')
parser.add_option('-o', '--outDir', dest='outDir', help='output directory with files', default=None, type='string')
parser.add_option('-c', '--cleanup', dest='cleanup', help='removes original crab directory', default =False, action='store_true')
parser.add_option( '--nocheck', dest='nocheck', help='do not prompt user', default=False, action='store_true')
parser.add_option( '--only', dest='only', help='only this tag', default=None, type='string')
(opt, args) = parser.parse_args()
Popen([eos_cmd, ' -b fuse mount', 'eos'],stdout=PIPE).communicate()
#prepare output directory
if opt.outDir is None: opt.outDir=opt.inDir
Popen([eos_cmd, 'mkdir', '/eos/cms/'+opt.outDir],stdout=PIPE).communicate()
dset_list=getEOSlslist(directory=opt.inDir,prepend='')
for dset in dset_list:
dsetname=dset.split('/')[-1]
pub_list=getEOSlslist(directory=dset,prepend='')
for pubDir in pub_list:
if not 'crab' in pubDir:
print 'Ambiguity found @ <publication-name> for <primary-dataset>=%s , bailing out'%dsetname
continue
#select if it doesn't match the required selection
pub=pubDir.split('/crab_')[-1]
if opt.only:
if pub!=opt.only:
continue
#check if it's an extension
pubExt=None
try:
extSplit=pub.split('_ext')
pubExt='ext'+extSplit[1]
pub=extSplit[0]
print 'Extension will be postfixed with ',pubExt
except:
print 'Core sample (no extension)'
time_list=getEOSlslist(directory=pubDir,prepend='')
if len(time_list)!=1:
print 'Ambiguity found @ <time-stamp> for <primary-dataset>=%s , bailing out'%dsetname
continue
time_stamp=time_list[0].split('/')[-1]
out_list=[]
count_list=getEOSlslist(directory=time_list[0],prepend='')
for count in count_list: out_list += getEOSlslist(directory=count,prepend='')
file_list=[x for x in out_list if '.root' in x]
newDir='%s/%s' % (opt.outDir,pub)
print '<primary-dataset>=%s <publication-name>=crab_%s <time-stamp>=%s has %d files' % (dsetname,pub,time_stamp,len(file_list) )
if not opt.nocheck:
choice = raw_input('Will move to %s current output directory. [y/n] ?' % newDir ).lower()
if not 'y' in choice : continue
Popen([eos_cmd, 'mkdir', '/eos/cms/'+newDir],stdout=PIPE).communicate()
moveIndividualFiles=True
if len(file_list)>0:
#subgroupMerge = int( raw_input('This set has %d files. Merge into groups? (enter 0 if no merging)' % len(file_list)) )
subgroupMerge=60 if 'Data' in dsetname else 20
if subgroupMerge>0:
moveIndividualFiles=False
splitFunc = lambda A, n=subgroupMerge: [A[i:i+n] for i in range(0, len(A), n)]
split_file_lists = splitFunc( file_list )
for ilist in xrange(0,len(split_file_lists)):
if pubExt:
mergedFileName='/tmp/MergedMiniEvents_%d_%s.root '%(ilist,pubExt)
else:
mergedFileName='/tmp/MergedMiniEvents_%d.root '%ilist
toAdd='%s ' % mergedFileName
for f in split_file_lists[ilist]:
toAdd += 'eos/cms/%s '%f
finalOutput='eos/cms/%s/%s'%(newDir,mergedFileName.replace('/tmp/',''))
fIn=ROOT.TFile.Open(finalOutput)
try:
if fIn or not fIn.IsZombie():
print '%s already in EOS, skipping'%finalOutput
fIn.Close()
continue
except:
pass
os.system('hadd -f %s'%toAdd)
os.system('cp %s eos/cms/%s/'%(mergedFileName,newDir))
os.system('rm %s'%mergedFileName)
#os.system('xrdcp -f %s root://eoscms//eos/cms/%s/MergedMiniEvents_%d.root' %(mergedFileName,newDir,ilist))
#if still needed copy individual files
if moveIndividualFiles:
for f in file_list :
#os.system('xrdcp -f %s eos/cms/%s/' % (f, newDir) )
newF=f
if pubExt:
newF=f.replace('.root','_%s.root'%pubExt)
os.system('cp %s eos/cms/%s/%s' % (f, newDir,newF) )
if not opt.nocheck and opt.cleanup :
choice = raw_input('Will remove output directory. [y/n] ?').lower()
if 'y' in choice:
Popen([eos_cmd, 'rm', '-r /eos/cms/'+dset],stdout=PIPE).communicate()
print 'Crab outputs may now be found in %s' % newDir
#Popen([eos_cmd, ' -b fuse umount', 'eos'],stdout=PIPE).communicate()
print '-'*50
print 'All done. In case errors were found check that the crab output structure is '
print '<outLFNDirBase>/<primary-dataset>/<publication-name>/<time-stamp>/<counter>/<file-name>'
print '-'*50
