batchManager = MyBatchManager()
batchManager.parser_.usage="""
%prog [options] <cfgFile>
Run Colin's python analysis system on the batch.
Job splitting is determined by your configuration file.
"""
options, args = batchManager.ParseOptions()
cfgFileName = args[0]
handle = open(cfgFileName, 'r')
cfo = imp.load_source("pycfg", cfgFileName, handle)
config = cfo.config
handle.close()
components = split( [comp for comp in config.components if len(comp.files)>0] )
listOfValues = range(0, len(components))
listOfNames = [comp.name for comp in components]
print 'Preparing jobs'
batchManager.PrepareJobs( listOfValues, listOfNames )
if '-t' not in options.batch:
waitingTime = 0.1
batchManager.SubmitJobs( waitingTime )
else:
print 'Submitting job array'
batchManager.SubmitJobArray(len(listOfNames))
# pickle.dump( cfo, cfgFile )
cfgFile.close()
if __name__ == '__main__':
batchManager = MyBatchManager()
batchManager.parser_.usage="""
%prog [options] <cfgFile>
Run Colin's python analysis system on the batch.
Job splitting is determined by your configuration file.
"""
options, args = batchManager.ParseOptions()
cfgFileName = args[0]
handle = open(cfgFileName, 'r')
cfo = imp.load_source("pycfg", cfgFileName, handle)
config = cfo.config
handle.close()
components = split( [comp for comp in config.components if len(comp.files)>0] )
listOfValues = range(0, len(components))
listOfNames = [comp.name for comp in components]
batchManager.PrepareJobs( listOfValues, listOfNames )
waitingTime = 0.1
batchManager.SubmitJobs( waitingTime )
print "job", job, " out of", total
# fetch config file
import imp
handle = open("heppy_config.py", 'r')
cfo = imp.load_source("heppy_config", "heppy_config.py", handle)
config = cfo.config
handle.close()
from PhysicsTools.HeppyCore.framework.heppy import split
# pick right component from dataset and file from jobID
selectedComponents = []
for comp in config.components:
if comp.name == dataset:
# this selects the files and events and changes the name to _ChunkX according to fineSplitFactor and splitFactor
newComp = split([comp])[job - 1] # first job number is 1
if useAAA:
newComp.files = [
x.replace("root://eoscms.cern.ch//eos/cms",
"root://cms-xrd-global.cern.ch/")
for x in newComp.files
]
selectedComponents.append(newComp)
# check selectedComponents
if len(selectedComponents) == 0:
print "No selected components found!!"
print " - dataset:", dataset
print " - components:", config.components
if len(selectedComponents) > 1:
print "More than one selected component:"
production_label = "prod74Xdata_noJSON"
cmg_version = 'MT2_CMGTools-from-CMSSW_7_4_3'
debug = False
useAAA = False
handle = open("heppy_config.py", 'r')
cfo = imp.load_source("heppy_config", "heppy_config.py", handle)
conf = cfo.config
handle.close()
os.system("scramv1 runtime -sh")
os.system("source /cvmfs/cms.cern.ch/crab3/crab.sh")
os.environ["PROD_LABEL"] = production_label
os.environ["CMG_VERSION"] = cmg_version
os.environ["DEBUG"] = str(debug)
os.environ["USEAAA"] = str(useAAA)
from PhysicsTools.HeppyCore.framework.heppy import split
for comp in conf.components:
# get splitting from config file according to splitFactor and fineSplitFactor (priority given to the latter)
NJOBS = len(split([comp]))
os.environ["NJOBS"] = str(NJOBS)
os.environ["DATASET"] = str(comp.name)
os.system("crab submit -c heppy_crab_config_env.py")
os.system("rm -f python.tar.gz")
os.system("rm -f cmgdataset.tar.gz")
os.system("rm -f cafpython.tar.gz")