batchManager = MyBatchManager() batchManager.parser_.usage=""" %prog [options] <cfgFile> Run Colin's python analysis system on the batch. Job splitting is determined by your configuration file. """ options, args = batchManager.ParseOptions() cfgFileName = args[0] handle = open(cfgFileName, 'r') cfo = imp.load_source("pycfg", cfgFileName, handle) config = cfo.config handle.close() components = split( [comp for comp in config.components if len(comp.files)>0] ) listOfValues = range(0, len(components)) listOfNames = [comp.name for comp in components] print 'Preparing jobs' batchManager.PrepareJobs( listOfValues, listOfNames ) if '-t' not in options.batch: waitingTime = 0.1 batchManager.SubmitJobs( waitingTime ) else: print 'Submitting job array' batchManager.SubmitJobArray(len(listOfNames))
# pickle.dump( cfo, cfgFile ) cfgFile.close() if __name__ == '__main__': batchManager = MyBatchManager() batchManager.parser_.usage=""" %prog [options] <cfgFile> Run Colin's python analysis system on the batch. Job splitting is determined by your configuration file. """ options, args = batchManager.ParseOptions() cfgFileName = args[0] handle = open(cfgFileName, 'r') cfo = imp.load_source("pycfg", cfgFileName, handle) config = cfo.config handle.close() components = split( [comp for comp in config.components if len(comp.files)>0] ) listOfValues = range(0, len(components)) listOfNames = [comp.name for comp in components] batchManager.PrepareJobs( listOfValues, listOfNames ) waitingTime = 0.1 batchManager.SubmitJobs( waitingTime )
print "job", job, " out of", total # fetch config file import imp handle = open("heppy_config.py", 'r') cfo = imp.load_source("heppy_config", "heppy_config.py", handle) config = cfo.config handle.close() from PhysicsTools.HeppyCore.framework.heppy import split # pick right component from dataset and file from jobID selectedComponents = [] for comp in config.components: if comp.name == dataset: # this selects the files and events and changes the name to _ChunkX according to fineSplitFactor and splitFactor newComp = split([comp])[job - 1] # first job number is 1 if useAAA: newComp.files = [ x.replace("root://eoscms.cern.ch//eos/cms", "root://cms-xrd-global.cern.ch/") for x in newComp.files ] selectedComponents.append(newComp) # check selectedComponents if len(selectedComponents) == 0: print "No selected components found!!" print " - dataset:", dataset print " - components:", config.components if len(selectedComponents) > 1: print "More than one selected component:"
production_label = "prod74Xdata_noJSON" cmg_version = 'MT2_CMGTools-from-CMSSW_7_4_3' debug = False useAAA = False handle = open("heppy_config.py", 'r') cfo = imp.load_source("heppy_config", "heppy_config.py", handle) conf = cfo.config handle.close() os.system("scramv1 runtime -sh") os.system("source /cvmfs/cms.cern.ch/crab3/crab.sh") os.environ["PROD_LABEL"] = production_label os.environ["CMG_VERSION"] = cmg_version os.environ["DEBUG"] = str(debug) os.environ["USEAAA"] = str(useAAA) from PhysicsTools.HeppyCore.framework.heppy import split for comp in conf.components: # get splitting from config file according to splitFactor and fineSplitFactor (priority given to the latter) NJOBS = len(split([comp])) os.environ["NJOBS"] = str(NJOBS) os.environ["DATASET"] = str(comp.name) os.system("crab submit -c heppy_crab_config_env.py") os.system("rm -f python.tar.gz") os.system("rm -f cmgdataset.tar.gz") os.system("rm -f cafpython.tar.gz")