def run(self): name = self.config.get('Configuration', 'channel') if self.config.has_option('Configuration', 'channel') else '_' timestamp = datetime.datetime.now().strftime("%y%m%d") tmpName = self.tmpDir + '/skim_' + name + '_' + region + '_' + timestamp + '_tmp.root' destName = self.pathOUT + '/skim_' + name + '_' + region + '_' + timestamp + '.root' sampleTree = SampleTree(self.fileNames, config=self.config) if self.config.has_option('Plot_general', 'controlSample'): controlSampleDict = eval(self.config.get('Plot_general', 'controlSample')) controlSample = controlSampleDict[self.region] if self.region in controlSampleDict else -1 sampleTree.addOutputBranch("controlSample", lambda x: controlSample, branchType="i") print("INFO: setting controlSample to", controlSample) sampleTree.addOutputTree(tmpName, cut='1', branches='*', friend=False) sampleTree.process() # copy to final destination if sampleTree.getNumberOfOutputTrees() > 0: try: self.fileLocator.cp(tmpName, destName, force=True) print('copy ', tmpName, destName) if not self.fileLocator.isValidRootFile(destName): print("\x1b[31mERROR: copy failed, output is broken!\x1b[0m") else: try: self.fileLocator.rm(tmpName) except Exception as e: print(e) except Exception as e: print("\x1b[31mERROR: copy failed!", e, "\x1b[0m")
def run(self): inputFileNames = [ "{path}/{sample}/{fileName}".format( path=self.config.get('Directories', 'HADDin'), sample=self.sampleIdentifier, fileName=self.fileLocator.getFilenameAfterPrep(fileName)) for fileName in self.fileNames ] outputFileName = self.getTemporaryFileName() self.fileLocator.makedirs('/'.join(outputFileName.split('/')[:-1])) command = self.commandTemplate.format(output=outputFileName, inputs=' '.join(inputFileNames), f="-f" if self.force else "") if self.debug: print("DEBUG: run \x1b[34m", command, "\x1b[0m") if self.useChain: # use sampleTree class (can e.g. drop branches at the same time) sampleTree = SampleTree(inputFileNames, config=self.config) try: removeBranches = eval( self.config.get('General', 'remove_branches')) for removeBranch in removeBranches: sampleTree.addBranchToBlacklist(removeBranch) print("DEBUG: disable branch ", removeBranch) except Exception as e: print("DEBUG: could not disable branch:", e) sampleTree.addOutputTree(outputFileName, cut='1', branches='*') sampleTree.process() result = 0 else: # standard hadd result = self.fileLocator.runCommand(command) print("INFO: hadd returned ", result) if result == 0: finalOutputFileName = self.getOutputFileName() print("move file to final destination: \x1b[34m", finalOutputFileName, "\x1b[0m") self.fileLocator.makedirs('/'.join( finalOutputFileName.split('/')[:-1])) resultCopy = self.fileLocator.cp(outputFileName, finalOutputFileName, self.force) if not resultCopy: print("\x1b[31mERROR: copy failed\n from:", outputFileName, "\n to:", finalOutputFileName, "\n force:", self.force, "\x1b[0m") raise Exception("FileCopyError") # try to delete temporary file try: self.fileLocator.rm(outputFileName) except Exception as e: print("ERROR: could not delete temporary file:", outputFileName, " => ", e) print("INFO: done.") else: raise Exception("HaddError")
def run(self): inputFileNames = ["{path}/{sample}/{fileName}".format(path=self.config.get('Directories','HADDin'), sample=self.sampleIdentifier, fileName=self.fileLocator.getFilenameAfterPrep(fileName)) for fileName in self.fileNames] outputFileName = self.getTemporaryFileName() self.fileLocator.makedirs('/'.join(outputFileName.split('/')[:-1])) command = self.commandTemplate.format(output=outputFileName, inputs=' '.join(inputFileNames), f="-f" if self.force else "") if self.debug: print ("DEBUG: run \x1b[34m", command, "\x1b[0m") if self.useChain: # use sampleTree class (can e.g. drop branches at the same time) sampleTree = SampleTree(inputFileNames, config=self.config) try: removeBranches = eval(self.config.get('General', 'remove_branches')) for removeBranch in removeBranches: sampleTree.addBranchToBlacklist(removeBranch) print("DEBUG: disable branch ", removeBranch) except Exception as e: print("DEBUG: could not disable branch:", e) sampleTree.addOutputTree(outputFileName, cut='1', branches='*') sampleTree.process() result = 0 else: # standard hadd result = self.fileLocator.runCommand(command) print ("INFO: hadd returned ", result) if result == 0: finalOutputFileName = self.getOutputFileName() print("move file to final destination: \x1b[34m", finalOutputFileName, "\x1b[0m") self.fileLocator.makedirs('/'.join(finalOutputFileName.split('/')[:-1])) resultCopy = self.fileLocator.cp(outputFileName, finalOutputFileName, self.force) if not resultCopy: print("\x1b[31mERROR: copy failed\n from:", outputFileName, "\n to:", finalOutputFileName, "\n force:", self.force, "\x1b[0m") raise Exception("FileCopyError") # try to delete temporary file try: self.fileLocator.rm(outputFileName) except Exception as e: print("ERROR: could not delete temporary file:", outputFileName, " => ", e) print("INFO: done.") else: raise Exception("HaddError")
def run(self): name = self.config.get('Configuration', 'channel') if self.config.has_option( 'Configuration', 'channel') else '_' timestamp = datetime.datetime.now().strftime("%y%m%d") tmpName = self.tmpDir + '/skim_' + name + '_' + region + '_' + timestamp + '_tmp.root' destName = self.pathOUT + '/skim_' + name + '_' + region + '_' + timestamp + '.root' sampleTree = SampleTree(self.fileNames, config=self.config) if self.config.has_option('Plot_general', 'controlSample'): controlSampleDict = eval( self.config.get('Plot_general', 'controlSample')) controlSample = controlSampleDict[ self.region] if self.region in controlSampleDict else -1 sampleTree.addOutputBranch("controlSample", lambda x: controlSample, branchType="i") print("INFO: setting controlSample to", controlSample) sampleTree.addOutputTree(tmpName, cut='1', branches='*', friend=False) sampleTree.process() # copy to final destination if sampleTree.getNumberOfOutputTrees() > 0: try: self.fileLocator.cp(tmpName, destName, force=True) print('copy ', tmpName, destName) if not self.fileLocator.isValidRootFile(destName): print( "\x1b[31mERROR: copy failed, output is broken!\x1b[0m") else: try: self.fileLocator.rm(tmpName) except Exception as e: print(e) except Exception as e: print("\x1b[31mERROR: copy failed!", e, "\x1b[0m")
def run(self): nFilesProcessed = 0 nFilesFailed = 0 for subJob in self.subJobs: # only process if output is non-existing/broken or --force was used if self.opts.force or not self.fileLocator.isValidRootFile(subJob['outputFileName']): # create directories outputFolder = '/'.join(subJob['outputFileName'].split('/')[:-1]) tmpFolder = '/'.join(subJob['tmpFileName'].split('/')[:-1]) self.fileLocator.makedirs(outputFolder) self.fileLocator.makedirs(tmpFolder) # load sample tree sampleTree = SampleTree(subJob['localInputFileNames'], config=self.config) if not sampleTree.tree: print "trying fallback...", len(subJob['inputFileNames']) if len(subJob['inputFileNames']) == 1: # try original naming scheme if reading directly from Heppy/Nano ntuples (without prep) fileNameOriginal = self.pathIN + '/' + subJob['inputFileNames'][0] print "FO:", fileNameOriginal xrootdRedirector = self.fileLocator.getRedirector(fileNameOriginal) sampleTree = SampleTree([fileNameOriginal], config=self.config, xrootdRedirector=xrootdRedirector) if not sampleTree.tree: print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED!\x1b[0m" nFilesFailed += 1 continue else: print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED! (old naming scheme not supported for joining multipel files)\x1b[0m" nFilesFailed += 1 continue # to use this syntax, use "--addCollections Sys.Vtype" for a config file entry like this: # [Sys] # Vtype = VtypeCorrector.VtypeCorrector(channel='Zll') # (instead of passing the tree in the constructor, the setTree method can be used) pyModules = [] versionTable = [] for collection in self.collections: if '.' in collection: section = collection.split('.')[0] key = collection.split('.')[1] if self.config.has_section(section) and self.config.has_option(section, key): pyCode = self.config.get(section, key) elif '(' in collection and collection.endswith(')'): print "WARNING: config option", collection, " not found, interpreting it as Python code!" pyCode = collection else: print "\x1b[31mERROR: config option not found:", collection, ". To specify Python code directly, pass a complete constructor, e.g. --addCollections 'Module.Class()'. Module has to be placed in python/myutils/ folder.\x1b[0m" raise Exception("ConfigError") # import module from myutils moduleName = pyCode.split('(')[0].split('.')[0].strip() if self.debug: print "DEBUG: import module:", moduleName print("\x1b[33mDEBUG: " + collection + ": run PYTHON code:\n"+pyCode+"\x1b[0m") globals()[moduleName] = importlib.import_module(".{module}".format(module=moduleName), package="myutils") # get object wObject = eval(pyCode) # pass the tree and other variables if needed to finalize initialization if hasattr(wObject, "customInit") and callable(getattr(wObject, "customInit")): wObject.customInit({'config': self.config, 'sampleTree': sampleTree, 'tree': sampleTree.tree, 'sample': self.sample, 'channel': self.channel, 'pathIN': self.pathIN, 'pathOUT': self.pathOUT, }) # add callbacks if the objects provides any if hasattr(wObject, "processEvent") and callable(getattr(wObject, "processEvent")): sampleTree.addCallback('event', wObject.processEvent) for cb in ["finish", "prepareOutput"]: if hasattr(wObject, cb) and callable(getattr(wObject, cb)): sampleTree.addCallback(cb, getattr(wObject, cb)) # add branches if hasattr(wObject, "getBranches") and callable(getattr(wObject, "getBranches")): sampleTree.addOutputBranches(wObject.getBranches()) pyModules.append(wObject) versionTable.append([moduleName, wObject.getVersion() if hasattr(wObject, "getVersion") else 0]) else: print "\x1b[31mERROR: config option not found:", collection, " the format should be: [Section].[Option]\x1b[0m" raise Exception("ConfigError") for moduleName, moduleVersion in versionTable: print " > {m}:{v}".format(m=moduleName, v=moduleVersion) # DEPRECATED, do not use anymore ---> use BranchTools.TreeFormulas() if 'addbranches' in self.collections: writeNewVariables = eval(self.config.get("Regression", "writeNewVariablesDict")) sampleTree.addOutputBranches(writeNewVariables) # DEPRECATED, do not use anymore ---> use BranchTools.Drop() if 'removebranches' in self.collections: bl_branch = eval(config.get('Branches', 'useless_branch')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) bl_branch = eval(config.get('Branches', 'useless_after_sys')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) # define output file sampleTree.addOutputTree(subJob['tmpFileName'], cut='1', branches='*', friend=self.opts.friend) # run processing for pyModule in pyModules: if hasattr(pyModule, "beforeProcessing"): getattr(pyModule, "beforeProcessing")() sampleTree.process() for pyModule in pyModules: if hasattr(pyModule, "afterProcessing"): getattr(pyModule, "afterProcessing")() # if output trees have been produced: copy temporary file to output folder if sampleTree.getNumberOfOutputTrees() > 0: try: self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True) print 'copy ', subJob['tmpFileName'], subJob['outputFileName'] if self.verifyCopy: if not self.fileLocator.isValidRootFile(subJob['outputFileName']): print 'INFO: output at final destination broken, try to copy again from scratch disk to final destination...' self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True) print 'INFO: second attempt copy done!' if not self.fileLocator.isValidRootFile(subJob['outputFileName']): print '\x1b[31mERROR: output still broken!\x1b[0m' nFilesFailed += 1 raise Exception("FileCopyError") else: print 'INFO: file is good after second attempt!' except Exception as e: print e print "\x1b[31mERROR: copy from scratch to final destination failed!!\x1b[0m" # delete temporary file try: self.fileLocator.rm(subJob['tmpFileName']) except Exception as e: print e print "WARNING: could not delete file on scratch!" # clean up if hasattr(wObject, "cleanUp") and callable(getattr(wObject, "cleanUp")): getattr(wObject, "cleanUp")() else: print 'SKIP:', subJob['inputFileNames'] if nFilesFailed > 0: raise Exception("ProcessingIncomplete")
class CacheTraining(object): def __init__(self, config, sampleIdentifier, trainingRegions, splitFilesChunks=1, chunkNumber=1, splitFilesChunkSize=-1, force=False): self.config = config self.force = force self.sampleIdentifier = sampleIdentifier self.trainingRegions = trainingRegions self.sampleTree = None self.samplesPath = self.config.get('Directories', 'MVAin') self.samplesDefinitions = self.config.get('Directories','samplesinfo') self.samplesInfo = ParseInfo(self.samplesDefinitions, self.samplesPath) self.sampleFilesFolder = self.config.get('Directories', 'samplefiles') self.backgroundSampleNames = list(set(sum([eval(self.config.get(trainingRegion, 'backgrounds')) for trainingRegion in self.trainingRegions], []))) self.signalSampleNames = list(set(sum([eval(self.config.get(trainingRegion, 'signals')) for trainingRegion in self.trainingRegions], []))) self.samples = self.samplesInfo.get_samples(list(set(self.backgroundSampleNames + self.signalSampleNames))) self.trainingRegionsDict = {} for trainingRegion in self.trainingRegions: treeCutName = config.get(trainingRegion, 'treeCut') treeVarSet = config.get(trainingRegion, 'treeVarSet').strip() #systematics = [x for x in config.get('systematics', 'systematics').split(' ') if len(x.strip())>0] systematics = eval(config.get(trainingRegion, 'systematics')) if config.has_option(trainingRegion, 'systematics') else [] mvaVars = config.get(treeVarSet, 'Nominal').split(' ') weightVars = [] #for systematic in systematics: for syst in systematics: systNameUp = syst+'_UP' if self.config.has_option('Weights',syst+'_UP') else syst+'_Up' systNameDown = syst+'_DOWN' if self.config.has_option('Weights',syst+'_DOWN') else syst+'_Down' weightVars += [self.config.get('Weights',systNameUp), self.config.get('Weights',systNameDown)] self.trainingRegionsDict[trainingRegion] = { 'cut': config.get('Cuts', treeCutName), 'vars': mvaVars, 'weightVars': weightVars, } self.TrainCut = config.get('Cuts', 'TrainCut') self.EvalCut = config.get('Cuts', 'EvalCut') self.splitFilesChunks = splitFilesChunks self.chunkNumber = chunkNumber self.splitFilesChunkSize = splitFilesChunkSize VHbbNameSpace=config.get('VHbbNameSpace','library') ROOT.gSystem.Load(VHbbNameSpace) def printInfo(self): print ("REGION:".ljust(24),"CUT:") for trainingRegion,trainingRegionInfo in self.trainingRegionsDict.iteritems(): print (" > ",trainingRegion.ljust(20), trainingRegionInfo['cut']) def run(self): # ---------------------------------------------------------------------------------------------------------------------- # cache samples # ---------------------------------------------------------------------------------------------------------------------- for sampleToCache in [self.sampleIdentifier]: print ('*'*80) print (' ',sampleToCache) print ('*'*80) # prepare caches for training and evaluation samples treeCaches = [] self.sampleTree = None # use all (sub)samples which come from the same files (sampleIdentifier) subsamples = [x for x in self.samples if x.identifier == sampleToCache] # list of branches to keep for use as MVA input variables branchListOfMVAVars = BranchList() for sample in subsamples: for trainingRegion,trainingRegionInfo in self.trainingRegionsDict.iteritems(): for additionalCut in [self.TrainCut, self.EvalCut]: branchListOfMVAVars.addCut(trainingRegionInfo['vars']) for weightVar in trainingRegionInfo['weightVars']: branchListOfMVAVars.addCut(weightVar) branchListOfMVAVars.addCut(self.config.get('Weights', 'weightF')) mvaBranches = branchListOfMVAVars.getListOfBranches() # loop over all samples for sample in subsamples: # add cuts for all training regions for trainingRegion,trainingRegionInfo in self.trainingRegionsDict.iteritems(): # add cuts for training and evaluation for additionalCut in [self.TrainCut, self.EvalCut]: # cuts sampleCuts = [sample.subcut] if additionalCut: sampleCuts.append(additionalCut) if trainingRegionInfo['cut']: sampleCuts.append(trainingRegionInfo['cut']) # add cache object tc = TreeCache.TreeCache( name='{region}_{sample}_{tr}'.format(region=trainingRegion, sample=sample.name, tr='TRAIN' if additionalCut==self.TrainCut else 'EVAL'), sample=sample.name, cutList=sampleCuts, inputFolder=self.samplesPath, splitFilesChunks=self.splitFilesChunks, chunkNumber=self.chunkNumber, splitFilesChunkSize=self.splitFilesChunkSize, branches=mvaBranches, config=self.config, debug=True ) # check if this part of the sample is already cached isCached = tc.partIsCached() if not isCached or self.force: if isCached: tc.deleteCachedFiles(chunkNumber=self.chunkNumber) # for the first sample which comes from this files, load the tree if not self.sampleTree: self.sampleTree = SampleTree({'name': sample.identifier, 'folder': self.samplesPath}, splitFilesChunkSize=self.splitFilesChunkSize, chunkNumber=self.chunkNumber, config=self.config, saveMemory=True) treeCaches.append(tc.setSampleTree(self.sampleTree).cache()) if len(treeCaches) > 0: # run on the tree self.sampleTree.process() else: print ("nothing to do!")
# TODO: this can also be made a separate module if 'addbranches' in collections: writeNewVariables = eval(config.get("Regression", "writeNewVariablesDict")) sampleTree.addOutputBranches(writeNewVariables) if 'removebranches' in collections: bl_branch = eval(config.get('Branches', 'useless_branch')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) bl_branch = eval(config.get('Branches', 'useless_after_sys')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) # define output file sampleTree.addOutputTree(tmpFileName, cut='1', branches='*') sampleTree.process() # copy temporary file to output folder if opts.force and fileLocator.exists(outputFileName): fileLocator.rm(outputFileName) try: fileLocator.cp(tmpFileName, outputFileName) except Exception as e: print "\x1b[31mERROR: copy from scratch to final destination failed!!\x1b[0m" print e try: fileLocator.rm(tmpFileName) except Exception as e: print "ERROR: could not delete file on scratch!"
# TODO: this can also be made a separate module if 'addbranches' in collections: writeNewVariables = eval(config.get("Regression", "writeNewVariablesDict")) sampleTree.addOutputBranches(writeNewVariables) if 'removebranches' in collections: bl_branch = eval(config.get('Branches', 'useless_branch')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) bl_branch = eval(config.get('Branches', 'useless_after_sys')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) # define output file sampleTree.addOutputTree(tmpFileName, cut='1', branches='*') sampleTree.process() # copy temporary file to output folder if opts.force and fileLocator.exists(outputFileName): fileLocator.rm(outputFileName) try: fileLocator.cp(tmpFileName, outputFileName) except Exception as e: print "\x1b[31mERROR: copy from scratch to final destination failed!!\x1b[0m" print e try: fileLocator.rm(tmpFileName) except Exception as e: print "ERROR: could not delete file on scratch!"
class CachePlot(object): def __init__(self, config, sampleIdentifier, regions, splitFilesChunks=1, chunkNumber=1, splitFilesChunkSize=-1, forceRedo=False, fileList=None): self.config = config self.sampleIdentifier = sampleIdentifier self.regions = list(set(regions)) self.forceRedo = forceRedo self.sampleTree = None self.samplesPath = self.config.get('Directories', 'plottingSamples') self.samplesInfo = ParseInfo(samples_path=self.samplesPath, config=self.config) self.sampleFilesFolder = self.config.get('Directories', 'samplefiles') self.sampleNames = list( eval(self.config.get('Plot_general', 'samples'))) self.dataNames = list(eval(self.config.get('Plot_general', 'Data'))) self.samples = self.samplesInfo.get_samples(self.sampleNames + self.dataNames) self.regionsDict = {} for region in self.regions: treeCut = config.get('Cuts', region) self.regionsDict[region] = {'cut': treeCut} self.splitFilesChunkSize = splitFilesChunkSize self.splitFilesChunks = splitFilesChunks self.chunkNumber = chunkNumber self.fileList = FileList.decompress(fileList) if fileList else None VHbbNameSpace = config.get('VHbbNameSpace', 'library') returnCode = ROOT.gSystem.Load(VHbbNameSpace) if returnCode != 0: print( "\x1b[31mERROR: loading VHbbNameSpace failed with code %d\x1b[0m" % returnCode) else: print("INFO: loaded VHbbNameSpace: %s" % VHbbNameSpace) def printInfo(self): print("REGION:".ljust(24), "CUT:") for region, regionInfo in self.regionsDict.iteritems(): print(" > ", region.ljust(20), regionInfo['cut']) def run(self): # keep additional branches for plotting try: keepBranchesPlot = eval( self.config.get('Branches', 'keep_branches_plot')) except: keepBranchesPlot = [] try: keepBranchesPlot += eval( self.config.get('Branches', 'keep_branches')) except: pass # also keep some branches which might be used later in variables definition and weights try: for section in self.config.sections(): try: if section.startswith( 'plotDef:') and self.config.has_option( section, 'relPath'): keepBranchesPlot.append( self.config.get(section, 'relPath')) except Exception as e: print("\x1b[31mWARNING: config error in:", section, "=>", e, "\x1b[0m") except Exception as e2: print( "\x1b[31mERROR: config file contains an error! automatic selection of branches to keep will not work!\x1b[0m" ) print(e2) try: keepBranchesPlot.append(self.config.get('Weights', 'weightF')) except: pass # plotting region cut for region, regionInfo in self.regionsDict.iteritems(): keepBranchesPlot.append(regionInfo['cut']) keepBranchesPlotFinal = BranchList( keepBranchesPlot).getListOfBranches() print("KEEP:", keepBranchesPlotFinal) # ---------------------------------------------------------------------------------------------------------------------- # cache samples # ---------------------------------------------------------------------------------------------------------------------- for sampleToCache in [self.sampleIdentifier]: print('*' * 80) print(' ', sampleToCache) print('*' * 80) # prepare caches for training and evaluation samples treeCaches = [] sampleTree = None # for all (sub)samples which come from the same files (sampleIdentifier) subsamples = [ x for x in self.samples if x.identifier == sampleToCache ] for sample in subsamples: # add cuts for all training regions for region, regionInfo in self.regionsDict.iteritems(): configSection = 'Plot:%s' % region # cuts sampleCuts = [sample.subcut] if regionInfo['cut']: sampleCuts.append(regionInfo['cut']) if self.config.has_option(configSection, 'Datacut'): sampleCuts.append( self.config.get(configSection, 'Datacut')) if self.config.has_option('Plot_general', 'addBlindingCut'): sampleCuts.append( self.config.has_option('Plot_general', 'addBlindingCut')) # arbitrary (optional) name for the output tree, used for print-out (the TreeCache object has no idea what it is doing, e.g. dc, plot etc.) cacheName = 'plot:{region}_{sample}'.format( region=region, sample=sample.name) # add cache object tc = TreeCache.TreeCache( name=cacheName, sample=sample.name, cutList=sampleCuts, inputFolder=self.samplesPath, splitFilesChunks=self.splitFilesChunks, chunkNumber=self.chunkNumber, splitFilesChunkSize=self.splitFilesChunkSize, fileList=self.fileList, branches=keepBranchesPlotFinal, config=self.config, debug=True) # check if this part of the sample is already cached isCached = tc.partIsCached() if not isCached or self.forceRedo: if isCached: tc.deleteCachedFiles(chunkNumber=self.chunkNumber) # for the first sample which comes from this files, load the tree if not self.sampleTree: self.sampleTree = SampleTree( { 'name': sample.identifier, 'folder': self.samplesPath }, splitFilesChunkSize=self.splitFilesChunkSize, chunkNumber=self.chunkNumber, config=self.config, saveMemory=True) if not self.sampleTree or not self.sampleTree.tree: print( "\x1b[31mERROR: creation of sample tree failed!!\x1b[0m" ) raise Exception("CreationOfSampleTreeFailed") # consistency check on the file list at submission time and now fileListNow = self.sampleTree.getSampleFileNameChunk( self.chunkNumber) if self.fileList and (sorted(self.fileList) != sorted(fileListNow)): print( "\x1b[31mERROR: sample files have changed between submission and run of the job!\x1b[0m" ) raise Exception("SampleFilesHaveChanged") treeCaches.append( tc.setSampleTree(self.sampleTree).cache()) else: print("INFO: already cached!", tc, "(", tc.hash, ")") if len(treeCaches) > 0: # run on the tree self.sampleTree.process() else: print("nothing to do!")
class CacheDatacards(object): def __init__(self, config, regions, sampleToCache, splitFilesChunks=1, chunkNumber=1, splitFilesChunkSize=-1, forceRedo=False, fileList=None, verbose=False): self.verbose = verbose or ('XBBDEBUG' in os.environ) self.config = config self.regions = regions self.treeCaches = [] self.sampleTree = None self.sampleToCache = sampleToCache self.forceRedo = forceRedo # settings which part of input files to process self.splitFilesChunkSize = splitFilesChunkSize self.splitFilesChunks = splitFilesChunks self.chunkNumber = chunkNumber self.fileList = FileList.decompress(fileList) if fileList else None # initialize Datacard objects self.dcMakers = [ Datacard(config=self.config, region=region) for region in self.regions ] # make a minimum list of samples which is needed to produce all the Datacard regions at the same time def getAllSamples(self): samples = [] for dcMaker in self.dcMakers: for sample in dcMaker.getAllSamples(): if len([x for x in samples if x.name == sample.name]) < 1: samples.append(sample) return samples def prepare(self): if len(self.dcMakers) > 0: self.treeCaches = [] self.sampleTree = None # cuts allSamples = self.getAllSamples() subsamples = [ x for x in allSamples if x.identifier == self.sampleToCache ] # loop over all datacard regions for dcMaker in self.dcMakers: # loop over all subsamples (which come from the same root tree files) for sample in subsamples: # combine subcut and systematics cut with logical AND # systematics cuts are combined with logical OR, such that 1 cache file can be used for all the systematics isData = (sample.type == 'DATA') systematicsCuts = sorted( list( set([ x['cachecut'] for x in dcMaker.getSystematicsList( isData=isData) ]))) sampleCuts = { 'AND': [sample.subcut, { 'OR': systematicsCuts }] } if self.verbose: print( json.dumps(sampleCuts, sort_keys=True, indent=8, default=str)) # make list of branches to keep in root file branchList = BranchList(sample.subcut) branchList.addCut( [x['cachecut'] for x in dcMaker.getSystematicsList()]) branchList.addCut( [x['cut'] for x in dcMaker.getSystematicsList()]) branchList.addCut( [x['var'] for x in dcMaker.getSystematicsList()]) branchList.addCut( [x['weight'] for x in dcMaker.getSystematicsList()]) branchList.addCut(self.config.get('Weights', 'weightF')) branchList.addCut( eval(self.config.get('Branches', 'keep_branches'))) branchesToKeep = branchList.getListOfBranches() # arbitrary (optional) name for the output tree, used for print-out (the TreeCache object has no idea what it is doing, e.g. dc, plot etc.) cacheName = 'dc:{region}_{sample}'.format( region=dcMaker.getRegion(), sample=sample.name) # add cache object tc = TreeCache.TreeCache( name=cacheName, sample=sample.name, cutList=sampleCuts, cutSequenceMode='TREE', branches=branchesToKeep, inputFolder=dcMaker.path, splitFilesChunks=self.splitFilesChunks, chunkNumber=self.chunkNumber, splitFilesChunkSize=self.splitFilesChunkSize, fileList=self.fileList, config=self.config, debug=self.verbose) # check if this part of the sample is already cached isCached = tc.partIsCached() print( "check if sample \x1b[34m{sample}\x1b[0m part {part} is cached:" .format(sample=sample.name, part=self.chunkNumber), isCached) if not isCached or self.forceRedo: if isCached: tc.deleteCachedFiles(chunkNumber=self.chunkNumber) # for the first sample which comes from this files, load the tree if not self.sampleTree: self.sampleTree = SampleTree( { 'name': sample.identifier, 'folder': dcMaker.path }, splitFilesChunkSize=self.splitFilesChunkSize, chunkNumber=self.chunkNumber, config=self.config, saveMemory=True) if not self.sampleTree or not self.sampleTree.tree: print( "\x1b[31mERROR: creation of sample tree failed!!\x1b[0m" ) raise Exception("CreationOfSampleTreeFailed") # consistency check on the file list at submission time and now fileListNow = self.sampleTree.getSampleFileNameChunk( self.chunkNumber) if self.fileList and (sorted(self.fileList) != sorted(fileListNow)): print( "\x1b[31mERROR: sample files have changed between submission and run of the job!\x1b[0m" ) raise Exception("SampleFilesHaveChanged") # connect the TreeCache object to the input sampleTree and add it to the list of cached trees self.treeCaches.append( tc.setSampleTree(self.sampleTree).cache()) else: print("WARNING: no datacard regions added, nothing to do.") return self def run(self): if len(self.treeCaches) > 0: # run on the tree self.sampleTree.process() else: print("nothing to do!")
class CacheTraining(object): def __init__(self, config, sampleIdentifier, trainingRegions, splitFilesChunks=1, chunkNumber=1, splitFilesChunkSize=-1, force=False): self.config = config self.force = force self.sampleIdentifier = sampleIdentifier self.trainingRegions = trainingRegions self.sampleTree = None self.samplesPath = self.config.get('Directories', 'MVAin') self.samplesDefinitions = self.config.get('Directories', 'samplesinfo') self.samplesInfo = ParseInfo(self.samplesDefinitions, self.samplesPath) self.sampleFilesFolder = self.config.get('Directories', 'samplefiles') self.backgroundSampleNames = list( set( sum([ eval(self.config.get(trainingRegion, 'backgrounds')) for trainingRegion in self.trainingRegions ], []))) self.signalSampleNames = list( set( sum([ eval(self.config.get(trainingRegion, 'signals')) for trainingRegion in self.trainingRegions ], []))) self.samples = self.samplesInfo.get_samples( list(set(self.backgroundSampleNames + self.signalSampleNames))) self.trainingRegionsDict = {} for trainingRegion in self.trainingRegions: treeCutName = config.get(trainingRegion, 'treeCut') treeVarSet = config.get(trainingRegion, 'treeVarSet').strip() systematics = [ x for x in config.get('systematics', 'systematics').split(' ') if len(x.strip()) > 0 ] mvaVars = [] for systematic in systematics: mvaVars += config.get(treeVarSet, systematic).strip().split(' ') self.trainingRegionsDict[trainingRegion] = { 'cut': config.get('Cuts', treeCutName), 'vars': mvaVars, } self.TrainCut = config.get('Cuts', 'TrainCut') self.EvalCut = config.get('Cuts', 'EvalCut') self.splitFilesChunks = splitFilesChunks self.chunkNumber = chunkNumber self.splitFilesChunkSize = splitFilesChunkSize VHbbNameSpace = config.get('VHbbNameSpace', 'library') ROOT.gSystem.Load(VHbbNameSpace) def printInfo(self): print("REGION:".ljust(24), "CUT:") for trainingRegion, trainingRegionInfo in self.trainingRegionsDict.iteritems( ): print(" > ", trainingRegion.ljust(20), trainingRegionInfo['cut']) def run(self): # ---------------------------------------------------------------------------------------------------------------------- # cache samples # ---------------------------------------------------------------------------------------------------------------------- for sampleToCache in [self.sampleIdentifier]: print('*' * 80) print(' ', sampleToCache) print('*' * 80) # prepare caches for training and evaluation samples treeCaches = [] self.sampleTree = None # use all (sub)samples which come from the same files (sampleIdentifier) subsamples = [ x for x in self.samples if x.identifier == sampleToCache ] # list of branches to keep for use as MVA input variables branchListOfMVAVars = BranchList() for sample in subsamples: for trainingRegion, trainingRegionInfo in self.trainingRegionsDict.iteritems( ): for additionalCut in [self.TrainCut, self.EvalCut]: branchListOfMVAVars.addCut(trainingRegionInfo['vars']) branchListOfMVAVars.addCut(self.config.get('Weights', 'weightF')) mvaBranches = branchListOfMVAVars.getListOfBranches() # loop over all samples for sample in subsamples: # add cuts for all training regions for trainingRegion, trainingRegionInfo in self.trainingRegionsDict.iteritems( ): # add cuts for training and evaluation for additionalCut in [self.TrainCut, self.EvalCut]: # cuts sampleCuts = [sample.subcut] if additionalCut: sampleCuts.append(additionalCut) if trainingRegionInfo['cut']: sampleCuts.append(trainingRegionInfo['cut']) # add cache object tc = TreeCache.TreeCache( name='{region}_{sample}_{tr}'.format( region=trainingRegion, sample=sample.name, tr='TRAIN' if additionalCut == self.TrainCut else 'EVAL'), sample=sample.name, cutList=sampleCuts, inputFolder=self.samplesPath, splitFilesChunks=self.splitFilesChunks, chunkNumber=self.chunkNumber, splitFilesChunkSize=self.splitFilesChunkSize, branches=mvaBranches, config=self.config, debug=True) # check if this part of the sample is already cached isCached = tc.partIsCached() if not isCached or self.force: if isCached: tc.deleteCachedFiles( chunkNumber=self.chunkNumber) # for the first sample which comes from this files, load the tree if not self.sampleTree: self.sampleTree = SampleTree( { 'name': sample.identifier, 'folder': self.samplesPath }, splitFilesChunkSize=self. splitFilesChunkSize, chunkNumber=self.chunkNumber, config=self.config, saveMemory=True) treeCaches.append( tc.setSampleTree(self.sampleTree).cache()) if len(treeCaches) > 0: # run on the tree self.sampleTree.process() else: print("nothing to do!")
def run(self): nFilesProcessed = 0 nFilesFailed = 0 for subJob in self.subJobs: # only process if output is non-existing/broken or --force was used if self.opts.force or not self.fileLocator.isValidRootFile(subJob['outputFileName']): # create directories outputFolder = '/'.join(subJob['outputFileName'].split('/')[:-1]) tmpFolder = '/'.join(subJob['tmpFileName'].split('/')[:-1]) self.fileLocator.makedirs(outputFolder) self.fileLocator.makedirs(tmpFolder) # load sample tree sampleTree = SampleTree(subJob['localInputFileNames'], config=self.config) if not sampleTree.tree: print "trying fallback...", len(subJob['inputFileNames']) if len(subJob['inputFileNames']) == 1: # try original naming scheme if reading directly from Heppy/Nano ntuples (without prep) fileNameOriginal = self.pathIN + '/' + subJob['inputFileNames'][0] print "FO:", fileNameOriginal xrootdRedirector = self.fileLocator.getRedirector(fileNameOriginal) sampleTree = SampleTree([fileNameOriginal], config=self.config, xrootdRedirector=xrootdRedirector) if not sampleTree.tree: print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED!\x1b[0m" nFilesFailed += 1 continue else: print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED! (old naming scheme not supported for joining multipel files)\x1b[0m" nFilesFailed += 1 continue # to use this syntax, use "--addCollections Sys.Vtype" for a config file entry like this: # [Sys] # Vtype = VtypeCorrector.VtypeCorrector(channel='Zll') # (instead of passing the tree in the constructor, the setTree method can be used) pyModules = [] for collection in self.collections: if '.' in collection: section = collection.split('.')[0] key = collection.split('.')[1] pyCode = self.config.get(section, key) # import module from myutils moduleName = pyCode.split('(')[0].split('.')[0].strip() if self.debug: print "DEBUG: import module:", moduleName print("\x1b[33mDEBUG: " + collection + ": run PYTHON code:\n"+pyCode+"\x1b[0m") globals()[moduleName] = importlib.import_module(".{module}".format(module=moduleName), package="myutils") # get object wObject = eval(pyCode) # pass the tree and other variables if needed to finalize initialization if hasattr(wObject, "customInit") and callable(getattr(wObject, "customInit")): wObject.customInit({'config': self.config, 'sampleTree': sampleTree, 'tree': sampleTree.tree, 'sample': self.sample, 'channel': self.channel, 'pathIN': self.pathIN, 'pathOUT': self.pathOUT, }) # add callbacks if the objects provides any if hasattr(wObject, "processEvent") and callable(getattr(wObject, "processEvent")): sampleTree.addCallback('event', wObject.processEvent) # add branches if hasattr(wObject, "getBranches") and callable(getattr(wObject, "getBranches")): sampleTree.addOutputBranches(wObject.getBranches()) pyModules.append(wObject) # DEPRECATED, do not use anymore ---> use BranchTools.TreeFormulas() if 'addbranches' in self.collections: writeNewVariables = eval(self.config.get("Regression", "writeNewVariablesDict")) sampleTree.addOutputBranches(writeNewVariables) # DEPRECATED, do not use anymore ---> use BranchTools.Drop() if 'removebranches' in self.collections: bl_branch = eval(config.get('Branches', 'useless_branch')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) bl_branch = eval(config.get('Branches', 'useless_after_sys')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) # define output file sampleTree.addOutputTree(subJob['tmpFileName'], cut='1', branches='*', friend=self.opts.friend) # run processing for pyModule in pyModules: if hasattr(pyModule, "beforeProcessing"): getattr(pyModule, "beforeProcessing")() sampleTree.process() for pyModule in pyModules: if hasattr(pyModule, "afterProcessing"): getattr(pyModule, "afterProcessing")() # if output trees have been produced: copy temporary file to output folder if sampleTree.getNumberOfOutputTrees() > 0: try: self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True) print 'copy ', subJob['tmpFileName'], subJob['outputFileName'] if self.verifyCopy: if not self.fileLocator.isValidRootFile(subJob['outputFileName']): print 'INFO: output at final destination broken, try to copy again from scratch disk to final destination...' self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True) print 'INFO: second attempt copy done!' if not self.fileLocator.isValidRootFile(subJob['outputFileName']): print '\x1b[31mERROR: output still broken!\x1b[0m' nFilesFailed += 1 raise Exception("FileCopyError") else: print 'INFO: file is good after second attempt!' except Exception as e: print e print "\x1b[31mERROR: copy from scratch to final destination failed!!\x1b[0m" # delete temporary file try: self.fileLocator.rm(subJob['tmpFileName']) except Exception as e: print e print "WARNING: could not delete file on scratch!" # clean up if hasattr(wObject, "cleanUp") and callable(getattr(wObject, "cleanUp")): getattr(wObject, "cleanUp")() else: print 'SKIP:', subJob['inputFileNames'] if nFilesFailed > 0: raise Exception("ProcessingIncomplete")
class CachePlot(object): def __init__(self, config, sampleIdentifier, regions, splitFilesChunks=1, chunkNumber=1, splitFilesChunkSize=-1, forceRedo=False, fileList=None): self.config = config self.sampleIdentifier = sampleIdentifier self.regions = list(set(regions)) self.forceRedo = forceRedo self.sampleTree = None self.samplesPath = self.config.get('Directories', 'plottingSamples') self.samplesDefinitions = self.config.get('Directories','samplesinfo') self.samplesInfo = ParseInfo(self.samplesDefinitions, self.samplesPath) self.sampleFilesFolder = self.config.get('Directories', 'samplefiles') self.sampleNames = eval(self.config.get('Plot_general', 'samples')) self.dataNames = eval(self.config.get('Plot_general', 'Data')) self.samples = self.samplesInfo.get_samples(self.sampleNames + self.dataNames) self.regionsDict = {} for region in self.regions: treeCut = config.get('Cuts', region) self.regionsDict[region] = {'cut': treeCut} self.splitFilesChunkSize = splitFilesChunkSize self.splitFilesChunks = splitFilesChunks self.chunkNumber = chunkNumber self.fileList = FileList.decompress(fileList) if fileList else None VHbbNameSpace=config.get('VHbbNameSpace','library') returnCode = ROOT.gSystem.Load(VHbbNameSpace) if returnCode != 0: print ("\x1b[31mERROR: loading VHbbNameSpace failed with code %d\x1b[0m"%returnCode) else: print ("INFO: loaded VHbbNameSpace: %s"%VHbbNameSpace) def printInfo(self): print ("REGION:".ljust(24),"CUT:") for region,regionInfo in self.regionsDict.iteritems(): print (" > ",region.ljust(20), regionInfo['cut']) def run(self): # keep additional branches for plotting try: keepBranchesPlot = eval(self.config.get('Branches', 'keep_branches_plot')) except: keepBranchesPlot = [] try: keepBranchesPlot += eval(self.config.get('Branches', 'keep_branches')) except: pass # also keep some branches which might be used later in variables definition and weights try: for section in self.config.sections(): if section.startswith('plotDef:') and self.config.has_option(section, 'relPath'): keepBranchesPlot.append(self.config.get(section, 'relPath')) except Exception as e: print("\x1b[31mERROR: config file contains an error! automatic selection of branches to keep will not work!\x1b[0m") print(e) try: keepBranchesPlot.append(self.config.get('Weights', 'weightF')) except: pass # plotting region cut for region,regionInfo in self.regionsDict.iteritems(): keepBranchesPlot.append(regionInfo['cut']) keepBranchesPlotFinal = BranchList(keepBranchesPlot).getListOfBranches() print("KEEP:", keepBranchesPlotFinal) # ---------------------------------------------------------------------------------------------------------------------- # cache samples # ---------------------------------------------------------------------------------------------------------------------- for sampleToCache in [self.sampleIdentifier]: print ('*'*80) print (' ',sampleToCache) print ('*'*80) # prepare caches for training and evaluation samples treeCaches = [] sampleTree = None # for all (sub)samples which come from the same files (sampleIdentifier) subsamples = [x for x in self.samples if x.identifier == sampleToCache] for sample in subsamples: # add cuts for all training regions for region,regionInfo in self.regionsDict.iteritems(): configSection = 'Plot:%s'%region # cuts sampleCuts = [sample.subcut] if regionInfo['cut']: sampleCuts.append(regionInfo['cut']) if self.config.has_option(configSection, 'Datacut'): sampleCuts.append(self.config.get(configSection, 'Datacut')) if self.config.has_option('Plot_general','addBlindingCut'): sampleCuts.append(self.config.has_option('Plot_general', 'addBlindingCut')) # arbitrary (optional) name for the output tree, used for print-out (the TreeCache object has no idea what it is doing, e.g. dc, plot etc.) cacheName = 'plot:{region}_{sample}'.format(region=region, sample=sample.name) # add cache object tc = TreeCache.TreeCache( name=cacheName, sample=sample.name, cutList=sampleCuts, inputFolder=self.samplesPath, splitFilesChunks=self.splitFilesChunks, chunkNumber=self.chunkNumber, splitFilesChunkSize=self.splitFilesChunkSize, fileList=self.fileList, branches=keepBranchesPlotFinal, config=self.config, debug=True ) # check if this part of the sample is already cached isCached = tc.partIsCached() if not isCached or self.forceRedo: if isCached: tc.deleteCachedFiles(chunkNumber=self.chunkNumber) # for the first sample which comes from this files, load the tree if not self.sampleTree: self.sampleTree = SampleTree({'name': sample.identifier, 'folder': self.samplesPath}, splitFilesChunkSize=self.splitFilesChunkSize, chunkNumber=self.chunkNumber, config=self.config, saveMemory=True) if not self.sampleTree or not self.sampleTree.tree: print ("\x1b[31mERROR: creation of sample tree failed!!\x1b[0m") raise Exception("CreationOfSampleTreeFailed") # consistency check on the file list at submission time and now fileListNow = self.sampleTree.getSampleFileNameChunk(self.chunkNumber) if self.fileList and (sorted(self.fileList) != sorted(fileListNow)): print ("\x1b[31mERROR: sample files have changed between submission and run of the job!\x1b[0m") raise Exception("SampleFilesHaveChanged") treeCaches.append(tc.setSampleTree(self.sampleTree).cache()) else: print ("INFO: already cached!",tc, "(",tc.hash,")") if len(treeCaches) > 0: # run on the tree self.sampleTree.process() else: print ("nothing to do!")
class CacheDatacards(object): def __init__(self, config, regions, sampleToCache, splitFilesChunks=1, chunkNumber=1, splitFilesChunkSize=-1, forceRedo=False, fileList=None, verbose=False): self.verbose = verbose self.config = config self.regions = regions self.treeCaches = [] self.sampleTree = None self.sampleToCache = sampleToCache self.forceRedo = forceRedo # settings which part of input files to process self.splitFilesChunkSize = splitFilesChunkSize self.splitFilesChunks = splitFilesChunks self.chunkNumber = chunkNumber self.fileList = FileList.decompress(fileList) if fileList else None # initialize Datacard objects self.dcMakers = [Datacard(config=self.config, region=region) for region in self.regions] # make a minimum list of samples which is needed to produce all the Datacard regions at the same time def getAllSamples(self): samples = [] for dcMaker in self.dcMakers: for sample in dcMaker.getAllSamples(): if len([x for x in samples if x.name == sample.name]) < 1: samples.append(sample) return samples def prepare(self): if len(self.dcMakers) > 0: self.treeCaches = [] self.sampleTree = None # cuts allSamples = self.getAllSamples() subsamples = [x for x in allSamples if x.identifier == self.sampleToCache] # loop over all datacard regions for dcMaker in self.dcMakers: # loop over all subsamples (which come from the same root tree files) for sample in subsamples: # combine subcut and systematics cut with logical AND # systematics cuts are combined with logical OR, such that 1 cache file can be used for all the systematics isData = (sample.type == 'DATA') systematicsCuts = sorted(list(set([x['cachecut'] for x in dcMaker.getSystematicsList(isData=isData)]))) sampleCuts = {'AND': [sample.subcut, {'OR': systematicsCuts}]} if True or self.verbose: print (json.dumps(sampleCuts, sort_keys=True, indent=8, default=str)) # make list of branches to keep in root file branchList = BranchList(sample.subcut) branchList.addCut([x['cachecut'] for x in dcMaker.getSystematicsList()]) branchList.addCut([x['cut'] for x in dcMaker.getSystematicsList()]) branchList.addCut([x['var'] for x in dcMaker.getSystematicsList()]) branchList.addCut([x['weight'] for x in dcMaker.getSystematicsList()]) branchList.addCut(self.config.get('Weights', 'weightF')) branchList.addCut(eval(self.config.get('Branches', 'keep_branches'))) branchesToKeep = branchList.getListOfBranches() # arbitrary (optional) name for the output tree, used for print-out (the TreeCache object has no idea what it is doing, e.g. dc, plot etc.) cacheName = 'dc:{region}_{sample}'.format(region=dcMaker.getRegion(), sample=sample.name) # add cache object tc = TreeCache.TreeCache( name=cacheName, sample=sample.name, cutList=sampleCuts, cutSequenceMode='TREE', branches=branchesToKeep, inputFolder=dcMaker.path, splitFilesChunks=self.splitFilesChunks, chunkNumber=self.chunkNumber, splitFilesChunkSize=self.splitFilesChunkSize, fileList=self.fileList, config=self.config, debug=self.verbose ) # check if this part of the sample is already cached isCached = tc.partIsCached() print ("check if sample \x1b[34m{sample}\x1b[0m part {part} is cached:".format(sample=sample.name, part=self.chunkNumber), isCached) if not isCached or self.forceRedo: if isCached: tc.deleteCachedFiles(chunkNumber=self.chunkNumber) # for the first sample which comes from this files, load the tree if not self.sampleTree: self.sampleTree = SampleTree({'name': sample.identifier, 'folder': dcMaker.path}, splitFilesChunkSize=self.splitFilesChunkSize, chunkNumber=self.chunkNumber, config=self.config, saveMemory=True) if not self.sampleTree or not self.sampleTree.tree: print ("\x1b[31mERROR: creation of sample tree failed!!\x1b[0m") raise Exception("CreationOfSampleTreeFailed") # consistency check on the file list at submission time and now fileListNow = self.sampleTree.getSampleFileNameChunk(self.chunkNumber) if self.fileList and (sorted(self.fileList) != sorted(fileListNow)): print ("\x1b[31mERROR: sample files have changed between submission and run of the job!\x1b[0m") raise Exception("SampleFilesHaveChanged") # connect the TreeCache object to the input sampleTree and add it to the list of cached trees self.treeCaches.append(tc.setSampleTree(self.sampleTree).cache()) else: print("WARNING: no datacard regions added, nothing to do.") return self def run(self): if len(self.treeCaches) > 0: # run on the tree self.sampleTree.process() else: print ("nothing to do!")