def __init__(self,samples_config,samples_path): try: os.stat(samples_config) except: raise Exception('config file is wrong/missing') if '/pnfs/psi.ch/cms/' in samples_path: T3 = True _,p2=samples_path.split('/pnfs/') t3_path = '/pnfs/'+p2.strip('\n') else: T3 = False config = BetterConfigParser() config.read(samples_config) newprefix=config.get('General','newprefix') lumi=float(config.get('General','lumi')) weightexpression=config.get('General','weightexpression') self._samplelist = [] self.__fileslist=[] # print 'T3',T3,'samples_path',samples_path,'t3_path',t3_path if T3: # ls = os.popen("lcg-ls -b -D srmv2 -l srm://t3se01.psi.ch:8443/srm/managerv2?SFN="+t3_path) ls = os.popen("ls "+t3_path) else: ls = os.popen("ls "+samples_path) for line in ls.readlines(): if('.root' in line): truncated_line = line[line.rfind('/')+1:] _p = findnth(truncated_line,'.',2) self.__fileslist.append(truncated_line[_p+1:truncated_line.rfind('.')]) print '@DEBUG: ' + str(self.__fileslist) run_on_fileList = eval(config.get('Samples_running','run_on_fileList')) if( not test_samples(run_on_fileList,self.__fileslist,config.sections()) ): # stop if it finds None as sample sys.exit('@ERROR: Sample == None. Check RunOnFileList flag in section General, the sample_config of the sample directory.') for _listed_file,_config_entry in map(None,self.__fileslist,config.sections()): if( run_on_fileList ): _sample = _listed_file self._list = self.__fileslist else: _sample = _config_entry self._list = config.sections() sample = self.checkSplittedSample(_sample) print 'sample',sample if not config.has_option(sample,'infile'): continue infile = _sample sampleName = config.get(sample,'sampleName') check_correspondency(sample,self._list,config) #Initialize samplecalss element sampleType = config.get(sample,'sampleType') cut = config.get(sample, 'cut') newsample = Sample(sampleName,sampleType) newsample.addtreecut = cut newsample.identifier=infile newsample.weightexpression=weightexpression newsample.lumi=lumi newsample.prefix=newprefix if eval(config.get(sample,'subsamples')): subnames = eval((config.get(sample, 'subnames'))) subcuts = eval((config.get(sample, 'subcuts'))) subgroups = eval((config.get(sample,'sampleGroup'))) if sampleType != 'DATA': subxsecs = eval((config.get(sample, 'xSec'))) subsfs = eval((config.get(sample, 'SF'))) newsamples = [] for i,cut in enumerate(subcuts): newsubsample = copy(newsample) newsubsample.subsample = True newsubsample.name = subnames[i] newsubsample.subcut = subcuts[i] newsubsample.group = subgroups[i] if sampleType != 'DATA': newsubsample.sf = float(subsfs[i]) newsubsample.xsec = float(subxsecs[i]) newsamples.append(newsubsample) self._samplelist.extend(newsamples) self._samplelist.append(newsample) else: if sampleType != 'DATA': newsample.xsec = eval((config.get(sample,'xSec'))) newsample.sf = eval((config.get(sample, 'SF'))) newsample.group = config.get(sample,'sampleGroup') self._samplelist.append(newsample)
def __init__(self,samples_config,samples_path): ''' Methode filling a list of Sample "self._samplelist = []" contained in the class. "sample_path" contains the path where the samples are stored (PREPin). "samples_config" is the "samples_nosplit.cfg" file. Depending of the variable "run_on_files" defined in "samples_nosplit.cfg", the sample list are generated from the input folder (PREPin) or the list in "samples_nosplit.cfg" ''' self.debug = 'XBBDEBUG' in os.environ if self.debug: print "Start getting infos on all the samples (ParseInfo)" print "==================================================\n" print 'samples_config is', samples_config try: os.stat(samples_config) except: raise Exception('config file is wrong/missing') if samples_path and '/pnfs/psi.ch/cms/' in samples_path: T3 = True _,p2=samples_path.split('/pnfs/') t3_path = '/pnfs/'+p2.strip('\n') else: T3 = False config = BetterConfigParser() config.read(samples_config) # TODO: 08.03.2018: newprefix and weightexpression needed? newprefix = config.get('General','newprefix') if config.has_option('General','newprefix') else '' lumi=float(config.get('General','lumi')) weightexpression=config.get('General','weightexpression') if config.has_option('General','weightexpression') else '' self._samplelist = [] self.__fileslist=[] # TODO: 08.03.2018: clean up this file !!!!!! if samples_path: #!! Store the list of input samples in __fileslist. Reads them directly from the folder defined in PREPin # print 'T3',T3,'samples_path',samples_path,'t3_path',t3_path if T3: ls = os.popen("ls "+t3_path) else: ls = os.popen("ls "+samples_path) #print 'will start the loop over the lines.' #print ls.read() for line in ls.readlines(): #print 'loop over the lines' if('.root' in line): truncated_line = line[line.rfind('/')+1:] _p = findnth(truncated_line,'.',2) self.__fileslist.append(truncated_line[_p+1:truncated_line.rfind('.')]) #print 'added a new line !' print '@DEBUG: ' + str(self.__fileslist) #Deleteme: Do a loop to check on __fileslist #Start the loop #for i in range(0,len(self.__fileslist)): #print 'Is the ',i ,'th file None ? Answer:', (self.__fileslist[i] == None) #End Deleteme run_on_fileList = eval(config.get('Samples_running','run_on_fileList'))#Evaluate run_on_fileList from samples_nosplit.cfg #print 'Is Sample None ? Answer: ', (self.__fileslist == None) if( not test_samples(run_on_fileList,self.__fileslist,config.sections()) ): # stop if it finds None as sample sys.exit('@ERROR: Sample == None. Check RunOnFileList flag in section General, the sample_config of the sample directory.') #!! Start to loop over the samples. If run_on_files list is true, use the sample from the PREPin folder (_listed_file). #!! Else use the sample from samples_nosplit.cfg (_config_entry). #!! The sample description from samples_nosplit.cfg are then applied. for _listed_file,_config_entry in map(None,self.__fileslist,config.sections()): if( run_on_fileList ): _sample = _listed_file self._list = self.__fileslist else: _sample = _config_entry self._list = config.sections() sample = self.checkSplittedSample(_sample)#Check if is splitted and remove the _ if not config.has_option(sample,'sampleName'): continue #Check if the sample has the infile parameter. If not skip infile = _sample # print 'infile',infile sampleName = config.get(sample,'sampleName') check_correspondency(sample,self._list,config)#Check if the sample exists, not fully understood yet #Initialize samplecalss element sampleType = config.get(sample,'sampleType') cut = config.get(sample, 'cut') if config.has_option(sample, 'specialweight'): specialweight = config.get(sample, 'specialweight') else: specialweight = "1" fullname = config.get(sample,'sampleName') mergeCachingSize = int(config.get(sample, 'mergeCachingSize')) if config.has_option(sample, 'mergeCachingSize') else -1 #fill the sample newsample = Sample(sampleName,sampleType) newsample.addtreecut = cut newsample.identifier=infile newsample.weightexpression=weightexpression newsample.specialweight=specialweight newsample.lumi=lumi newsample.prefix=newprefix newsample.FullName = fullname if mergeCachingSize > 0: newsample.mergeCachingSize = mergeCachingSize if config.has_option(sample, 'skipParts'): newsample.skipParts = eval(config.get(sample, 'skipParts')) #add and fills all the subsamples if eval(config.get(sample,'subsamples')): subgroups = eval((config.get(sample,'sampleGroup'))) try: subnames = eval((config.get(sample, 'subnames'))) except: # create subnames automatically based on subgroup name to avoid duplication try: shortname = config.get(sample, 'shortname').strip() except: # use full name if no short name given shortname = sampleName subnames = [shortname + '_' + x for x in subgroups] subcuts = eval((config.get(sample, 'subcuts'))) if sampleType != 'DATA': subxsecs = eval((config.get(sample, 'xSec'))) subsfs = eval((config.get(sample, 'SF'))) try: subspecialweights = eval((config.get(sample, 'specialweight'))) #print 'specialweights=', subspecialweights if len(subspecialweights) < 2: subspecialweights = [] except: subspecialweights = [] newsamples = [] for i,cut in enumerate(subcuts): newsubsample = copy(newsample) newsubsample.subsample = True newsubsample.name = subnames[i] newsubsample.subcut = subcuts[i] newsubsample.group = subgroups[i] if sampleType != 'DATA': newsubsample.sf = float(subsfs[i]) newsubsample.xsec = float(subxsecs[i]) if len(subspecialweights) == len(subcuts): newsubsample.specialweight = subspecialweights[i] newsamples.append(newsubsample) #print 'newsubsample:', newsubsample self._samplelist.extend(newsamples) self._samplelist.append(newsample) else: if sampleType != 'DATA': newsample.xsec = eval((config.get(sample,'xSec'))) newsample.sf = eval((config.get(sample, 'SF'))) newsample.group = config.get(sample,'sampleGroup') self._samplelist.append(newsample) if self.debug: print "Finished getting infos on all the samples (ParseInfo)" print "=====================================================\n"
'').replace('dcap://t3se01.psi.ch:22125/', '').replace('root://t3dcachedb03.psi.ch:1094/', '') # print "opts.task",opts.task samplefiles = config.get('Directories', 'samplefiles') hash = '' if opts.region: print 'opts.region', opts.region region = opts.region print 'evaluating cuts for region', region section = 'Plot:%s' % region vars = (config.get(section, 'vars')).split( ',') #get the variables to be ploted in each region print 'vars', vars SignalRegion = False if config.has_option(section, 'Signal'): # mc.append(config.get(section,'Signal')) SignalRegion = True # # #GETALL AT ONCE options = [] Stacks = [] #print "Start Loop over the list of variables(to fill the StackMaker )" print "==============================================================\n" for i in range( len(vars) ): # loop over the list of variables to be ploted in each reagion # print "The variable is ", vars[i], "\n" Stacks.append( StackMaker.StackMaker(config, vars[i], region, SignalRegion) ) # defined in myutils DoubleStackMaker. The StackMaker merge together all the informations necessary to perform the plot (plot region, variables, samples and signal region ). "options" contains the variables information, including the cuts. options.append(Stacks[i].options) cuts = []
pathIN_orig = pathIN_orig.replace('gsidcap://t3se01.psi.ch:22128/','').replace('dcap://t3se01.psi.ch:22125/','').replace('root://t3dcachedb03.psi.ch:1094/','') # print "opts.task",opts.task samplefiles = config.get('Directories','samplefiles') print 'pathIN_orig',pathIN_orig print 'pathOUT_orig',pathOUT_orig hash = '' if opts.region: # print 'opts.region',opts.region region = opts.region print 'evaluating cuts for region',region section='Plot:%s'%region vars = (config.get(section, 'vars')).split(',')#get the variables to be ploted in each region # print 'vars',vars SignalRegion = False if config.has_option(section,'Signal'): # mc.append(config.get(section,'Signal')) SignalRegion = True # # #GETALL AT ONCE options = [] Stacks = [] #print "Start Loop over the list of variables(to fill the StackMaker )" print "==============================================================\n" for i in range(len(vars)):# loop over the list of variables to be ploted in each reagion # print "The variable is ", vars[i], "\n" Stacks.append(StackMaker.StackMaker(config,vars[i],region,SignalRegion))# defined in myutils DoubleStackMaker. The StackMaker merge together all the informations necessary to perform the plot (plot region, variables, samples and signal region ). "options" contains the variables information, including the cuts. options.append(Stacks[i].options) cuts = [] for option in options: cuts.append(option['cut']) _cutList = [] #! Make the cut lists from inputs
def __init__(self, samples_config, samples_path): try: os.stat(samples_config) except: raise Exception('config file is wrong/missing') if '/pnfs/psi.ch/cms/' in samples_path: T3 = True _, p2 = samples_path.split('/pnfs/') t3_path = '/pnfs/' + p2.strip('\n') else: T3 = False config = BetterConfigParser() config.read(samples_config) newprefix = config.get('General', 'newprefix') lumi = float(config.get('General', 'lumi')) weightexpression = config.get('General', 'weightexpression') self._samplelist = [] self.__fileslist = [] if T3: ls = os.popen( "lcg-ls -b -D srmv2 -l srm://t3se01.psi.ch:8443/srm/managerv2?SFN=" + t3_path) else: ls = os.popen("ls -l " + samples_path) for line in ls.readlines(): if ('.root' in line): truncated_line = line[line.rfind('/') + 1:] _p = findnth(truncated_line, '.', 2) self.__fileslist.append( truncated_line[_p + 1:truncated_line.rfind('.')]) print '@DEBUG: ' + str(self.__fileslist) run_on_fileList = eval(config.get('Samples_running', 'run_on_fileList')) if (not test_samples( run_on_fileList, self.__fileslist, config.sections())): # stop if it finds None as sample sys.exit( '@ERROR: Sample == None. Check RunOnFileList flag in section General, the sample_config of the sample directory.' ) for _listed_file, _config_entry in map(None, self.__fileslist, config.sections()): if (run_on_fileList): _sample = _listed_file self._list = self.__fileslist else: _sample = _config_entry self._list = config.sections() sample = self.checkSplittedSample(_sample) if not config.has_option(sample, 'infile'): continue infile = _sample sampleName = config.get(sample, 'sampleName') check_correspondency(sample, self._list, config) #Initialize samplecalss element sampleType = config.get(sample, 'sampleType') cut = config.get(sample, 'cut') newsample = Sample(sampleName, sampleType) newsample.addtreecut = cut newsample.identifier = infile newsample.weightexpression = weightexpression newsample.lumi = lumi newsample.prefix = newprefix if eval(config.get(sample, 'subsamples')): subnames = eval((config.get(sample, 'subnames'))) subcuts = eval((config.get(sample, 'subcuts'))) subgroups = eval((config.get(sample, 'sampleGroup'))) if sampleType != 'DATA': subxsecs = eval((config.get(sample, 'xSec'))) subsfs = eval((config.get(sample, 'SF'))) newsamples = [] for i, cut in enumerate(subcuts): newsubsample = copy(newsample) newsubsample.subsample = True newsubsample.name = subnames[i] newsubsample.subcut = subcuts[i] newsubsample.group = subgroups[i] if sampleType != 'DATA': newsubsample.sf = float(subsfs[i]) newsubsample.xsec = float(subxsecs[i]) newsamples.append(newsubsample) self._samplelist.extend(newsamples) self._samplelist.append(newsample) else: if sampleType != 'DATA': newsample.xsec = eval((config.get(sample, 'xSec'))) newsample.sf = eval((config.get(sample, 'SF'))) newsample.group = config.get(sample, 'sampleGroup') self._samplelist.append(newsample)
def __init__(self,samples_config,samples_path,config=None): ''' Methode filling a list of Sample "self._samplelist = []" contained in the class. "sample_path" contains the path where the samples are stored (PREPin). "samples_config" is the "samples_nosplit.cfg" file. Depending of the variable "run_on_files" defined in "samples_nosplit.cfg", the sample list are generated from the input folder (PREPin) or the list in "samples_nosplit.cfg" ''' self.debug = 'XBBDEBUG' in os.environ if not config: if self.debug: print "Start getting infos on all the samples (ParseInfo)" print "==================================================\n" print 'samples_config is', samples_config try: os.stat(samples_config) except: raise Exception('config file is wrong/missing') # if no config object given, read full config again # this should be avoided by passing config object to ParseInfo class pathConfig = '/'.join(samples_config.split('/')[:-1]) + '/paths.ini' if os.path.isfile(pathConfig): config = BetterConfigParser() config.read(pathConfig) configList = config.get('Configuration', 'List').split(' ') for configFileName in configList: configFileName = '/'.join(samples_config.split('/')[:-1]) + '/' + configFileName if os.path.isfile(configFileName): config.read(configFileName) if self.debug: print "WARNING (performance): whole config read again, pass the config obect to ParseInfo class to avoid this." else: # old behavior: read only one file, config might be inconsistent config = BetterConfigParser() config.read(samples_config) elif self.debug: print "DEBUG: config object passed to sample parser." lumi = float(config.get('General','lumi')) self._samplelist = [] self.__fileslist = [] configSamples = [x for x in config.sections() if config.has_option(x, 'sampleName')] if self.debug: print "DEBUG:", len(configSamples), " samples found." for sample in configSamples: sampleName = config.get(sample, 'sampleName') sampleType = config.get(sample,'sampleType') cut = config.get(sample, 'cut') specialweight = config.get(sample, 'specialweight') if config.has_option(sample, 'specialweight') else "1" fullname = config.get(sample, 'sampleName') mergeCachingSize = int(config.get(sample, 'mergeCachingSize')) if config.has_option(sample, 'mergeCachingSize') else -1 #fill the sample newsample = Sample(sampleName, sampleType) newsample.addtreecut = cut newsample.identifier = sample newsample.weightexpression = '' newsample.specialweight = specialweight newsample.lumi = lumi newsample.prefix = '' newsample.FullName = sampleName if mergeCachingSize > 0: newsample.mergeCachingSize = mergeCachingSize if config.has_option(sample, 'skipParts'): newsample.skipParts = eval(config.get(sample, 'skipParts')) newsample.index = eval(config.get(sample, 'sampleIndex')) if config.has_option(sample, 'sampleIndex') else -999999 # add and fill all the subsamples if config.has_option(sample,'subsamples') and eval(config.get(sample,'subsamples')): subgroups = eval((config.get(sample,'sampleGroup'))) try: subnames = eval((config.get(sample, 'subnames'))) except: # create subnames automatically based on subgroup name to avoid duplication try: shortname = config.get(sample, 'shortname').strip() except: # use full name if no short name given shortname = sampleName subnames = [shortname + '_' + x for x in subgroups] subcuts = eval((config.get(sample, 'subcuts'))) if sampleType != 'DATA': subxsecs = eval((config.get(sample, 'xSec'))) if len(list(set(subxsecs))) == 1: newsample.xsec = [subxsecs[0]] else: print "\x1b[31mWARNING: different cross sections for the sub-samples of", sampleName, " are you sure you want to do this?\x1b[0m" subsfs = eval((config.get(sample, 'SF'))) if config.has_option(sample, 'SF') else [1.0]*len(subxsecs) try: subspecialweights = eval((config.get(sample, 'specialweight'))) #print 'specialweights=', subspecialweights if len(subspecialweights) < 2: subspecialweights = [] print "\x1b[31mWARNING: specialweight not defined for subsamples but for full sample only!\x1b[0m" except: subspecialweights = [] subindices = None newsamples = [] for i,cut in enumerate(subcuts): newsubsample = copy(newsample) newsubsample.subsample = True newsubsample.name = subnames[i] newsubsample.subcut = subcuts[i] newsubsample.group = subgroups[i] if sampleType != 'DATA': newsubsample.sf = float(subsfs[i]) newsubsample.xsec = float(subxsecs[i]) if len(subspecialweights) == len(subcuts): newsubsample.specialweight = subspecialweights[i] if type(newsample.index) == list: newsubsample.index = newsample.index[i] newsamples.append(newsubsample) self._samplelist.extend(newsamples) self._samplelist.append(newsample) else: if sampleType != 'DATA': newsample.xsec = eval((config.get(sample,'xSec'))) newsample.sf = eval((config.get(sample, 'SF'))) if config.has_option(sample, 'SF') else 1.0 newsample.group = config.get(sample,'sampleGroup') self._samplelist.append(newsample)
def __init__(self, samples_config, samples_path): ''' Methode filling a list of Sample "self._samplelist = []" contained in the class. "sample_path" contains the path where the samples are stored (PREPin). "samples_config" is the "samples_nosplit.cfg" file. Depending of the variable "run_on_files" defined in "samples_nosplit.cfg", the sample list are generated from the input folder (PREPin) or the list in "samples_nosplit.cfg" ''' print "Start getting infos on all the samples (ParseInfo)" print "==================================================\n" print 'samples_config is', samples_config try: os.stat(samples_config) except: raise Exception('config file is wrong/missing') if '/pnfs/psi.ch/cms/' in samples_path: T3 = True _, p2 = samples_path.split('/pnfs/') t3_path = '/pnfs/' + p2.strip('\n') else: T3 = False config = BetterConfigParser() config.read(samples_config) newprefix = config.get('General', 'newprefix') lumi = float(config.get('General', 'lumi')) weightexpression = config.get('General', 'weightexpression') self._samplelist = [] #!! Store the list of input samples in __fileslist. Reads them directly from the folder defined in PREPin self.__fileslist = [] # print 'T3',T3,'samples_path',samples_path,'t3_path',t3_path if T3: ls = os.popen("ls " + t3_path) else: ls = os.popen("ls " + samples_path) #print 'will start the loop over the lines.' #print ls.read() for line in ls.readlines(): #print 'loop over the lines' if ('.root' in line): truncated_line = line[line.rfind('/') + 1:] _p = findnth(truncated_line, '.', 2) self.__fileslist.append( truncated_line[_p + 1:truncated_line.rfind('.')]) #print 'added a new line !' print '@DEBUG: ' + str(self.__fileslist) #Deleteme: Do a loop to check on __fileslist #Start the loop #for i in range(0,len(self.__fileslist)): #print 'Is the ',i ,'th file None ? Answer:', (self.__fileslist[i] == None) #End Deleteme run_on_fileList = eval( config.get('Samples_running', 'run_on_fileList' )) #Evaluate run_on_fileList from samples_nosplit.cfg #print 'Is Sample None ? Answer: ', (self.__fileslist == None) if (not test_samples( run_on_fileList, self.__fileslist, config.sections())): # stop if it finds None as sample sys.exit( '@ERROR: Sample == None. Check RunOnFileList flag in section General, the sample_config of the sample directory.' ) #!! Start to loop over the samples. If run_on_files list is true, use the sample from the PREPin folder (_listed_file). #!! Else use the sample from samples_nosplit.cfg (_config_entry). #!! The sample description from samples_nosplit.cfg are then applied. for _listed_file, _config_entry in map(None, self.__fileslist, config.sections()): if (run_on_fileList): _sample = _listed_file self._list = self.__fileslist else: _sample = _config_entry self._list = config.sections() sample = self.checkSplittedSample( _sample) #Check if is splitted and remove the _ if not config.has_option(sample, 'sampleName'): continue #Check if the sample has the infile parameter. If not skip infile = _sample # print 'infile',infile sampleName = config.get(sample, 'sampleName') check_correspondency( sample, self._list, config) #Check if the sample exists, not fully understood yet #Initialize samplecalss element sampleType = config.get(sample, 'sampleType') cut = config.get(sample, 'cut') if config.has_option(sample, 'specialweight'): specialweight = config.get(sample, 'specialweight') else: specialweight = "1" #fill the sample newsample = Sample(sampleName, sampleType) newsample.addtreecut = cut newsample.identifier = infile newsample.weightexpression = weightexpression newsample.specialweight = specialweight newsample.lumi = lumi newsample.prefix = newprefix #add and fills all the subsamples if eval(config.get(sample, 'subsamples')): subnames = eval((config.get(sample, 'subnames'))) subcuts = eval((config.get(sample, 'subcuts'))) subgroups = eval((config.get(sample, 'sampleGroup'))) if sampleType != 'DATA': subxsecs = eval((config.get(sample, 'xSec'))) subsfs = eval((config.get(sample, 'SF'))) newsamples = [] for i, cut in enumerate(subcuts): newsubsample = copy(newsample) newsubsample.subsample = True newsubsample.name = subnames[i] newsubsample.subcut = subcuts[i] newsubsample.group = subgroups[i] if sampleType != 'DATA': newsubsample.sf = float(subsfs[i]) newsubsample.xsec = float(subxsecs[i]) newsamples.append(newsubsample) self._samplelist.extend(newsamples) self._samplelist.append(newsample) else: if sampleType != 'DATA': newsample.xsec = eval((config.get(sample, 'xSec'))) newsample.sf = eval((config.get(sample, 'SF'))) newsample.group = config.get(sample, 'sampleGroup') self._samplelist.append(newsample) print "Finished getting infos on all the samples (ParseInfo)" print "=====================================================\n"