def __init__(self,samples_config,samples_path):
'''
Methode filling a list of Sample "self._samplelist = []" contained in the class.
"sample_path" contains the path where the samples are stored (PREPin).
"samples_config" is the "samples_nosplit.cfg" file. Depending of the variable "run_on_files" defined in "samples_nosplit.cfg",
the sample list are generated from the input folder (PREPin) or the list in "samples_nosplit.cfg" '''
self.debug = 'XBBDEBUG' in os.environ
if self.debug:
print "Start getting infos on all the samples (ParseInfo)"
print "==================================================\n"
print 'samples_config is', samples_config
try:
os.stat(samples_config)
except:
raise Exception('config file is wrong/missing')
if samples_path and '/pnfs/psi.ch/cms/' in samples_path:
T3 = True
_,p2=samples_path.split('/pnfs/')
t3_path = '/pnfs/'+p2.strip('\n')
else:
T3 = False
config = BetterConfigParser()
config.read(samples_config)
# TODO: 08.03.2018: newprefix and weightexpression needed?
newprefix = config.get('General','newprefix') if config.has_option('General','newprefix') else ''
lumi=float(config.get('General','lumi'))
weightexpression=config.get('General','weightexpression') if config.has_option('General','weightexpression') else ''
self._samplelist = []
self.__fileslist=[]
# TODO: 08.03.2018: clean up this file !!!!!!
if samples_path:
#!! Store the list of input samples in __fileslist. Reads them directly from the folder defined in PREPin
# print 'T3',T3,'samples_path',samples_path,'t3_path',t3_path
if T3:
ls = os.popen("ls "+t3_path)
else:
ls = os.popen("ls "+samples_path)
#print 'will start the loop over the lines.'
#print ls.read()
for line in ls.readlines():
#print 'loop over the lines'
if('.root' in line):
truncated_line = line[line.rfind('/')+1:]
_p = findnth(truncated_line,'.',2)
self.__fileslist.append(truncated_line[_p+1:truncated_line.rfind('.')])
#print 'added a new line !'
print '@DEBUG: ' + str(self.__fileslist)
#Deleteme: Do a loop to check on __fileslist
#Start the loop
#for i in range(0,len(self.__fileslist)):
#print 'Is the ',i ,'th file None ? Answer:', (self.__fileslist[i] == None)
#End Deleteme
run_on_fileList = eval(config.get('Samples_running','run_on_fileList'))#Evaluate run_on_fileList from samples_nosplit.cfg
#print 'Is Sample None ? Answer: ', (self.__fileslist == None)
if( not test_samples(run_on_fileList,self.__fileslist,config.sections()) ): # stop if it finds None as sample
sys.exit('@ERROR: Sample == None. Check RunOnFileList flag in section General, the sample_config of the sample directory.')
#!! Start to loop over the samples. If run_on_files list is true, use the sample from the PREPin folder (_listed_file).
#!! Else use the sample from samples_nosplit.cfg (_config_entry).
#!! The sample description from samples_nosplit.cfg are then applied.
for _listed_file,_config_entry in map(None,self.__fileslist,config.sections()):
if( run_on_fileList ):
_sample = _listed_file
self._list = self.__fileslist
else:
_sample = _config_entry
self._list = config.sections()
sample = self.checkSplittedSample(_sample)#Check if is splitted and remove the _
if not config.has_option(sample,'sampleName'): continue #Check if the sample has the infile parameter. If not skip
infile = _sample
# print 'infile',infile
sampleName = config.get(sample,'sampleName')
check_correspondency(sample,self._list,config)#Check if the sample exists, not fully understood yet
#Initialize samplecalss element
sampleType = config.get(sample,'sampleType')
cut = config.get(sample, 'cut')
if config.has_option(sample, 'specialweight'):
specialweight = config.get(sample, 'specialweight')
else:
specialweight = "1"
fullname = config.get(sample,'sampleName')
mergeCachingSize = int(config.get(sample, 'mergeCachingSize')) if config.has_option(sample, 'mergeCachingSize') else -1
#fill the sample
newsample = Sample(sampleName,sampleType)
newsample.addtreecut = cut
newsample.identifier=infile
newsample.weightexpression=weightexpression
newsample.specialweight=specialweight
newsample.lumi=lumi
newsample.prefix=newprefix
newsample.FullName = fullname
if mergeCachingSize > 0:
newsample.mergeCachingSize = mergeCachingSize
if config.has_option(sample, 'skipParts'):
newsample.skipParts = eval(config.get(sample, 'skipParts'))
#add and fills all the subsamples
if eval(config.get(sample,'subsamples')):
subgroups = eval((config.get(sample,'sampleGroup')))
try:
subnames = eval((config.get(sample, 'subnames')))
except:
# create subnames automatically based on subgroup name to avoid duplication
try:
shortname = config.get(sample, 'shortname').strip()
except:
# use full name if no short name given
shortname = sampleName
subnames = [shortname + '_' + x for x in subgroups]
subcuts = eval((config.get(sample, 'subcuts')))
if sampleType != 'DATA':
subxsecs = eval((config.get(sample, 'xSec')))
subsfs = eval((config.get(sample, 'SF')))
try:
subspecialweights = eval((config.get(sample, 'specialweight')))
#print 'specialweights=', subspecialweights
if len(subspecialweights) < 2:
subspecialweights = []
except:
subspecialweights = []
newsamples = []
for i,cut in enumerate(subcuts):
newsubsample = copy(newsample)
newsubsample.subsample = True
newsubsample.name = subnames[i]
newsubsample.subcut = subcuts[i]
newsubsample.group = subgroups[i]
if sampleType != 'DATA':
newsubsample.sf = float(subsfs[i])
newsubsample.xsec = float(subxsecs[i])
if len(subspecialweights) == len(subcuts):
newsubsample.specialweight = subspecialweights[i]
newsamples.append(newsubsample)
#print 'newsubsample:', newsubsample
self._samplelist.extend(newsamples)
self._samplelist.append(newsample)
else:
if sampleType != 'DATA':
newsample.xsec = eval((config.get(sample,'xSec')))
newsample.sf = eval((config.get(sample, 'SF')))
newsample.group = config.get(sample,'sampleGroup')
self._samplelist.append(newsample)
if self.debug:
print "Finished getting infos on all the samples (ParseInfo)"
print "=====================================================\n"
def __init__(self, samples_config, samples_path):
'''
Methode filling a list of Sample "self._samplelist = []" contained in the class.
"sample_path" contains the path where the samples are stored (PREPin).
"samples_config" is the "samples_nosplit.cfg" file. Depending of the variable "run_on_files" defined in "samples_nosplit.cfg",
the sample list are generated from the input folder (PREPin) or the list in "samples_nosplit.cfg" '''
self.debug = 'XBBDEBUG' in os.environ
if self.debug:
print "Start getting infos on all the samples (ParseInfo)"
print "==================================================\n"
print 'samples_config is', samples_config
try:
os.stat(samples_config)
except:
raise Exception('config file is wrong/missing')
if samples_path and '/pnfs/psi.ch/cms/' in samples_path:
T3 = True
_, p2 = samples_path.split('/pnfs/')
t3_path = '/pnfs/' + p2.strip('\n')
else:
T3 = False
config = BetterConfigParser()
config.read(samples_config)
# TODO: 08.03.2018: newprefix and weightexpression needed?
newprefix = config.get('General', 'newprefix') if config.has_option(
'General', 'newprefix') else ''
lumi = float(config.get('General', 'lumi'))
weightexpression = config.get(
'General', 'weightexpression') if config.has_option(
'General', 'weightexpression') else ''
self._samplelist = []
self.__fileslist = []
# TODO: 08.03.2018: clean up this file !!!!!!
if samples_path:
#!! Store the list of input samples in __fileslist. Reads them directly from the folder defined in PREPin
# print 'T3',T3,'samples_path',samples_path,'t3_path',t3_path
if T3:
ls = os.popen("ls " + t3_path)
else:
ls = os.popen("ls " + samples_path)
#print 'will start the loop over the lines.'
#print ls.read()
for line in ls.readlines():
#print 'loop over the lines'
if ('.root' in line):
truncated_line = line[line.rfind('/') + 1:]
_p = findnth(truncated_line, '.', 2)
self.__fileslist.append(
truncated_line[_p + 1:truncated_line.rfind('.')])
#print 'added a new line !'
print '@DEBUG: ' + str(self.__fileslist)
#Deleteme: Do a loop to check on __fileslist
#Start the loop
#for i in range(0,len(self.__fileslist)):
#print 'Is the ',i ,'th file None ? Answer:', (self.__fileslist[i] == None)
#End Deleteme
run_on_fileList = eval(
config.get('Samples_running', 'run_on_fileList'
)) #Evaluate run_on_fileList from samples_nosplit.cfg
#print 'Is Sample None ? Answer: ', (self.__fileslist == None)
if (not test_samples(
run_on_fileList, self.__fileslist,
config.sections())): # stop if it finds None as sample
sys.exit(
'@ERROR: Sample == None. Check RunOnFileList flag in section General, the sample_config of the sample directory.'
)
#!! Start to loop over the samples. If run_on_files list is true, use the sample from the PREPin folder (_listed_file).
#!! Else use the sample from samples_nosplit.cfg (_config_entry).
#!! The sample description from samples_nosplit.cfg are then applied.
for _listed_file, _config_entry in map(None, self.__fileslist,
config.sections()):
if (run_on_fileList):
_sample = _listed_file
self._list = self.__fileslist
else:
_sample = _config_entry
self._list = config.sections()
sample = self.checkSplittedSample(
_sample) #Check if is splitted and remove the _
if not config.has_option(sample, 'sampleName'):
continue #Check if the sample has the infile parameter. If not skip
infile = _sample
# print 'infile',infile
sampleName = config.get(sample, 'sampleName')
check_correspondency(
sample, self._list,
config) #Check if the sample exists, not fully understood yet
#Initialize samplecalss element
sampleType = config.get(sample, 'sampleType')
cut = config.get(sample, 'cut')
if config.has_option(sample, 'specialweight'):
specialweight = config.get(sample, 'specialweight')
else:
specialweight = "1"
fullname = config.get(sample, 'sampleName')
mergeCachingSize = int(config.get(
sample, 'mergeCachingSize')) if config.has_option(
sample, 'mergeCachingSize') else -1
#fill the sample
newsample = Sample(sampleName, sampleType)
newsample.addtreecut = cut
newsample.identifier = infile
newsample.weightexpression = weightexpression
newsample.specialweight = specialweight
newsample.lumi = lumi
newsample.prefix = newprefix
newsample.FullName = fullname
if mergeCachingSize > 0:
newsample.mergeCachingSize = mergeCachingSize
if config.has_option(sample, 'skipParts'):
newsample.skipParts = eval(config.get(sample, 'skipParts'))
#add and fills all the subsamples
if eval(config.get(sample, 'subsamples')):
subgroups = eval((config.get(sample, 'sampleGroup')))
try:
subnames = eval((config.get(sample, 'subnames')))
except:
# create subnames automatically based on subgroup name to avoid duplication
try:
shortname = config.get(sample, 'shortname').strip()
except:
# use full name if no short name given
shortname = sampleName
subnames = [shortname + '_' + x for x in subgroups]
subcuts = eval((config.get(sample, 'subcuts')))
if sampleType != 'DATA':
subxsecs = eval((config.get(sample, 'xSec')))
subsfs = eval((config.get(sample, 'SF')))
try:
subspecialweights = eval(
(config.get(sample, 'specialweight')))
#print 'specialweights=', subspecialweights
if len(subspecialweights) < 2:
subspecialweights = []
except:
subspecialweights = []
newsamples = []
for i, cut in enumerate(subcuts):
newsubsample = copy(newsample)
newsubsample.subsample = True
newsubsample.name = subnames[i]
newsubsample.subcut = subcuts[i]
newsubsample.group = subgroups[i]
if sampleType != 'DATA':
newsubsample.sf = float(subsfs[i])
newsubsample.xsec = float(subxsecs[i])
if len(subspecialweights) == len(subcuts):
newsubsample.specialweight = subspecialweights[i]
newsamples.append(newsubsample)
#print 'newsubsample:', newsubsample
self._samplelist.extend(newsamples)
self._samplelist.append(newsample)
else:
if sampleType != 'DATA':
newsample.xsec = eval((config.get(sample, 'xSec')))
newsample.sf = eval((config.get(sample, 'SF')))
newsample.group = config.get(sample, 'sampleGroup')
self._samplelist.append(newsample)
if self.debug:
print "Finished getting infos on all the samples (ParseInfo)"
print "=====================================================\n"
def __init__(self,samples_config,samples_path,config=None):
'''
Methode filling a list of Sample "self._samplelist = []" contained in the class.
"sample_path" contains the path where the samples are stored (PREPin).
"samples_config" is the "samples_nosplit.cfg" file. Depending of the variable "run_on_files" defined in "samples_nosplit.cfg",
the sample list are generated from the input folder (PREPin) or the list in "samples_nosplit.cfg" '''
self.debug = 'XBBDEBUG' in os.environ
if not config:
if self.debug:
print "Start getting infos on all the samples (ParseInfo)"
print "==================================================\n"
print 'samples_config is', samples_config
try:
os.stat(samples_config)
except:
raise Exception('config file is wrong/missing')
# if no config object given, read full config again
# this should be avoided by passing config object to ParseInfo class
pathConfig = '/'.join(samples_config.split('/')[:-1]) + '/paths.ini'
if os.path.isfile(pathConfig):
config = BetterConfigParser()
config.read(pathConfig)
configList = config.get('Configuration', 'List').split(' ')
for configFileName in configList:
configFileName = '/'.join(samples_config.split('/')[:-1]) + '/' + configFileName
if os.path.isfile(configFileName):
config.read(configFileName)
if self.debug:
print "WARNING (performance): whole config read again, pass the config obect to ParseInfo class to avoid this."
else:
# old behavior: read only one file, config might be inconsistent
config = BetterConfigParser()
config.read(samples_config)
elif self.debug:
print "DEBUG: config object passed to sample parser."
lumi = float(config.get('General','lumi'))
self._samplelist = []
self.__fileslist = []
configSamples = [x for x in config.sections() if config.has_option(x, 'sampleName')]
if self.debug:
print "DEBUG:", len(configSamples), " samples found."
for sample in configSamples:
sampleName = config.get(sample, 'sampleName')
sampleType = config.get(sample,'sampleType')
cut = config.get(sample, 'cut')
specialweight = config.get(sample, 'specialweight') if config.has_option(sample, 'specialweight') else "1"
fullname = config.get(sample, 'sampleName')
mergeCachingSize = int(config.get(sample, 'mergeCachingSize')) if config.has_option(sample, 'mergeCachingSize') else -1
#fill the sample
newsample = Sample(sampleName, sampleType)
newsample.addtreecut = cut
newsample.identifier = sample
newsample.weightexpression = ''
newsample.specialweight = specialweight
newsample.lumi = lumi
newsample.prefix = ''
newsample.FullName = sampleName
if mergeCachingSize > 0:
newsample.mergeCachingSize = mergeCachingSize
if config.has_option(sample, 'skipParts'):
newsample.skipParts = eval(config.get(sample, 'skipParts'))
newsample.index = eval(config.get(sample, 'sampleIndex')) if config.has_option(sample, 'sampleIndex') else -999999
# add and fill all the subsamples
if config.has_option(sample,'subsamples') and eval(config.get(sample,'subsamples')):
subgroups = eval((config.get(sample,'sampleGroup')))
try:
subnames = eval((config.get(sample, 'subnames')))
except:
# create subnames automatically based on subgroup name to avoid duplication
try:
shortname = config.get(sample, 'shortname').strip()
except:
# use full name if no short name given
shortname = sampleName
subnames = [shortname + '_' + x for x in subgroups]
subcuts = eval((config.get(sample, 'subcuts')))
if sampleType != 'DATA':
subxsecs = eval((config.get(sample, 'xSec')))
if len(list(set(subxsecs))) == 1:
newsample.xsec = [subxsecs[0]]
else:
print "\x1b[31mWARNING: different cross sections for the sub-samples of", sampleName, " are you sure you want to do this?\x1b[0m"
subsfs = eval((config.get(sample, 'SF'))) if config.has_option(sample, 'SF') else [1.0]*len(subxsecs)
try:
subspecialweights = eval((config.get(sample, 'specialweight')))
#print 'specialweights=', subspecialweights
if len(subspecialweights) < 2:
subspecialweights = []
print "\x1b[31mWARNING: specialweight not defined for subsamples but for full sample only!\x1b[0m"
except:
subspecialweights = []
subindices = None
newsamples = []
for i,cut in enumerate(subcuts):
newsubsample = copy(newsample)
newsubsample.subsample = True
newsubsample.name = subnames[i]
newsubsample.subcut = subcuts[i]
newsubsample.group = subgroups[i]
if sampleType != 'DATA':
newsubsample.sf = float(subsfs[i])
newsubsample.xsec = float(subxsecs[i])
if len(subspecialweights) == len(subcuts):
newsubsample.specialweight = subspecialweights[i]
if type(newsample.index) == list:
newsubsample.index = newsample.index[i]
newsamples.append(newsubsample)
self._samplelist.extend(newsamples)
self._samplelist.append(newsample)
else:
if sampleType != 'DATA':
newsample.xsec = eval((config.get(sample,'xSec')))
newsample.sf = eval((config.get(sample, 'SF'))) if config.has_option(sample, 'SF') else 1.0
newsample.group = config.get(sample,'sampleGroup')
self._samplelist.append(newsample)
def __init__(self, samples_config=None, samples_path=None, config=None):
'''
Methode filling a list of Sample "self._samplelist = []" contained in the class.
"sample_path" contains the path where the samples are stored (PREPin).
"samples_config" is the "samples_nosplit.cfg" file. Depending of the variable "run_on_files" defined in "samples_nosplit.cfg",
the sample list are generated from the input folder (PREPin) or the list in "samples_nosplit.cfg" '''
self.debug = 'XBBDEBUG' in os.environ
# legacy behavior not supported anymore (INSTEAD: pass config object)
if not config:
print(
"\x1b[41m\x1b[37mERROR: ParseInfo() is now required to have config= argument passed\x1b[0m"
)
traceback.print_exc(file=sys.stdout)
raise Exception("Error")
elif self.debug:
print "DEBUG: config object passed to sample parser."
lumi = float(config.get('General', 'lumi'))
self._samplelist = []
self.__fileslist = []
configSamples = [
x for x in config.sections() if config.has_option(x, 'sampleName')
]
if self.debug:
print "DEBUG:", len(configSamples), " samples found."
for sample in configSamples:
sampleName = config.get(sample, 'sampleName')
sampleType = config.get(sample, 'sampleType')
cut = config.get(sample, 'cut') if config.has_option(
sample, 'cut') else '1'
specialweight = config.get(sample,
'specialweight') if config.has_option(
sample, 'specialweight') else "1"
fullname = config.get(sample, 'sampleName')
mergeCachingSize = int(config.get(
sample, 'mergeCachingSize')) if config.has_option(
sample, 'mergeCachingSize') else -1
#fill the sample
newsample = Sample(sampleName, sampleType)
newsample.addtreecut = cut
newsample.identifier = sample
newsample.weightexpression = ''
newsample.specialweight = specialweight
newsample.lumi = lumi
newsample.prefix = ''
newsample.FullName = sampleName
if mergeCachingSize > 0:
newsample.mergeCachingSize = mergeCachingSize
if config.has_option(sample, 'skipParts'):
newsample.skipParts = eval(config.get(sample, 'skipParts'))
newsample.index = eval(config.get(
sample, 'sampleIndex')) if config.has_option(
sample, 'sampleIndex') else -999999
# add and fill all the subsamples
if config.has_option(sample, 'subsamples') and eval(
config.get(sample, 'subsamples')):
subgroups = eval((config.get(sample, 'sampleGroup')))
try:
subnames = eval((config.get(sample, 'subnames')))
except:
# create subnames automatically based on subgroup name to avoid duplication
try:
shortname = config.get(sample, 'shortname').strip()
except:
# use full name if no short name given
shortname = sampleName
# if suffixes explicitly given use them
if config.has_option(sample, 'suffixes'):
suffixes = eval(config.get(sample, 'suffixes'))
subnames = [shortname + '_' + x for x in suffixes]
# otherwise use sample group as suffix
else:
subnames = [shortname + '_' + x for x in subgroups]
subcuts = eval((config.get(sample, 'subcuts')))
if sampleType != 'DATA':
subxsecs = eval((config.get(sample, 'xSec')))
if type(subxsecs) != list:
subxsecs = [subxsecs] * len(subcuts)
if len(list(set(subxsecs))) == 1:
newsample.xsec = [subxsecs[0]]
else:
print "\x1b[31mWARNING: different cross sections for the sub-samples of", sampleName, " are you sure you want to do this?\x1b[0m"
subsfs = eval(
(config.get(sample, 'SF'))) if config.has_option(
sample, 'SF') else [1.0] * len(subxsecs)
if type(subsfs) != list:
subsfs = [subsfs] * len(subcuts)
try:
subspecialweights = eval(
(config.get(sample, 'specialweight')))
if len(subspecialweights) < 2:
subspecialweights = []
print "\x1b[31mWARNING: specialweight not defined for subsamples but for full sample only!\x1b[0m"
except:
subspecialweights = []
subindices = None
newsamples = []
for i, cut in enumerate(subcuts):
newsubsample = copy(newsample)
newsubsample.subsample = True
newsubsample.name = subnames[i]
newsubsample.subcut = subcuts[i]
newsubsample.group = subgroups[i]
if sampleType != 'DATA':
newsubsample.sf = float(subsfs[i])
newsubsample.xsec = float(subxsecs[i])
if len(subspecialweights) == len(subcuts):
newsubsample.specialweight = subspecialweights[i]
if type(newsample.index) == list:
newsubsample.index = newsample.index[i]
if config.has_option(sample, 'offsets'):
sampleIndexOffsets = eval(config.get(
sample, 'offsets'))
if type(sampleIndexOffsets) == list and len(
sampleIndexOffsets) == len(subcuts):
newsubsample.index += sampleIndexOffsets[i]
else:
print "sample:", newsample.name
print "=>", type(sampleIndexOffsets), len(
sampleIndexOffsets), len(subcuts)
print "\x1b[31mERROR: sampleIndex offset does not match subcuts\x1b[0m"
raise Exception("ConfigError")
newsamples.append(newsubsample)
self._samplelist.extend(newsamples)
self._samplelist.append(newsample)
else:
if sampleType != 'DATA':
newsample.xsec = eval((config.get(sample, 'xSec')))
newsample.sf = eval(
(config.get(sample, 'SF'))) if config.has_option(
sample, 'SF') else 1.0
newsample.group = config.get(sample, 'sampleGroup')
self._samplelist.append(newsample)