def test_check_input_path_existance():
package_path = ml.__path__[0].replace('python', 'src')
res_wildcard = os.path.join(
package_path, 'info', 'HH', '*', 'res', '*', 'info.json')
nonRes_wildcard = os.path.join(
package_path, 'info', 'HH', '*', 'nonRes', 'info.json')
missing_inputPaths = []
faulty_info = []
for resInfo in glob.glob(res_wildcard):
info_dict = ut.read_json_cfg(resInfo)
tauID_wps = info_dict['tauID_training']
for wp in tauID_wps.keys():
input_paths = tauID_wps[wp]
for inPath in input_paths.keys():
if not os.path.exists(input_paths[inPath]):
missing_inputPaths.append(input_paths[inPath])
faulty_info.append(resInfo)
for nonResInfo in glob.glob(nonRes_wildcard):
info_dict = ut.read_json_cfg(nonResInfo)
tauID_wps = info_dict['tauID_training']
for wp in tauID_wps.keys():
input_paths = tauID_wps[wp]
for inPath in input_paths.keys():
if not os.path.exists(input_paths[inPath]):
missing_inputPaths.append(input_paths[inPath])
faulty_info.append(nonResInfo)
assert len(missing_inputPaths) == 0, "Missing ntuple directories: \n" \
+ str(set(missing_inputPaths)) + " in the files: \n" + str(set(faulty_info))
def main(to_continue, opt_dir, bbww):
if not to_continue:
settings_dir = os.path.join(
os.path.expandvars('$CMSSW_BASE'),
'src/machineLearning/machineLearning/settings')
global_settings = ut.read_settings(settings_dir, 'global')
output_dir = os.path.expandvars(global_settings['output_dir'])
else:
settings_dir = os.path.join(opt_dir, 'run_settings')
global_settings = ut.read_settings(settings_dir, 'global')
output_dir = opt_dir
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
if not to_continue:
if not os.path.exists(os.path.join(output_dir, 'run_settings')):
ut.save_run_settings(output_dir)
if not os.path.exists(os.path.join(output_dir, 'run_info')):
ut.save_info_dir(output_dir)
# use_scratch_for_data(global_settings)
print("::::::: Reading parameters :::::::")
if global_settings['ml_method'] == 'xgb':
param_file = os.path.join(settings_dir, 'xgb_parameters.json')
else:
param_file = os.path.join(settings_dir, 'nn_parameters.json')
hyperparameter_info = ut.read_json_cfg(param_file)
pso_settings = ut.read_settings(settings_dir, 'pso')
pso_settings.update(global_settings)
addition = ut.create_infoPath_addition(global_settings)
channel_dir = os.path.join(output_dir, 'run_info')
info_dir = os.path.join(channel_dir, addition)
scenario = global_settings['scenario']
reader = hpr.HHParameterReader(channel_dir, scenario)
preferences = reader.parameters
normalizer = hht.HHDataNormalizer if not bbww else bbwwt.bbWWDataNormalizer
if os.path.exists(preferences['data_csv']):
print(':::::::: Loading data from .csv file ::::::::')
data = pandas.read_csv(preferences['data_csv'])
else:
print('::::::: Loading data to be saved to pandas.DataFrame :::::::')
if not bbww:
loader = hht.HHDataLoader(normalizer, preferences, global_settings)
else:
loader = bbwwt.bbWWLoader(normalizer, preferences, global_settings)
data = loader.data
if global_settings['ml_method'] in ['lbn', 'nn']:
use_Wjets = True
if 'bb2l' in global_settings['channel']:
use_Wjets = False
data = mt.multiclass_encoding(data, use_Wjets)
loader.save_to_csv()
print("\n============ Starting hyperparameter optimization ==========\n")
swarm = pt.ParticleSwarm(pso_settings, st.get_fitness_score,
hyperparameter_info, to_continue, output_dir)
optimal_hyperparameters = swarm.particleSwarmOptimization()[0]
print("\n============ Saving results ================\n")
best_parameters_path = os.path.join(output_dir,
'best_hyperparameters.json')
ut.save_dict_to_json(optimal_hyperparameters, best_parameters_path)
print("Results saved to " + str(output_dir))
def test_check_info_files():
package_path = ml.__path__[0].replace('python', 'src')
res_wildcard = os.path.join(
package_path, 'info', 'HH', '*', 'res', '*', 'info.json')
nonRes_wildcard = os.path.join(
package_path, 'info', 'HH', '*', 'nonRes', 'info.json')
faulty_files = []
for resInfo in glob.glob(res_wildcard):
try:
ut.read_json_cfg(resInfo)
except:
faulty_files.append(resInfo)
for nonResInfo in glob.glob(nonRes_wildcard):
try:
ut.read_json_cfg(nonResInfo)
except:
faulty_files.append(nonResInfo)
assert len(faulty_files) == 0, "Faulty files: " + str(faulty_files)
def test_check_weights_dir_existance():
package_path = ml.__path__[0].replace('python', 'src')
res_wildcard = os.path.join(
package_path, 'info', 'HH', '*', 'res', '*', 'info.json')
missing_directories = []
for path in glob.glob(res_wildcard):
info_dict = ut.read_json_cfg(path)
weight_dir = info_dict['weight_dir']
if not os.path.exists(weight_dir):
missing_directories.append(path)
assert len(missing_directories) == 0, "Missing weight_dirs in: " + str(missing_directories)
def renew_data_paths(global_settings):
addition = ut.create_infoPath_addition(global_settings)
channel_dir = os.path.expandvars(
os.path.join(global_settings['output_dir'], 'run_info'))
info_file = os.path.join(channel_dir, addition, 'info.json')
info_dict = ut.read_json_cfg(info_file)
paths = info_dict['tauID_training'][global_settings['tauID_training']]
for key in paths:
path = paths[key]
paths[key] = path.replace(
path.split('/hhAnalysis')[0], '/scratch-persistent')
ut.save_dict_to_json(info_dict, info_file)
def get_original_input_paths(global_settings):
info_dir = os.path.join(os.path.expandvars('$CMSSW_BASE'),
'src/machineLearning/machineLearning/info/HH')
addition = ut.create_infoPath_addition(global_settings)
info_file = os.path.join(info_dir, global_settings['channel'], addition,
'info.json')
info_dict = ut.read_json_cfg(info_file)
paths = info_dict['tauID_training'][global_settings['tauID_training']]
for key in paths.keys():
path = paths[key]
paths[key] = path.replace(
path.split('/hhAnalysis')[0],
path.split('/hhAnalysis')[0] + '/.')
return paths
def interpret_info_file(self):
info_path = os.path.join(self.info_dir, 'info.json')
info_dict = ut.read_json_cfg(info_path)
self.parameters = {}
tau_id_application = info_dict.pop('tauID_application')
default_tauID = info_dict['default_tauID_application']
tau_id_training = info_dict['tauID_training_key']
self.parameters['tauID_application'] = tau_id_application[
default_tauID]
self.parameters.update(
self.find_input_paths(info_dict, tau_id_training))
self.parameters.update(self.load_era_keys(info_dict.pop('keys')))
self.parameters['trainvars'], trainvars_info = self.load_trainvars()
self.parameters['trainvar_info'] = trainvars_info
self.parameters['all_trainvar_info'] = self.read_trainvar_info(
self.all_trainvars_path)
self.parameters.update(info_dict)
def set_background_sample_info(self, path):
target = 0
background_catfile = os.path.join(
os.path.expandvars('$CMSSW_BASE'),
'src/machineLearning/machineLearning/info',
'HH',
'background_categories.json'
)
background_categories = ut.read_json_cfg(background_catfile)
possible_processes = []
sample_dict = {}
for category in background_categories:
possible_samples = background_categories[category]
sample_dict.update(possible_samples)
for sample in possible_samples.keys():
if sample in path:
possible_processes.append(sample)
process = sample_dict[max(possible_processes, key=len)]
return process, target
def data_cutting(self, data):
package_dir = os.path.join(
os.path.expandvars('$CMSSW_BASE'),
'src/machineLearning/machineLearning/'
)
if 'nonres' in self.global_settings['scenario']:
addition = self.global_settings['scenario']
else:
addition = 'res/%s' %(self.global_settings['scenario'])
if self.global_settings['dataCuts'] == 1:
cut_file = os.path.join(
package_dir, 'info', self.global_settings['process'],
self.global_settings['channel'], addition, 'cuts.json'
)
else:
cut_file = os.path.join(
package_dir,
'info',
self.global_settings['process'],
self.global_settings['channel'],
addition,
self.global_settings['dataCuts']
)
if os.path.exists(cut_file):
cut_dict = ut.read_json_cfg(cut_file)
if cut_dict == {}:
print('No cuts given in the cut file %s' % cut_file)
else:
cut_keys = list(cut_dict.keys())
for key in cut_keys:
try:
min_value = cut_dict[key]['min']
data = data.loc[(data[key] >= min_value)]
except KeyError:
print('Minimum condition for %s not implemented' % key)
try:
max_value = cut_dict[key]['max']
data = data.loc[(data[key] <= max_value)]
except KeyError:
print('Maximum condition for %s not implemented' % key)
else:
print('Cut file %s does not exist' % cut_file)
return data
def main(hyperparameter_file, output_dir):
settings_dir = os.path.join(output_dir, 'run_settings')
global_settings = ut.read_settings(settings_dir, 'global')
nthread = global_settings['nthread']
path = Path(hyperparameter_file)
save_dir = str(path.parent)
hyperparameters = ut.read_json_cfg(hyperparameter_file)
addition = ut.create_infoPath_addition(global_settings)
channel_dir = os.path.join(output_dir, 'run_info')
info_dir = os.path.join(channel_dir, addition)
scenario = global_settings['scenario']
reader = hpr.HHParameterReader(channel_dir, scenario)
preferences = reader.parameters
global_settings['debug'] = False
data_file = os.path.join(output_dir, 'data.csv')
data = pandas.read_csv(data_file)
if bool(global_settings['use_kfold']):
score, train, test = et.kfold_cv(
xt.model_evaluation_main,
data,
preferences['trainvars'],
global_settings,
hyperparameters
)
else:
score, train, test = et.get_evaluation(
xt.model_evaluation_main,
data,
preferences['trainvars'],
global_settings,
hyperparameters
)
score_path = os.path.join(save_dir, 'score.json')
score_dict = {
global_settings['fitness_fn']: score,
'train': train,
'test': test
}
with open(score_path, 'w') as score_file:
json.dump(score_dict, score_file)
def get_ntuple_paths(self, input_path, folder_name, file_type='hadd*Tight.root'):
paths = []
background_catfile = os.path.join(
os.path.expandvars('$CMSSW_BASE'),
'src/machineLearning/machineLearning/info',
'HH',
'background_categories.json'
)
background_categories = ut.read_json_cfg(background_catfile)
if 'signal' not in folder_name and folder_name in background_categories.keys():
bkg_elements = background_categories[folder_name]
for bkg_element in bkg_elements:
bkg_element_paths = self.find_paths_both_conventions(
input_path, bkg_element, file_type=file_type)
print('--------------')
print(bkg_element)
print(bkg_element_paths)
paths.extend(bkg_element_paths)
else:
paths = self.find_paths_both_conventions(
input_path, folder_name + '*', file_type=file_type)
return paths
def main(output_dir, settings_dir, hyperparameter_file, debug):
if settings_dir == 'None':
settings_dir = os.path.join(
os.path.expandvars('$CMSSW_BASE'),
'src/machineLearning/machineLearning/settings')
global_settings = ut.read_settings(settings_dir, 'global')
if output_dir == 'None':
output_dir = global_settings['output_dir']
else:
global_settings['output_dir'] = output_dir
global_settings['output_dir'] = os.path.expandvars(
global_settings['output_dir'])
if not os.path.exists(global_settings['output_dir']):
os.makedirs(global_settings['output_dir'])
channel_dir, info_dir, _ = ut.find_settings()
scenario = global_settings['scenario']
reader = hpr.HHParameterReader(channel_dir, scenario)
preferences = reader.parameters
if hyperparameter_file == 'None':
hyperparameter_file = os.path.join(info_dir, 'hyperparameters.json')
hyperparameters = ut.read_json_cfg(hyperparameter_file)
evaluation_main(global_settings, preferences, hyperparameters, debug)
def main(output_dir, settings_dir, hyperparameter_file, debug):
if settings_dir == 'None':
settings_dir = os.path.join(
os.path.expandvars('$CMSSW_BASE'),
'src/machineLearning/machineLearning/settings')
global_settings = settings_dir + '/' + 'global_%s_%s_%s_settings.json' % (
channel, mode, res_nonres)
command = 'rsync %s ~/machineLearning/CMSSW_11_2_0_pre1/src/machineLearning/machineLearning/settings/global_settings.json' % global_settings
p = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
global_settings = ut.read_settings(settings_dir, 'global')
if output_dir == 'None':
output_dir = global_settings['channel']+'/'+global_settings['ml_method']+'/'+\
res_nonres + '/' + mode +'/' + era
global_settings['output_dir'] = output_dir
else:
global_settings['output_dir'] = output_dir
global_settings['output_dir'] = os.path.expandvars(
global_settings['output_dir'])
if not os.path.exists(global_settings['output_dir']):
os.makedirs(global_settings['output_dir'])
channel_dir, info_dir, _ = ut.find_settings()
scenario = global_settings['scenario']
reader = hpr.HHParameterReader(channel_dir, scenario)
preferences = reader.parameters
if not BM == 'None':
preferences["nonResScenarios"] = [BM]
print('BM point to be considered: ' + str(preferences["nonResScenarios"]))
if not era == '0':
preferences['included_eras'] = [era.replace('20', '')]
print('era: ' + str(preferences['included_eras']))
preferences = define_trainvars(global_settings, preferences, info_dir)
if hyperparameter_file == 'None':
hyperparameter_file = os.path.join(info_dir, 'hyperparameters.json')
hyperparameters = ut.read_json_cfg(hyperparameter_file)
print('hyperparametrs ' + str(hyperparameters))
evaluation_main(global_settings, preferences, hyperparameters, debug)
def read_fitness(output_dir, fitness_key='d_roc'):
"""Creates the list of score dictionaries of each sample. List is ordered
according to the number of the sample
Parameters:
----------
output_dir : str
Path to the directory of output
Returns:
-------
scores : list of floats
List of fitnesses
"""
samples = os.path.join(output_dir, 'samples')
wild_card_path = os.path.join(samples, '*', 'score.json')
number_samples = len(glob.glob(wild_card_path))
score_dicts = []
for number in range(number_samples):
path = os.path.join(samples, str(number), 'score.json')
score_dict = ut.read_json_cfg(path)
score_dicts.append(score_dict)
scores = [score_dict[fitness_key] for score_dict in score_dicts]
return scores
def test_read_parameters():
path_to_test_file = os.path.join(resources_dir, 'best_parameters.json')
result = ut.read_json_cfg(path_to_test_file)
expected = {'a': 1, 'b': 2, 'c': 3}
assert result == expected