def test_make_concatenate_and_shuffle_scripts(self):
#main
#repeat first 4 steps
self.cfg = read_config(config_file)
self.ip_group_keys = mds.get_all_ip_group_keys(self.cfg)
self.cfg, self.n_evts_total = update_cfg(self.cfg)
self.cfg['n_evts_total'] = self.n_evts_total
mds.print_input_statistics(self.cfg, self.ip_group_keys)
for key in self.ip_group_keys:
mds.add_fpaths_for_data_split_to_cfg(self.cfg, key)
mds.make_dsplit_list_files(self.cfg)
#create the bash job scripts and test their content
mds.make_concatenate_and_shuffle_scripts(self.cfg)
assert os.path.exists(concatenate_bash_script_train) == 1
with open(concatenate_bash_script_train) as f:
for line in f:
pass #yay, awesome style! ^^
last_line = line
self.assertIn(last_line, self.contents_concatenate_script)
f.close
assert os.path.exists(shuffle_bash_script_train) == 1
with open(shuffle_bash_script_train) as f2:
for line in f2:
pass
last_line = line
self.assertIn(last_line, self.contents_shuffle_script)
f2.close
def test_make_split(self):
#main
#repeat first 3 steps
self.cfg = read_config(config_file)
self.ip_group_keys = mds.get_all_ip_group_keys(self.cfg)
self.cfg, self.n_evts_total = update_cfg(self.cfg)
self.cfg['n_evts_total'] = self.n_evts_total
mds.print_input_statistics(self.cfg, self.ip_group_keys)
for key in self.ip_group_keys:
mds.add_fpaths_for_data_split_to_cfg(self.cfg, key)
mds.make_dsplit_list_files(self.cfg)
#assert the single output lists
assert os.path.exists(list_output_val) == 1
with open(list_output_val) as f:
for line in f:
self.assertIn(line, self.file_path_list_val)
f.close
assert os.path.exists(list_output_train) == 1
with open(list_output_train) as f2:
for line in f2:
self.assertIn(line, self.file_path_list)
f2.close
def test_get_filepath_and_n_events(self):
#repeat first 2 steps
self.cfg = read_config(config_file)
self.ip_group_keys = mds.get_all_ip_group_keys(self.cfg)
self.cfg, self.n_evts_total = update_cfg(self.cfg)
for key in self.ip_group_keys:
self.assertIn(self.cfg[key]['fpaths'][0], self.file_path_list)
self.assertIn(self.cfg[key]['n_evts'], self.n_events_list)
def update_cfg(cfg):
''' Update the cfg with file paths and also return the total number of events'''
#get input groups and compare
ip_group_keys = mds.get_all_ip_group_keys(cfg)
os.chdir(test_data_dir)
n_evts_total = 0
for key in ip_group_keys:
print('Collecting information from input group ' + key)
cfg[key]['fpaths'] = mds.get_h5_filepaths(cfg[key]['dir'])
cfg[key]['n_files'] = len(cfg[key]['fpaths'])
cfg[key]['n_evts'], cfg[key]['n_evts_per_file_mean'], cfg[key][
'run_ids'] = mds.get_number_of_evts_and_run_ids(cfg[key]['fpaths'],
dataset_key='y')
n_evts_total += cfg[key]['n_evts']
return cfg, n_evts_total
def update_cfg(cfg):
""" Update the cfg with file paths and also return the total number of events"""
# get input groups and compare
ip_group_keys = mds.get_all_ip_group_keys(cfg)
os.chdir(test_data_dir)
n_evts_total = 0
for key in ip_group_keys:
print("Collecting information from input group " + key)
cfg[key]["fpaths"] = mds.get_h5_filepaths(cfg[key]["dir"])
cfg[key]["n_files"] = len(cfg[key]["fpaths"])
(
cfg[key]["n_evts"],
cfg[key]["n_evts_per_file_mean"],
cfg[key]["run_ids"],
) = mds.get_number_of_evts_and_run_ids(cfg[key]["fpaths"],
dataset_key="y")
n_evts_total += cfg[key]["n_evts"]
return cfg, n_evts_total
def test_read_keys_off_config(self):
self.cfg = read_config(config_file)
#get input groups and compare
self.ip_group_keys = mds.get_all_ip_group_keys(self.cfg)
self.assertSequenceEqual(self.ip_group_keys,
self.input_categories_list)