def test__read():
testing_set = get_testing_set()
assert os.path.isfile(testing_set['badparameters_in_fn'])
assert os.path.isfile(testing_set['config_fn'])
if os.path.isfile(testing_set['badparameters_out_fn']):
m = "removing the badparameter_out_file:{}".format(
testing_set['badparameters_out_fn'])
print(m)
os.remove(testing_set['badparameters_out_fn'])
o = PyposmatBadParametersFile(
filename=testing_set['badparameters_out_fn'],
config_fn=testing_set['config_fn'])
o.read(filename=testing_set['badparameters_in_fn'])
assert isinstance(o.df,pd.DataFrame)
def dev__write_header_section():
print(80 * '-')
print('{:^80}'.format('write_header_section'))
print(80 * '-')
testing_set = get_testing_set()
if os.path.isfile(testing_set['filename']):
os.remove(testing_set['filename'])
f = PyposmatBadParametersFile()
f.write_header_section(parameter_names=testing_set['parameter_names'],
filename=testing_set['filename'])
with open(testing_set['filename'], 'r') as f:
print(f.read())
def merge_error_files(self, i_iteration):
""" merge the pyposmat data files
Args:
i_iteration(int): the current iteration which just finished
last_datafile_fn(str,optional): the filename of the last dataset in the data directory.
new_datafile_fn(str,optional): where to output the file results
"""
badparameters_fn = os.path.join(self.data_directory,
'pyposmat.badparameters.out')
data_dir = self.data_directory
rank_dirs = [
v for v in os.listdir(self.root_directory) if v.startswith('rank_')
]
filenames = [
os.path.join(self.root_directory, v, 'pyposmat.badparameters.out')
for v in rank_dirs
]
# consolidate rank directories
badparameters_new = None
badparameters_next = None
for i, v in enumerate(filenames):
if badparameters_new is None:
try:
badparameters_new = PyposmatBadParametersFile(
o_config=self.configuration)
badparameters_new.read(filename=v)
except FileNotFoundError as e:
self.log("no bad parameters file at {}".format(v))
else:
try:
badparameters_next = PyposmatBadParametersFile(
o_config=self.configuration)
badparameters_next.read(filename=v)
badparameters_new.df = pd.concat(
[badparameters_new.df, badparameters_next.df])
except FileNotFoundError as e:
self.log("no bad parameters file as {}".format(v))
# determine the sim_id for bad parameters of the sim_id
if badparameters_new.df is None:
# no previous bad paramters found
# TODO: need to implement something here to deal with bad parameters
pass
else:
nrows = len(badparameters_new.df)
sim_id_fmt = '{:0>2}_{:0>6}'
sim_id_str = [
sim_id_fmt.format(i_iteration, i) for i in range(nrows)
]
badparameters_new.df['sim_id'] = sim_id_str
if self.configuration.sampling_type[i_iteration][
'type'] == "from_file":
badparameters_new.write(filename=badparameters_fn)
else:
self.log(
"merging with bad candidates from previous simulations")
self.log("\tfilename:{}".format(badparameters_fn))
badparameters = PyposmatBadParametersFile(
o_config=self.configuration)
try:
badparameters.read(filename=badparameters_fn)
badparameters.df = pd.concat(
[badparameters.df, badparameters_new.df])
badparameters.write(filename=badparameters_fn)
except FileNotFoundError as e:
if i_iteration == 0:
badparameters_new.write(filename=badparameters_fn)
else:
raise