def test__read(): testing_set = get_testing_set() assert os.path.isfile(testing_set['badparameters_in_fn']) assert os.path.isfile(testing_set['config_fn']) if os.path.isfile(testing_set['badparameters_out_fn']): m = "removing the badparameter_out_file:{}".format( testing_set['badparameters_out_fn']) print(m) os.remove(testing_set['badparameters_out_fn']) o = PyposmatBadParametersFile( filename=testing_set['badparameters_out_fn'], config_fn=testing_set['config_fn']) o.read(filename=testing_set['badparameters_in_fn']) assert isinstance(o.df,pd.DataFrame)
def dev__write_header_section(): print(80 * '-') print('{:^80}'.format('write_header_section')) print(80 * '-') testing_set = get_testing_set() if os.path.isfile(testing_set['filename']): os.remove(testing_set['filename']) f = PyposmatBadParametersFile() f.write_header_section(parameter_names=testing_set['parameter_names'], filename=testing_set['filename']) with open(testing_set['filename'], 'r') as f: print(f.read())
def merge_error_files(self, i_iteration): """ merge the pyposmat data files Args: i_iteration(int): the current iteration which just finished last_datafile_fn(str,optional): the filename of the last dataset in the data directory. new_datafile_fn(str,optional): where to output the file results """ badparameters_fn = os.path.join(self.data_directory, 'pyposmat.badparameters.out') data_dir = self.data_directory rank_dirs = [ v for v in os.listdir(self.root_directory) if v.startswith('rank_') ] filenames = [ os.path.join(self.root_directory, v, 'pyposmat.badparameters.out') for v in rank_dirs ] # consolidate rank directories badparameters_new = None badparameters_next = None for i, v in enumerate(filenames): if badparameters_new is None: try: badparameters_new = PyposmatBadParametersFile( o_config=self.configuration) badparameters_new.read(filename=v) except FileNotFoundError as e: self.log("no bad parameters file at {}".format(v)) else: try: badparameters_next = PyposmatBadParametersFile( o_config=self.configuration) badparameters_next.read(filename=v) badparameters_new.df = pd.concat( [badparameters_new.df, badparameters_next.df]) except FileNotFoundError as e: self.log("no bad parameters file as {}".format(v)) # determine the sim_id for bad parameters of the sim_id if badparameters_new.df is None: # no previous bad paramters found # TODO: need to implement something here to deal with bad parameters pass else: nrows = len(badparameters_new.df) sim_id_fmt = '{:0>2}_{:0>6}' sim_id_str = [ sim_id_fmt.format(i_iteration, i) for i in range(nrows) ] badparameters_new.df['sim_id'] = sim_id_str if self.configuration.sampling_type[i_iteration][ 'type'] == "from_file": badparameters_new.write(filename=badparameters_fn) else: self.log( "merging with bad candidates from previous simulations") self.log("\tfilename:{}".format(badparameters_fn)) badparameters = PyposmatBadParametersFile( o_config=self.configuration) try: badparameters.read(filename=badparameters_fn) badparameters.df = pd.concat( [badparameters.df, badparameters_new.df]) badparameters.write(filename=badparameters_fn) except FileNotFoundError as e: if i_iteration == 0: badparameters_new.write(filename=badparameters_fn) else: raise