def test____init____using_path_args(): testing_set = get_testing_set() o = PyposmatDataAnalyzer( config_fn=testing_set['config_fn'], results_data_fn=testing_set['results_data_fn'] ) config = PyposmatConfigurationFile() config.read(filename=testing_set['config_fn']) assert isinstance(o,PyposmatDataAnalyzer) assert o.config_fn == testing_set['config_fn'] assert isinstance(o.configuration,PyposmatConfigurationFile) assert isinstance(o.parameter_names,list) assert set(o.parameter_names) == set(config.parameter_names) assert isinstance(o.error_names, list) assert set(o.error_names) == set(config.error_names) assert isinstance(o.qoi_names, list) assert set(o.qoi_names) == set(config.qoi_names) assert o.results_data_fn == testing_set['results_data_fn'] assert isinstance(o.results_data,PyposmatDataFile) assert isinstance(o.results_df,pd.DataFrame)
def do_attribute_tests(sampler, config_fn): from pypospack.pyposmat.engines import PyposmatBaseSampler from pypospack.pyposmat.data import PyposmatConfigurationFile # test arguments assert type(sampler) is PyposmatBaseSampler assert type(config_fn) is str config = PyposmatConfigurationFile() config.read(filename=config_fn) assert sampler.structure_directory == config.structures[ 'structure_directory'] assert sampler.n_iterations == config.sampling_type['n_iterations'] assert sampler.parameter_names == config.parameter_names assert sampler.qoi_names == config.qoi_names assert sampler.error_names == config.error_names assert sampler.free_parameter_names == config.free_parameter_names assert set(sampler.parameter_constraints.keys()) == set( config.sampling_constraints.keys()) assert all([ sampler.parameter_constraints[k] == config.sampling_constraints[k] for k in sampler.parameter_constraints ]) assert sampler.parameter_constraints == config.sampling_constraints assert sampler.constrained_parameter_names == \ [p for p in sampler.parameter_names if p not in sampler.free_parameter_names]
def dev__defect_calculation(): print(80*'-') print('{:^80}'.format('defect_calculation')) print(80*'-') simulation_dir = 'rank_test' testing_set = testing_set_Si_sw.get_testing_set_Si() configuration = PyposmatConfigurationFile() configuration.qois = testing_set['qoi_db'].qois configuration.structures = testing_set['structure_db'] qoi_manager = QoiManager( qoi_database=configuration.qois, fullauto=False ) qoi_manager.configure() qoi_manager.determine_tasks() print('qoi_manager.tasks') print(len('qoi_manager.tasks')*'-') for k,v in qoi_manager.tasks.items(): print(k,v) task_manager = TaskManager(base_directory=simulation_dir) task_manager.configure( tasks = qoi_manager.tasks, structures = testing_set['structure_db'] ) task_managger.evaluate_tasks() qoi_manager.calculate_qois( task_results=task_manager.results ) qoi_manager.qois
def calculate_kld_parameters(config,data_directory,kld_param_fn='pyposmat.kld_param.out'): assert isinstance(config,str) or isinstance(config,PyposmatConfigurationFile) assert os.path.isdir(data_directory) assert isinstance(kld_param_fn,str) # process the the configuration argument, the configuration argument has two # options for processing # (1) if config is a str, the config is assumed to be a path to the # the path to the configuration file, and o_config is initialized from it # (2) if config is PyposmatConfigurationFile object, then o_config is set to it if isinstance(config,str): o_config = PyposmatConfigurationFile() o_config.read(filename=config) else: assert isinstance(config,PyposmatConfigurationFile) o_config = config kld = OrderedDict() for i in range(o_config.n_iterations): kld[i] = OrderedDict() if i == 0: kld[i]['results'] = None kld[i]['kde'] = None kld[i]['filter'] = calculate_kld( data_1_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i)), data_2_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i+1)), names = o_config.free_parameter_names, n_samples=n_samples) else: kld[i]['results'] = calculate_kld( data_1_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i-1)), data_2_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i)), names = o_config.free_parameter_names, n_samples=n_samples) kld[i]['kde'] = calculate_kld( data_1_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i)), data_2_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i+1)), names = o_config.free_parameter_names, n_samples=n_samples) kld[i]['filter'] = calculate_kld( data_1_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i)), data_2_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i+1)), names = o_config.free_parameter_names, n_samples=n_samples) print(i) # write out the kld_parameters file with open(kld_param_fn,'w') as f: f.write(",".join(['iteration','results','kde','filter'])+"\n") for kld_iteration,kld_row in kld.items(): s_list = [] s_list.append(kld_iteration) for k in ['results','kde','filter']: if kld_row[k] is None: s_list.append(float('NaN')) else: s_list.append(kld_row[k][0]) f.write(",".join([str(s) for s in s_list])+"\n")
class FileSampler(): def __init__(self, configuration, data, structure_name, structure_path, workflow_type, workflow_definition): self.initialize_configuration(configuration) self.initialize_data(data) self.structure_name = structure_name self.strucutre_path = structure_path self.workflow_type = workflow_type self.workflow_definition = workflow_definition self.potential_definition = self.configuration.potential def initialize_configuration(self,configuration): if isinstance(configuration,PyposmatConfigurationFile): self.configuration = configuration elif isinstance(configuration,str): self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=configuration) else: msg = ("configuration must be a path to a configuration file or an " "instance of the PyposmatConfigurationFile,") raise TypeError(msg) def initialize_data(self,data): if isinstance(data,PyposmatDataFile): self.data = data elif isinstance(data,str): self.data = PyposmatDataFile() self.data.read(filename=data) else: msg = ("data must be a path to a data file or an instance of " "PyposmatDataFile.") raise TypeError(msg) def run(self): for index,row in self.data.df.iterrows(): sim_id = row['sim_id'] print('working on sim_id:{}'.format(sim_id)) parameters = OrderedDict([(k,row[k]) for k in self.configuration.parameter_names]) original_path = os.getcwd() os.mkdir(sim_id) os.chdir(sim_id) if workflow_type == 'lmps_thermal_expansion': workflow = LammpsThermalExpansion( structure_name=Si_structure_definition['name'], structure_path=Si_structure_definition['filename'], **workflow_definition) workflow.create_task_configurations() workflow.create_tasks() workflow.prepare_tasks( potential_definition = self.potential_definition, potential_parameters = parameters) workflow.run() os.chdir(original_path)
def make_latex_table(config, data, qoi_type=None, param_type=None): qoi_types = ['by_qoi_target'] param_type = [] assert isinstance(config,str) \ or isinstance(config,PyposmatConfigurationFile) assert isinstance(data,str) \ or isinstance(data,PyposmatDataFile) if isinstance(config, str): o_config = PyposmatConfigurationFile() o_config.read(filename=config) elif isinstance(config, PyposmatConfigurationFile): o_config = config else: raise TypeError() if isinstance(data, str): o_data = PyposmatDataFile() o_data.read(filename=data) elif isinstance(data, PyposmatDataFile): o_data = data else: raise TypeError() if qoi_type == 'by_qoi_target': o_data.create_normalized_errors(normalize_type='by_qoi_target', qoi_targets=o_config.qoi_targets) df = o_data.df[o_data.normalized_error_names]
def test__initialize_configuration__with_object(): testing_set = get_testing_set() o_config = PyposmatConfigurationFile() o_config.read(filename=testing_set['config_fn']) o = PyposmatDataAnalyzer() o.initialize_configuration(o_config=o_config)
def initialize_configuration(self,pyposmat_configuration): if isinstance(pyposmat_configuration,PyposmatConfigurationFile): self.configuration = pyposmat_configuration elif isinstance(pyposmat_configuration,str): self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=pyposmat_configuration) else: raise TypeError('pyposmat_configuration must be either a path or PyposmatConfigurationFile')
def config(self, config): if isinstance(config, str): self._config = PyposmatConfigurationFile() self._config.read(filename=config) elif isinstance(config, PyposmatConfigurationFile): self._config = config else: m = "type(config)={}".format(str(type(config))) raise TypeError(m)
def read_configuration(self, filename): """read in pyposmat configuration file Args: filename(str): path of the configuraiton file """ self.configuration_fn = filename self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=filename)
def _initialize_configuration(self, configuration): if isinstance(configuration, str): assert os.path.isfile(configuration) self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=configuration) elif isinstance(configuration, PyposmatConfigurationFile): self.configuration = configuration else: raise TypeError('configuration cannot be type:{}'.format(str(type(configuration))))
def initialize_configuration(self,configuration): if isinstance(configuration,PyposmatConfigurationFile): self.configuration = configuration elif isinstance(configuration,str): self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=configuration) else: msg = ("configuration must be a path to a configuration file or an " "instance of the PyposmatConfigurationFile,") raise TypeError(msg)
def __init__(self, configuration_fn, datafile_fn): self.configuration_fn = configuration_fn self.datafile_fn = datafile_fn if configuration_fn is not None: self.configuration = PyposmatConfigurationFile() self.configuration.read(configuration_fn) if datafile_fn is not None: self.datafile = PyposmatDataFile() self.datafile.read(filename=datafile_fn)
def get_qoi_database_from_PyposmatConfigurationFile(config_fn): config = PyposmatConfigurationFile() config.read(filename=config_fn) assert type(config.qois) is OrderedDict for qoi_id, qoi_info in config.qois.items(): assert type(qoi_id) is str assert set(qoi_info.keys()) == set(['qoi_type','structures','target']) return config.qois
def test__initialize_configuration__with_object_and_path(): testing_set = get_testing_set() o_config = PyposmatConfigurationFile() o_config.read(filename=testing_set['config_fn']) o = PyposmatDataAnalyzer() with pytest.raises(TypeError) as e: o.initialize_configuration(config_fn=testing_set['config_fn'], o_config=o_config)
def gmm_analysis(config_fn, data_fn, names, output_directory='gmm_analysis', max_components=20): assert isinstance(config_fn, str) assert isinstance(data_fn, str) assert os.path.isfile(config_fn) assert os.path.isfile(data_fn) if not os.path.isdir(output_directory): os.mkdir(output_directory) o_config = PyposmatConfigurationFile() o_config.read(filename=config_fn) o_data = PyposmatDataFile() o_data.read(filename=data_fn) o_data.create_normalized_errors(normalize_type='by_qoi_target', qoi_targets=o_config.qoi_targets) o_data.df['score'] = o_data.df[o_config.normalized_error_names].abs().sum( axis=1) data = o_data.df[names] n_components = np.arange(1, max_components) models = [ GaussianMixture(n_components=n, covariance_type='full', random_state=0).fit(data) for n in n_components ] # AIC analysis aic, aic_idx = min( (val, idx) for (idx, val) in enumerate([m.aic(data) for m in models])) aic_n_components = n_components[aic_idx] aic_criteria = [m.aic(data) for m in models] # BIC analysis bic, bic_idx = min( (val, idx) for (idx, val) in enumerate([m.bic(data) for m in models])) bic_n_components = n_components[bic_idx] bic_criteria = [m.bic(data) for m in models] #plot the criteria print('bic_n_components:{}'.format(bic_n_components)) print('aic_n_components:{}'.format(aic_n_components)) plot_fn = os.path.join(output_directory, 'aic_bic_plot.jpg') plot_gmm_aic_bic(filename=plot_fn, n_components=n_components, aic_criteria=aic_criteria, bic_criteria=bic_criteria, aic_n_components=aic_n_components, bic_n_components=bic_n_components) filename = os.path.join('gmm_analysis', 'gmm_analysis.jpg') plot_gmm(models[bic_n_components], data, filename=filename)
def test__read_configuration(): from pypospack.pyposmat.data import PyposmatConfigurationFile o_config = PyposmatConfigurationFile() o_config.read(filename=config_fn) o_rugplot = PyposmatParetoRugplot() o_rugplot.read_configuration(filename=config_fn) assert type(o_rugplot.parameter_names) is list assert type(o_rugplot.qoi_names) is list assert type(o_rugplot.error_names) is list assert type(o_rugplot.qoi_validation_names) is list assert type(o_rugplot.error_validation_names) is list assert isinstance(o_rugplot.qoi_targets, dict)
class PyposmatPostProcessorTestHarness(object): def __init__(self, configuration_fn, datafile_fn): self.configuration_fn = configuration_fn self.datafile_fn = datafile_fn if configuration_fn is not None: self.configuration = PyposmatConfigurationFile() self.configuration.read(configuration_fn) if datafile_fn is not None: self.datafile = PyposmatDataFile() self.datafile.read(filename=datafile_fn) def get_parameter_names(self): return self.configuration.parameter_names
def read_configuration(self,filename=None): if filename is not None: self.configuration_fn = filename _filename = self.configuration_fn self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=filename) self._parameter_names = self.configuration.parameter_names # determine_qoi_names_from_configuration_file self.qoi_fitting_names = self.configuration.qoi_names self.qoi_testing_names = self.configuration.qoi_validation_names self.qoi_names = self.qoi_fitting_names + self.qoi_testing_names # determine_error_names_from_configuration_file self.error_fitting_names = self.configuration.error_names self.error_testing_names = self.configuration.qoi_validation_names self.error_names = self.error_fitting_names + self.error_testing_names # determine_absolute_error_names_from_configuration_file self.abs_error_fitting_names = ['{}.abserr'.format(v) for self.qoi_fitting_names] self.abs_error_testing_names = ['{}.abserr'.format(v) for self.qoi_testing_names] self.abs_error_names = self.abs_error_fitting_names + self.abs_error_testing_names # determine_normalized_error_names_from_configuration_file self.norm_error_fitting_names = ['{}.nerr'.format(v) for self.qoi_fitting_names] self.norm_error_testing_names = ['{}.nerr'.format(v) for self.qoi_testing_names] self.norm_error_names = self.norm_error_fitting_names + self.norm_error_testing_names if self.qoi_fitting_names is not None: _config_qois_fitting = self.configuration.qois self.qoi_fitting_targets = OrderedDict( [(q,_config_qois_fitting[q]['target']) for q in self.qoi_fitting_names] ) if self.qoi_testing_names is not None: _config_qoi_testing = self.configuration.qois_validation self.qoi_testing_targets = OrderedDict( [(q,_config_qoi_testing[q]['target']) for q in self.qoi_testing_names] ) self.qoi_targets = OrderedDict() for k,v in self.qoi_fitting_targets.items(): self.qoi_targets[k] = v for k,v in self.qoi_testing_targets.items(): self.qoi_targets[k] = v # old naming convention [DEPRECATED] self.qoi_validation_names = self.configuration.qoi_validation_names self.error_validation_names = self.configuration.qoi_validation_names
def initialize_configuration(self, config): assert isinstance(config,str) \ or isinstance(config,PyposmatConfigurationFile) \ or config is None if isinstance(config, str): self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=config) elif isinstance(config, PyposmatConfigurationFile): self.configuration = config elif config is None: self.configuration = None else: m = 'config arguement must either be a path string of a PyposmatConfigurationFile object' raise TypeError(m)
def show_qoi_targets(config_fn, data_fn): o_config = PyposmatConfigurationFile() o_config.read(filename=config_fn) o_data = PyposmatDataFile() o_data.read(filename=data_fn) for qoi_name, qoi_target in o_config.qoi_targets.items(): try: qoi_avg = o_data.df[qoi_name].mean() except KeyError as e: qoi_avg = 'no value' s = "{:20} {:10} {:10}".format(qoi_name,qoi_target,qoi_avg) print(s)
def test____init____config_as_obj(): o = Pyposmat2DDensityPlot(config=PyposmatConfigurationFile()) assert isinstance(o, Pyposmat2DDensityPlot) assert isinstance(o.configuration, PyposmatConfigurationFile) assert o.data is None assert o.fig is None assert o.ax is None
class BaseAnalysis(object): def __init__(self, configuration, data, output_path=None): self.configuration = None self.data = None self.output_path = None self._initialize_configuration(configuration=configuration) self._initialize_data(data=data) self._initialize_output_path(path=output_path) def _initialize_configuration(self, configuration): if isinstance(configuration, str): assert os.path.isfile(configuration) self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=configuration) elif isinstance(configuration, PyposmatConfigurationFile): self.configuration = configuration else: raise TypeError('configuration cannot be type:{}'.format( str(type(configuration)))) def _initialize_data(self, data): if isinstance(data, str): assert os.path.isfile(data) self.data = PyposmatDataFile() self.data.read(filename=data) elif isinstance(data, PyposmatDataFile): self.data = deepcopy(data) else: raise TypeError('data cannot be type:{}'.format(str(type(data)))) self.data.create_normalized_errors( normalize_type='by_qoi_target', qoi_targets=self.configuration.qoi_targets) def _initialize_output_path(self, path): if path is None: self.output_path = None elif isinstance(path, str): if os.path.isdir(path): shutil.rmtree(path) os.mkdir(path) self.output_path = path else: raise TypeError
def read_configuration(self, filename=None): if filename is not None: self.configuration_fn = filename _filename = self.configuration_fn self.configuration = PyposmatConfigurationFile() self.configuration.read(_filename) self.parameter_names = list(self.configuration.parameter_names) self.qoi_names = list(self.configuration.qoi_names) self.error_names = list(self.configuration.error_names) self.qoi_validation_names = list( self.configuration.qoi_validation_names) self.error_validation_names = list( self.configuration.error_validation_names) self.qoi_targets = self.configuration.qoi_targets self.qoi_validation_targets = self.configuration.qoi_validation_targets
def test__calculate_stacking_fault(potential,qoi_db,structure_db): simulation_directory = 'rank_test' configuration = PyposmatConfigurationFile() configuration.qois = qoi_db.qois configuration.structures = structure_db qoi_manager = QoiManager(qoi_database=configuration.qois,fullauto=False) qoi_manager.configure() qoi_manager.determine_tasks() task_manager = TaskManager(base_directory='rank_test') task_manager.configure(tasks=qoi_manager.tasks,structures=structure_db) task_manager.evaluate_tasks(parameters=potential,potential=potential) qoi_manager.calculate_qois( task_results=task_manager.results) qoi_manager.qois
def write_configuration_file( filename, qois, potential_definition, structures): configuration = PyposmatConfigurationFile() configuration.qois = qois configuration.potential = potential_definition configuration.sampling_distribution = parameter_distribution configuration.structures = structures configuration.write(filename=filename)
def initialize_configuration(self,config_fn=None,o_config=None): """ read the configuration file Must provide either the path of the configuration file through the config_fn argument, or provide an instance of a PyposmatConfigurationFile object. If no arguments are provided, it will read the config_fn attribute as the path of the configuration file. Args: config_fn(str,None): the path of the configuration file to read. By default, this attribute is None. o_config(str,None): an instance of a PyposmatConfigurationFile object. By default, this attribute is None. """ assert config_fn is None or isinstance(config_fn,str) assert o_config is None or isinstance(o_config,PyposmatConfigurationFile) if config_fn is not None and o_config is not None: m = ( "must either provide the path to config_fn or a PyposmatConfigurationFile " "instance to to o_config" ) raise TypeError(m) # default behavior elif (config_fn is None) and (o_config is None): self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=self.config_fn) # a path is provided elif isinstance(config_fn,str): self.config_fn = config_fn self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=self.config_fn) # an object is provided elif isinstance(o_config,PyposmatConfigurationFile): self.config_fn = None self.configuration = o_config else: m = ( "must either provide the path to config_fn or a PyposmatConfigurationFile " "instance to to o_config" ) raise TypeError(m)
def read_configuration_file(self, filename=None): assert type(filename) in [type(None), str] assert type(self.configuration_filename) in [type(None), str] if filename is not None: self.configuration_filename = filename if not os.path.isabs(self.configuration_filename): self.configuration_filename = os.path.abspath( self.configuration_filename) self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=self.configuration_filename) if self.mpi_rank == 0: self._write_parameter_names() self._write_qoi_names() self._write_error_names()
class Manifold(object): def __init__(self,pyposmat_configuration,pyposmat_data,manifold_config=None): self.configuration = None self.data = None self.manifold_configuration = None self.initialize_configuration(pyposmat_configuration=pyposmat_configuration) self.initialize_data(pyposmat_data=pyposmat_data) self.initialize_manifold_config(manifold_config=manifold_config) def initialize_configuration(self,pyposmat_configuration): if isinstance(pyposmat_configuration,PyposmatConfigurationFile): self.configuration = pyposmat_configuration elif isinstance(pyposmat_configuration,str): self.configuration = PyposmatConfigurationFile() self.configuration.read(filename=pyposmat_configuration) else: raise TypeError('pyposmat_configuration must be either a path or PyposmatConfigurationFile') def initialize_data(self,pyposmat_data): if isinstance(pyposmat_data, PyposmatDataFile): self.data = pyposmat_data elif isinstance(pyposmat_data, str): self.data = PyposmatDataFile self.data.read(filename=pyposmat_data) else: raise TypeError('pyposmat_data must either be a path or a PyposmatDataFile') def initialize_manifold_configuration(self,manifold_configuration=None): if manifold_configuration is None: self.manifold_configuration = None else: raise NotImplementedError def learn_manifold(self,names,scaling_type): raise NotImplementedError def scale_data(self,X,scaling_type): if scaling_type='standard': X_scaled = preprocessing.scale(X) elif scaling_type='none': X_scaled = X
def make_rug_plot(config_fn, data_fn, ax=None, plot_fn='rugplot.png'): o_config = PyposmatConfigurationFile() o_config.read(filename=config_fn) o_data = PyposmatDataFile() o_data.read(filename=data_fn) qoi_targets = o_config.qoi_targets #qoi_targets = get_qoi_targets(o_config) error_names = o_data.error_names qoi_names = o_data.qoi_names # create normalized error df = copy.deepcopy(o_data.df[error_names]) for qn in qoi_names: en = "{}.err".format(qn) nen = "{}.nerr".format(qn) q = qoi_targets[qn] df[nen]=o_data.df[en]/q-q (_nrows,_ncols) = o_data.df.shape if ax is None: fig, ax = plt.subplots(nrows=1,ncols=1) for iq,qn in enumerate(qoi_names): _yloc = [iq+1] ax.scatter( df["{}.nerr".format(qn)], _nrows*[iq+1], marker='|', s=100., color='k' ) plt.sca(ax) plt.yticks(range(len(qoi_names)+1),['']+qoi_names) fig.savefig(plot_fn)