def test____init____using_path_args():
testing_set = get_testing_set()
o = PyposmatDataAnalyzer(
config_fn=testing_set['config_fn'],
results_data_fn=testing_set['results_data_fn']
)
config = PyposmatConfigurationFile()
config.read(filename=testing_set['config_fn'])
assert isinstance(o,PyposmatDataAnalyzer)
assert o.config_fn == testing_set['config_fn']
assert isinstance(o.configuration,PyposmatConfigurationFile)
assert isinstance(o.parameter_names,list)
assert set(o.parameter_names) == set(config.parameter_names)
assert isinstance(o.error_names, list)
assert set(o.error_names) == set(config.error_names)
assert isinstance(o.qoi_names, list)
assert set(o.qoi_names) == set(config.qoi_names)
assert o.results_data_fn == testing_set['results_data_fn']
assert isinstance(o.results_data,PyposmatDataFile)
assert isinstance(o.results_df,pd.DataFrame)
def do_attribute_tests(sampler, config_fn):
from pypospack.pyposmat.engines import PyposmatBaseSampler
from pypospack.pyposmat.data import PyposmatConfigurationFile
# test arguments
assert type(sampler) is PyposmatBaseSampler
assert type(config_fn) is str
config = PyposmatConfigurationFile()
config.read(filename=config_fn)
assert sampler.structure_directory == config.structures[
'structure_directory']
assert sampler.n_iterations == config.sampling_type['n_iterations']
assert sampler.parameter_names == config.parameter_names
assert sampler.qoi_names == config.qoi_names
assert sampler.error_names == config.error_names
assert sampler.free_parameter_names == config.free_parameter_names
assert set(sampler.parameter_constraints.keys()) == set(
config.sampling_constraints.keys())
assert all([
sampler.parameter_constraints[k] == config.sampling_constraints[k]
for k in sampler.parameter_constraints
])
assert sampler.parameter_constraints == config.sampling_constraints
assert sampler.constrained_parameter_names == \
[p for p in sampler.parameter_names if p not in sampler.free_parameter_names]
def dev__defect_calculation():
print(80*'-')
print('{:^80}'.format('defect_calculation'))
print(80*'-')
simulation_dir = 'rank_test'
testing_set = testing_set_Si_sw.get_testing_set_Si()
configuration = PyposmatConfigurationFile()
configuration.qois = testing_set['qoi_db'].qois
configuration.structures = testing_set['structure_db']
qoi_manager = QoiManager(
qoi_database=configuration.qois,
fullauto=False
)
qoi_manager.configure()
qoi_manager.determine_tasks()
print('qoi_manager.tasks')
print(len('qoi_manager.tasks')*'-')
for k,v in qoi_manager.tasks.items():
print(k,v)
task_manager = TaskManager(base_directory=simulation_dir)
task_manager.configure(
tasks = qoi_manager.tasks,
structures = testing_set['structure_db']
)
task_managger.evaluate_tasks()
qoi_manager.calculate_qois(
task_results=task_manager.results
)
qoi_manager.qois
def calculate_kld_parameters(config,data_directory,kld_param_fn='pyposmat.kld_param.out'):
assert isinstance(config,str) or isinstance(config,PyposmatConfigurationFile)
assert os.path.isdir(data_directory)
assert isinstance(kld_param_fn,str)
# process the the configuration argument, the configuration argument has two
# options for processing
# (1) if config is a str, the config is assumed to be a path to the
# the path to the configuration file, and o_config is initialized from it
# (2) if config is PyposmatConfigurationFile object, then o_config is set to it
if isinstance(config,str):
o_config = PyposmatConfigurationFile()
o_config.read(filename=config)
else:
assert isinstance(config,PyposmatConfigurationFile)
o_config = config
kld = OrderedDict()
for i in range(o_config.n_iterations):
kld[i] = OrderedDict()
if i == 0:
kld[i]['results'] = None
kld[i]['kde'] = None
kld[i]['filter'] = calculate_kld(
data_1_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i)),
data_2_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i+1)),
names = o_config.free_parameter_names,
n_samples=n_samples)
else:
kld[i]['results'] = calculate_kld(
data_1_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i-1)),
data_2_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i)),
names = o_config.free_parameter_names,
n_samples=n_samples)
kld[i]['kde'] = calculate_kld(
data_1_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i)),
data_2_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i+1)),
names = o_config.free_parameter_names,
n_samples=n_samples)
kld[i]['filter'] = calculate_kld(
data_1_fn=os.path.join(data_directory,'pyposmat.results.{}.out'.format(i)),
data_2_fn=os.path.join(data_directory,'pyposmat.kde.{}.out'.format(i+1)),
names = o_config.free_parameter_names,
n_samples=n_samples)
print(i)
# write out the kld_parameters file
with open(kld_param_fn,'w') as f:
f.write(",".join(['iteration','results','kde','filter'])+"\n")
for kld_iteration,kld_row in kld.items():
s_list = []
s_list.append(kld_iteration)
for k in ['results','kde','filter']:
if kld_row[k] is None:
s_list.append(float('NaN'))
else:
s_list.append(kld_row[k][0])
f.write(",".join([str(s) for s in s_list])+"\n")
class FileSampler():
def __init__(self,
configuration,
data,
structure_name,
structure_path,
workflow_type,
workflow_definition):
self.initialize_configuration(configuration)
self.initialize_data(data)
self.structure_name = structure_name
self.strucutre_path = structure_path
self.workflow_type = workflow_type
self.workflow_definition = workflow_definition
self.potential_definition = self.configuration.potential
def initialize_configuration(self,configuration):
if isinstance(configuration,PyposmatConfigurationFile):
self.configuration = configuration
elif isinstance(configuration,str):
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=configuration)
else:
msg = ("configuration must be a path to a configuration file or an "
"instance of the PyposmatConfigurationFile,")
raise TypeError(msg)
def initialize_data(self,data):
if isinstance(data,PyposmatDataFile):
self.data = data
elif isinstance(data,str):
self.data = PyposmatDataFile()
self.data.read(filename=data)
else:
msg = ("data must be a path to a data file or an instance of "
"PyposmatDataFile.")
raise TypeError(msg)
def run(self):
for index,row in self.data.df.iterrows():
sim_id = row['sim_id']
print('working on sim_id:{}'.format(sim_id))
parameters = OrderedDict([(k,row[k]) for k in self.configuration.parameter_names])
original_path = os.getcwd()
os.mkdir(sim_id)
os.chdir(sim_id)
if workflow_type == 'lmps_thermal_expansion':
workflow = LammpsThermalExpansion(
structure_name=Si_structure_definition['name'],
structure_path=Si_structure_definition['filename'],
**workflow_definition)
workflow.create_task_configurations()
workflow.create_tasks()
workflow.prepare_tasks(
potential_definition = self.potential_definition,
potential_parameters = parameters)
workflow.run()
os.chdir(original_path)
def make_latex_table(config, data, qoi_type=None, param_type=None):
qoi_types = ['by_qoi_target']
param_type = []
assert isinstance(config,str) \
or isinstance(config,PyposmatConfigurationFile)
assert isinstance(data,str) \
or isinstance(data,PyposmatDataFile)
if isinstance(config, str):
o_config = PyposmatConfigurationFile()
o_config.read(filename=config)
elif isinstance(config, PyposmatConfigurationFile):
o_config = config
else:
raise TypeError()
if isinstance(data, str):
o_data = PyposmatDataFile()
o_data.read(filename=data)
elif isinstance(data, PyposmatDataFile):
o_data = data
else:
raise TypeError()
if qoi_type == 'by_qoi_target':
o_data.create_normalized_errors(normalize_type='by_qoi_target',
qoi_targets=o_config.qoi_targets)
df = o_data.df[o_data.normalized_error_names]
def test__initialize_configuration__with_object():
testing_set = get_testing_set()
o_config = PyposmatConfigurationFile()
o_config.read(filename=testing_set['config_fn'])
o = PyposmatDataAnalyzer()
o.initialize_configuration(o_config=o_config)
def initialize_configuration(self,pyposmat_configuration):
if isinstance(pyposmat_configuration,PyposmatConfigurationFile):
self.configuration = pyposmat_configuration
elif isinstance(pyposmat_configuration,str):
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=pyposmat_configuration)
else:
raise TypeError('pyposmat_configuration must be either a path or PyposmatConfigurationFile')
def config(self, config):
if isinstance(config, str):
self._config = PyposmatConfigurationFile()
self._config.read(filename=config)
elif isinstance(config, PyposmatConfigurationFile):
self._config = config
else:
m = "type(config)={}".format(str(type(config)))
raise TypeError(m)
def read_configuration(self, filename):
"""read in pyposmat configuration file
Args:
filename(str): path of the configuraiton file
"""
self.configuration_fn = filename
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=filename)
def _initialize_configuration(self, configuration):
if isinstance(configuration, str):
assert os.path.isfile(configuration)
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=configuration)
elif isinstance(configuration, PyposmatConfigurationFile):
self.configuration = configuration
else:
raise TypeError('configuration cannot be type:{}'.format(str(type(configuration))))
def initialize_configuration(self,configuration):
if isinstance(configuration,PyposmatConfigurationFile):
self.configuration = configuration
elif isinstance(configuration,str):
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=configuration)
else:
msg = ("configuration must be a path to a configuration file or an "
"instance of the PyposmatConfigurationFile,")
raise TypeError(msg)
def __init__(self, configuration_fn, datafile_fn):
self.configuration_fn = configuration_fn
self.datafile_fn = datafile_fn
if configuration_fn is not None:
self.configuration = PyposmatConfigurationFile()
self.configuration.read(configuration_fn)
if datafile_fn is not None:
self.datafile = PyposmatDataFile()
self.datafile.read(filename=datafile_fn)
def get_qoi_database_from_PyposmatConfigurationFile(config_fn):
config = PyposmatConfigurationFile()
config.read(filename=config_fn)
assert type(config.qois) is OrderedDict
for qoi_id, qoi_info in config.qois.items():
assert type(qoi_id) is str
assert set(qoi_info.keys()) == set(['qoi_type','structures','target'])
return config.qois
def test__initialize_configuration__with_object_and_path():
testing_set = get_testing_set()
o_config = PyposmatConfigurationFile()
o_config.read(filename=testing_set['config_fn'])
o = PyposmatDataAnalyzer()
with pytest.raises(TypeError) as e:
o.initialize_configuration(config_fn=testing_set['config_fn'],
o_config=o_config)
def gmm_analysis(config_fn,
data_fn,
names,
output_directory='gmm_analysis',
max_components=20):
assert isinstance(config_fn, str)
assert isinstance(data_fn, str)
assert os.path.isfile(config_fn)
assert os.path.isfile(data_fn)
if not os.path.isdir(output_directory):
os.mkdir(output_directory)
o_config = PyposmatConfigurationFile()
o_config.read(filename=config_fn)
o_data = PyposmatDataFile()
o_data.read(filename=data_fn)
o_data.create_normalized_errors(normalize_type='by_qoi_target',
qoi_targets=o_config.qoi_targets)
o_data.df['score'] = o_data.df[o_config.normalized_error_names].abs().sum(
axis=1)
data = o_data.df[names]
n_components = np.arange(1, max_components)
models = [
GaussianMixture(n_components=n, covariance_type='full',
random_state=0).fit(data) for n in n_components
]
# AIC analysis
aic, aic_idx = min(
(val, idx) for (idx, val) in enumerate([m.aic(data) for m in models]))
aic_n_components = n_components[aic_idx]
aic_criteria = [m.aic(data) for m in models]
# BIC analysis
bic, bic_idx = min(
(val, idx) for (idx, val) in enumerate([m.bic(data) for m in models]))
bic_n_components = n_components[bic_idx]
bic_criteria = [m.bic(data) for m in models]
#plot the criteria
print('bic_n_components:{}'.format(bic_n_components))
print('aic_n_components:{}'.format(aic_n_components))
plot_fn = os.path.join(output_directory, 'aic_bic_plot.jpg')
plot_gmm_aic_bic(filename=plot_fn,
n_components=n_components,
aic_criteria=aic_criteria,
bic_criteria=bic_criteria,
aic_n_components=aic_n_components,
bic_n_components=bic_n_components)
filename = os.path.join('gmm_analysis', 'gmm_analysis.jpg')
plot_gmm(models[bic_n_components], data, filename=filename)
def test__read_configuration():
from pypospack.pyposmat.data import PyposmatConfigurationFile
o_config = PyposmatConfigurationFile()
o_config.read(filename=config_fn)
o_rugplot = PyposmatParetoRugplot()
o_rugplot.read_configuration(filename=config_fn)
assert type(o_rugplot.parameter_names) is list
assert type(o_rugplot.qoi_names) is list
assert type(o_rugplot.error_names) is list
assert type(o_rugplot.qoi_validation_names) is list
assert type(o_rugplot.error_validation_names) is list
assert isinstance(o_rugplot.qoi_targets, dict)
class PyposmatPostProcessorTestHarness(object):
def __init__(self, configuration_fn, datafile_fn):
self.configuration_fn = configuration_fn
self.datafile_fn = datafile_fn
if configuration_fn is not None:
self.configuration = PyposmatConfigurationFile()
self.configuration.read(configuration_fn)
if datafile_fn is not None:
self.datafile = PyposmatDataFile()
self.datafile.read(filename=datafile_fn)
def get_parameter_names(self):
return self.configuration.parameter_names
def read_configuration(self,filename=None):
if filename is not None:
self.configuration_fn = filename
_filename = self.configuration_fn
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=filename)
self._parameter_names = self.configuration.parameter_names
# determine_qoi_names_from_configuration_file
self.qoi_fitting_names = self.configuration.qoi_names
self.qoi_testing_names = self.configuration.qoi_validation_names
self.qoi_names = self.qoi_fitting_names + self.qoi_testing_names
# determine_error_names_from_configuration_file
self.error_fitting_names = self.configuration.error_names
self.error_testing_names = self.configuration.qoi_validation_names
self.error_names = self.error_fitting_names + self.error_testing_names
# determine_absolute_error_names_from_configuration_file
self.abs_error_fitting_names = ['{}.abserr'.format(v) for self.qoi_fitting_names]
self.abs_error_testing_names = ['{}.abserr'.format(v) for self.qoi_testing_names]
self.abs_error_names = self.abs_error_fitting_names + self.abs_error_testing_names
# determine_normalized_error_names_from_configuration_file
self.norm_error_fitting_names = ['{}.nerr'.format(v) for self.qoi_fitting_names]
self.norm_error_testing_names = ['{}.nerr'.format(v) for self.qoi_testing_names]
self.norm_error_names = self.norm_error_fitting_names + self.norm_error_testing_names
if self.qoi_fitting_names is not None:
_config_qois_fitting = self.configuration.qois
self.qoi_fitting_targets = OrderedDict(
[(q,_config_qois_fitting[q]['target']) for q in self.qoi_fitting_names]
)
if self.qoi_testing_names is not None:
_config_qoi_testing = self.configuration.qois_validation
self.qoi_testing_targets = OrderedDict(
[(q,_config_qoi_testing[q]['target']) for q in self.qoi_testing_names]
)
self.qoi_targets = OrderedDict()
for k,v in self.qoi_fitting_targets.items(): self.qoi_targets[k] = v
for k,v in self.qoi_testing_targets.items(): self.qoi_targets[k] = v
# old naming convention [DEPRECATED]
self.qoi_validation_names = self.configuration.qoi_validation_names
self.error_validation_names = self.configuration.qoi_validation_names
def initialize_configuration(self, config):
assert isinstance(config,str) \
or isinstance(config,PyposmatConfigurationFile) \
or config is None
if isinstance(config, str):
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=config)
elif isinstance(config, PyposmatConfigurationFile):
self.configuration = config
elif config is None:
self.configuration = None
else:
m = 'config arguement must either be a path string of a PyposmatConfigurationFile object'
raise TypeError(m)
def show_qoi_targets(config_fn,
data_fn):
o_config = PyposmatConfigurationFile()
o_config.read(filename=config_fn)
o_data = PyposmatDataFile()
o_data.read(filename=data_fn)
for qoi_name, qoi_target in o_config.qoi_targets.items():
try:
qoi_avg = o_data.df[qoi_name].mean()
except KeyError as e:
qoi_avg = 'no value'
s = "{:20} {:10} {:10}".format(qoi_name,qoi_target,qoi_avg)
print(s)
def test____init____config_as_obj():
o = Pyposmat2DDensityPlot(config=PyposmatConfigurationFile())
assert isinstance(o, Pyposmat2DDensityPlot)
assert isinstance(o.configuration, PyposmatConfigurationFile)
assert o.data is None
assert o.fig is None
assert o.ax is None
class BaseAnalysis(object):
def __init__(self, configuration, data, output_path=None):
self.configuration = None
self.data = None
self.output_path = None
self._initialize_configuration(configuration=configuration)
self._initialize_data(data=data)
self._initialize_output_path(path=output_path)
def _initialize_configuration(self, configuration):
if isinstance(configuration, str):
assert os.path.isfile(configuration)
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=configuration)
elif isinstance(configuration, PyposmatConfigurationFile):
self.configuration = configuration
else:
raise TypeError('configuration cannot be type:{}'.format(
str(type(configuration))))
def _initialize_data(self, data):
if isinstance(data, str):
assert os.path.isfile(data)
self.data = PyposmatDataFile()
self.data.read(filename=data)
elif isinstance(data, PyposmatDataFile):
self.data = deepcopy(data)
else:
raise TypeError('data cannot be type:{}'.format(str(type(data))))
self.data.create_normalized_errors(
normalize_type='by_qoi_target',
qoi_targets=self.configuration.qoi_targets)
def _initialize_output_path(self, path):
if path is None:
self.output_path = None
elif isinstance(path, str):
if os.path.isdir(path):
shutil.rmtree(path)
os.mkdir(path)
self.output_path = path
else:
raise TypeError
def read_configuration(self, filename=None):
if filename is not None: self.configuration_fn = filename
_filename = self.configuration_fn
self.configuration = PyposmatConfigurationFile()
self.configuration.read(_filename)
self.parameter_names = list(self.configuration.parameter_names)
self.qoi_names = list(self.configuration.qoi_names)
self.error_names = list(self.configuration.error_names)
self.qoi_validation_names = list(
self.configuration.qoi_validation_names)
self.error_validation_names = list(
self.configuration.error_validation_names)
self.qoi_targets = self.configuration.qoi_targets
self.qoi_validation_targets = self.configuration.qoi_validation_targets
def test__calculate_stacking_fault(potential,qoi_db,structure_db):
simulation_directory = 'rank_test'
configuration = PyposmatConfigurationFile()
configuration.qois = qoi_db.qois
configuration.structures = structure_db
qoi_manager = QoiManager(qoi_database=configuration.qois,fullauto=False)
qoi_manager.configure()
qoi_manager.determine_tasks()
task_manager = TaskManager(base_directory='rank_test')
task_manager.configure(tasks=qoi_manager.tasks,structures=structure_db)
task_manager.evaluate_tasks(parameters=potential,potential=potential)
qoi_manager.calculate_qois(
task_results=task_manager.results)
qoi_manager.qois
def write_configuration_file(
filename,
qois,
potential_definition,
structures):
configuration = PyposmatConfigurationFile()
configuration.qois = qois
configuration.potential = potential_definition
configuration.sampling_distribution = parameter_distribution
configuration.structures = structures
configuration.write(filename=filename)
def initialize_configuration(self,config_fn=None,o_config=None):
""" read the configuration file
Must provide either the path of the configuration file through the config_fn
argument, or provide an instance of a PyposmatConfigurationFile object. If no
arguments are provided, it will read the config_fn attribute as the path of the
configuration file.
Args:
config_fn(str,None): the path of the configuration file to read. By default,
this attribute is None.
o_config(str,None): an instance of a PyposmatConfigurationFile object. By default,
this attribute is None.
"""
assert config_fn is None or isinstance(config_fn,str)
assert o_config is None or isinstance(o_config,PyposmatConfigurationFile)
if config_fn is not None and o_config is not None:
m = (
"must either provide the path to config_fn or a PyposmatConfigurationFile "
"instance to to o_config"
)
raise TypeError(m)
# default behavior
elif (config_fn is None) and (o_config is None):
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=self.config_fn)
# a path is provided
elif isinstance(config_fn,str):
self.config_fn = config_fn
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=self.config_fn)
# an object is provided
elif isinstance(o_config,PyposmatConfigurationFile):
self.config_fn = None
self.configuration = o_config
else:
m = (
"must either provide the path to config_fn or a PyposmatConfigurationFile "
"instance to to o_config"
)
raise TypeError(m)
def read_configuration_file(self, filename=None):
assert type(filename) in [type(None), str]
assert type(self.configuration_filename) in [type(None), str]
if filename is not None:
self.configuration_filename = filename
if not os.path.isabs(self.configuration_filename):
self.configuration_filename = os.path.abspath(
self.configuration_filename)
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=self.configuration_filename)
if self.mpi_rank == 0:
self._write_parameter_names()
self._write_qoi_names()
self._write_error_names()
class Manifold(object):
def __init__(self,pyposmat_configuration,pyposmat_data,manifold_config=None):
self.configuration = None
self.data = None
self.manifold_configuration = None
self.initialize_configuration(pyposmat_configuration=pyposmat_configuration)
self.initialize_data(pyposmat_data=pyposmat_data)
self.initialize_manifold_config(manifold_config=manifold_config)
def initialize_configuration(self,pyposmat_configuration):
if isinstance(pyposmat_configuration,PyposmatConfigurationFile):
self.configuration = pyposmat_configuration
elif isinstance(pyposmat_configuration,str):
self.configuration = PyposmatConfigurationFile()
self.configuration.read(filename=pyposmat_configuration)
else:
raise TypeError('pyposmat_configuration must be either a path or PyposmatConfigurationFile')
def initialize_data(self,pyposmat_data):
if isinstance(pyposmat_data, PyposmatDataFile):
self.data = pyposmat_data
elif isinstance(pyposmat_data, str):
self.data = PyposmatDataFile
self.data.read(filename=pyposmat_data)
else:
raise TypeError('pyposmat_data must either be a path or a PyposmatDataFile')
def initialize_manifold_configuration(self,manifold_configuration=None):
if manifold_configuration is None:
self.manifold_configuration = None
else:
raise NotImplementedError
def learn_manifold(self,names,scaling_type):
raise NotImplementedError
def scale_data(self,X,scaling_type):
if scaling_type='standard':
X_scaled = preprocessing.scale(X)
elif scaling_type='none':
X_scaled = X
def make_rug_plot(config_fn,
data_fn,
ax=None,
plot_fn='rugplot.png'):
o_config = PyposmatConfigurationFile()
o_config.read(filename=config_fn)
o_data = PyposmatDataFile()
o_data.read(filename=data_fn)
qoi_targets = o_config.qoi_targets
#qoi_targets = get_qoi_targets(o_config)
error_names = o_data.error_names
qoi_names = o_data.qoi_names
# create normalized error
df = copy.deepcopy(o_data.df[error_names])
for qn in qoi_names:
en = "{}.err".format(qn)
nen = "{}.nerr".format(qn)
q = qoi_targets[qn]
df[nen]=o_data.df[en]/q-q
(_nrows,_ncols) = o_data.df.shape
if ax is None:
fig, ax = plt.subplots(nrows=1,ncols=1)
for iq,qn in enumerate(qoi_names):
_yloc = [iq+1]
ax.scatter(
df["{}.nerr".format(qn)],
_nrows*[iq+1],
marker='|',
s=100.,
color='k'
)
plt.sca(ax)
plt.yticks(range(len(qoi_names)+1),['']+qoi_names)
fig.savefig(plot_fn)