# run gPC algorithm
session, coeffs, results = session.run()
#%%
# Postprocessing
# --------------
# read session
session = pygpc.read_session(fname=session.fn_session,
folder=session.fn_session_folder)
# Post-process gPC and add results to .hdf5 file
pygpc.get_sensitivities_hdf5(fn_gpc=session.fn_results,
output_idx=None,
calc_sobol=True,
calc_global_sens=True,
calc_pdf=True,
n_samples=1e4)
#%%
# Validation
# ----------
# Validate gPC vs original model function (2D-surface)
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# Validate gPC vs original model function
pygpc.validate_gpc_plot(session=session,
coeffs=coeffs,
random_vars=["Rd", "n_Qd"],
n_grid=[51, 51],
output_idx=500,
fn_out=None,
# run gPC algorithm
session, coeffs, results = session.run()
#%%
# Postprocessing
# --------------
# read session
session = pygpc.read_session(fname=session.fn_session, folder=session.fn_session_folder)
# Post-process gPC
pygpc.get_sensitivities_hdf5(fn_gpc=options["fn_results"],
output_idx=None,
calc_sobol=True,
calc_global_sens=True,
calc_pdf=True,
algorithm="sampling",
n_samples=1e3)
#%%
# Validation
# ----------
# Validate gPC vs original model function (2D-surface)
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pygpc.validate_gpc_plot(session=session,
coeffs=coeffs,
random_vars=list(problem.parameters_random.keys()),
n_grid=[51, 51],
output_idx=[0],
fn_out=None,
def run(self, withLesion=False):
# define the Problem
parameters = OrderedDict(
) # conductivity values adapted according to (Saturnino et al. 2019)
parameters["scalp_cond"] = Beta(pdf_shape=[3, 3],
pdf_limits=[.2,
.5]) # before: [.28,.87]
parameters["skull_cond"] = Beta(
pdf_shape=[3, 3],
pdf_limits=[.003, .012]) # compact bone, before: [.0016,.033]
parameters["csf_cond"] = Beta(
pdf_shape=[3, 3],
pdf_limits=[1.2, 1.8]) # before: 1.65 (fixed not random variable)
parameters["gm_cond"] = Beta(pdf_shape=[3, 3],
pdf_limits=[.1, .6]) # before: [.22, .67]
parameters["wm_cond"] = Beta(pdf_shape=[3, 3],
pdf_limits=[.1, .4]) # before: [.09, .29]
parameters["air_cond"] = [1e-15]
parameters["electrode_cond"] = [1.4]
if withLesion:
print(
"Performing sensitivity analysis with lesioned white matter tissue as input variable."
)
parameters["lesion_cond"] = Beta(
pdf_shape=[3, 3], pdf_limits=[
.1, 1.8
]) # range: [lowest-WM,highest-CSF], before: [.04, 1.5]
else:
print(
"Performing sensitivity analysis without lesioned white matter tissue."
)
parameters["OF_CASE_DIR_BASE"] = self.of_case_dir_base
wmlSensitivity = Problem(FOAMSim(), parameters)
# gPC options
options = dict()
options["order_start"] = 1 # default
options["matrix_ratio"] = 1.5 # default
options["eps"] = 1e-3 # default
options["order_max_norm"] = 1 # default
options["order_end"] = 5
options["interaction_order"] = 3 # Auf 2 oder 3 setzen
options["seed"] = 42
options["n_cpu"] = 6
options["settings"] = None # ?
options["fn_results"] = self.results_filename
options["solver"] = "Moore-Penrose" # default
options["error_norm"] = "relative" # default
options["method"] = "Reg" # not necessary when using "RegAdaptive" ?
options["print_func_time"] = True # default
options["gradient_enhanced"] = False # default
options["projection"] = False # default
options["adaptive_sampling"] = False # default
options["GPU"] = False # default
# define the kind of gPC algorithm
algorithm = pygpc.RegAdaptive(problem=wmlSensitivity, options=options)
# run gPC algorithm
print("(1) RUN GPC")
gpc_phi, coeffs_phi, results_phi = algorithm.run()
######### DEBUG START
'''
print("Loading:", self.results_filename+".hdf5")
print("Loading:", self.results_filename+".pkl")
results_phi = []
try:
with h5py.File( self.results_filename+".hdf5","r") as f:
results_phi = f['model_evaluations/results'][:]
except (KeyError, ValueError):
return None
gpc_phi=None
with open( self.results_filename+".pkl", "rb") as p:
gpc_phi=pickle.load(p)
'''
######### DEBUG END
# post process
print("(2) COMPUTE SOBOL INDICES")
pygpc.get_sensitivities_hdf5(
fn_gpc=options["fn_results"],
output_idx=
None, # Indices of output quantities (QOIs) to consider in postprocessind
calc_sobol=True, # Calculate Sobol indices
calc_global_sens=
True, # Calculate global derivative based sensitivities
calc_pdf=False
) # Calculate probability density functions of output quantities
print("(3) WRITE POSTPORCESSED DATA AS OF FIELD")
structures = np.array(["scalp", "skull", "csf", "gm", "wm"])
if withLesion:
structures = np.append(structures, ["lesion"])
print("Output will contain Sobol Indices of the following structures:",
structures)
# read hdf5 files with post-processed results and write as OpenFoam field
with h5py.File(self.results_filename + ".hdf5", "r") as f:
try:
FOAMSim.write_result_field(parameters["OF_CASE_DIR_BASE"],
"ElPot_mean", f["sens/mean"][0])
FOAMSim.write_result_field(parameters["OF_CASE_DIR_BASE"],
"ElPot_var", f["sens/var"][0])
FOAMSim.write_result_field(parameters["OF_CASE_DIR_BASE"],
"ElPot_rstd", f["sens/rstd"][0])
# "sobol_idx_order" is a boolean array representing which of the random input variables
# is represented by the data on "sens/sobol" of the same index.
# e.g. given 3 input random variables A,B,C, sobol_idx_bool = [True, False, False]
# means that sobol[0] represents the contribution of A to the variance
for sobol_order in range(0, len(f["sens/sobol_idx_bool"])):
sobol_structures = structures[np.array(
f["sens/sobol_idx_bool"][sobol_order])]
structures_string = ""
for structure in sobol_structures:
structures_string += structure + "_"
structures_string = structures_string[:-1]
FOAMSim.write_result_field(
parameters["OF_CASE_DIR_BASE"], "ElPot_sobol_" +
str(sobol_order) + "_" + structures_string,
f["sens/sobol"][sobol_order])
except KeyError:
print("Cannot read result data.")
# Determine gPC coefficients again for the E field on the midlayer.
# -> We use the before determined expansion (from ElPot).
# -> We determine the coefficients based on the E field sampled on the midlayer.
total_num_iterations = len(results_phi)
#total_num_iterations = 51
E_midlayer = [0] * total_num_iterations
# The results are each written into a separate directory.
# Each process creates one sub-case-folder and stores its results in that folder.
# The results of each process are stored in a sub-directory of the corresponding case folder.
# This result directory is names results_'Iterationnumber'_iter_'NumderOfOpenFOAMIterationsToConverge'
# e.g. result_0_iter8 = is the result of the very first pygpc-iteration and OpenFOAM took 8
# iterations to converge below the global residual.
# 1st: collect the sub-process-case-directories
process_directories = [parameters["OF_CASE_DIR_BASE"] + "_0"]
for i in range(1, options["n_cpu"]):
process_directories.append(parameters["OF_CASE_DIR_BASE"] + "_" +
str(i))
print("(4) COLLECT RESULTS OF MIDLAYER")
# 2nd: collect the results from the respective result-folders of each sub-process-directory
# and add them to the result-array (E_midlayer) to the correct position (according to
# the iteration number when that result was calculated)
for root, dirs, files in chain.from_iterable(
walk(path) for path in process_directories):
iteration_number = search("result_([0-9]+)_iter.*", root)
if iteration_number:
with open(root + "/E_midlayer.csv") as midlayer_data_csv:
midayer_data = []
reader = csv.reader(midlayer_data_csv, delimiter=',')
next(reader) # skip first line
for row in reader:
midayer_data.append(float(row[0]))
E_midlayer[int(iteration_number.group(1))] = midayer_data
'''
# DEBUG: for debugging the later steps without having to re-compute the gPC for Phi
with open( self.results_filename+".pkl","rb") as pickle_file:
gpc_phi = pickle.load( pickle_file )
gpc_phi.n_cpu=6
'''
print("(5) COMPUTE NEW GPC COEFFICIENTS FROM MIDLAYER DATA")
# Compute new coefficients according to the E field at the midlayer
# (+ store for later/repeated use)
print(len(E_midlayer))
print(len(E_midlayer[0]))
coeffs_E = gpc_phi.solve(E_midlayer, solver=options["solver"])
print("(6) WRITE E MIDLAYER")
print("Writing to '" + self.results_filename + "_coeffs_E.hdf'")
with h5py.File(self.results_filename + "_coeffs_E.hdf5", "a") as f:
if "coeffs_E" in f.keys():
del f['coeffs_E']
f.create_dataset("coeffs_E",
data=coeffs_E,
maxshape=None,
dtype="float64")
'''
# DEBUG: for debugging the later stages without having to re-calculate the above again...
coeffs_E = []
try:
with h5py.File( self.results_filename+"_coeffs_E.hdf5","r") as f:
coeffs_E = f['coeffs_E'][:]
except (KeyError, ValueError):
return None
# determine error
'''
print("(7) DETERMINE LOOCV ERROR")
eps = gpc_phi.loocv(
results=np.array(E_midlayer), coeffs=coeffs_E
) # I do not need a nonNanMask since my results won't become NaN
'''
pygpc.validate_gpc_mc( gpc_phi, coeffs_E, n_samples=int(6), n_cpu=6, fn_out=self.results_filename+"_MC_validation_result")
exit()
'''
print("*** Error in E field estimation = " + str(eps) + " ***")
# postprocessing of E-field gPC
mean_E = gpc_phi.get_mean(coeffs_E[:, np.arange(coeffs_E.shape[1])])[
0] # in case of multi-part gpce this might be > 0
std_E = gpc_phi.get_std(coeffs_E[:, np.arange(coeffs_E.shape[1])])[0]
var_E = std_E**2
sobol_E, sobol_idx_E, sobol_idx_bool_E = gpc_phi.get_sobol_indices(
coeffs_E[:, np.arange(coeffs_E.shape[1])])
structures_strings = []
for sobol_order in range(0, len(sobol_idx_bool_E)):
sobol_structures = structures[np.array(
sobol_idx_bool_E[sobol_order])]
structures_string = ""
for structure in sobol_structures:
structures_string += structure + "_"
structures_strings.append(structures_string[:-1])
with open(self.results_filename + "_postprocessing_E.csv",
"w",
newline="") as midlayer_gpc_results_csv:
writer = csv.writer(midlayer_gpc_results_csv, delimiter=",")
sobol_indices_columnnames = []
for sobol_order in range(0, len(sobol_E)):
sobol_indices_columnnames.append(
"E_sobol_" + str(sobol_order) + "_" +
structures_strings[sobol_order])
writer.writerow(["E_mean", "E_var"] + sobol_indices_columnnames)
for i in range(0, len(mean_E)):
sobol_indices = []
for sobol_order in range(0, len(sobol_E)):
sobol_indices.append(sobol_E[sobol_order][i])
writer.writerow([mean_E[i], var_E[i]] + sobol_indices)
def test_12_Matlab_gpc(self):
"""
Algorithm: RegAdaptive
Method: Regression
Solver: Moore-Penrose
Grid: RandomGrid
"""
global folder, plot, matlab
test_name = 'pygpc_test_12_Matlab_gpc'
print(test_name)
if matlab:
import matlab.engine
from templates.MyModel_matlab import MyModel_matlab
# define model
model = MyModel_matlab(fun_path=os.path.join(pygpc.__path__[0], "testfunctions"))
# define problem (the parameter names have to be the same as in the model)
parameters = OrderedDict()
parameters["x1"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[-np.pi, np.pi])
parameters["x2"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[-np.pi, np.pi])
parameters["x3"] = 0.
parameters["a"] = 7.
parameters["b"] = 0.1
problem = pygpc.Problem(model, parameters)
# gPC options
options = dict()
options["order_start"] = 5
options["order_end"] = 20
options["solver"] = "LarsLasso"
options["interaction_order"] = 2
options["order_max_norm"] = 0.7
options["n_cpu"] = 0
options["adaptive_sampling"] = True
options["gradient_enhanced"] = True
options["fn_results"] = os.path.join(folder, test_name)
options["eps"] = 0.0075
options["matlab_model"] = True
# define algorithm
algorithm = pygpc.RegAdaptive(problem=problem, options=options)
# run gPC algorithm
gpc, coeffs, results = algorithm.run()
if plot:
# Validate gPC vs original model function (2D-surface)
pygpc.validate_gpc_plot(gpc=gpc,
coeffs=coeffs,
random_vars=list(problem.parameters_random.keys()),
n_grid=[51, 51],
output_idx=0,
fn_out=None,
n_cpu=options["n_cpu"])
# Post-process gPC
pygpc.get_sensitivities_hdf5(fn_gpc=options["fn_results"],
output_idx=None,
calc_sobol=True,
calc_global_sens=True,
calc_pdf=True,
algorithm="sampling",
n_samples=1e3)
# Validate gPC vs original model function (Monte Carlo)
nrmsd = pygpc.validate_gpc_mc(gpc=gpc,
coeffs=coeffs,
n_samples=int(1e4),
output_idx=0,
n_cpu=options["n_cpu"],
smooth_pdf=True,
fn_out=None,
plot=plot)
files_consistent, error_msg = pygpc.check_file_consistency(options["fn_results"] + ".hdf5")
print("> Maximum NRMSD (gpc vs original): {:.2}%".format(np.max(nrmsd)))
# self.expect_true(np.max(nrmsd) < 0.1, 'gPC test failed with NRMSD error = {:1.2f}%'.format(np.max(nrmsd)*100))
print("> Checking file consistency...")
self.expect_true(files_consistent, error_msg)
print("done!\n")
else:
print("Skipping Matlab test...")
def test_7_MERegAdaptiveProjection_gpc(self):
"""
Algorithm: MERegAdaptiveProjection
Method: Regression
Solver: Moore-Penrose
Grid: RandomGrid
"""
global folder, plot
test_name = 'pygpc_test_7_MERegAdaptiveProjection_gpc'
print(test_name)
# define model
model = pygpc.testfunctions.DiscontinuousRidgeManufactureDecay()
# define problem
parameters = OrderedDict()
parameters["x1"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[0, 1])
parameters["x2"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[0, 1])
problem = pygpc.Problem(model, parameters)
# gPC options
options = dict()
options["method"] = "reg"
options["solver"] = "Moore-Penrose"
options["settings"] = None
options["order_start"] = 0
options["order_end"] = 15
options["interaction_order"] = 2
options["matrix_ratio"] = 2
options["projection"] = True
options["n_cpu"] = 0
options["gradient_enhanced"] = True
options["gradient_calculation"] = "standard_forward"
options["error_type"] = "loocv"
options["n_samples_validation"] = 1e4
options["qoi"] = "all"
options["classifier"] = "learning"
options["classifier_options"] = {"clusterer": "KMeans",
"n_clusters": 2,
"classifier": "MLPClassifier",
"classifier_solver": "lbfgs"}
options["n_samples_discontinuity"] = 10
options["adaptive_sampling"] = True
options["eps"] = 0.01
options["n_grid_init"] = 10
options["GPU"] = False
options["fn_results"] = os.path.join(folder, test_name)
# define algorithm
algorithm = pygpc.MERegAdaptiveProjection(problem=problem, options=options)
# run gPC algorithm
gpc, coeffs, results = algorithm.run()
if plot:
# Validate gPC vs original model function (2D-surface)
pygpc.validate_gpc_plot(gpc=gpc,
coeffs=coeffs,
random_vars=list(problem.parameters_random.keys()),
n_grid=[51, 51],
output_idx=0,
fn_out=None,
n_cpu=options["n_cpu"])
# Post-process gPC
pygpc.get_sensitivities_hdf5(fn_gpc=options["fn_results"],
output_idx=None,
calc_sobol=True,
calc_global_sens=True,
calc_pdf=True,
algorithm="sampling",
n_samples=1e3)
# Validate gPC vs original model function (Monte Carlo)
nrmsd = pygpc.validate_gpc_mc(gpc=gpc,
coeffs=coeffs,
n_samples=int(1e4),
output_idx=0,
n_cpu=options["n_cpu"],
smooth_pdf=True,
fn_out=None,
plot=plot)
files_consistent, error_msg = pygpc.check_file_consistency(options["fn_results"] + ".hdf5")
print("> Maximum NRMSD (gpc vs original): {:.2}%".format(np.max(nrmsd)))
# self.expect_true(np.max(nrmsd) < 0.1, 'gPC test failed with NRMSD error = {:1.2f}%'.format(np.max(nrmsd)*100))
print("> Checking file consistency...")
self.expect_true(files_consistent, error_msg)
print("done!\n")
def test_0_Static_gpc_quad(self):
"""
Algorithm: Static
Method: Quadrature
Solver: NumInt
Grid: TensorGrid
"""
global folder, plot
test_name = 'pygpc_test_0_Static_gpc_quad'
print(test_name)
# define model
model = pygpc.testfunctions.Peaks()
# define problem
parameters = OrderedDict()
parameters["x1"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[1.2, 2])
parameters["x2"] = 1.25
parameters["x3"] = pygpc.Beta(pdf_shape=[1, 1], pdf_limits=[0, 0.6])
problem = pygpc.Problem(model, parameters)
# gPC options
options = dict()
options["method"] = "quad"
options["solver"] = "NumInt"
options["settings"] = None
options["order"] = [9, 9]
options["order_max"] = 9
options["interaction_order"] = 2
options["error_type"] = "nrmsd"
options["n_cpu"] = 0
options["fn_results"] = os.path.join(folder, test_name)
options["GPU"] = False
# generate grid
grid = pygpc.TensorGrid(parameters_random=problem.parameters_random,
options={"grid_type": ["jacobi", "jacobi"], "n_dim": [9, 9]})
# define algorithm
algorithm = pygpc.Static(problem=problem, options=options, grid=grid)
# run gPC algorithm
gpc, coeffs, results = algorithm.run()
# Post-process gPC
pygpc.get_sensitivities_hdf5(fn_gpc=options["fn_results"],
output_idx=None,
calc_sobol=True,
calc_global_sens=True,
calc_pdf=True,
algorithm="standard",
n_samples=1e3)
# Validate gPC vs original model function (Monte Carlo)
nrmsd = pygpc.validate_gpc_mc(gpc=gpc,
coeffs=coeffs,
n_samples=int(1e4),
output_idx=0,
fn_out=None,
plot=plot)
files_consistent, error_msg = pygpc.check_file_consistency(options["fn_results"] + ".hdf5")
print("> Maximum NRMSD (gpc vs original): {:.2}%".format(np.max(nrmsd)))
# self.expect_true(np.max(nrmsd) < 0.1, 'gPC test failed with NRMSD error = {:1.2f}%'.format(np.max(nrmsd)*100))
print("> Checking file consistency...")
self.expect_true(files_consistent, error_msg)
print("done!\n")
def test_3_StaticProjection_gpc(self):
"""
Algorithm: StaticProjection
Method: Regression
Solver: Moore-Penrose
Grid: RandomGrid
"""
global folder, plot
test_name = 'pygpc_test_3_StaticProjection_gpc'
print(test_name)
# define model
model = pygpc.testfunctions.GenzOscillatory()
# define problem
parameters = OrderedDict()
parameters["x1"] = pygpc.Beta(pdf_shape=[1., 1.], pdf_limits=[0., 1.])
parameters["x2"] = pygpc.Beta(pdf_shape=[1., 1.], pdf_limits=[0., 1.])
problem = pygpc.Problem(model, parameters)
# gPC options
options = dict()
options["method"] = "reg"
options["solver"] = "Moore-Penrose"
options["settings"] = None
options["order"] = [10]
options["order_max"] = 10
options["interaction_order"] = 1
options["n_cpu"] = 0
options["error_type"] = "nrmsd"
options["error_norm"] = "relative"
options["matrix_ratio"] = 2
options["qoi"] = 0
options["n_grid_gradient"] = 50
options["fn_results"] = os.path.join(folder, test_name)
options["gradient_enhanced"] = True
# define algorithm
algorithm = pygpc.StaticProjection(problem=problem, options=options)
# run gPC algorithm
gpc, coeffs, results = algorithm.run()
if plot:
# Validate gPC vs original model function (2D-surface)
pygpc.validate_gpc_plot(gpc=gpc,
coeffs=coeffs,
random_vars=list(problem.parameters_random.keys()),
n_grid=[51, 51],
output_idx=0,
fn_out=None,
n_cpu=options["n_cpu"])
# Post-process gPC
pygpc.get_sensitivities_hdf5(fn_gpc=options["fn_results"],
output_idx=None,
calc_sobol=True,
calc_global_sens=True,
calc_pdf=True,
algorithm="sampling",
n_samples=1e3)
# Validate gPC vs original model function (Monte Carlo)
nrmsd = pygpc.validate_gpc_mc(gpc=gpc,
coeffs=coeffs,
n_samples=int(1e4),
output_idx=0,
n_cpu=options["n_cpu"],
smooth_pdf=False,
fn_out=None,
plot=plot)
files_consistent, error_msg = pygpc.check_file_consistency(options["fn_results"] + ".hdf5")
print("> Maximum NRMSD (gpc vs original): {:.2}%".format(np.max(nrmsd)))
# self.expect_true(np.max(nrmsd) < 0.1, 'gPC test failed with NRMSD error = {:1.2f}%'.format(np.max(nrmsd)*100))
print("> Checking file consistency...")
self.expect_true(files_consistent, error_msg)
print("done!\n")
