def get_cf_params_local_sa(lca, write_dir, const_factors=(0.1, 10)):
"""Local SA for characterization factors."""
path_lsa_cf = Path(write_dir) / "LSA_scores_cf.pickle"
if not path_lsa_cf.exists():
bio_x_tech_x_demand = lca.biosphere_matrix * spsolve(
lca.technosphere_matrix, lca.demand_array
)
path_lsa_include_inds_cf = Path(write_dir) / "include_inds_cf.pickle"
inds_uncertain = np.where(lca.cf_params["uncertainty_type"] > 1)[0]
if not path_lsa_include_inds_cf.exists():
uncertain_cf_params_temp = lca.cf_params[inds_uncertain]
# Exclude characterization factors that are not affected by given demand
exclude_flows = np.where(bio_x_tech_x_demand == 0)[0]
exclude_inds = np.array([])
for flow in exclude_flows:
exclude_inds = np.hstack(
[exclude_inds, np.where(uncertain_cf_params_temp["row"] == flow)[0]]
)
include_inds_temp = np.setdiff1d(
np.arange(len(uncertain_cf_params_temp)), exclude_inds
)
write_pickle(include_inds_temp, path_lsa_include_inds_cf)
else:
include_inds_temp = read_pickle(path_lsa_include_inds_cf)
include_inds = inds_uncertain[include_inds_temp]
uncertain_cf_params = lca.cf_params[include_inds]
flows = uncertain_cf_params["row"]
bio_reverse_dict = lca.reverse_dict()[WHERE_BIO_REVERSE_DICT]
lsa_scores_cf = {}
for i, param in enumerate(uncertain_cf_params):
flow = flows[i]
input_ = bio_reverse_dict[flow]
scores = []
for const_factor in const_factors:
characterization_vector = sum(deepcopy(lca.characterization_matrix))
characterization_vector[0, flow] *= const_factor
score = characterization_vector * bio_x_tech_x_demand
scores.append(score[0])
lsa_scores_cf[include_inds[i]] = {
"input": input_,
"scores": np.array(scores),
}
write_pickle(lsa_scores_cf, path_lsa_cf)
else:
lsa_scores_cf = read_pickle(path_lsa_cf)
return lsa_scores_cf
def generate_model_output_from_chunks(gsa, n_workers):
Y = np.zeros(shape=(0, ))
for i in range(n_workers):
filepath_Y_chunk = gsa.dirpath_Y / "{}.{}.pickle".format(i, n_workers)
Y_chunk = read_pickle(filepath_Y_chunk)
Y = np.hstack([Y, Y_chunk
]) # TODO change to vstack for multidimensional output
write_hdf5_array(Y, gsa.filepath_Y)
return Y
def get_graph_traversal_params(self, cutoff=1e-16, max_calc=1e16):
fp_graph_traversal = self.write_dir / self.create_graph_traversal_filename(
cutoff, max_calc
)
if fp_graph_traversal.exists():
res = read_pickle(fp_graph_traversal)
else:
res = bc.GraphTraversal().calculate(
self.lca, cutoff=cutoff, max_calc=max_calc
)
write_pickle(res, fp_graph_traversal)
return res
def run_convergence(self, parameter_inds=None, fig_format=()):
"""Calls function that computes converging indices, saves them and plots for model inputs ``parameter_inds``."""
t0 = time.time()
filepath_convergence_dict = self.create_convergence_dict_filepath()
if filepath_convergence_dict.exists():
print("{} already exists".format(filepath_convergence_dict.name))
sa_convergence_dict = read_pickle(filepath_convergence_dict)
else:
sa_convergence_dict = self.generate_converging_gsa_indices()
write_pickle(sa_convergence_dict, filepath_convergence_dict)
t1 = time.time()
print("Total convergence time -> {:8.3f} s".format(t1 - t0))
fig = self.plot_convergence(sa_convergence_dict, parameter_inds, fig_format)
return fig
def get_tech_params_local_sa(
where_tech_lsa, lca, write_dir, const_factors=(0.1, 10), tag=None
):
"""Local SA for technosphere exchanges."""
path_lsa_tech = Path(write_dir) / "LSA_scores_tech_{}.pickle".format(tag)
if not path_lsa_tech.exists():
# 1. lca related
d = lca.demand_array
B = lca.biosphere_matrix
C = sum(lca.characterization_matrix)
reverse_dict = lca.reverse_dict()[0]
# 2. Find zero indices using Local SA
where_tech_lsa.sort()
num_params = len(where_tech_lsa)
tech_params_lsa = lca.tech_params[where_tech_lsa]
rows = tech_params_lsa["row"]
cols = tech_params_lsa["col"]
# 3. Constant factor
const_factor = np.tile(const_factors, (num_params, 1))
N = const_factor.shape[1]
# 4. Preparation for saving of the results
lsa_scores_tech = {}
# 5. Run LSA
for i, where in enumerate(where_tech_lsa):
scores = np.empty(N)
scores[:] = np.nan
for j in range(N):
A = deepcopy(lca.technosphere_matrix)
A[rows[i], cols[i]] *= const_factor[i, j]
scores[j] = C * B * spsolve(A, d)
del A
lsa_scores_tech[int(where)] = dict(
input=reverse_dict[rows[i]],
output=reverse_dict[cols[i]],
scores=deepcopy(scores),
)
# 6. Save results
write_pickle(lsa_scores_tech, path_lsa_tech)
else:
lsa_scores_tech = read_pickle(path_lsa_tech)
return lsa_scores_tech
def compute_scores_per_worker(option, num_params, iterations, i_worker,
n_workers, setup_lca_model, path_base):
if option == "corr":
gsa = setup_corr(num_params, iterations, setup_lca_model, path_base)
elif option == "salt":
gsa = setup_salt(num_params, iterations, setup_lca_model, path_base)
elif option == "delt":
gsa = setup_delt(num_params, iterations, setup_lca_model, path_base)
elif option == "xgbo":
gsa = setup_xgbo(num_params, iterations, setup_lca_model, path_base)
gsa.dirpath_Y.mkdir(parents=True, exist_ok=True)
filepath_X_chunk = gsa.dirpath_Y / "X.unitcube.{}.{}.pickle".format(
i_worker, n_workers)
X_chunk_unitcube = read_pickle(filepath_X_chunk)
X_chunk_rescaled = gsa.model.rescale(X_chunk_unitcube)
del X_chunk_unitcube
scores = gsa.model(X_chunk_rescaled)
Y_filename = "{}.{}.pickle".format(i_worker, n_workers)
filepath = gsa.dirpath_Y / Y_filename
write_pickle(scores, filepath)
return scores
filepath_val_dict = {
"all": filepath_val_all,
"corr": filepath_val_corr,
"salt": filepath_val_salt,
"delt": filepath_val_delt,
"xgbo": filepath_val_gain,
}
Y_dict, S_dict = {}, {}
Y_arr, S_arr = np.zeros((0, iterations)), np.zeros((0, num_params))
stability_dicts = []
for k in filepath_val_dict.keys():
Y_dict[k] = read_hdf5_array(filepath_val_dict[k]).flatten()
if k != "all":
Y_arr = np.vstack([Y_arr, Y_dict[k]])
S_dict[k] = read_pickle(
filepath_gsa_dict[k][0])[filepath_gsa_dict[k][1]]
S_arr = np.vstack([S_arr, S_dict[k]])
stability_dict = read_pickle(filepath_stability_dict[k][0])
stability_dicts.append(stability_dict)
S_sorted = np.sort(np.abs(S_dict["xgbo"]))[-1::-1]
th = (S_sorted[99] - S_sorted[1000]) / 4
bootstrap_ranking_tag = "paper1"
st = Robustness(
stability_dicts,
write_dir,
num_ranks=num_ranks,
bootstrap_ranking_tag=bootstrap_ranking_tag,
num_params_screening=int(0.90 * num_params),
)
def get_bootstrap_rankings_2(
self, bootstrap_data, sa_mean_results, tag, num_ranks=10, rho_choice="spearmanr"
):
"""Get clustered rankings from bootstrap sensitivity indices.
Parameters
----------
bootstrap_data : dict
Dictionary where keys are sensitivity methods names and values are arrays with sensitivity indices from
bootstrapping in rows for each model input in columns.
sa_mean_results : dict
Dictionary where keys are sensitivity methods names and values are mean results for each model input
over all bootstrap samples.
tag : str
Tag to save clustered rankings.
num_ranks : int
Number of clusters.
Returns
-------
bootstrap_rankings : dict
Dictionary where keys are sensitivity methods names and values are clustered ranks for all model inputs.
"""
bootstrap_rankings = {}
for sa_name in self.sa_names:
num_bootstrap = bootstrap_data[sa_name][0].shape[0]
filepath_bootstrap_rankings = self.create_bootstrap_rankings_filepath(
num_ranks,
"{}_v2".format(tag),
sa_name,
num_bootstrap,
)
if filepath_bootstrap_rankings.exists():
bootstrap_rankings_arr = read_pickle(filepath_bootstrap_rankings)
else:
num_combinations = num_bootstrap * (num_bootstrap - 1) // 2
bootstrap_rankings_arr = np.zeros((0, num_combinations))
bootstrap_rankings_arr[:] = np.nan
# TODO change smth here
if sa_name == "total_gain":
means = self.bootstrap_data[sa_name][-1][0, :]
else:
means = sa_mean_results[sa_name][-1, :]
breaks = jenkspy.jenks_breaks(means, nb_class=num_ranks)
# mean_ranking = self.get_one_clustered_ranking(means, num_ranks, breaks)
# mean_ranking = mean_ranking.astype(int)
for i in range(len(self.iterations[sa_name])):
bootstrap_data_sa = bootstrap_data[sa_name][i]
rankings = np.zeros((0, self.num_params))
for data in bootstrap_data_sa:
rankings = np.vstack(
[
rankings,
self.get_one_clustered_ranking(data, num_ranks, breaks),
]
)
rankings = rankings.astype(int)
rho_arr = np.zeros(num_combinations)
rho_arr[:] = np.nan
k = 0
for i, r1 in enumerate(rankings[:-1]):
rho, _ = compute_rho_choice(
rankings[i + 1 :, :], r1, rho_choice=rho_choice
)
rho_arr[k : k + num_bootstrap - i - 1] = rho
k += num_bootstrap - i - 1
bootstrap_rankings_arr = np.vstack(
[bootstrap_rankings_arr, rho_arr]
)
write_pickle(bootstrap_rankings_arr, filepath_bootstrap_rankings)
bootstrap_rankings[sa_name] = bootstrap_rankings_arr
return bootstrap_rankings
lca.lcia()
ei_tech_group = [
g for g in lca.technosphere_mm.groups
if "tech" in g.label and "ecoinvent" in g.label
]
assert len(ei_tech_group) == 1
ei_tech_group = ei_tech_group[0]
tech_params = ei_tech_group.package.data[2]
cutoff = 0.01
max_calc = 1e3
sct_tech_params_filename = "tech_params_cutoff{}_maxcalc{}.pickle".format(
cutoff, int(max_calc))
sct_tech_params_filepath = write_dir_sct / sct_tech_params_filename
if not sct_tech_params_filepath.exists():
tech_params_sct = filter_uncertain_technosphere_exchanges(lca,
cutoff=1e-6)
write_pickle(tech_params_sct, sct_tech_params_filepath)
else:
tech_params_sct = read_pickle(sct_tech_params_filepath)
# Local SA
model = LCAModel(
demand,
uncertain_method,
write_dir,
)
print()
"delt": get_file("delt"),
"xgbo": get_file("xgbo"),
}
return files_dict
path_base = Path(
"/Users/akim/PycharmProjects/gsa-framework-master/dev/write_files/")
num_params = 1000
write_dir = path_base / "{}_morris4".format(num_params)
write_dir_fig = path_base / "paper_figures_review1"
write_dir_arr = write_dir / "arrays"
# Read correlations GSA results
filepath_S = write_dir_arr / "S.correlationsGsa.randomSampling.4000.3407.pickle"
S = read_pickle(filepath_S)
sa_name = "spearman"
# Read stability dicts
files_stability = [
x for x in write_dir_arr.iterdir()
if x.is_file() and "S." in x.name and "stability" in x.name
]
files_stability_dict = get_files_dict_sorted(files_stability)
filepath_stability_dict = {
"corr": (files_stability_dict["corr"], "spearman"),
"salt": (files_stability_dict["salt"], "Total order"),
"delt": (files_stability_dict["delt"], "delta"),
"xgbo": (files_stability_dict["xgbo"], "total_gain"),
}
stability_dicts = []
iterations_const = saltelli_lcm * input_step
inputs_const = 1000
# saltelli_Mstep = saltelli_lcm // (inputs_const // input_step) // 10
# saltelli_Mlast = saltelli_lcm // (inputs_const // input_step)
saltelli_Mstep = 2
saltelli_Mlast = 20
saltelli_M = np.arange(saltelli_Mstep, saltelli_Mlast + 1, saltelli_Mstep)
# 1. wrt number of inputs
# -> salib
filename = "saltelli.sampling.salib.k{}.kstep{}.N{}.pickle".format(
input_last, input_step, iterations_const)
filepath = path_base / filename
if filepath.exists():
saltelli_sampling_salib_inputs = read_pickle(filepath)
else:
saltelli_sampling_salib_inputs = {}
for ind, k in enumerate(inputs):
problem = {"num_vars": k, "bounds": [[0, 1] * k]}
t1 = time.time()
x = salib_saltelli_sample(problem, saltelli_lcm // (ind + 1),
calc_second_order, seed, skip_values)
t2 = time.time()
saltelli_sampling_salib_inputs[k] = t2 - t1
write_pickle(saltelli_sampling_salib_inputs, filepath)
# -> gsa_framework
filename = "saltelli.sampling.gsafr.k{}.kstep{}.N{}.pickle".format(
input_last, input_step, iterations_const)
filepath = path_base / filename
if filepath.exists():
num_params: {
"corr": (files_stability_dict["corr"], "spearman"),
"salt": (files_stability_dict["salt"], "Total order"),
"delt": (files_stability_dict["delt"], "delta"),
"xgbo": (files_stability_dict["xgbo"], "total_gain"),
},
})
# Read GSA files
S_dict_all = {}
S_arr_all = {}
for num_params in nums_params:
S_arr = np.zeros((0, num_params))
S_dict = {}
filepath_gsa_dict_ = filepath_gsa_dict[num_params]
for k, v in filepath_gsa_dict_.items():
S_dict[k] = read_pickle(v[0])[v[1]]
S_arr = np.vstack([S_arr, S_dict[k]])
S_dict_all[num_params] = S_dict
S_arr_all[num_params] = S_arr
# Read stability files
st_classes = {}
for num_params in nums_params:
stability_dicts = []
write_dir = path_base / "{}_morris4".format(num_params)
for k, v in filepath_stability_dict[num_params].items():
stability_dict = read_pickle(v[0])
stability_dicts.append(stability_dict)
st_classes[num_params] = Robustness(
stability_dicts,
write_dir,
def get_bio_params_local_sa(lca, write_dir, const_factors=(0.1, 10)):
"""Local SA for biosphere parameters."""
path_lsa_bio = Path(write_dir) / "LSA_scores_bio.pickle"
if not path_lsa_bio.exists():
tech_x_demand = spsolve(lca.technosphere_matrix, lca.demand_array)
characterization_vector = sum(lca.characterization_matrix)
path_include_inds_bio = Path(write_dir) / "include_inds_bio.pickle"
inds_uncertain = np.where(lca.bio_params["uncertainty_type"] > 1)[0]
if not path_include_inds_bio.exists():
uncertain_bio_params_temp = lca.bio_params[inds_uncertain]
# Exclude bio exchanges that are not selected with the demand vector
exclude_cols = np.where(tech_x_demand == 0)[0]
exclude_inds = np.array([])
for col in exclude_cols:
exclude_inds = np.hstack(
[exclude_inds, np.where(uncertain_bio_params_temp["col"] == col)[0]]
)
# Exclude bio exchanges that are not included in the given lcia method
exclude_rows = np.where(characterization_vector.toarray()[0] == 0)[0]
for row in exclude_rows:
exclude_inds = np.hstack(
[exclude_inds, np.where(uncertain_bio_params_temp["row"] == row)[0]]
)
print(
"Excluding {}/{} biosphere exchanges".format(
len(exclude_inds), len(uncertain_bio_params_temp)
)
)
exclude_inds = np.sort(exclude_inds)
include_inds_temp = np.setdiff1d(
np.arange(len(uncertain_bio_params_temp)), exclude_inds
)
write_pickle(include_inds_temp, path_include_inds_bio)
else:
include_inds_temp = read_pickle(path_include_inds_bio)
include_inds = inds_uncertain[include_inds_temp]
uncertain_bio_params = lca.bio_params[include_inds]
nbio = len(uncertain_bio_params)
rows = uncertain_bio_params["row"]
cols = uncertain_bio_params["col"]
bio_reverse_dict = lca.reverse_dict()[2]
tech_reverse_dict = lca.reverse_dict()[0]
lsa_scores_bio = {}
for i, param in enumerate(uncertain_bio_params):
if i % 1000 == 0:
print("{}/{}".format(i, nbio))
row, col = rows[i], cols[i]
input_ = bio_reverse_dict[row]
output_ = tech_reverse_dict[col]
scores = []
for const_factor in const_factors:
biosphere_matrix = deepcopy(lca.biosphere_matrix)
biosphere_matrix[row, col] *= const_factor
score = characterization_vector * (biosphere_matrix * tech_x_demand)
scores.append(score[0])
lsa_scores_bio[include_inds[i]] = {
"input": input_,
"output": output_,
"scores": np.array(scores),
}
write_pickle(lsa_scores_bio, path_lsa_bio)
else:
lsa_scores_bio = read_pickle(path_lsa_bio)
return lsa_scores_bio