'SkewNormal',
"estimator":
estimator,
"adaptive_noise_fn":
"polynomial_rate_2_2.00" if estimator == "KernelMixtureNetwork"
else ("polynomial_rate_2_1.00" if estimator ==
"NormalizingFlowEstimator" else "polynomial_rate_3_1.00"),
},
},
) for estimator in estimators])
colors = iter(['#1f77b4', '#2ca02c', '#9467bd', '#ff7f0e', '#d62728'])
fig = gof_result.plot_metric(plot_dict,
metric="score",
figsize=(12, 6),
layout=(2, 3),
log_scale_y=False,
color=colors)
# -- add the bayesian data from csv --
axarr = fig.axes
if isinstance(axarr, np.ndarray):
axarr = axarr.flatten()
df = pd.read_csv(os.path.join(DATA_DIR_LOCAL, EXP_PREFIX, BAYES_RESULTS_FILE))
df["param_kl_weight_scale"] = df.param_kl_weight_scale * df.n_datapoints
test_score_columns = [
column for column in df.columns
if column.startswith("split") and column.endswith("_test_score")
]
"simulator": simulator,
"estimator": estimator,
"adaptive_noise_fn": "rule_of_thumb_0.70"
},
"rule_of_thumb_0.5": {
"simulator": simulator,
"estimator": estimator,
"adaptive_noise_fn": "rule_of_thumb_0.50"
},
},
)
for simulator in simulators
]
)
fig = gof_result.plot_metric(
plot_dict, metric="score", figsize=(12, 4.5), layout=(1, 3), log_scale_y=False
)
plt.suptitle('noise schedules %s'%estimator)
fig.savefig(os.path.join(os.path.join(DATA_DIR_LOCAL, EXP_PREFIX), "noise_schedules_rule_of_thumb_%s.png"%estimator))
#
for estimator in estimators:
plot_dict = dict(
[
(
simulator,
{
"fixed_rate_0.40": {
"simulator": simulator,
"estimator": estimator,
"adaptive_noise_fn": "fixed_rate_0.40"
"y_noise_std": 0.1,
},
"NF_all": {
"simulator": simulator,
"estimator": "NormalizingFlowEstimator",
"weight_normalization": False,
"dropout": 0.0,
"weight_decay": 5e-5,
"x_noise_std": 0.1,
"y_noise_std": 0.1,
},
},
) for simulator in simulators
])
fig = gof_result.plot_metric(plot_dict,
metric="hellinger_distance",
figsize=(14, 10),
layout=(2, 2))
fig.savefig(os.path.join(EXP_PREFIX, "nf_compare.pdf"))
fig.savefig(os.path.join(EXP_PREFIX, "nf_compare.png"))
plot_dict = dict([(
simulator,
{
"MDN_baseline": {
"simulator": simulator,
"estimator": "MixtureDensityNetwork",
"weight_normalization": True,
"dropout": 0.0,
"weight_decay": 0.0,
"x_noise_std": 0.2,
"y_noise_std": 0.1,
},
) for estimator in estimators]
from collections import OrderedDict
plot_list = [
plot_list[0], plot_list[3], plot_list[1], plot_list[4], plot_list[2],
plot_list[5]
]
plot_dict = OrderedDict(plot_list)
colors = iter(['#1f77b4', '#2ca02c', '#9467bd', '#ff7f0e', '#d62728'])
fig = gof_result.plot_metric(plot_dict,
metric="score",
figsize=(7, 8.5),
layout=(3, 2),
log_scale_y=False,
color=colors)
# -- add the bayesian data from csv --
axarr = fig.axes
if isinstance(axarr, np.ndarray):
axarr = axarr.flatten()
df = pd.read_csv(os.path.join(DATA_DIR_LOCAL, EXP_PREFIX, BAYES_RESULTS_FILE))
df["param_kl_weight_scale"] = df.param_kl_weight_scale * df.n_datapoints
test_score_columns = [
column for column in df.columns
if column.startswith("split") and column.endswith("_test_score")
]
base_experiment.KEYS_OF_INTEREST)
graph_dicts = [{
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 0.001,
"n_centers": 20
}, {
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 0.01,
"n_centers": 20
}, {
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 0.1,
"n_centers": 20
}, {
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 1,
"n_centers": 20
}, {
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 10,
"n_centers": 20
}, {
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 100,
"n_centers": 20
}]
gof_result.plot_metric(graph_dicts, metric="js_divergence")
print(results_df)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(base_experiment.KEYS_OF_INTEREST + ['bandwidth'])
plot_dict = dict([(simulator,
{"MDN": {"simulator": simulator, "estimator": "MixtureDensityNetwork", "x_noise_std": 0.1},
"KMN": {"simulator": simulator, "estimator": "KernelMixtureNetwork", "x_noise_std": 0.1},
"LSCDE": {"simulator": simulator, "estimator": "LSConditionalDensityEstimation"},
"CKDE": {"simulator": simulator, "estimator": "ConditionalKernelDensityEstimation", "bandwidth": "normal_reference"},
"CKDE_CV": {"simulator": simulator, "estimator": "ConditionalKernelDensityEstimation", "bandwidth": "cv_ml"},
"NKDE": {"simulator": simulator, "estimator": "NeighborKernelDensityEstimation"},
"NF": {"simulator": simulator, "estimator": "NormalizingFlowEstimator",}
}) for simulator in ["EconDensity", "ArmaJump", "SkewNormal"]])
fig = gof_result.plot_metric(plot_dict, metric="hellinger_distance", figsize=(15, 5.5))
## ARMA JUMP
EXP_PREFIX = 'question4_benchmark_arma_jump'
logger.configure(
'/home/simon/Documents/KIT/Informatik/Bachelorarbeit/Conditional_Density_Estimation/data/cluster',
EXP_PREFIX)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(base_experiment.KEYS_OF_INTEREST + ['bandwidth'])
plot_dict = dict([(simulator,
"n_centers": n_centers,
"simulator": simulator
},
"x_noise_std=0.0": {
"estimator": estimator,
"x_noise_std": None,
"y_noise_std": None,
"n_centers": n_centers,
"simulator": simulator
}
}) for simulator in
["EconDensity", "ArmaJump", "GaussianMixture", "SkewNormal"]
])
fig = gof_result.plot_metric(plot_dict,
metric="js_divergence",
figsize=(14, 10),
layout=(2, 2))
plt.suptitle(title, fontsize=TITLE_SIZE)
plt.tight_layout(h_pad=2, rect=[0, 0, 1, 0.95])
plt.savefig(
os.path.join(EXP_PREFIX, "%s_%i_x_noise.png" % (estimator, n_centers)))
plt.clf()
""" Y-Noise Regularization"""
for estimator, n_centers in [("MixtureDensityNetwork", 10),
("KernelMixtureNetwork", 20)]:
title = "%s (%i kernels) - Y-Noise Regularization" % (estimator, n_centers)
plot_dict = dict([
(
simulator,
{
"y_noise_std=0.2": {
plot_dict = dict([(simulator, {
"KDE-normal-reference": {
"simulator": simulator,
"estimator": "ConditionalKernelDensityEstimation",
"bandwidth_selection": "normal_reference"
},
"KDE-ml-cv": {
"simulator": simulator,
"estimator": "ConditionalKernelDensityEstimation",
"bandwidth_selection": "cv_ml"
}
}) for simulator in ["EconDensity", "ArmaJump", "SkewNormal"]])
fig = gof_result.plot_metric(plot_dict,
metric="js_divergence",
figsize=(14, 5),
layout=(1, 3),
color=color)
plt.suptitle(title, fontsize=TITLE_SIZE)
plt.tight_layout(h_pad=2, rect=[0, 0, 1, 0.95])
# add KMN and MDN plots
logger.configure(
'/home/jonasrothfuss/Dropbox/Eigene_Dateien/Uni/WS17_18/Density_Estimation/Nonparametric_Density_Estimation/data/cluster',
"question1_noise_reg_x_v1")
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
_ = gof_result.generate_results_dataframe(base_experiment.KEYS_OF_INTEREST)
plot_dict = dict([(simulator, {
"estimator": "NeighborKernelDensityEstimation",
"param_selection": "normal_reference"
},
"LSCDE": {
"simulator": simulator,
"estimator": "LSConditionalDensityEstimation",
},
},
)
for simulator in simulators
]
)
FIGSIZE = (10, 4.5)
fig = gof_result.plot_metric(
plot_dict, metric="score", figsize=FIGSIZE, layout=(1, 2), log_scale_y=False
)
fig.tight_layout()
for i in range(len(fig.axes)):
fig.axes[i].set_title('')
fig.axes[i].set_xlabel('')
fig.axes[i].set_ylabel('')
fig.axes[0].set_xlabel('number of train observations')
fig.axes[0].set_ylabel('log-likelihood')
fig.axes[0].get_legend().remove()
fig.axes[1].set_ylim((0.9, 1.62))
fig.axes[0].set_title('Gaussian Mixture')