def _dump_current_state(self):
#if self.export_csv:
# self._export_results(task=task, gof_result=gof_single_result, file_handle_results=self.file_handle_results_csv)
#if self.export_pickle:
with open(self.results_pickle_path, "wb") as f:
intermediate_gof_results = GoodnessOfFitResults(
single_results_dict=self.gof_single_res_collection)
io.dump_as_pickle(f, intermediate_gof_results, verbose=False)
def __init__(self,
exp_prefix,
est_params,
sim_params,
observations,
keys_of_interest,
n_mc_samples=10**7,
n_x_cond=5,
n_seeds=5,
use_gpu=True,
tail_measures=True):
assert est_params and exp_prefix and sim_params and keys_of_interest
assert observations.all()
# every simulator configuration will be run multiple times with different randomness seeds
sim_params = _add_seeds_to_sim_params(n_seeds, sim_params)
self.observations = observations
self.n_mc_samples = n_mc_samples
self.n_x_cond = n_x_cond
self.keys_of_interest = keys_of_interest
self.exp_prefix = exp_prefix
self.use_gpu = use_gpu
self.tail_measures = tail_measures
logger.configure(log_directory=config.DATA_DIR,
prefix=exp_prefix,
color='green')
''' ---------- Either load or generate the configs ----------'''
config_pkl_path = os.path.join(logger.log_directory, logger.prefix,
EXP_CONFIG_FILE)
if os.path.isfile(config_pkl_path):
logger.log("{:<70s} {:<30s}".format(
"Loading experiment previous configs from file: ",
config_pkl_path))
self.configs = logger.load_pkl(EXP_CONFIG_FILE)
else:
logger.log("{:<70s} {:<30s}".format(
"Generating and storing experiment configs under: ",
config_pkl_path))
self.configs = self._generate_configuration_variants(
est_params, sim_params)
logger.dump_pkl(data=self.configs, path=EXP_CONFIG_FILE)
''' ---------- Either load already existing results or start a new result collection ---------- '''
results_pkl_path = os.path.join(logger.log_directory, logger.prefix,
RESULTS_FILE)
if os.path.isfile(results_pkl_path):
logger.log_line("{:<70s} {:<30s}".format("Continue with: ",
results_pkl_path))
self.gof_single_res_collection = dict(
logger.load_pkl_log(RESULTS_FILE))
else: # start from scratch
self.gof_single_res_collection = {}
self.gof_results = GoodnessOfFitResults(self.gof_single_res_collection)
def get_density_plots(estimators_list,
simulators_dict,
path_to_results,
exp_prefix="question1_noise_reg_x",
task_ids=None):
"""
This function allows to compare plots from estimators and simulators (i.e. fitted and true densities). Two modes are currently available:
1) by specifying estimators and simulator, the function picks one result pair randomly that matches the given simulator/estimator
selection
2) by specifying the task_ids as list, it is possible to pick specific plots to compare
Args:
estimators: a list containing strings of estimators to be evaluated, e.g. ['KernelMixtureNetwork', 'MixtureDensityNetwork']
simulators: a dict containing specifications of a simulator under which the estimators shall be compared, e.g.
{'heteroscedastic': True, 'random_seed': 20, 'std': 1, 'simulator': 'EconDensity'}
path_to_results: absolute path to where the dumped model files are stored
exp_prefix: specifies the task question
Returns:
A list of figures for fitted and true densities.
"""
if task_ids is not None:
assert type(task_ids) == list
assert len(task_ids) == len(estimators_list)
RESULTS_FILE = 'results.pkl'
logger.configure(path_to_results, exp_prefix)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(
single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(
base_experiment.KEYS_OF_INTEREST)
""" load model's estimators """
if task_ids is None:
models_of_interest = {
k: v
for k, v in gof_result.single_results_dict.items()
if v.probabilistic_model_params == simulators_dict and v.ndim_x +
v.ndim_y == 2
}
models = [
ConfigRunner.load_dumped_estimator(
take_of_type(1, estimator_str, models_of_interest))
for estimator_str in estimators_list
]
else:
models = [
ConfigRunner.load_dumped_estimators(gof_result, task_id=task_ids)
]
""" load model's simulators """
# todo: implement when simulator dumps exist
figs = []
for model in models:
graph = model.estimator.sess.graph
sess = tf.Session(graph=graph)
with sess:
sess.run(tf.global_variables_initializer())
model.estimator.sess = sess
""" fitted density figures"""
plt.suptitle(model.estimator.name)
fig_fitted = model.estimator.plot3d()
figs.append(fig_fitted)
""" true density figures """
# todo: use newly dumped simulators
sess.close()
return figs
'y_noise_std': [0.1],
},
}
simulators_params = {'LinearStudentT': {'ndim_x': [10]}}
observations = 100 * np.logspace(2, 6, num=8, base=2.0, dtype=np.int32)
return estimator_params, simulators_params, observations
if __name__ == '__main__':
estimator_params, simulators_params, observations = question4()
load = base_experiment.launch_experiment(estimator_params,
simulators_params,
observations,
EXP_PREFIX,
n_mc_samples=N_MC_SAMPLES,
tail_measures=False)
if load:
logger.configure(config.DATA_DIR, EXP_PREFIX)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(
single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(
base_experiment.KEYS_OF_INTEREST)
gof_result = ConfigRunner.load_dumped_estimators(gof_result)
import os
EXP_PREFIX = "question6_noise_schedules"
RESULTS_FILE = "results.pkl"
CLUSTER_DIR = "/local/rojonas/cde/data/local"
LOCATION = "{}/{}/{}".format(CLUSTER_DIR, EXP_PREFIX, RESULTS_FILE)
DATA_DIR_LOCAL = "/home/jonasrothfuss/Dropbox/Eigene_Dateien/ETH/02_Projects/02_Noise_Regularization/02_Code_Conditional_Density_Estimation/data/cluster"
logger.configure(
#"/local/rojonas/cde/data/local",
DATA_DIR_LOCAL,
EXP_PREFIX,
)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(base_experiment.KEYS_OF_INTEREST_LOGPROB)
results_df.replace(to_replace=[None], value="None", inplace=True)
estimators = [
"MixtureDensityNetwork",
"KernelMixtureNetwork",
"NormalizingFlowEstimator"
]
simulators = ["EconDensity", "GaussianMixture", "SkewNormal"]
# rules of thumb
for estimator in estimators:
plot_dict = dict(
[
(
import pickle
import os
EXP_PREFIX = "question5_benchmark"
RESULTS_FILE = "results.pkl"
CLUSTER_DIR = "/home/simon/Documents/KIT/Informatik/Bachelorarbeit/Conditional_Density_Estimation/data/cluster"
LOCATION = "{}/{}/{}".format(CLUSTER_DIR, EXP_PREFIX, RESULTS_FILE)
logger.configure(
"/home/simon/Documents/KIT/Informatik/Bachelorarbeit/Conditional_Density_Estimation/data/cluster",
EXP_PREFIX,
)
with open(LOCATION, "rb") as fh:
results_from_pkl_file = pickle.load(fh)
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(
base_experiment.KEYS_OF_INTEREST)
results_df.replace(to_replace=[None], value="None", inplace=True)
estimators = [
"MixtureDensityNetwork",
"KernelMixtureNetwork",
"NormalizingFlowEstimator",
]
simulators = ["ArmaJump", "EconDensity", "GaussianMixture", "SkewNormal"]
plot_dict = dict([
(
simulator,
{
#
def _resize_plots(fig):
fig.axes[0].set_ylim((-3, -1.9))
#fig.axes[1].set_ylim((-7, -4.5))
fig.axes[3].set_ylim((1.0, 1.63))
# results_df.to_csv('/home/jonasrothfuss/Dropbox/Eigene_Dateien/ETH/02_Projects/02_Noise_Regularization/02_Code_Conditional_Density_Estimation/'
# 'data/local/question7_regularization_logprob/results.csv')
results_df = pd.read_csv(
'/home/jonasrothfuss/Dropbox/Eigene_Dateien/ETH/02_Projects/02_Noise_Regularization/02_Code_Conditional_Density_Estimation/data/local/question7_regularization_logprob/results.csv'
)
gof_result = GoodnessOfFitResults(single_results_dict=[])
gof_result.results_df = results_df
# rules of thumb
# for estimator in estimators:
# plot_dict = dict(
# [
# (
# simulator,
# {
# "rule_of_thumb_1.0": {
# "simulator": simulator,
# "estimator": estimator,
# "adaptive_noise_fn": "rule_of_thumb_1.00"
# },
# "rule_of_thumb_0.7": {
}
observations = 100 * np.logspace(0, 6, num=7, base=2.0, dtype=np.int32)
return estimator_params, simulators_params, observations
if __name__ == '__main__':
estimator_params, simulators_params, observations = question2()
load = base_experiment.launch_experiment(estimator_params,
simulators_params, observations,
EXP_PREFIX)
if load:
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(
single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(
base_experiment.KEYS_OF_INTEREST)
graph_dicts = [{
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 0.001,
"n_centers": 20
}, {
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 0.01,
"n_centers": 20
}, {
"estimator": "KernelMixtureNetwork",
"entropy_reg_coef": 0.1,
"n_centers": 20
plt.rc('legend', fontsize=MEDIUM_SIZE) # legend fontsize
## SKEW Normal
EXP_PREFIX = 'question4_benchmark_skew_NF'
RESULTS_FILE = 'results.pkl'
if not os.path.isdir(EXP_PREFIX):
os.makedirs(EXP_PREFIX)
logger.configure(
'/home/simon/Documents/KIT/Informatik/Bachelorarbeit/Conditional_Density_Estimation/data/cluster',
EXP_PREFIX)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(base_experiment.KEYS_OF_INTEREST + ['bandwidth'])
plot_dict = dict([(simulator,
{"MDN": {"simulator": simulator, "estimator": "MixtureDensityNetwork", "x_noise_std": 0.1},
"KMN": {"simulator": simulator, "estimator": "KernelMixtureNetwork", "x_noise_std": 0.1},
"LSCDE": {"simulator": simulator, "estimator": "LSConditionalDensityEstimation"},
"CKDE": {"simulator": simulator, "estimator": "ConditionalKernelDensityEstimation", "bandwidth": "normal_reference"},
"CKDE_CV": {"simulator": simulator, "estimator": "ConditionalKernelDensityEstimation", "bandwidth": "cv_ml"},
"NKDE": {"simulator": simulator, "estimator": "NeighborKernelDensityEstimation"},
"NF": {"simulator": simulator, "estimator": "NormalizingFlowEstimator",}
}) for simulator in ["EconDensity", "ArmaJump", "SkewNormal"]])
fig = gof_result.plot_metric(plot_dict, metric="hellinger_distance", figsize=(15, 5.5))
from cde.model_fitting.GoodnessOfFitResults import GoodnessOfFitResults
from cde.evaluation.simulation_eval import base_experiment
import cde.model_fitting.ConfigRunner as ConfigRunner
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
EXP_PREFIX = 'question2_entropy_reg'
RESULTS_FILE = 'results.pkl'
logger.configure(
'/home/jonasrothfuss/Dropbox/Eigene_Dateien/Uni/WS17_18/Density_Estimation/Nonparametric_Density_Estimation/data/cluster',
EXP_PREFIX)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(
base_experiment.KEYS_OF_INTEREST + ['entropy_reg_coef'])
#gof_result = ConfigRunner.load_dumped_estimators(gof_result, task_id=[5])
SMALL_SIZE = 11
MEDIUM_SIZE = 12
LARGE_SIZE = 16
TITLE_SIZE = 20
LINEWIDTH = 6
plt.rc('font', size=SMALL_SIZE) # controls default text sizes
plt.rc('axes', titlesize=LARGE_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
import cde.model_fitting.ConfigRunner as ConfigRunner
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.pyplot import cm
import pandas as pd
EXP_PREFIX = 'question3_KDE'
RESULTS_FILE = 'results.pkl'
logger.configure(
'/home/jonasrothfuss/Dropbox/Eigene_Dateien/Uni/WS17_18/Density_Estimation/Nonparametric_Density_Estimation/data/cluster',
EXP_PREFIX)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(
base_experiment.KEYS_OF_INTEREST + ['bandwidth_selection'])
#gof_result = ConfigRunner.load_dumped_estimators(gof_result, task_id=[5])
SMALL_SIZE = 11
MEDIUM_SIZE = 12
LARGE_SIZE = 16
TITLE_SIZE = 20
LINEWIDTH = 6
plt.rc('font', size=SMALL_SIZE) # controls default text sizes
plt.rc('axes', titlesize=LARGE_SIZE) # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels
EXP_PREFIX = "question8_benchmark"
RESULTS_FILE = "results.pkl"
CLUSTER_DIR = "/local/rojonas/cde/data/local"
LOCATION = "{}/{}/{}".format(CLUSTER_DIR, EXP_PREFIX, RESULTS_FILE)
DATA_DIR_LOCAL = "/home/jonasrothfuss/Dropbox/Eigene_Dateien/ETH/02_Projects/02_Noise_Regularization/02_Code_Conditional_Density_Estimation/data/cluster"
logger.configure(
#"/local/rojonas/cde/data/local",
DATA_DIR_LOCAL,
#CLUSTER_DIR,
EXP_PREFIX,
)
results_from_pkl_file = dict(logger.load_pkl_log(RESULTS_FILE))
gof_result = GoodnessOfFitResults(single_results_dict=results_from_pkl_file)
results_df = gof_result.generate_results_dataframe(base_experiment.KEYS_OF_INTEREST_LOGPROB + ["bandwidth", "param_selection"])
results_df.replace(to_replace=[None], value="None", inplace=True)
# seprarate 2d and 4d GMM
results_df.index = list(range(len(results_df)))
for i, row in results_df[['simulator', 'ndim_y']].iterrows():
if row['simulator'] == 'GaussianMixture':
results_df.at[i, 'simulator'] = '%s_%id'%(row['simulator'], row['ndim_y'])
estimators = [
"MixtureDensityNetwork",
"KernelMixtureNetwork",
"NormalizingFlowEstimator",
"ConditionalKernelDensityEstimation",