def __init__(self, config):
self.config = config
_check_valid_config(self.config)
config['param_str'] = '_'.join(
['{}_{}'.format(filesafe(k), v) for k, v in self.config['mc_opts'].items()])
config['param_str'] += '_' + '_'.join(
['{}_{}'.format(filesafe(k), v) for k, v in self.config['dgp_opts'].items()])
config['param_str'] += '_' + '_'.join(
['{}_{}'.format(filesafe(k), v) for k, v in self.config['method_opts'].items()])
return
def plot_metric_comparisons(param_estimates, metric_results, config):
for dgp_name, mdgp in metric_results.items():
n_methods = len(list(mdgp.keys()))
for metric_name in next(iter(mdgp.values())).keys():
plt.figure(figsize=(1.5 * n_methods, 2.5))
plt.violinplot([
mdgp[method_name][metric_name] -
mdgp[config['proposed_method']][metric_name]
for method_name in mdgp.keys()
if method_name != config['proposed_method']
],
showmedians=True)
plt.xticks(np.arange(1, n_methods), [
method_name for method_name in mdgp.keys()
if method_name != config['proposed_method']
])
plt.ylabel('decrease in {}'.format(metric_name))
plt.tight_layout()
plt.savefig(os.path.join(
config['target_dir'],
'{}_decrease_dgp_{}_{}.png'.format(filesafe(metric_name),
dgp_name,
config['param_str'])),
dpi=300)
plt.close()
return
def instance_plot(plot_name, param_estimates, metric_results, config,
plot_config):
methods = plot_config['methods'] if 'methods' in plot_config else list(
config['methods'].keys())
metrics = plot_config['metrics'] if 'metrics' in plot_config else list(
config['metrics'].keys())
dgps = plot_config['dgps'] if 'dgps' in plot_config else list(
config['dgps'].keys())
metric_transforms = plot_config[
'metric_transforms'] if 'metric_transforms' in plot_config else {
'': mcpy.metrics.transform_identity
}
for tr_name, tr_fn in metric_transforms.items():
for dgp_name in dgps:
for metric_name in metrics:
plt.figure(figsize=(1.5 * len(methods), 2.5))
plt.violinplot([
tr_fn(metric_results, dgp_name, method_name, metric_name,
config) for method_name in methods
],
showmedians=True)
plt.xticks(np.arange(1, len(methods) + 1), methods)
plt.ylabel('{}({})'.format(tr_name, metric_name))
plt.tight_layout()
plt.savefig(os.path.join(
config['target_dir'],
'{}_{}_{}_dgp_{}_{}.png'.format(plot_name,
filesafe(metric_name),
tr_name, dgp_name,
config['param_str'])),
dpi=300)
plt.close()
return
def plot_raw(param_estimates, metric_results, config):
for dgp_name, mdgp in metric_results.items():
metric_name = 'raw'
plt.figure()
for it, method_name in enumerate(mdgp.keys()):
if it == 0:
true_params = np.array(mdgp[method_name][metric_name][0, :, 1])
plt.plot(np.arange(true_params.shape[0]),
true_params,
'--',
label='true')
res = np.array(mdgp[method_name][metric_name][:, :, 0])
med_res = np.median(res, axis=0)
lb_res = np.percentile(res, 5, axis=0)
ub_res = np.percentile(res, 95, axis=0)
line = plt.plot(np.arange(med_res.shape[0]),
med_res,
label=method_name)
plt.fill_between(np.arange(med_res.shape[0]),
lb_res,
ub_res,
alpha=.4,
color=line[0].get_color())
plt.legend()
plt.tight_layout()
plt.savefig(os.path.join(
config['target_dir'],
'raw_{}_dgp_{}_{}.png'.format(filesafe(metric_name), dgp_name,
config['param_str'])),
dpi=300)
plt.close()
return
def sweep_plot_marginal_transformed_metric(transform_fn,
transform_name,
dgps,
methods,
metrics,
plot_name,
sweep_keys,
sweep_params,
sweep_metrics,
config,
param_subset={},
select_vals={},
filter_vals={}):
sweeps = {}
for dgp_key, dgp_val in config['dgp_opts'].items():
if hasattr(dgp_val, "__len__"):
sweeps[dgp_key] = dgp_val
mask = _select_config_keys(sweep_keys, select_vals, filter_vals)
if np.sum(mask) == 0:
print("Filtering resulted in no valid configurations!")
return
for dgp in dgps:
for metric in metrics:
for param, param_vals in sweeps.items():
if param_subset is not None and param not in param_subset:
continue
plt.figure(figsize=(5, 3))
for method in methods:
medians = []
mins = []
maxs = []
for val in param_vals:
subset = [
transform_fn(metrics, dgp, method, metric, config)
for key, metrics, ms in zip(
sweep_keys, sweep_metrics, mask)
if (param, val) in key and ms
]
if len(subset) > 0:
grouped_results = np.concatenate(subset)
medians.append(np.median(grouped_results))
mins.append(np.percentile(grouped_results, 5))
maxs.append(np.percentile(grouped_results, 95))
line = plt.plot(param_vals, medians, label=method)
plt.fill_between(param_vals,
maxs,
mins,
alpha=0.3,
color=line[0].get_color())
plt.legend()
plt.xlabel(param)
plt.ylabel('{}({})'.format(transform_name, metric))
plt.tight_layout()
plt.savefig(os.path.join(
config['target_dir'],
'{}_{}_{}_dgp_{}_growing_{}_{}.png'.format(
plot_name, filesafe(metric), transform_name, dgp,
filesafe(param), config['param_str'])),
dpi=300)
plt.close()
for param1, param2 in itertools.combinations(sweeps.keys(), 2):
if param_subset is not None and (
param1, param2) not in param_subset and (
param2, param1) not in param_subset:
continue
x, y, z = [], [], []
for method_it, method in enumerate(methods):
x.append([]), y.append([]), z.append([])
for val1, val2 in itertools.product(
*[sweeps[param1], sweeps[param2]]):
subset = [
transform_fn(metrics, dgp, method, metric, config)
for key, metrics, ms in zip(
sweep_keys, sweep_metrics, mask)
if (param1, val1) in key and (param2,
val2) in key and ms
]
if len(subset) > 0:
grouped_results = np.concatenate(subset)
x[method_it].append(val1)
y[method_it].append(val2)
z[method_it].append(np.median(grouped_results))
vmin = np.min(z)
vmax = np.max(z)
fig, axes = plt.subplots(nrows=1,
ncols=len(methods),
figsize=(4 * len(methods), 3))
if not hasattr(axes, '__len__'):
axes = [axes]
for method_it, (method, ax) in enumerate(zip(methods, axes)):
xi = np.linspace(np.min(x[method_it]),
np.max(x[method_it]),
5 * len(x[method_it]))
yi = np.linspace(np.min(y[method_it]),
np.max(y[method_it]),
5 * len(y[method_it]))
xi, yi = np.meshgrid(xi, yi)
zi = griddata(
(np.array(x[method_it]), np.array(y[method_it])),
np.array(z[method_it]), (xi, yi),
method='linear')
ax.contour(xi, yi, zi, 15, linewidths=0.2, colors='k')
im = ax.pcolormesh(xi,
yi,
zi,
cmap=plt.cm.Reds,
vmin=vmin,
vmax=vmax)
ax.contourf(xi,
yi,
zi,
15,
cmap=plt.cm.Reds,
vmin=vmin,
vmax=vmax)
ax.scatter(np.array(x[method_it]),
np.array(y[method_it]),
alpha=0.5,
s=3,
c='b')
ax.set_xlabel(param1)
ax.set_ylabel(param2)
ax.set_title(method)
plt.tight_layout()
fig.subplots_adjust(right=0.8)
cbar_ax = fig.add_axes([0.81, 0.15, 0.02, 0.72])
cbar = fig.colorbar(im, cax=cbar_ax)
cbar.ax.tick_params(labelsize=8)
cbar.ax.set_ylabel('median {}({})'.format(
transform_name, metric))
plt.savefig(os.path.join(
config['target_dir'],
'{}_{}_{}_dgp_{}_growing_{}_and_{}_{}.png'.format(
plot_name, filesafe(metric), transform_name, dgp,
filesafe(param1), filesafe(param2),
config['param_str'])),
dpi=300)
plt.close()