def error_vs_cost(model, generate_random_samples, validation_levels,
num_samples=10):
validation_levels = np.asarray(validation_levels)
assert len(validation_levels) == model.base_model.num_config_vars
config_vars = pya.cartesian_product(
[np.arange(ll) for ll in validation_levels])
random_samples = generate_random_samples(num_samples)
samples = pya.get_all_sample_combinations(random_samples, config_vars)
reference_samples = samples[:, ::config_vars.shape[1]].copy()
reference_samples[-model.base_model.num_config_vars:, :] =\
validation_levels[:, np.newaxis]
reference_values = model(reference_samples)
reference_mean = reference_values[:, 0].mean()
values = model(samples)
# put keys in order returned by cartesian product
keys = sorted(model.work_tracker.costs.keys(), key=lambda x: x[::-1])
# remove validation key associated with validation samples
keys = keys[:-1]
costs, ndofs, means, errors = [], [], [], []
for ii in range(len(keys)):
key = keys[ii]
costs.append(np.median(model.work_tracker.costs[key]))
# nx,ny,dt = model.base_model.get_degrees_of_freedom_and_timestep(
# np.asarray(key))
nx, ny = model.base_model.get_mesh_resolution(np.asarray(key)[:2])
dt = model.base_model.get_timestep(key[2])
ndofs.append(nx*ny*model.base_model.final_time/dt)
means.append(np.mean(values[ii::config_vars.shape[1], 0]))
errors.append(abs(means[-1]-reference_mean)/abs(reference_mean))
times = costs.copy()
# make costs relative
costs /= costs[-1]
n1, n2, n3 = validation_levels
indices = np.reshape(
np.arange(len(keys), dtype=int), (n1, n2, n3), order='F')
costs = np.reshape(np.array(costs), (n1, n2, n3), order='F')
ndofs = np.reshape(np.array(ndofs), (n1, n2, n3), order='F')
errors = np.reshape(np.array(errors), (n1, n2, n3), order='F')
times = np.reshape(np.array(times), (n1, n2, n3), order='F')
validation_index = reference_samples[-model.base_model.num_config_vars:, 0]
validation_time = np.median(
model.work_tracker.costs[tuple(validation_levels)])
validation_cost = validation_time/costs[-1]
validation_ndof = np.prod(reference_values[:, -2:], axis=1)
data = {"costs": costs, "errors": errors, "indices": indices,
"times": times, "validation_index": validation_index,
"validation_cost": validation_cost, "validation_ndof": validation_ndof,
"validation_time": validation_time, "ndofs": ndofs}
return data
def test_rsquared_mfmc(self):
functions = ShortColumnModelEnsemble()
model_ensemble = pya.ModelEnsemble(
[functions.m0, functions.m3, functions.m4])
univariate_variables = [
uniform(5, 10),
uniform(15, 10),
norm(500, 100),
norm(2000, 400),
lognorm(s=0.5, scale=np.exp(5))
]
variable = pya.IndependentMultivariateRandomVariable(
univariate_variables)
generate_samples = partial(pya.generate_independent_random_samples,
variable)
npilot_samples = int(1e4)
pilot_samples = generate_samples(npilot_samples)
config_vars = np.arange(model_ensemble.nmodels)[np.newaxis, :]
pilot_samples = pya.get_all_sample_combinations(
pilot_samples, config_vars)
pilot_values = model_ensemble(pilot_samples)
pilot_values = np.reshape(pilot_values,
(npilot_samples, model_ensemble.nmodels))
cov = np.cov(pilot_values, rowvar=False)
nhf_samples = 10
nsample_ratios = np.asarray([2, 4])
nsamples_per_model = np.concatenate([[nhf_samples],
nsample_ratios * nhf_samples])
eta = pya.get_mfmc_control_variate_weights(cov)
cor = pya.get_correlation_from_covariance(cov)
var_mfmc = cov[0, 0] / nsamples_per_model[0]
for k in range(1, model_ensemble.nmodels):
var_mfmc += (1 / nsamples_per_model[k - 1] -
1 / nsamples_per_model[k]) * (
eta[k - 1]**2 * cov[k, k] + 2 * eta[k - 1] *
cor[0, k] * np.sqrt(cov[0, 0] * cov[k, k]))
assert np.allclose(var_mfmc / cov[0, 0] * nhf_samples,
1 - pya.get_rsquared_mfmc(cov, nsample_ratios))
from pyapprox.benchmarks.benchmarks import setup_benchmark
benchmark = setup_benchmark('multi_level_advection_diffusion',
nvars=nrandom_vars,
corr_len=corr_len,
max_eval_concurrency=max_eval_concurrency)
model = benchmark.fun
variable = benchmark.variable
#%%
#Now lets us plot each model as a function of the random variable
lb, ub = variable.get_statistics('interval', alpha=1)[0]
nsamples = 10
random_samples = np.linspace(lb, ub, nsamples)[np.newaxis, :]
config_vars = np.arange(nmodels)[np.newaxis, :]
samples = pya.get_all_sample_combinations(random_samples, config_vars)
values = model(samples)
values = np.reshape(values, (nsamples, nmodels))
import dolfin as dl
plt.figure(figsize=(nmodels * 8, 2 * 6))
config_samples = benchmark['multi_level_model'].map_to_multidimensional_index(
config_vars)
for ii in range(nmodels):
nx, ny = model.base_model.get_mesh_resolution(config_samples[:2, ii])
dt = model.base_model.get_timestep(config_samples[2, ii])
mesh = dl.RectangleMesh(dl.Point(0, 0), dl.Point(1, 1), nx, ny)
plt.subplot(2, nmodels, ii + 1)
dl.plot(mesh)
label = r'$f_%d$' % ii