def error_vs_cost(model, generate_random_samples, validation_levels, num_samples=10): validation_levels = np.asarray(validation_levels) assert len(validation_levels) == model.base_model.num_config_vars config_vars = pya.cartesian_product( [np.arange(ll) for ll in validation_levels]) random_samples = generate_random_samples(num_samples) samples = pya.get_all_sample_combinations(random_samples, config_vars) reference_samples = samples[:, ::config_vars.shape[1]].copy() reference_samples[-model.base_model.num_config_vars:, :] =\ validation_levels[:, np.newaxis] reference_values = model(reference_samples) reference_mean = reference_values[:, 0].mean() values = model(samples) # put keys in order returned by cartesian product keys = sorted(model.work_tracker.costs.keys(), key=lambda x: x[::-1]) # remove validation key associated with validation samples keys = keys[:-1] costs, ndofs, means, errors = [], [], [], [] for ii in range(len(keys)): key = keys[ii] costs.append(np.median(model.work_tracker.costs[key])) # nx,ny,dt = model.base_model.get_degrees_of_freedom_and_timestep( # np.asarray(key)) nx, ny = model.base_model.get_mesh_resolution(np.asarray(key)[:2]) dt = model.base_model.get_timestep(key[2]) ndofs.append(nx*ny*model.base_model.final_time/dt) means.append(np.mean(values[ii::config_vars.shape[1], 0])) errors.append(abs(means[-1]-reference_mean)/abs(reference_mean)) times = costs.copy() # make costs relative costs /= costs[-1] n1, n2, n3 = validation_levels indices = np.reshape( np.arange(len(keys), dtype=int), (n1, n2, n3), order='F') costs = np.reshape(np.array(costs), (n1, n2, n3), order='F') ndofs = np.reshape(np.array(ndofs), (n1, n2, n3), order='F') errors = np.reshape(np.array(errors), (n1, n2, n3), order='F') times = np.reshape(np.array(times), (n1, n2, n3), order='F') validation_index = reference_samples[-model.base_model.num_config_vars:, 0] validation_time = np.median( model.work_tracker.costs[tuple(validation_levels)]) validation_cost = validation_time/costs[-1] validation_ndof = np.prod(reference_values[:, -2:], axis=1) data = {"costs": costs, "errors": errors, "indices": indices, "times": times, "validation_index": validation_index, "validation_cost": validation_cost, "validation_ndof": validation_ndof, "validation_time": validation_time, "ndofs": ndofs} return data
def test_rsquared_mfmc(self): functions = ShortColumnModelEnsemble() model_ensemble = pya.ModelEnsemble( [functions.m0, functions.m3, functions.m4]) univariate_variables = [ uniform(5, 10), uniform(15, 10), norm(500, 100), norm(2000, 400), lognorm(s=0.5, scale=np.exp(5)) ] variable = pya.IndependentMultivariateRandomVariable( univariate_variables) generate_samples = partial(pya.generate_independent_random_samples, variable) npilot_samples = int(1e4) pilot_samples = generate_samples(npilot_samples) config_vars = np.arange(model_ensemble.nmodels)[np.newaxis, :] pilot_samples = pya.get_all_sample_combinations( pilot_samples, config_vars) pilot_values = model_ensemble(pilot_samples) pilot_values = np.reshape(pilot_values, (npilot_samples, model_ensemble.nmodels)) cov = np.cov(pilot_values, rowvar=False) nhf_samples = 10 nsample_ratios = np.asarray([2, 4]) nsamples_per_model = np.concatenate([[nhf_samples], nsample_ratios * nhf_samples]) eta = pya.get_mfmc_control_variate_weights(cov) cor = pya.get_correlation_from_covariance(cov) var_mfmc = cov[0, 0] / nsamples_per_model[0] for k in range(1, model_ensemble.nmodels): var_mfmc += (1 / nsamples_per_model[k - 1] - 1 / nsamples_per_model[k]) * ( eta[k - 1]**2 * cov[k, k] + 2 * eta[k - 1] * cor[0, k] * np.sqrt(cov[0, 0] * cov[k, k])) assert np.allclose(var_mfmc / cov[0, 0] * nhf_samples, 1 - pya.get_rsquared_mfmc(cov, nsample_ratios))
from pyapprox.benchmarks.benchmarks import setup_benchmark benchmark = setup_benchmark('multi_level_advection_diffusion', nvars=nrandom_vars, corr_len=corr_len, max_eval_concurrency=max_eval_concurrency) model = benchmark.fun variable = benchmark.variable #%% #Now lets us plot each model as a function of the random variable lb, ub = variable.get_statistics('interval', alpha=1)[0] nsamples = 10 random_samples = np.linspace(lb, ub, nsamples)[np.newaxis, :] config_vars = np.arange(nmodels)[np.newaxis, :] samples = pya.get_all_sample_combinations(random_samples, config_vars) values = model(samples) values = np.reshape(values, (nsamples, nmodels)) import dolfin as dl plt.figure(figsize=(nmodels * 8, 2 * 6)) config_samples = benchmark['multi_level_model'].map_to_multidimensional_index( config_vars) for ii in range(nmodels): nx, ny = model.base_model.get_mesh_resolution(config_samples[:2, ii]) dt = model.base_model.get_timestep(config_samples[2, ii]) mesh = dl.RectangleMesh(dl.Point(0, 0), dl.Point(1, 1), nx, ny) plt.subplot(2, nmodels, ii + 1) dl.plot(mesh) label = r'$f_%d$' % ii