def test_calc_error_sum_of_sq(self): """ Applied Statistics and Probability for Engineers 3rd Ed. (2003), online (Example 11-3, section 11-5.2) Douglas C. Montgomery & George C. Runger """ SS_E_act = 21.25 tol = 0.2 # small difference due to rounding differences responses = np.array([ 90.1, 89.05, 91.43, 93.74, 96.73, 94.45, 87.59, 91.77, 99.42, 93.65, 93.54, 92.52, 90.56, 89.54, 89.85, 90.39, 93.25, 93.41, 94.98, 87.33 ]) var_basis = np.array([ [1, 0.99], [1, 1.02], [1, 1.15], [1, 1.29], [1, 1.46], [1, 1.36], [1, 0.87], [1, 1.23], [1, 1.55], [1, 1.40], [1, 1.19], [1, 1.15], [1, 0.98], [1, 1.01], [1, 1.11], [1, 1.20], [1, 1.26], [1, 1.32], [1, 1.43], [1, 0.95] ]) matrix_coeffs = solve_coeffs(var_basis, responses) SS_E_calc = calc_error_sum_of_sq(var_basis, matrix_coeffs, responses) self.assertAlmostEqual( SS_E_act, SS_E_calc, delta=tol, msg='calc_error_sum_of_sq is not working correctly.' )
def test_solve_coeffs(self): """ Applied Statistics and Probability for Engineers 3rd Ed. online (2003), (section 12-1.2) Douglas C. Montgomery & George C. Runger """ tol = 1e-5 responses = np.array([ 9.95, 24.45, 31.75, 35.0, 25.02, 16.86, 14.38, 9.6, 24.35, 27.5, 17.08, 37.0, 41.95, 11.66, 21.65, 17.89, 69.0, 10.3, 34.93, 46.59, 44.88, 54.12, 56.63, 22.13, 21.15 ]) var_basis = np.array([[1, 2, 50], [1, 8, 110], [1, 11, 120], [1, 10, 550], [1, 8, 295], [1, 4, 200], [1, 2, 375], [1, 2, 52], [1, 9, 100], [1, 8, 300], [1, 4, 412], [1, 11, 400], [1, 12, 500], [1, 2, 360], [1, 4, 205], [1, 4, 400], [1, 20, 600], [1, 1, 585], [1, 10, 540], [1, 15, 250], [1, 15, 290], [1, 16, 510], [1, 17, 590], [1, 6, 100], [1, 5, 400]]) coeffs_act = np.array([2.26379, 2.74427, 0.01253]) coeffs_calc = solve_coeffs(var_basis, responses) self.assertTrue((np.abs(coeffs_calc - coeffs_act) < tol).all(), msg='statistics function solve_coeffs is not correct')
def test_calc_error_variance(self): """ Design and Analysis of Experiments 8th ed. (pg 469, 258) Douglas Montgomery """ responses = np.array([45, 100, 45, 65, 75, 60, 80, 96]) var_basis = np.array([ [1, -1, -1, -1, 1, 1], [1, 1, -1, 1, -1, 1], [1, -1, -1, 1, 1, -1], [1, 1, -1, -1, -1, -1], [1, -1, 1, 1, -1, -1], [1, 1, 1, -1, 1, -1], [1, -1, 1, -1, -1, 1], [1, 1, 1, 1, 1, 1] ]) matrix_coeffs = solve_coeffs(var_basis, responses) calc_err_var = calc_error_variance(var_basis, matrix_coeffs, responses) act_err_var = 3.25 self.assertEqual( calc_err_var, act_err_var, msg='calc_error_variance is not working correctly.' )
def test_calc_mean_sq_err(self): """ Applied Statistics and Probability for Engineers 4th Ed. (2007), (pg. 462) Douglas C. Montgomery & George C. Runger """ tol = 1e-3 MSE_act = 5.2352 responses = np.array([ 9.95, 24.45, 31.75, 35.0, 25.02, 16.86, 14.38, 9.6, 24.35, 27.5, 17.08, 37.0, 41.95, 11.66, 21.65, 17.89, 69.0, 10.3, 34.93, 46.59, 44.88, 54.12, 56.63, 22.13, 21.15 ]) var_basis = np.array([ [1, 2, 50], [1, 8, 110], [1, 11, 120], [1, 10, 550], [1, 8, 295], [1, 4, 200], [1, 2, 375], [1, 2, 52], [1, 9, 100], [1, 8, 300], [1, 4, 412], [1, 11, 400], [1, 12, 500], [1, 2, 360], [1, 4, 205], [1, 4, 400], [1, 20, 600], [1, 1, 585], [1, 10, 540], [1, 15, 250], [1, 15, 290], [1, 16, 510], [1, 17, 590], [1, 6, 100], [1, 5, 400] ]) matrix_coeffs = solve_coeffs(var_basis, responses) MSE_calc = calc_mean_sq_err(responses, matrix_coeffs, var_basis) self.assertAlmostEqual( MSE_act, MSE_calc, delta=tol, msg='calc_mean_sq_err is not working correctly.' )
def solve(self): """ Uses the matrix system to solve for the matrix coefficients. """ comm = MPI_COMM_WORLD rank = comm.rank if rank == 0: self.matrix_coeffs = solve_coeffs(self.var_basis_sys_eval, self.responses) else: self.matrix_coeffs = np.zeros(self.var_basis_sys_eval.shape[1]) comm.Bcast([self.matrix_coeffs, MPI_DOUBLE], root=0) return self.matrix_coeffs
def calc_PRESS_res(var_basis, responses): """ Inputs: var_basis- evaluated variable basis responses- the responses var_basis_func- the function that calculates the model variable basis from the input point An efficient way to calculate the PRESS residual for a given model. """ # TODO: cite resp_count = len(responses) err_ver_sq = np.zeros(resp_count) comm = MPI_COMM_WORLD size = comm.size rank = comm.rank base = resp_count // size rem = resp_count % size beg = base * rank + (rank >= rem) * rem + (rank < rem) * rank count = base + (rank < rem) end = beg + count ranks = np.arange(0, size, dtype=int) seq_count = (ranks < rem) + base seq_disp = base * ranks + (ranks >= rem) * rem + (ranks < rem) * ranks err_ver_sq_temp = np.zeros(count) for idx in range(beg, end): temp_basis = np.delete(var_basis, idx, axis=0) temp_resps = np.delete( responses, idx, ) matrix_coeffs = solve_coeffs(temp_basis, temp_resps) ver_pred = np.matmul(var_basis[idx], matrix_coeffs) err_ver_sq_temp[idx - beg] = (responses[idx] - ver_pred)**2 comm.Allgatherv([err_ver_sq_temp, count, MPI_DOUBLE], [err_ver_sq, seq_count, seq_disp, MPI_DOUBLE]) press_res = np.sum(err_ver_sq) return press_res
def test_calc_mean_conf_int(self): """ Applied Statistics and Probability for Engineers 4th Ed. (2007), (pg. 13, 440, 467) Douglas C. Montgomery & George C. Runger Adapted to work for the format required by the ProbabilityBoxes method calc_mean_conf_int. """ signif = 0.05 mean_act = 27.66 uncert_act = 1 responses = np.array([ 9.95, 24.45, 31.75, 35.0, 25.02, 16.86, 14.38, 9.6, 24.35, 27.5, 17.08, 37.0, 41.95, 11.66, 21.65, 17.89, 69.0, 10.3, 34.93, 46.59, 44.88, 54.12, 56.63, 22.13, 21.15, 27.66 ]) var_basis = np.array([[1, 2, 50], [1, 8, 110], [1, 11, 120], [1, 10, 550], [1, 8, 295], [1, 4, 200], [1, 2, 375], [1, 2, 52], [1, 9, 100], [1, 8, 300], [1, 4, 412], [1, 11, 400], [1, 12, 500], [1, 2, 360], [1, 4, 205], [1, 4, 400], [1, 20, 600], [1, 1, 585], [1, 10, 540], [1, 15, 250], [1, 15, 290], [1, 16, 510], [1, 17, 590], [1, 6, 100], [1, 5, 400], [1, 8, 275]]) var_list = [ UniformVariable(2, 18, order=1, number=0), UniformVariable(50, 600, order=1, number=1) ] pbox = ProbabilityBoxes(var_list) pbox.matrix_coeffs = solve_coeffs(var_basis, responses) pbox.var_basis_resamp = var_basis mean_calc, uncert_calc = pbox.calc_mean_conf_int( var_basis, responses, signif) self.assertTrue( np.isclose(mean_calc[-1], mean_act, rtol=0, atol=0.01), msg='ProbabilityBoxes calc_mean_conf_int is not correct.') self.assertTrue( np.isclose(uncert_calc[-1], uncert_act, rtol=0, atol=0.05), msg='ProbabilityBoxes calc_mean_conf_int is not correct.')
def test_calc_var_conf_int(self): """ Design and Analysis of Experiments, 8th Ed. (pg 468) Douglas Montgomery """ tol = 5e-3 # round off error in hand calculation # Hand calculated using the matrix coefficients minus and plus # the coefficient uncertainty for the lower and upper CI, respectively. var_conf_int_act = ( 1 / 3 * (6.285) ** 2 + 1 / 3 * (3.316) ** 2, 1 / 3 * (8.957) ** 2 + 1 / 3 * (13.853) ** 2 ) responses = np.array([ 2256, 2340, 2426, 2293, 2330, 2368, 2250, 2409, 2364, 2379, 2440, 2364, 2404, 2317, 2309, 2328 ]) signif = 0.05 var_basis = np.array([ [1, 80, 8], [1, 93, 9], [1, 100, 10], [1, 82, 12], [1, 90, 11], [1, 99, 8], [1, 81, 8], [1, 96, 10], [1, 94, 12], [1, 93, 11], [1, 97, 13], [1, 95, 11], [1, 100, 8], [1, 85, 12], [1, 86, 9], [1, 87, 12] ]) norm_sq = np.array([[1], [1 / 3], [1 / 3]]) matrix_coeffs = solve_coeffs(var_basis, responses) coeff_uncert = calc_coeff_conf_int( var_basis, matrix_coeffs, responses, signif ) var_conf_int_calc = calc_var_conf_int( matrix_coeffs, coeff_uncert, norm_sq ) self.assertTrue( (np.abs(np.array(var_conf_int_act) - np.array(var_conf_int_calc)) < tol).all(), msg='calc_var_conf_int is not correct.' )
def test_calc_pred_conf_int(self): """ Design and Analysis of Experiments 8th Ed. (pg 469, 258) """ signif = 0.05 tol = 0.15 # the book example rounds much more than UQOCE # Design and Analysis of Experiments 8th ed, pg 469, 258 responses = np.array([45, 100, 45, 65, 75, 60, 80, 96]) var_basis = np.array([ [1, -1, -1, -1, 1, 1], [1, 1, -1, 1, -1, 1], [1, -1, -1, 1, 1, -1], [1, 1, -1, -1, -1, -1], [1, -1, 1, 1, -1, -1], [1, 1, 1, -1, 1, -1], [1, -1, 1, -1, -1, 1], [1, 1, 1, 1, 1, 1] ]) basis_eval_ver = np.array([[1, 1, -1, 1, -1, 1]]) matrix_coeffs = solve_coeffs(var_basis, responses) approx_mean, mean_uncert = ( calc_pred_conf_int( var_basis, matrix_coeffs, responses, signif, basis_eval_ver ) ) act_mean = 100.25 act_uncert = 10.25 self.assertEqual( approx_mean, act_mean, msg='calc_pred_conf_int is calculating the wrong mean' ) self.assertTrue( np.isclose(mean_uncert, act_uncert, rtol=0, atol=tol), msg='calc_pred_conf_int is calculating the mean uncertainty' )
def test_calc_R_sq_adj(self): """ Design and Analysis of Experiments 8th ed, pg 454, 464 Douglas C. Montgomery """ tol = 1e-5 # book rounds, so this is a higher value than default responses = np.array([ 2256, 2340, 2426, 2293, 2330, 2368, 2250, 2409, 2364, 2379, 2440, 2364, 2404, 2317, 2309, 2328 ]) interval_low = -10 interval_high = 10 order = 2 var_list = [ UniformVariable(interval_low, interval_high, order=order, number=0), UniformVariable(interval_low, interval_high, order=order, number=1) ] var_basis = np.array([ [1, 80, 8], [1, 93, 9], [1, 100, 10], [1, 82, 12], [1, 90, 11], [1, 99, 8], [1, 81, 8], [1, 96, 10], [1, 94, 12], [1, 93, 11], [1, 97, 13], [1, 95, 11], [1, 100, 8], [1, 85, 12], [1, 86, 9], [1, 87, 12] ]) for i in range(1, 3): var_list[i - 1].std_vals = var_basis[:, i] var_list[i - 1].vals = var_basis[:, i] matrix_coeffs = solve_coeffs(var_basis, responses) R_sq_adj_calc = calc_R_sq_adj(var_basis, matrix_coeffs, responses) R_sq_adj_act = 0.915735 self.assertTrue( np.isclose(R_sq_adj_calc, R_sq_adj_act, rtol=0, atol=tol), msg='statistics function calc_R_sq_adj is not correct' )
def _build_alt_model(self, responses, var_basis, norm_sq, idx): """ Inputs: responses- the array of responses var_basis- the evaluated variable basis norm_sq- the norm squared idx- the index of the point that will be omitted Creates a model for the input combination; the mean, variance, and errors are calculated and returned. """ incr_idx = idx + 1 eval_count = 1 # remove the test point to solve for constants and build model var_basis = np.append(var_basis[:idx], var_basis[incr_idx:], axis=0) responses = np.append(responses[:idx], responses[incr_idx:]) matrix_coeffs = solve_coeffs(var_basis, responses) # create a model for each of the subsystems; check model error temp_model = SurrogateModel(responses, matrix_coeffs) err, pred = temp_model.calc_error(var_basis) mean_err = calc_mean_err(err) var, mean = temp_model.calc_var(norm_sq) # evaluate with the test point resp_ver = ( temp_model.verify( self.var_basis_vect_symb, self.var_list, eval_count, self.var_list_symb, 'std_vals', idx ) )[0] # diff between actual point and calculated value err_ver = np.abs(calc_difference(self.responses[idx], resp_ver)[0]) return err_ver, mean_err, mean, var
def test_get_sobol_bounds(self): """ First Tests: Design and Analysis of Experiments, 8th Ed. (pg 468) Douglas Montgomery Values compared to hand-calculated values from this data. Second Tests: Calculates the smallest and largest of all possible Sobols- ensures that the lowest corresponds to the low and the highest corresponds to the high. """ # Design and Analysis of Experiments 8th ed, pg 468 responses = np.array([ 2256, 2340, 2426, 2293, 2330, 2368, 2250, 2409, 2364, 2379, 2440, 2364, 2404, 2317, 2309, 2328 ]) signif = 0.05 tol = 1e-8 var_basis = np.array([ [1, 80, 8], [1, 93, 9], [1, 100, 10], [1, 82, 12], [1, 90, 11], [1, 99, 8], [1, 81, 8], [1, 96, 10], [1, 94, 12], [1, 93, 11], [1, 97, 13], [1, 95, 11], [1, 100, 8], [1, 85, 12], [1, 86, 9], [1, 87, 12] ]) # Hand calculated from coefficients and coefficient uncertainty sobol_CIL_act = np.array([0.17070561225122574, 0.12055447113345258]) sobol_CIH_act = np.array([0.8794455288665474, 0.8292943877487742]) matrix_coeffs = solve_coeffs(var_basis, responses) coeff_uncert = calc_coeff_conf_int( var_basis, matrix_coeffs, responses, signif ) norm_sq = np.array([[1], [1 / 3], [1 / 3]]) mod = SurrogateModel(responses, matrix_coeffs) mod.calc_var(norm_sq) sobols = mod.get_sobols(norm_sq) sobol_CIL_calc, sobol_CIH_calc = get_sobol_bounds( matrix_coeffs, sobols, coeff_uncert, norm_sq ) self.assertTrue( np.isclose(sobol_CIL_calc, sobol_CIL_act, rtol=0, atol=tol).all(), msg='get_sobol_bounds is not correct.' ) self.assertTrue( np.isclose(sobol_CIH_calc, sobol_CIH_act, rtol=0, atol=tol).all(), msg='get_sobol_bounds is not correct.' ) # Sanity check on the methodology- the low Sobol for first variable and # high for the second variable should sum to 1 and vice versa. self.assertTrue( np.isclose( np.sum([sobol_CIL_calc[0], sobol_CIH_calc[1]]), 1, rtol=0, atol=tol ), msg='get_sobol_bounds is not correct.' ) self.assertTrue( np.isclose( np.sum([sobol_CIL_calc[1], sobol_CIH_calc[0]]), 1, rtol=0, atol=tol ), msg='get_sobol_bounds is not correct.' ) # Special case example- one matrix coefficient of 0 with larg matrix_coeffs = np.array([1, 2, 0, 1.5]) sobols = np.array([0.24615385, 0, 0.08307692]) coeff_uncert = np.array([0.1, 2, 0.1, 0.1]) norm_sq = np.array([[1], [1 / 3], [1 / 3], [1 / 3]]) sobol_CIL_calc, sobol_CIH_calc = get_sobol_bounds( matrix_coeffs, sobols, coeff_uncert, norm_sq ) # Sobol 0 should have low bound of 0 since the uncertainty can lead to # a coefficient of 0 and therefore Sobol of zero. # Sobol 0 should have a high bound of > 0.5 since Sobol 1 can be 0 and # coeff 2 at its smallest is smaller than coeff 0 at its largest. self.assertTrue( sobol_CIL_calc[0] == 0, msg='get_sobol_bounds is not correct.' ) self.assertTrue( sobol_CIH_calc[0] > 0.5, msg='get_sobol_bounds is not correct.' ) # Sobol 1 should have low bound of 0 since the calculated Sobol is 0. # Sobol 1 should have a high bound that is relatively small since the # largest magnitude of coeff 1 is 0.1 and smallest coeff 2 is much # larger. self.assertTrue( sobol_CIL_calc[1] == 0, msg='get_sobol_bounds is not correct.' ) self.assertTrue( sobol_CIH_calc[1] < 0.1, msg='get_sobol_bounds is not correct.' ) # Sobol 2 should have a low bound that is small but relatively large # since the lowest coeff 2 is 1.4, and highest coeff 0 and coeff 1 are # 1.1 and 4, respectively. # Sobol 2 should have a high bound of 1 since the other Sobols can # both be 0. self.assertTrue( sobol_CIL_calc[2] > 0 and sobol_CIL_calc[2] < 0.2, msg='get_sobol_bounds is not correct.' ) self.assertTrue( sobol_CIH_calc[2] == 1, msg='get_sobol_bounds is not correct.' )
def test_calc_coeff_conf_int(self): """ Design and Analysis of Experiments, 8th Ed. (pg 468) Douglas Montgomery """ tol = 1e-3 # book rounds, so this is a higher value than default # Design and Analysis of Experiments 8th ed, pg 468 responses = np.array([ 2256, 2340, 2426, 2293, 2330, 2368, 2250, 2409, 2364, 2379, 2440, 2364, 2404, 2317, 2309, 2328 ]) order = 2 signif = 0.05 interval_low = -10 interval_high = 10 var_list = [ UniformVariable(interval_low, interval_high, order=order, number=0), UniformVariable(interval_low, interval_high, order=order, number=1) ] var_basis = np.array([ [1, 80, 8], [1, 93, 9], [1, 100, 10], [1, 82, 12], [1, 90, 11], [1, 99, 8], [1, 81, 8], [1, 96, 10], [1, 94, 12], [1, 93, 11], [1, 97, 13], [1, 95, 11], [1, 100, 8], [1, 85, 12], [1, 86, 9], [1, 87, 12] ]) for i in range(1, 3): var_list[i - 1].std_vals = var_basis[:, i] var_list[i - 1].vals = var_basis[:, i] matrix_coeffs = solve_coeffs(var_basis, responses) coeff_uncert = calc_coeff_conf_int( var_basis, matrix_coeffs, responses, signif ) calc_coeff = matrix_coeffs[1] act_coeff = 7.62129 calc_low_bound = matrix_coeffs[1] - coeff_uncert[1] act_low_bound = 6.2855 calc_high_bound = matrix_coeffs[1] + coeff_uncert[1] act_high_bound = 8.9570 self.assertTrue( np.isclose(calc_coeff, act_coeff, rtol=0, atol=tol), msg='calc_coeff_conf_int is not correct.' ) self.assertTrue( np.isclose(calc_low_bound, act_low_bound, rtol=0, atol=tol), msg='calc_coeff_conf_int is not correct.' ) self.assertTrue( np.isclose(calc_high_bound, act_high_bound, rtol=0, atol=tol), msg='calc_coeff_conf_int is not correct.' )
def test_calc_mean_conf_int(self): """ Applied Statistics and Probability for Engineers 4th Ed. (2007), (pg. 13, 440, 467) Douglas C. Montgomery & George C. Runger """ signif = 0.05 tol = 1e-2 # book rounds to 2 decimal place responses = np.array([ 9.95, 24.45, 31.75, 35.0, 25.02, 16.86, 14.38, 9.6, 24.35, 27.5, 17.08, 37.0, 41.95, 11.66, 21.65, 17.89, 69.0, 10.3, 34.93, 46.59, 44.88, 54.12, 56.63, 22.13, 21.15 ]) var_basis = np.array([ [1, 2, 50], [1, 8, 110], [1, 11, 120], [1, 10, 550], [1, 8, 295], [1, 4, 200], [1, 2, 375], [1, 2, 52], [1, 9, 100], [1, 8, 300], [1, 4, 412], [1, 11, 400], [1, 12, 500], [1, 2, 360], [1, 4, 205], [1, 4, 400], [1, 20, 600], [1, 1, 585], [1, 10, 540], [1, 15, 250], [1, 15, 290], [1, 16, 510], [1, 17, 590], [1, 6, 100], [1, 5, 400] ]) matrix_coeffs = solve_coeffs(var_basis, responses) var_basis_ver = np.array([[1, 8, 275]]) mean_calc, uncert_calc = calc_mean_conf_int( var_basis, matrix_coeffs, responses, signif, var_basis_ver ) mean_act = 27.66 uncert_act = 1 self.assertTrue( np.isclose(mean_calc, mean_act, rtol=0, atol=tol), msg='calc_mean_conf_int is not correct.' ) self.assertTrue( np.isclose(uncert_calc, uncert_act, rtol=0, atol=tol), msg='calc_mean_conf_int is not correct.' )
def get_press_stats(self): """ Calculates the PRESS statistic of the model. """ comm = MPI_COMM_WORLD size = comm.size rank = comm.rank base = self.act_model_size // size rem = self.act_model_size % size beg = base * rank + (rank >= rem) * rem + (rank < rem) * rank count = base + (rank < rem) end = beg + count ranks = np.arange(0, size, dtype=int) seq_count = (ranks < rem) + base seq_disp = base * ranks + (ranks >= rem) * rem + (ranks < rem) * ranks temp_eval = np.zeros([self.act_model_size, self.min_model_size]) mean_err = np.zeros(count) var = np.zeros(count) mean = np.zeros(count) ver = np.zeros(count) tot_mean_err = np.zeros(self.act_model_size) tot_var = np.zeros(self.act_model_size) tot_mean = np.zeros(self.act_model_size) press = np.zeros(1) temp_eval = np.copy(self.var_basis_sys_eval) for i in range(beg, end): idx = i - beg temp = np.delete(temp_eval, i, axis=0) responses = np.delete(self.responses, i) matrix_coeffs = solve_coeffs(temp, responses) # create a model for each of the subsystems; check model error temp_model = SurrogateModel(responses, matrix_coeffs) err = temp_model.calc_error(temp)[0] mean_err[idx] = calc_mean_err(err) var[idx], mean[idx] = temp_model.calc_var(self.norm_sq) ver[idx] = np.matmul(temp_eval[i, :], matrix_coeffs) ver = np.sum((ver - self.responses[beg:end]) ** 2) comm.Allreduce( [ver, MPI_DOUBLE], [press, MPI_DOUBLE], op=MPI_SUM ) comm.Allgatherv( [mean_err, count, MPI_DOUBLE], [tot_mean_err, seq_count, seq_disp, MPI_DOUBLE] ) comm.Allgatherv( [mean, count, MPI_DOUBLE], [tot_mean, seq_count, seq_disp, MPI_DOUBLE] ) comm.Allgatherv( [var, count, MPI_DOUBLE], [tot_var, seq_count, seq_disp, MPI_DOUBLE] ) mean_err_avg = float(np.mean(tot_mean_err)) mean_err_var = float(np.var(tot_mean_err)) mean_avg = float(np.mean(tot_mean)) mean_var = float(np.var(tot_mean)) var_avg = float(np.mean(tot_var)) var_var = float(np.var(tot_var)) outputs = { 'PRESS':float(press), 'mean_of_model_mean_err':mean_err_avg, 'variance_of_model_mean_err':mean_err_var, 'mean_of_model_mean':mean_avg, 'variance_of_model_mean':mean_var, 'mean_of_model_variance':var_avg, 'variance_of_model_variance':var_var } return outputs