def test_mismatched_cost_and_variance_raises_error(algorithm):
covariance = np.array([[1, 0.9], [0.9, 1]])
model_costs = np.array([1, 2, 3])
with pytest.raises(ValueError):
optimizer = Optimizer(model_costs, covariance)
_ = optimizer.optimize(algorithm=algorithm, target_cost=30)
def test_input_asymmetry(algorithm):
covariance = np.array([[0, 2], [1, 0]])
model_costs = np.array([1, 1, 1])
covariance[0, 1] -= 1
with pytest.raises(ValueError):
optimizer = Optimizer(model_costs, covariance)
_ = optimizer.optimize(algorithm=algorithm, target_cost=30)
def test_mismatched_cost_and_covariance_raises_error():
model_costs = np.array([1, 2])
covariance = np.array([[dummy_var, 1, dummy_var],
[1, dummy_var, 1],
[dummy_var, 1, 2]])
with pytest.raises(ValueError):
optimizer = Optimizer(model_costs, covariance)
_ = optimizer.optimize(algorithm="mlmc", target_cost=30)
def test_opt_result_variance_and_cost_match_allocation(algorithm):
exponents = [4, 3, 2, 1]
covariance = _monomial_model_covariance(exponents)
model_costs = _monomial_model_costs(exponents)
target_cost = 10
optimizer = Optimizer(model_costs, covariance=covariance)
opt_result = optimizer.optimize(algorithm, target_cost)
_assert_opt_result_is_consistent(covariance, model_costs, opt_result)
def test_mfmc_and_acvmfmc_have_about_equal_variance():
exponents = [4, 3, 2, 1]
covariance = _monomial_model_covariance(exponents)
model_costs = _monomial_model_costs(exponents)
target_cost = 10
optimizer = Optimizer(model_costs, covariance=covariance)
analytical_result = optimizer.optimize("mfmc", target_cost)
numerical_result = optimizer.optimize("acvmfmc", target_cost)
assert analytical_result.variance \
== pytest.approx(numerical_result.variance)
def test_opt_results_are_correct_sizes_using_model_selection(
num_models, algorithm):
covariance = np.eye(num_models)
covariance[0] = np.linspace(1.0, 0.6, num_models)
covariance[:, 0] = np.linspace(1.0, 0.6, num_models)
model_costs = np.arange(num_models, 0, -1)
optimizer = Optimizer(model_costs, covariance)
opt_result = optimizer.optimize(algorithm=algorithm,
target_cost=20,
auto_model_selection=True)
opt_sample_array = opt_result.allocation.compressed_allocation
assert opt_sample_array.shape[1] == num_models * 2
def test_optimize_results_are_correct_sizes(algorithm, num_models):
covariance = np.eye(num_models)
covariance[0] = np.linspace(1.0, 0.6, num_models)
covariance[:, 0] = np.linspace(1.0, 0.6, num_models)
model_costs = np.arange(num_models, 0, -1)
optimizer = Optimizer(model_costs, covariance)
opt_result = optimizer.optimize(algorithm=algorithm, target_cost=20)
opt_sample_array = opt_result.allocation.compressed_allocation
if algorithm in ["mfmc", "mlmc", "acvis"]:
assert opt_sample_array.shape[0] == num_models
assert opt_sample_array.shape[1] == num_models * 2
def test_raises_error_if_first_model_is_not_highest_cost():
'''
MXMC assumes first model is high fidelity. For MLMC, the high fidelity
model is finest discretization and therefore is the one with highest cost
'''
model_costs = np.array([11, 1, 12])
cov_matrix = np.ones([3, 3])
with pytest.raises(ValueError):
optimizer = Optimizer(model_costs=model_costs,
covariance=cov_matrix)
_ = optimizer.optimize(algorithm="mlmc", target_cost=50)
def test_mfmc_with_model_selection_hifi_fastest():
covariance = np.array([[1, 0.0], [0.0, 1]])
model_costs = np.array([1, 2])
optimizer = Optimizer(model_costs, covariance)
opt_result = optimizer.optimize(algorithm="mfmc",
target_cost=30,
auto_model_selection=True)
expected_cost = 30
expected_variance = 1 / 30
expected_sample_array = np.array([[30, 1, 0, 0]], dtype=int)
assert isinstance(opt_result.allocation, ACVSampleAllocation)
assert_opt_result_equal(opt_result, expected_cost, expected_variance,
expected_sample_array)
def test_case_mfmc(target_cost_multiplier, covariance_multiplier):
covariance = np.array([[1, 0.5], [0.5, 1]]) * covariance_multiplier
model_costs = np.array([4800, 4])
optimizer = Optimizer(model_costs, covariance)
target_cost = 14640 * target_cost_multiplier
opt_result = optimizer.optimize(algorithm="mfmc", target_cost=target_cost)
expected_cost = 14640 * target_cost_multiplier
expected_variance = 61 / 240 * covariance_multiplier \
/ target_cost_multiplier
expected_sample_array = np.array([[3 * target_cost_multiplier, 1, 1, 1],
[57 * target_cost_multiplier, 0, 0, 1]],
dtype=int)
assert_opt_result_equal(opt_result, expected_cost, expected_variance,
expected_sample_array)
def test_three_models_out_of_order():
target_cost = 24
cov_matrix = np.array([[1.5, dummy_var, 1], [dummy_var, 4, 2], [1, 2, 1]])
model_costs = np.array([5, 1, 3])
optimizer = Optimizer(model_costs=model_costs, covariance=cov_matrix)
sample_array_expected = np.array([[1, 1, 0, 0, 1, 0],
[2, 0, 1, 0, 0, 1],
[8, 0, 0, 1, 0, 0]])
var_expected = 1.5
cost_expected = target_cost
opt_result = optimizer.optimize(algorithm="mlmc",
target_cost=target_cost)
assert_opt_result_equal(opt_result, cost_expected, var_expected,
sample_array_expected)
def optimizer_four_model():
model_costs = np.array([11, 5, 3, 1])
cov_matrix = np.array([[0.75, 1, dummy_var, dummy_var],
[1, 1.5, 1, dummy_var],
[dummy_var, 1, 1, 2],
[dummy_var, dummy_var, 2, 4]])
return Optimizer(model_costs=model_costs, covariance=cov_matrix)
def test_mfmc_with_model_selection_no_best_result(mocker):
mocker.patch('mxmc.optimizer.AutoModelSelection.' +
'_get_subsets_of_model_indices',
return_value=[])
covariance = np.array([[1, 0.0], [0.0, 1]])
model_costs = np.array([1, 2])
optimizer = Optimizer(model_costs, covariance)
opt_result = optimizer.optimize(algorithm="mfmc",
target_cost=30,
auto_model_selection=True)
expected_cost = 0
expected_variance = np.inf
expected_sample_array = np.array([[1, 1, 0, 0]], dtype=int)
assert_opt_result_equal(opt_result, expected_cost, expected_variance,
expected_sample_array)
def test_mlmc_with_model_selection():
model_costs = np.array([3, 2, 1])
cov_matrix = np.array([[8, 6, 11/2.], [6, 6, 7/2.], [11/2., 7/2., 4.]])
optimizer = Optimizer(model_costs, covariance=cov_matrix)
target_cost = 17
sample_array_expected = np.array([[2, 1, 0, 0, 1, 0],
[8, 0, 0, 0, 0, 1]])
variance_expected = 1.
cost_expected = 16
opt_result = optimizer.optimize(algorithm="mlmc", target_cost=target_cost,
auto_model_selection=True)
assert_opt_result_equal(opt_result, cost_expected, variance_expected,
sample_array_expected)
def test_optimizer_returns_monte_carlo_result_for_one_model(algorithm):
covariance = np.array([[12.]])
model_costs = np.array([2.])
target_cost = 8.
optimizer = Optimizer(model_costs, covariance)
opt_result = optimizer.optimize(algorithm=algorithm,
target_cost=target_cost)
N_ref = target_cost / model_costs[0]
variance_ref = covariance[0] / N_ref
cost_ref = target_cost
opt_sample_array = opt_result.allocation.compressed_allocation
assert np.isclose(variance_ref, opt_result.variance)
assert np.isclose(cost_ref, opt_result.cost)
assert N_ref == opt_sample_array[0, 0]
def test_acvmfmc_known_solution(cost_factor, covariance_factor, mocker):
covariance = np.array([[1, 0.5], [0.5, 1]]) * covariance_factor
model_costs = np.array([4800, 4])
optimizer = Optimizer(model_costs, covariance)
ratios_for_opt = np.array([20])
mocker.patch.object(Optimizer.get_algorithm("acvmfmc"),
'_solve_opt_problem',
return_value=ratios_for_opt)
cost_ref = 14640 * cost_factor
var_ref = 61 / 240 * covariance_factor / cost_factor
allocation_ref = np.array(
[[3 * cost_factor, 1, 1, 1], [57 * cost_factor, 0, 0, 1]], dtype=int)
target_cost = 14640 * cost_factor
opt_result = optimizer.optimize(algorithm="acvmfmc",
target_cost=target_cost)
assert_opt_result_equal(opt_result, cost_ref, var_ref, allocation_ref)
def test_mr_enumeration(mocker, num_models, num_combinations):
covariance = np.random.random((num_models, num_models))
covariance *= covariance.transpose()
model_costs = np.arange(num_models, 0, -1)
optimizer = Optimizer(model_costs, covariance)
mocked_optimizer = mocker.Mock()
dummy_samples = np.array([[1, 1] + [0] * (num_models * 2 - 2)], dtype=int)
mocked_optimizer.optimize.return_value = OptimizationResult(
10, 0.1, dummy_samples)
mocker.patch(
'mxmc.optimizers.approximate_control_variates.'
'generalized_multifidelity.impl_optimizers.GMFUnordered',
return_value=mocked_optimizer)
target_cost = 100
_ = optimizer.optimize("gmfmr", target_cost)
assert impl_optimizers.GMFUnordered.call_count == num_combinations
def test_four_models_out_of_order():
target_cost = 64
model_costs = np.array([11, 1, 5, 3])
cov_matrix = np.array([[0.75, dummy_var, 1, dummy_var],
[dummy_var, 4, dummy_var, 2],
[1, dummy_var, 1.5, 1],
[dummy_var, 2, 1, 1]])
optimizer = Optimizer(model_costs=model_costs, covariance=cov_matrix)
sample_array_expected = np.array([[1, 1, 0, 0, 1, 0, 0, 0],
[2, 0, 0, 0, 0, 1, 1, 0],
[4, 0, 1, 0, 0, 0, 0, 1],
[16, 0, 0, 1, 0, 0, 0, 0]])
var_expected = 1.
cost_expected = target_cost
opt_result = optimizer.optimize(algorithm="mlmc",
target_cost=target_cost)
assert_opt_result_equal(opt_result, cost_expected, var_expected,
sample_array_expected)
def test_basic_acv_optimizers_give_consistent_output(acv_optimizer, mocker):
exponents = [4, 3, 2, 1]
covariance = _monomial_model_covariance(exponents)
model_costs = _monomial_model_costs(exponents)
target_cost = 10
optimizer = Optimizer.get_algorithm(acv_optimizer)(model_costs,
covariance=covariance)
constraints = optimizer._get_constraints(target_cost)
np.random.seed(0)
for i in range(25):
valid_ratios = \
_generate_random_ratios_fulfilling_constraints(constraints,
model_costs)
mocker.patch.object(Optimizer.get_algorithm(acv_optimizer),
'_solve_opt_problem',
return_value=valid_ratios)
opt_result = optimizer.optimize(target_cost)
_assert_opt_result_is_consistent(covariance, model_costs, opt_result)
def test_mlmc_with_model_selection_zero_q():
model_costs = np.array([3, 2, 1])
cov_matrix = np.array([[8, 6, 11/2.], [6, 6, 7/2.], [11/2., 7/2., 4.]])
optimizer = Optimizer(model_costs, covariance=cov_matrix)
target_cost = 8
sample_array_expected = np.array([[1, 1, 0, 0, 1, 0],
[4, 0, 0, 0, 0, 1]])
variance_expected = 2.
cost_expected = 8.
with warnings.catch_warnings(record=True) as warning_log:
opt_result = optimizer.optimize(algorithm="mlmc",
target_cost=target_cost,
auto_model_selection=True)
assert_opt_result_equal(opt_result, cost_expected, variance_expected,
sample_array_expected)
assert len(warning_log) == 1
assert issubclass(warning_log[-1].category, UserWarning)
def test_acvmf_three_models_known_solution(cost_factor, covariance_factor,
mocker):
covariance = np.array([[1, 0.75, 0.25], [0.75, 1., 0.5], [0.25, 0.5, 1.]
]) * covariance_factor
model_costs = np.array([3, 2, 1])
optimizer = Optimizer(model_costs, covariance)
ratios_for_opt = np.array([1, 2])
mocker.patch.object(Optimizer.get_algorithm("wrdiff"),
'_solve_opt_problem',
return_value=ratios_for_opt)
cost_ref = 10. * cost_factor
var_ref = 0.6931818181818182 * covariance_factor / cost_factor
allocation_ref = np.array(
[[1 * cost_factor, 1, 1, 0, 0, 0], [1 * cost_factor, 0, 0, 1, 1, 0],
[2 * cost_factor, 0, 0, 0, 0, 1]],
dtype=int)
target_cost = 10 * cost_factor
opt_result = optimizer.optimize(algorithm="wrdiff",
target_cost=target_cost)
assert_opt_result_equal(opt_result, cost_ref, var_ref, allocation_ref)
def optimizer_three_model():
model_costs = np.array([5, 3, 1])
cov_matrix = np.array([[1.5, 1, dummy_var], [1, 1, 2], [dummy_var, 2, 4]])
return Optimizer(model_costs=model_costs, covariance=cov_matrix)
import numpy as np
import pytest
from mxmc.optimizer import Optimizer
from mxmc.util.testing import assert_opt_result_equal
ALGORITHMS = Optimizer.get_algorithm_names()
DUMMY_VAR = 999
@pytest.fixture
def ui_optimizer():
model_costs = np.array([100, 1])
covariance = np.array([[1, 0.9], [0.9, 1]])
return Optimizer(model_costs, covariance=covariance)
@pytest.mark.parametrize("algorithm", ALGORITHMS)
@pytest.mark.parametrize("target_cost", [-1, 0.5, 0])
def test_target_cost_too_low_to_run_models(ui_optimizer, algorithm,
target_cost):
opt_result = ui_optimizer.optimize(algorithm=algorithm,
target_cost=target_cost)
assert_opt_result_equal(opt_result, 0, np.inf, np.array([[1, 1, 0, 0]]))
@pytest.mark.parametrize("algorithm", ALGORITHMS)
def test_optimizer_returns_tuple_with(ui_optimizer, algorithm):
opt_result = ui_optimizer.optimize(algorithm="mfmc", target_cost=10)
for member in ["cost", "variance", "allocation"]:
assert member in dir(opt_result)
def optimizer_two_model():
model_costs = np.array([3, 1])
cov = np.array([[1, 2], [2, 4]])
return Optimizer(model_costs=model_costs, covariance=cov)
def ui_optimizer():
model_costs = np.array([100, 1])
covariance = np.array([[1, 0.9], [0.9, 1]])
return Optimizer(model_costs, covariance=covariance)
def test_optimizer_can_initialize_with_extra_inputs():
covariance = np.array([[1, 0.5], [0.5, 1]])
model_costs = np.array([4800, 4])
_ = Optimizer(model_costs, covariance, 0, abc=0)