def test_to_robust_problem(self, mocked_platypus):
mocked_model = Model('test', function=mock.Mock())
mocked_model.levers = [
RealParameter('a', 0, 1),
RealParameter('b', 0, 1)
]
mocked_model.uncertainties = [
RealParameter('c', 0, 1),
RealParameter('d', 0, 1)
]
mocked_model.outcomes = [ScalarOutcome('x'), ScalarOutcome('y')]
scenarios = 5
robustness_functions = [
ScalarOutcome('mean x', variable_name='x', function=mock.Mock()),
ScalarOutcome('mean y', variable_name='y', function=mock.Mock())
]
problem = to_robust_problem(mocked_model, scenarios,
robustness_functions)
self.assertEqual('robust', problem.searchover)
for entry in problem.parameters:
self.assertIn(entry.name, mocked_model.levers.keys())
self.assertEqual(['a', 'b'], problem.parameter_names)
self.assertEqual(['mean x', 'mean y'], problem.outcome_names)
def test_pf_sampler(self):
uncs = [RealParameter('a', 0, 5, resolution=(0, 2.5,5), pff=True),
RealParameter('b', 0, 1, resolution=(0,1), pff=True),
RealParameter('c', 0, 1),
RealParameter('d', 1, 2),
]
sampler = PartialFactorialSampler()
designs = sampler.generate_designs(uncs, 10)
designs.kind = Scenario
expected = 60
self.assertEqual(expected, designs.n)
self.assertEqual(expected, len([design for design in designs]))
ff, other = sampler._sort_parameters(uncs)
received = {u.name for u in ff}
expected = {'a', 'b'}
self.assertEqual(received, expected)
received = {u.name for u in other}
expected = {'c', 'd'}
self.assertEqual(received, expected)
def test_to_problem(self, mocked_platypus):
mocked_model = Model('test', function=mock.Mock())
mocked_model.levers = [
RealParameter('a', 0, 1),
RealParameter('b', 0, 1)
]
mocked_model.uncertainties = [
RealParameter('c', 0, 1),
RealParameter('d', 0, 1)
]
mocked_model.outcomes = [ScalarOutcome('x'), ScalarOutcome('y')]
mocked_model.constraints = []
searchover = 'levers'
problem = to_problem(mocked_model, searchover)
self.assertEqual(searchover, problem.searchover)
for entry in problem.parameters:
self.assertIn(entry.name, mocked_model.levers.keys())
self.assertIn(entry, list(mocked_model.levers))
for entry in problem.outcome_names:
self.assertIn(entry.name, mocked_model.outcomes.keys())
searchover = 'uncertainties'
problem = to_problem(mocked_model, searchover)
self.assertEqual(searchover, problem.searchover)
for entry in problem.parameters:
self.assertIn(entry.name, mocked_model.uncertainties.keys())
self.assertIn(entry, list(mocked_model.uncertainties))
for entry in problem.outcome_names:
self.assertIn(entry.name, mocked_model.outcomes.keys())
def test_init(self):
# let's add some uncertainties to this
uncs = [RealParameter("a", 0, 1),
RealParameter("b", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
constraints = []
callback = DefaultCallback(uncs, [], outcomes, constraints,
nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = rf.get_names(callback.cases.dtype)
names = set(names)
self.assertEqual(names, {'a', 'b', 'policy', 'model', 'scenario_id'})
self.assertEqual(callback.results, {})
# with levers
levers = [RealParameter('c', 0, 10)]
callback = DefaultCallback(uncs, levers, outcomes, constraints,
nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = rf.get_names(callback.cases.dtype)
names = set(names)
self.assertEqual(names, {'a', 'b', 'c','policy', 'model', 'scenario_id'})
self.assertEqual(callback.results, {})
def test_determine_parameters(self):
function = mock.Mock()
model_a = Model("A", function)
model_a.uncertainties = [
RealParameter('a', 0, 1),
RealParameter('b', 0, 1),
]
function = mock.Mock()
model_b = Model("B", function)
model_b.uncertainties = [
RealParameter('b', 0, 1),
RealParameter('c', 0, 1),
]
models = [model_a, model_b]
parameters = determine_parameters(models, 'uncertainties', union=True)
for model in models:
for unc in model.uncertainties:
self.assertIn(unc.name, parameters.keys())
parameters = determine_parameters(models, 'uncertainties', union=False)
self.assertIn('b', parameters.keys())
self.assertNotIn('c', parameters.keys())
self.assertNotIn('a', parameters.keys())
def test_store_results(self):
nr_experiments = 3
uncs = [RealParameter("a", 0, 1),
RealParameter("b", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
model = NamedObject('test')
experiment = Experiment(0, model, Policy('policy'), Scenario(a=1, b=0), 0)
# case 1 scalar shape = (1)
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: 1}
callback(experiment, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,))
# case 2 time series shape = (1, nr_time_steps)
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(10)}
callback(experiment, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,10))
# case 3 maps etc. shape = (x,y)
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(2,2)}
callback(experiment,result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,2,2))
# case 4 assert raises EMAError
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(2,2,2)}
self.assertRaises(EMAError, callback, experiment, result)
# KeyError
with mock.patch('ema_workbench.util.ema_logging.debug') as mocked_logging:
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
result = {'incorrect': np.random.rand(2,)}
callback(experiment, result)
for outcome in outcomes:
mocked_logging.assert_called_with("%s not specified as outcome in msi" % outcome.name)
def test_run_experiment(self):
mockMSI = mock.Mock(spec=Model)
mockMSI.name = 'test'
mockMSI.uncertainties = [RealParameter("a", 0, 10),
RealParameter("b", 0, 10)]
msis = NamedObjectMap(AbstractModel)
msis['test'] = mockMSI
runner = ExperimentRunner(msis)
experiment = Experiment('test',
mockMSI.name,
Policy('none'),
Scenario(a=1, b=2), 0)
runner.run_experiment(experiment)
sc, p = mockMSI.run_model.call_args[0]
self.assertEqual(sc.name, experiment.scenario.name)
self.assertEqual(p, experiment.policy)
mockMSI.reset_model.assert_called_once_with()
# assert handling of case error
mockMSI = mock.Mock(spec=Model)
mockMSI.name = 'test'
mockMSI.run_model.side_effect = Exception('some exception')
msis = NamedObjectMap(AbstractModel)
msis['test'] = mockMSI
runner = ExperimentRunner(msis)
experiment = Experiment('test',mockMSI.name,Policy('none'),
Scenario(a=1, b=2),0)
with self.assertRaises(EMAError):
runner.run_experiment(experiment)
# assert handling of case error
mockMSI = mock.Mock(spec=Model)
mockMSI.name = 'test'
mockMSI.run_model.side_effect = CaseError("message", {})
msis = NamedObjectMap(AbstractModel)
msis['test'] = mockMSI
runner = ExperimentRunner(msis)
experiment = Experiment('test',mockMSI.name,Policy('none'),
Scenario(a=1, b=2),0)
runner.run_experiment(experiment)
def test_store_results(self):
nr_experiments = 3
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
model = NamedObject('test')
experiment = Case(0, model, Policy('policy'), Scenario(a=1, b=0), 0)
# case 1 scalar shape = (1)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: 1}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, ))
# case 2 time series shape = (1, nr_time_steps)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(10)}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, 10))
# case 3 maps etc. shape = (x,y)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(2, 2)}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, 2, 2))
# case 4 assert raises EMAError
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(2, 2, 2)}
self.assertRaises(EMAError, callback, experiment, model_outcomes)
def test_run_model(self):
model_name = 'modelname'
function = mock.Mock()
model = Model(model_name, function)
model.uncertainties = [RealParameter('a', 0, 1)]
model.run_model(Scenario(**{'a': 0.1, 'b': 1}), Policy('test'))
function.assert_called_once_with(a=0.1)
# test complete translation of scenario
model = Model(model_name, function)
model.uncertainties = [
RealParameter('a', 0, 1, variable_name=['a', 'b'])
]
scenario = Scenario(**{'a': 0.1})
model.run_model(scenario, Policy('test'))
self.assertIn('a', scenario.keys())
self.assertIn('b', scenario.keys())
model = Model(model_name, function)
cats = [
Category('some name', [1, 2], multivalue=True),
Category('some other name', [3, 4], multivalue=True)
]
model.uncertainties = [
CategoricalParameter('a', cats, variable_name=['a', 'b'])
]
scenario = Scenario(**{'a': 'some name'})
model.run_model(scenario, Policy('test'))
self.assertIn('a', scenario.keys())
self.assertIn('b', scenario.keys())
self.assertEqual(scenario['a'], 1)
self.assertEqual(scenario['b'], 2)
scenario = Scenario(**{'a': 'some other name'})
model.run_model(scenario, Policy('test'))
self.assertIn('a', scenario.keys())
self.assertIn('b', scenario.keys())
self.assertEqual(scenario['a'], 3)
self.assertEqual(scenario['b'], 4)
def test_init(self):
# let's add some uncertainties to this
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
outcomes = [
ScalarOutcome("scalar"),
ArrayOutcome("array", shape=(10, )),
TimeSeriesOutcome("timeseries")
]
callback = DefaultCallback(uncs, [], outcomes, nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = callback.cases.columns.values.tolist()
names = set(names)
self.assertEqual(names, {'a', 'b', 'policy', 'model', 'scenario'})
self.assertNotIn('scalar', callback.results)
self.assertNotIn('timeseries', callback.results)
self.assertIn('array', callback.results)
a = np.all(np.isnan(callback.results['array']))
self.assertTrue(a)
# with levers
levers = [RealParameter('c', 0, 10)]
callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = callback.cases.columns.values.tolist()
names = set(names)
self.assertEqual(names, {'a', 'b', 'c', 'policy', 'model', 'scenario'})
self.assertNotIn('scalar', callback.results)
self.assertNotIn('timeseries', callback.results)
self.assertIn('array', callback.results)
a = np.all(np.isnan(callback.results['array']))
self.assertTrue(a)
def test_model_uncertainties(self):
model_name = 'modelname'
model = Model(model_name, lambda x:x)
self.assertTrue(len(model.uncertainties.keys())==0)
unc_a = RealParameter('a', 0, 1)
model.uncertainties = unc_a
self.assertTrue(len(model.uncertainties.keys())==1)
self.assertTrue(unc_a.name in model.uncertainties)
def test_to_platypus_types(self, mocked_platypus):
dv = [
RealParameter("real", 0, 1),
IntegerParameter("integer", 0, 10),
CategoricalParameter("categorical", ["a", "b"])
]
types = to_platypus_types(dv)
self.assertTrue(str(types[0]).find("platypus.Real") != -1)
self.assertTrue(str(types[1]).find("platypus.Integer") != -1)
self.assertTrue(str(types[2]).find("platypus.Permutation") != -1)
def test_store_cases(self):
nr_experiments = 3
uncs = [RealParameter("a", 0, 1),
RealParameter("b", 0, 1),
CategoricalParameter('c', [0, 1, 2]),
IntegerParameter("d", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
constraints = []
case = {unc.name:random.random() for unc in uncs}
case["c"] = int(round(case["c"]*2))
case["d"] = int(round(case["d"]))
model = NamedObject('test')
policy = Policy('policy')
scenario = Scenario(**case)
experiment = Case(0, model.name, policy, scenario, 0)
callback = DefaultCallback(uncs, [],outcomes, constraints,
nr_experiments=nr_experiments,
reporting_interval=1)
model_outcomes = {outcomes[0].name: 1}
model_constraints = {}
callback(experiment, model_outcomes, model_constraints)
experiments, _ = callback.get_results()
design = case
design['policy'] = policy.name
design['model'] = model.name
design['scenario_id'] = scenario.name
names = rf.get_names(experiments.dtype)
for name in names:
entry_a = experiments[name][0]
entry_b = design[name]
self.assertEqual(entry_a, entry_b, "failed for "+name)
# with levers
nr_experiments = 3
uncs = [RealParameter("a", 0, 1),
RealParameter("b", 0, 1)]
levers = [RealParameter("c", 0, 1),
RealParameter("d", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
case = {unc.name:random.random() for unc in uncs}
model = NamedObject('test')
policy = Policy('policy', c=1, d=1)
scenario = Scenario(**case)
experiment = Case(0, model.name, policy, scenario, 0)
callback = DefaultCallback(uncs, levers,outcomes,constraints,
nr_experiments=nr_experiments,
reporting_interval=1)
model_outcomes = {outcomes[0].name: 1}
model_constraints = {}
callback(experiment, model_outcomes, model_constraints)
experiments, _ = callback.get_results()
design = case
design['c'] = 1
design['d'] = 1
design['policy'] = policy.name
design['model'] = model.name
design['scenario_id'] = scenario.name
names = rf.get_names(experiments.dtype)
for name in names:
self.assertEqual(experiments[name][0], design[name])
class SamplerTestCase(unittest.TestCase):
uncertainties = [RealParameter("1", 0, 10),
IntegerParameter("2", 0, 10),
CategoricalParameter('3', ['a','b', 'c'])]
def _test_generate_designs(self, sampler):
designs = sampler.generate_designs(self.uncertainties, 10)
designs.kind = Scenario
msg = 'tested for {}'.format(type(sampler))
actual_nr_designs = 0
for design in designs:
actual_nr_designs +=1
self.assertIn('1', design, msg)
self.assertIn('2', design, msg)
self.assertIn('3', design, msg)
self.assertEqual(designs.n, actual_nr_designs, msg)
def test_lhs_sampler(self):
sampler = LHSSampler()
self._test_generate_designs(sampler)
def test_mc_sampler(self):
sampler = MonteCarloSampler()
self._test_generate_designs(sampler)
def test_ff_sampler(self):
sampler = FullFactorialSampler()
self._test_generate_designs(sampler)
def test_pf_sampler(self):
uncs = [RealParameter('a', 0, 5, resolution=(0, 2.5,5), pff=True),
RealParameter('b', 0, 1, resolution=(0,1), pff=True),
RealParameter('c', 0, 1),
RealParameter('d', 1, 2),
]
sampler = PartialFactorialSampler()
designs = sampler.generate_designs(uncs, 10)
designs.kind = Scenario
expected = 60
self.assertEqual(expected, designs.n)
self.assertEqual(expected, len([design for design in designs]))
ff, other = sampler._sort_parameters(uncs)
received = {u.name for u in ff}
expected = {'a', 'b'}
self.assertEqual(received, expected)
received = {u.name for u in other}
expected = {'c', 'd'}
self.assertEqual(received, expected)
def test_determine_parameters(self):
function = mock.Mock()
model_a = Model("A", function)
model_a.uncertainties = [RealParameter('a', 0, 1),
RealParameter('b', 0, 1),]
function = mock.Mock()
model_b = Model("B", function)
model_b.uncertainties = [RealParameter('b', 0, 1),
RealParameter('c', 0, 1),]
models = [model_a, model_b]
parameters = determine_parameters(models, 'uncertainties', union=True)
for model in models:
for unc in model.uncertainties:
self.assertIn(unc.name, parameters.keys())
parameters = determine_parameters(models, 'uncertainties', union=False)
self.assertIn('b', parameters.keys())
self.assertNotIn('c', parameters.keys())
self.assertNotIn('a', parameters.keys())
