def test_store_cases(self):
nr_experiments = 3
uncs = [ParameterUncertainty((0,1), "a"),
ParameterUncertainty((0,1), "b"),
CategoricalUncertainty([0, 1, 2], "c"),
ParameterUncertainty((0,1), "d", integer=True),]
outcomes = [Outcome("test", time=True)]
case = {unc.name:random.random() for unc in uncs}
case["c"] = int(round(case["c"]*2))
case["d"] = int(round(case["d"]))
policy = {'name':'none'}
name = "test"
callback = DefaultCallback(uncs,
[outcome.name for outcome in outcomes],
nr_experiments=nr_experiments,
reporting_interval=1)
result = {outcomes[0].name: 1}
callback(0, case, policy, name, result)
experiments, _ = callback.get_results()
design = case
design['policy'] = policy['name']
design['model'] = name
names = rf.get_names(experiments.dtype)
for name in names:
self.assertEqual(experiments[name][0], design[name])
def test_init(self):
# let's add some uncertainties to this
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
callback = DefaultCallback(uncs, [], outcomes, nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = rf.get_names(callback.cases.dtype)
names = set(names)
self.assertEqual(names, {'a', 'b', 'policy', 'model'})
self.assertEqual(callback.results, {})
# with levers
levers = [RealParameter('c', 0, 10)]
callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = rf.get_names(callback.cases.dtype)
names = set(names)
self.assertEqual(names, {'a', 'b', 'c', 'policy', 'model'})
self.assertEqual(callback.results, {})
def test_store_results(self):
nr_experiments = 3
uncs = [ParameterUncertainty((0,1), "a"),
ParameterUncertainty((0,1), "b")]
outcomes = [Outcome("test", time=True)]
case = {unc.name:random.random() for unc in uncs}
policy = {'name':'none'}
name = "test"
# case 1 scalar shape = (1)
callback = DefaultCallback(uncs,
[outcome.name for outcome in outcomes],
nr_experiments=nr_experiments)
result = {outcomes[0].name: 1}
callback(0, case, policy, name, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,))
# case 2 time series shape = (1, nr_time_steps)
callback = DefaultCallback(uncs,
[outcome.name for outcome in outcomes],
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(10)}
callback(0, case, policy, name, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,10))
# case 3 maps etc. shape = (x,y)
callback = DefaultCallback(uncs,
[outcome.name for outcome in outcomes],
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(2,2)}
callback(0, case, policy, name, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,2,2))
# case 4 assert raises EMAError
callback = DefaultCallback(uncs,
[outcome.name for outcome in outcomes],
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(2,2,2)}
self.assertRaises(EMAError, callback, 0, case, policy, name, result)
# KeyError
with mock.patch('ema_workbench.util.ema_logging.debug') as mocked_logging:
callback = DefaultCallback(uncs,
[outcome.name for outcome in outcomes],
nr_experiments=nr_experiments)
result = {'incorrect': np.random.rand(2,)}
callback(0, case, policy, name, result)
for outcome in outcomes:
mocked_logging.assert_called_with("%s not specified as outcome in msi" % outcome.name)
def test_init(self):
# let's add some uncertainties to this
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
outcomes = [
ScalarOutcome("scalar"),
ArrayOutcome("array", shape=(10, )),
TimeSeriesOutcome("timeseries")
]
callback = DefaultCallback(uncs, [], outcomes, nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = callback.cases.columns.values.tolist()
names = set(names)
self.assertEqual(names, {'a', 'b', 'policy', 'model', 'scenario'})
self.assertNotIn('scalar', callback.results)
self.assertNotIn('timeseries', callback.results)
self.assertIn('array', callback.results)
a = np.all(np.isnan(callback.results['array']))
self.assertTrue(a)
# with levers
levers = [RealParameter('c', 0, 10)]
callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=100)
self.assertEqual(callback.i, 0)
self.assertEqual(callback.nr_experiments, 100)
self.assertEqual(callback.cases.shape[0], 100)
self.assertEqual(callback.outcomes, [o.name for o in outcomes])
names = callback.cases.columns.values.tolist()
names = set(names)
self.assertEqual(names, {'a', 'b', 'c', 'policy', 'model', 'scenario'})
self.assertNotIn('scalar', callback.results)
self.assertNotIn('timeseries', callback.results)
self.assertIn('array', callback.results)
a = np.all(np.isnan(callback.results['array']))
self.assertTrue(a)
def test_store_results(self):
nr_experiments = 3
uncs = [RealParameter("a", 0, 1),
RealParameter("b", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
model = NamedObject('test')
experiment = Case(0, model, Policy('policy'),
Scenario(a=1, b=0), 0)
# case 1 scalar shape = (1)
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: 1}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,))
# case 2 time series shape = (1, nr_time_steps)
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(10)}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,10))
# case 3 maps etc. shape = (x,y)
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(2,2)}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3,2,2))
# case 4 assert raises EMAError
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(2,2,2)}
self.assertRaises(EMAError, callback, experiment, model_outcomes)
# KeyError
with mock.patch('ema_workbench.util.ema_logging.debug') as mocked_logging:
callback = DefaultCallback(uncs, [], outcomes,
nr_experiments=nr_experiments)
model_outcomes = {'incorrect': np.random.rand(2,)}
callback(experiment, model_outcomes)
for outcome in outcomes:
mocked_logging.assert_called_with("%s not specified as outcome in msi" % outcome.name)
def test_store_results(self):
nr_experiments = 3
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
model = NamedObject('test')
experiment = Experiment(0, model, Policy('policy'), Scenario(a=1, b=0),
0)
# case 1 scalar shape = (1)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: 1}
callback(experiment, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, ))
# case 2 time series shape = (1, nr_time_steps)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(10)}
callback(experiment, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, 10))
# case 3 maps etc. shape = (x,y)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(2, 2)}
callback(experiment, result)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, 2, 2))
# case 4 assert raises EMAError
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
result = {outcomes[0].name: np.random.rand(2, 2, 2)}
self.assertRaises(EMAError, callback, experiment, result)
# KeyError
with mock.patch(
'ema_workbench.util.ema_logging.debug') as mocked_logging:
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
result = {'incorrect': np.random.rand(2, )}
callback(experiment, result)
for outcome in outcomes:
mocked_logging.assert_called_with(
"%s not specified as outcome in msi" % outcome.name)
def test_store_cases(self):
nr_experiments = 3
uncs = [
RealParameter("a", 0, 1),
RealParameter("b", 0, 1),
CategoricalParameter('c', [0, 1, 2]),
IntegerParameter("d", 0, 1)
]
outcomes = [TimeSeriesOutcome("test")]
case = {unc.name: random.random() for unc in uncs}
case["c"] = int(round(case["c"] * 2))
case["d"] = int(round(case["d"]))
model = NamedObject('test')
policy = Policy('policy')
scenario = Scenario(**case)
experiment = Experiment(0, model, policy, scenario, 0)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments,
reporting_interval=1)
result = {outcomes[0].name: 1}
callback(experiment, result)
experiments, _ = callback.get_results()
design = case
design['policy'] = policy.name
design['model'] = model.name
names = rf.get_names(experiments.dtype)
for name in names:
self.assertEqual(experiments[name][0], design[name])
# with levers
nr_experiments = 3
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
levers = [RealParameter("c", 0, 1), RealParameter("d", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
case = {unc.name: random.random() for unc in uncs}
model = NamedObject('test')
policy = Policy('policy', c=1, d=1)
scenario = Scenario(**case)
experiment = Experiment(0, model, policy, scenario, 0)
callback = DefaultCallback(uncs,
levers,
outcomes,
nr_experiments=nr_experiments,
reporting_interval=1)
result = {outcomes[0].name: 1}
callback(experiment, result)
experiments, _ = callback.get_results()
design = case
design['c'] = 1
design['d'] = 1
design['policy'] = policy.name
design['model'] = model.name
names = rf.get_names(experiments.dtype)
print(experiments[0])
for name in names:
self.assertEqual(experiments[name][0], design[name])
def test_store_cases(self):
nr_experiments = 3
uncs = [RealParameter("a", 0, 1),
RealParameter("b", 0, 1),
CategoricalParameter('c', [0, 1, 2]),
IntegerParameter("d", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
constraints = []
case = {unc.name:random.random() for unc in uncs}
case["c"] = int(round(case["c"]*2))
case["d"] = int(round(case["d"]))
model = NamedObject('test')
policy = Policy('policy')
scenario = Scenario(**case)
experiment = Case(0, model.name, policy, scenario, 0)
callback = DefaultCallback(uncs, [],outcomes, constraints,
nr_experiments=nr_experiments,
reporting_interval=1)
model_outcomes = {outcomes[0].name: 1}
model_constraints = {}
callback(experiment, model_outcomes, model_constraints)
experiments, _ = callback.get_results()
design = case
design['policy'] = policy.name
design['model'] = model.name
design['scenario_id'] = scenario.name
names = rf.get_names(experiments.dtype)
for name in names:
entry_a = experiments[name][0]
entry_b = design[name]
self.assertEqual(entry_a, entry_b, "failed for "+name)
# with levers
nr_experiments = 3
uncs = [RealParameter("a", 0, 1),
RealParameter("b", 0, 1)]
levers = [RealParameter("c", 0, 1),
RealParameter("d", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
case = {unc.name:random.random() for unc in uncs}
model = NamedObject('test')
policy = Policy('policy', c=1, d=1)
scenario = Scenario(**case)
experiment = Case(0, model.name, policy, scenario, 0)
callback = DefaultCallback(uncs, levers,outcomes,constraints,
nr_experiments=nr_experiments,
reporting_interval=1)
model_outcomes = {outcomes[0].name: 1}
model_constraints = {}
callback(experiment, model_outcomes, model_constraints)
experiments, _ = callback.get_results()
design = case
design['c'] = 1
design['d'] = 1
design['policy'] = policy.name
design['model'] = model.name
design['scenario_id'] = scenario.name
names = rf.get_names(experiments.dtype)
for name in names:
self.assertEqual(experiments[name][0], design[name])
def test_store_cases(self):
nr_experiments = 3
uncs = [
RealParameter("a", 0, 1),
RealParameter("b", 0, 1),
CategoricalParameter('c', [0, 1, 2]),
IntegerParameter("d", 0, 1)
]
outcomes = [TimeSeriesOutcome("test")]
case = {unc.name: random.random() for unc in uncs}
case["c"] = int(round(case["c"] * 2))
case["d"] = int(round(case["d"]))
model = NamedObject('test')
policy = Policy('policy')
scenario = Scenario(**case)
experiment = Case(0, model.name, policy, scenario, 0)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments,
reporting_interval=1)
model_outcomes = {outcomes[0].name: 1}
callback(experiment, model_outcomes)
experiments, _ = callback.get_results()
design = case
design['policy'] = policy.name
design['model'] = model.name
design['scenario'] = scenario.name
names = experiments.columns.values.tolist()
for name in names:
entry_a = experiments[name][0]
entry_b = design[name]
self.assertEqual(entry_a, entry_b, "failed for " + name)
# with levers
nr_experiments = 3
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
levers = [RealParameter("c", 0, 1), RealParameter("d", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
case = {unc.name: random.random() for unc in uncs}
model = NamedObject('test')
policy = Policy('policy', c=1, d=1)
scenario = Scenario(**case)
experiment = Case(0, model.name, policy, scenario, 0)
callback = DefaultCallback(uncs,
levers,
outcomes,
nr_experiments=nr_experiments,
reporting_interval=1)
model_outcomes = {outcomes[0].name: 1}
callback(experiment, model_outcomes)
experiments, _ = callback.get_results()
design = case
design['c'] = 1
design['d'] = 1
design['policy'] = policy.name
design['model'] = model.name
design['scenario'] = scenario.name
names = experiments.columns.values.tolist()
for name in names:
self.assertEqual(experiments[name][0], design[name])
def test_store_results(self):
nr_experiments = 3
uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
outcomes = [TimeSeriesOutcome("test")]
model = NamedObject('test')
experiment = Case(0, model, Policy('policy'), Scenario(a=1, b=0), 0)
# case 1 scalar shape = (1)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: 1}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, ))
# case 2 time series shape = (1, nr_time_steps)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(10)}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, 10))
# case 3 maps etc. shape = (x,y)
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(2, 2)}
callback(experiment, model_outcomes)
_, out = callback.get_results()
self.assertIn(outcomes[0].name, out.keys())
self.assertEqual(out[outcomes[0].name].shape, (3, 2, 2))
# case 4 assert raises EMAError
callback = DefaultCallback(uncs, [],
outcomes,
nr_experiments=nr_experiments)
model_outcomes = {outcomes[0].name: np.random.rand(2, 2, 2)}
self.assertRaises(EMAError, callback, experiment, model_outcomes)