def test_store_cases(self): nr_experiments = 3 uncs = [ParameterUncertainty((0,1), "a"), ParameterUncertainty((0,1), "b"), CategoricalUncertainty([0, 1, 2], "c"), ParameterUncertainty((0,1), "d", integer=True),] outcomes = [Outcome("test", time=True)] case = {unc.name:random.random() for unc in uncs} case["c"] = int(round(case["c"]*2)) case["d"] = int(round(case["d"])) policy = {'name':'none'} name = "test" callback = DefaultCallback(uncs, [outcome.name for outcome in outcomes], nr_experiments=nr_experiments, reporting_interval=1) result = {outcomes[0].name: 1} callback(0, case, policy, name, result) experiments, _ = callback.get_results() design = case design['policy'] = policy['name'] design['model'] = name names = rf.get_names(experiments.dtype) for name in names: self.assertEqual(experiments[name][0], design[name])
def test_init(self): # let's add some uncertainties to this uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] outcomes = [TimeSeriesOutcome("test")] callback = DefaultCallback(uncs, [], outcomes, nr_experiments=100) self.assertEqual(callback.i, 0) self.assertEqual(callback.nr_experiments, 100) self.assertEqual(callback.cases.shape[0], 100) self.assertEqual(callback.outcomes, [o.name for o in outcomes]) names = rf.get_names(callback.cases.dtype) names = set(names) self.assertEqual(names, {'a', 'b', 'policy', 'model'}) self.assertEqual(callback.results, {}) # with levers levers = [RealParameter('c', 0, 10)] callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=100) self.assertEqual(callback.i, 0) self.assertEqual(callback.nr_experiments, 100) self.assertEqual(callback.cases.shape[0], 100) self.assertEqual(callback.outcomes, [o.name for o in outcomes]) names = rf.get_names(callback.cases.dtype) names = set(names) self.assertEqual(names, {'a', 'b', 'c', 'policy', 'model'}) self.assertEqual(callback.results, {})
def test_store_results(self): nr_experiments = 3 uncs = [ParameterUncertainty((0,1), "a"), ParameterUncertainty((0,1), "b")] outcomes = [Outcome("test", time=True)] case = {unc.name:random.random() for unc in uncs} policy = {'name':'none'} name = "test" # case 1 scalar shape = (1) callback = DefaultCallback(uncs, [outcome.name for outcome in outcomes], nr_experiments=nr_experiments) result = {outcomes[0].name: 1} callback(0, case, policy, name, result) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3,)) # case 2 time series shape = (1, nr_time_steps) callback = DefaultCallback(uncs, [outcome.name for outcome in outcomes], nr_experiments=nr_experiments) result = {outcomes[0].name: np.random.rand(10)} callback(0, case, policy, name, result) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3,10)) # case 3 maps etc. shape = (x,y) callback = DefaultCallback(uncs, [outcome.name for outcome in outcomes], nr_experiments=nr_experiments) result = {outcomes[0].name: np.random.rand(2,2)} callback(0, case, policy, name, result) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3,2,2)) # case 4 assert raises EMAError callback = DefaultCallback(uncs, [outcome.name for outcome in outcomes], nr_experiments=nr_experiments) result = {outcomes[0].name: np.random.rand(2,2,2)} self.assertRaises(EMAError, callback, 0, case, policy, name, result) # KeyError with mock.patch('ema_workbench.util.ema_logging.debug') as mocked_logging: callback = DefaultCallback(uncs, [outcome.name for outcome in outcomes], nr_experiments=nr_experiments) result = {'incorrect': np.random.rand(2,)} callback(0, case, policy, name, result) for outcome in outcomes: mocked_logging.assert_called_with("%s not specified as outcome in msi" % outcome.name)
def test_init(self): # let's add some uncertainties to this uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] outcomes = [ ScalarOutcome("scalar"), ArrayOutcome("array", shape=(10, )), TimeSeriesOutcome("timeseries") ] callback = DefaultCallback(uncs, [], outcomes, nr_experiments=100) self.assertEqual(callback.i, 0) self.assertEqual(callback.nr_experiments, 100) self.assertEqual(callback.cases.shape[0], 100) self.assertEqual(callback.outcomes, [o.name for o in outcomes]) names = callback.cases.columns.values.tolist() names = set(names) self.assertEqual(names, {'a', 'b', 'policy', 'model', 'scenario'}) self.assertNotIn('scalar', callback.results) self.assertNotIn('timeseries', callback.results) self.assertIn('array', callback.results) a = np.all(np.isnan(callback.results['array'])) self.assertTrue(a) # with levers levers = [RealParameter('c', 0, 10)] callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=100) self.assertEqual(callback.i, 0) self.assertEqual(callback.nr_experiments, 100) self.assertEqual(callback.cases.shape[0], 100) self.assertEqual(callback.outcomes, [o.name for o in outcomes]) names = callback.cases.columns.values.tolist() names = set(names) self.assertEqual(names, {'a', 'b', 'c', 'policy', 'model', 'scenario'}) self.assertNotIn('scalar', callback.results) self.assertNotIn('timeseries', callback.results) self.assertIn('array', callback.results) a = np.all(np.isnan(callback.results['array'])) self.assertTrue(a)
def test_store_results(self): nr_experiments = 3 uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] outcomes = [TimeSeriesOutcome("test")] model = NamedObject('test') experiment = Case(0, model, Policy('policy'), Scenario(a=1, b=0), 0) # case 1 scalar shape = (1) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: 1} callback(experiment, model_outcomes) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3,)) # case 2 time series shape = (1, nr_time_steps) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: np.random.rand(10)} callback(experiment, model_outcomes) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3,10)) # case 3 maps etc. shape = (x,y) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: np.random.rand(2,2)} callback(experiment, model_outcomes) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3,2,2)) # case 4 assert raises EMAError callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: np.random.rand(2,2,2)} self.assertRaises(EMAError, callback, experiment, model_outcomes) # KeyError with mock.patch('ema_workbench.util.ema_logging.debug') as mocked_logging: callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {'incorrect': np.random.rand(2,)} callback(experiment, model_outcomes) for outcome in outcomes: mocked_logging.assert_called_with("%s not specified as outcome in msi" % outcome.name)
def test_store_results(self): nr_experiments = 3 uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] outcomes = [TimeSeriesOutcome("test")] model = NamedObject('test') experiment = Experiment(0, model, Policy('policy'), Scenario(a=1, b=0), 0) # case 1 scalar shape = (1) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) result = {outcomes[0].name: 1} callback(experiment, result) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3, )) # case 2 time series shape = (1, nr_time_steps) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) result = {outcomes[0].name: np.random.rand(10)} callback(experiment, result) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3, 10)) # case 3 maps etc. shape = (x,y) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) result = {outcomes[0].name: np.random.rand(2, 2)} callback(experiment, result) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3, 2, 2)) # case 4 assert raises EMAError callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) result = {outcomes[0].name: np.random.rand(2, 2, 2)} self.assertRaises(EMAError, callback, experiment, result) # KeyError with mock.patch( 'ema_workbench.util.ema_logging.debug') as mocked_logging: callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) result = {'incorrect': np.random.rand(2, )} callback(experiment, result) for outcome in outcomes: mocked_logging.assert_called_with( "%s not specified as outcome in msi" % outcome.name)
def test_store_cases(self): nr_experiments = 3 uncs = [ RealParameter("a", 0, 1), RealParameter("b", 0, 1), CategoricalParameter('c', [0, 1, 2]), IntegerParameter("d", 0, 1) ] outcomes = [TimeSeriesOutcome("test")] case = {unc.name: random.random() for unc in uncs} case["c"] = int(round(case["c"] * 2)) case["d"] = int(round(case["d"])) model = NamedObject('test') policy = Policy('policy') scenario = Scenario(**case) experiment = Experiment(0, model, policy, scenario, 0) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments, reporting_interval=1) result = {outcomes[0].name: 1} callback(experiment, result) experiments, _ = callback.get_results() design = case design['policy'] = policy.name design['model'] = model.name names = rf.get_names(experiments.dtype) for name in names: self.assertEqual(experiments[name][0], design[name]) # with levers nr_experiments = 3 uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] levers = [RealParameter("c", 0, 1), RealParameter("d", 0, 1)] outcomes = [TimeSeriesOutcome("test")] case = {unc.name: random.random() for unc in uncs} model = NamedObject('test') policy = Policy('policy', c=1, d=1) scenario = Scenario(**case) experiment = Experiment(0, model, policy, scenario, 0) callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=nr_experiments, reporting_interval=1) result = {outcomes[0].name: 1} callback(experiment, result) experiments, _ = callback.get_results() design = case design['c'] = 1 design['d'] = 1 design['policy'] = policy.name design['model'] = model.name names = rf.get_names(experiments.dtype) print(experiments[0]) for name in names: self.assertEqual(experiments[name][0], design[name])
def test_store_cases(self): nr_experiments = 3 uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1), CategoricalParameter('c', [0, 1, 2]), IntegerParameter("d", 0, 1)] outcomes = [TimeSeriesOutcome("test")] constraints = [] case = {unc.name:random.random() for unc in uncs} case["c"] = int(round(case["c"]*2)) case["d"] = int(round(case["d"])) model = NamedObject('test') policy = Policy('policy') scenario = Scenario(**case) experiment = Case(0, model.name, policy, scenario, 0) callback = DefaultCallback(uncs, [],outcomes, constraints, nr_experiments=nr_experiments, reporting_interval=1) model_outcomes = {outcomes[0].name: 1} model_constraints = {} callback(experiment, model_outcomes, model_constraints) experiments, _ = callback.get_results() design = case design['policy'] = policy.name design['model'] = model.name design['scenario_id'] = scenario.name names = rf.get_names(experiments.dtype) for name in names: entry_a = experiments[name][0] entry_b = design[name] self.assertEqual(entry_a, entry_b, "failed for "+name) # with levers nr_experiments = 3 uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] levers = [RealParameter("c", 0, 1), RealParameter("d", 0, 1)] outcomes = [TimeSeriesOutcome("test")] case = {unc.name:random.random() for unc in uncs} model = NamedObject('test') policy = Policy('policy', c=1, d=1) scenario = Scenario(**case) experiment = Case(0, model.name, policy, scenario, 0) callback = DefaultCallback(uncs, levers,outcomes,constraints, nr_experiments=nr_experiments, reporting_interval=1) model_outcomes = {outcomes[0].name: 1} model_constraints = {} callback(experiment, model_outcomes, model_constraints) experiments, _ = callback.get_results() design = case design['c'] = 1 design['d'] = 1 design['policy'] = policy.name design['model'] = model.name design['scenario_id'] = scenario.name names = rf.get_names(experiments.dtype) for name in names: self.assertEqual(experiments[name][0], design[name])
def test_store_cases(self): nr_experiments = 3 uncs = [ RealParameter("a", 0, 1), RealParameter("b", 0, 1), CategoricalParameter('c', [0, 1, 2]), IntegerParameter("d", 0, 1) ] outcomes = [TimeSeriesOutcome("test")] case = {unc.name: random.random() for unc in uncs} case["c"] = int(round(case["c"] * 2)) case["d"] = int(round(case["d"])) model = NamedObject('test') policy = Policy('policy') scenario = Scenario(**case) experiment = Case(0, model.name, policy, scenario, 0) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments, reporting_interval=1) model_outcomes = {outcomes[0].name: 1} callback(experiment, model_outcomes) experiments, _ = callback.get_results() design = case design['policy'] = policy.name design['model'] = model.name design['scenario'] = scenario.name names = experiments.columns.values.tolist() for name in names: entry_a = experiments[name][0] entry_b = design[name] self.assertEqual(entry_a, entry_b, "failed for " + name) # with levers nr_experiments = 3 uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] levers = [RealParameter("c", 0, 1), RealParameter("d", 0, 1)] outcomes = [TimeSeriesOutcome("test")] case = {unc.name: random.random() for unc in uncs} model = NamedObject('test') policy = Policy('policy', c=1, d=1) scenario = Scenario(**case) experiment = Case(0, model.name, policy, scenario, 0) callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=nr_experiments, reporting_interval=1) model_outcomes = {outcomes[0].name: 1} callback(experiment, model_outcomes) experiments, _ = callback.get_results() design = case design['c'] = 1 design['d'] = 1 design['policy'] = policy.name design['model'] = model.name design['scenario'] = scenario.name names = experiments.columns.values.tolist() for name in names: self.assertEqual(experiments[name][0], design[name])
def test_store_results(self): nr_experiments = 3 uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)] outcomes = [TimeSeriesOutcome("test")] model = NamedObject('test') experiment = Case(0, model, Policy('policy'), Scenario(a=1, b=0), 0) # case 1 scalar shape = (1) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: 1} callback(experiment, model_outcomes) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3, )) # case 2 time series shape = (1, nr_time_steps) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: np.random.rand(10)} callback(experiment, model_outcomes) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3, 10)) # case 3 maps etc. shape = (x,y) callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: np.random.rand(2, 2)} callback(experiment, model_outcomes) _, out = callback.get_results() self.assertIn(outcomes[0].name, out.keys()) self.assertEqual(out[outcomes[0].name].shape, (3, 2, 2)) # case 4 assert raises EMAError callback = DefaultCallback(uncs, [], outcomes, nr_experiments=nr_experiments) model_outcomes = {outcomes[0].name: np.random.rand(2, 2, 2)} self.assertRaises(EMAError, callback, experiment, model_outcomes)