def test_store_cases(self):
     nr_experiments = 3
     uncs = [ParameterUncertainty((0,1), "a"),
            ParameterUncertainty((0,1), "b"),
            CategoricalUncertainty([0, 1, 2], "c"),
            ParameterUncertainty((0,1), "d", integer=True),]
     outcomes = [Outcome("test", time=True)]
     case = {unc.name:random.random() for unc in uncs}
     case["c"] = int(round(case["c"]*2))
     case["d"] = int(round(case["d"]))
     policy = {'name':'none'}
     name = "test"
  
     callback = DefaultCallback(uncs, 
                                [outcome.name for outcome in outcomes], 
                                nr_experiments=nr_experiments,
                                reporting_interval=1)
     result = {outcomes[0].name: 1}
     callback(0, case, policy, name, result)
      
     experiments, _ = callback.get_results()
     design = case
     design['policy'] = policy['name']
     design['model'] = name
     
     names = rf.get_names(experiments.dtype)
     for name in names:
         self.assertEqual(experiments[name][0], design[name])
Example #2
0
    def test_init(self):
        # let's add some uncertainties to this
        uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
        outcomes = [TimeSeriesOutcome("test")]
        callback = DefaultCallback(uncs, [], outcomes, nr_experiments=100)

        self.assertEqual(callback.i, 0)
        self.assertEqual(callback.nr_experiments, 100)
        self.assertEqual(callback.cases.shape[0], 100)
        self.assertEqual(callback.outcomes, [o.name for o in outcomes])

        names = rf.get_names(callback.cases.dtype)
        names = set(names)
        self.assertEqual(names, {'a', 'b', 'policy', 'model'})
        self.assertEqual(callback.results, {})

        # with levers
        levers = [RealParameter('c', 0, 10)]

        callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=100)

        self.assertEqual(callback.i, 0)
        self.assertEqual(callback.nr_experiments, 100)
        self.assertEqual(callback.cases.shape[0], 100)
        self.assertEqual(callback.outcomes, [o.name for o in outcomes])

        names = rf.get_names(callback.cases.dtype)
        names = set(names)
        self.assertEqual(names, {'a', 'b', 'c', 'policy', 'model'})
        self.assertEqual(callback.results, {})
    def test_store_results(self):
        nr_experiments = 3
        uncs = [ParameterUncertainty((0,1), "a"),
               ParameterUncertainty((0,1), "b")]
        outcomes = [Outcome("test", time=True)]
        case = {unc.name:random.random() for unc in uncs}
        policy = {'name':'none'}
        name = "test"
     
        # case 1 scalar shape = (1)
        callback = DefaultCallback(uncs, 
                                   [outcome.name for outcome in outcomes], 
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: 1}
        callback(0, case, policy, name, result)
         
        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3,))
     
        # case 2 time series shape = (1, nr_time_steps)
        callback = DefaultCallback(uncs, 
                                   [outcome.name for outcome in outcomes], 
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: np.random.rand(10)}
        callback(0, case, policy, name, result)
          
        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3,10))

        # case 3 maps etc. shape = (x,y)
        callback = DefaultCallback(uncs, 
                                   [outcome.name for outcome in outcomes], 
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: np.random.rand(2,2)}
        callback(0, case, policy, name, result)
          
        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3,2,2))

        # case 4 assert raises EMAError
        callback = DefaultCallback(uncs, 
                                   [outcome.name for outcome in outcomes], 
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: np.random.rand(2,2,2)}
        self.assertRaises(EMAError, callback, 0, case, policy, name, result)
        
        # KeyError
        with mock.patch('ema_workbench.util.ema_logging.debug') as mocked_logging:
            callback = DefaultCallback(uncs, 
                           [outcome.name for outcome in outcomes], 
                           nr_experiments=nr_experiments)
            result = {'incorrect': np.random.rand(2,)}
            callback(0, case, policy, name, result)
            
            for outcome in outcomes:
                mocked_logging.assert_called_with("%s not specified as outcome in msi" % outcome.name)
Example #4
0
    def test_init(self):
        # let's add some uncertainties to this
        uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
        outcomes = [
            ScalarOutcome("scalar"),
            ArrayOutcome("array", shape=(10, )),
            TimeSeriesOutcome("timeseries")
        ]
        callback = DefaultCallback(uncs, [], outcomes, nr_experiments=100)

        self.assertEqual(callback.i, 0)
        self.assertEqual(callback.nr_experiments, 100)
        self.assertEqual(callback.cases.shape[0], 100)
        self.assertEqual(callback.outcomes, [o.name for o in outcomes])

        names = callback.cases.columns.values.tolist()
        names = set(names)
        self.assertEqual(names, {'a', 'b', 'policy', 'model', 'scenario'})

        self.assertNotIn('scalar', callback.results)
        self.assertNotIn('timeseries', callback.results)
        self.assertIn('array', callback.results)

        a = np.all(np.isnan(callback.results['array']))
        self.assertTrue(a)

        # with levers
        levers = [RealParameter('c', 0, 10)]

        callback = DefaultCallback(uncs, levers, outcomes, nr_experiments=100)

        self.assertEqual(callback.i, 0)
        self.assertEqual(callback.nr_experiments, 100)
        self.assertEqual(callback.cases.shape[0], 100)
        self.assertEqual(callback.outcomes, [o.name for o in outcomes])

        names = callback.cases.columns.values.tolist()
        names = set(names)
        self.assertEqual(names, {'a', 'b', 'c', 'policy', 'model', 'scenario'})

        self.assertNotIn('scalar', callback.results)
        self.assertNotIn('timeseries', callback.results)
        self.assertIn('array', callback.results)

        a = np.all(np.isnan(callback.results['array']))
        self.assertTrue(a)
Example #5
0
    def test_store_results(self):
        nr_experiments = 3
        uncs = [RealParameter("a", 0, 1),
               RealParameter("b", 0, 1)]
        outcomes = [TimeSeriesOutcome("test")]
        model = NamedObject('test')

        experiment = Case(0, model, Policy('policy'), 
                                Scenario(a=1, b=0), 0)
     
        # case 1 scalar shape = (1)
        callback = DefaultCallback(uncs, [], outcomes, 
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: 1}
        callback(experiment, model_outcomes)
         
        _, out = callback.get_results()
        
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3,))
     
        # case 2 time series shape = (1, nr_time_steps)
        callback = DefaultCallback(uncs, [], outcomes, 
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: np.random.rand(10)}
        callback(experiment, model_outcomes)
          
        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3,10))

        # case 3 maps etc. shape = (x,y)
        callback = DefaultCallback(uncs, [], outcomes,
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: np.random.rand(2,2)}
        callback(experiment, model_outcomes)
          
        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3,2,2))

        # case 4 assert raises EMAError
        callback = DefaultCallback(uncs, [], outcomes,
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: np.random.rand(2,2,2)}
        self.assertRaises(EMAError, callback, experiment, model_outcomes)
        
        # KeyError
        with mock.patch('ema_workbench.util.ema_logging.debug') as mocked_logging:
            callback = DefaultCallback(uncs, [], outcomes,
                                       nr_experiments=nr_experiments)
            model_outcomes = {'incorrect': np.random.rand(2,)}
            callback(experiment, model_outcomes)
            
            for outcome in outcomes:
                mocked_logging.assert_called_with("%s not specified as outcome in msi" % outcome.name)
Example #6
0
    def test_store_results(self):
        nr_experiments = 3
        uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
        outcomes = [TimeSeriesOutcome("test")]
        model = NamedObject('test')

        experiment = Experiment(0, model, Policy('policy'), Scenario(a=1, b=0),
                                0)

        # case 1 scalar shape = (1)
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: 1}
        callback(experiment, result)

        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3, ))

        # case 2 time series shape = (1, nr_time_steps)
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: np.random.rand(10)}
        callback(experiment, result)

        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3, 10))

        # case 3 maps etc. shape = (x,y)
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: np.random.rand(2, 2)}
        callback(experiment, result)

        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3, 2, 2))

        # case 4 assert raises EMAError
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        result = {outcomes[0].name: np.random.rand(2, 2, 2)}
        self.assertRaises(EMAError, callback, experiment, result)

        # KeyError
        with mock.patch(
                'ema_workbench.util.ema_logging.debug') as mocked_logging:
            callback = DefaultCallback(uncs, [],
                                       outcomes,
                                       nr_experiments=nr_experiments)
            result = {'incorrect': np.random.rand(2, )}
            callback(experiment, result)

            for outcome in outcomes:
                mocked_logging.assert_called_with(
                    "%s not specified as outcome in msi" % outcome.name)
Example #7
0
    def test_store_cases(self):
        nr_experiments = 3
        uncs = [
            RealParameter("a", 0, 1),
            RealParameter("b", 0, 1),
            CategoricalParameter('c', [0, 1, 2]),
            IntegerParameter("d", 0, 1)
        ]
        outcomes = [TimeSeriesOutcome("test")]
        case = {unc.name: random.random() for unc in uncs}
        case["c"] = int(round(case["c"] * 2))
        case["d"] = int(round(case["d"]))

        model = NamedObject('test')
        policy = Policy('policy')
        scenario = Scenario(**case)
        experiment = Experiment(0, model, policy, scenario, 0)

        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments,
                                   reporting_interval=1)
        result = {outcomes[0].name: 1}
        callback(experiment, result)

        experiments, _ = callback.get_results()
        design = case
        design['policy'] = policy.name
        design['model'] = model.name

        names = rf.get_names(experiments.dtype)
        for name in names:
            self.assertEqual(experiments[name][0], design[name])

        # with levers
        nr_experiments = 3
        uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
        levers = [RealParameter("c", 0, 1), RealParameter("d", 0, 1)]
        outcomes = [TimeSeriesOutcome("test")]
        case = {unc.name: random.random() for unc in uncs}

        model = NamedObject('test')
        policy = Policy('policy', c=1, d=1)
        scenario = Scenario(**case)
        experiment = Experiment(0, model, policy, scenario, 0)

        callback = DefaultCallback(uncs,
                                   levers,
                                   outcomes,
                                   nr_experiments=nr_experiments,
                                   reporting_interval=1)
        result = {outcomes[0].name: 1}
        callback(experiment, result)

        experiments, _ = callback.get_results()
        design = case
        design['c'] = 1
        design['d'] = 1
        design['policy'] = policy.name
        design['model'] = model.name

        names = rf.get_names(experiments.dtype)

        print(experiments[0])

        for name in names:
            self.assertEqual(experiments[name][0], design[name])
Example #8
0
 def test_store_cases(self):
     nr_experiments = 3
     uncs = [RealParameter("a", 0, 1),
             RealParameter("b", 0, 1),
             CategoricalParameter('c', [0, 1, 2]),
             IntegerParameter("d", 0, 1)]
     outcomes = [TimeSeriesOutcome("test")]
     constraints = []
     case = {unc.name:random.random() for unc in uncs}
     case["c"] = int(round(case["c"]*2))
     case["d"] = int(round(case["d"]))
     
     model = NamedObject('test')
     policy  = Policy('policy')
     scenario = Scenario(**case)
     experiment = Case(0, model.name, policy, scenario, 0)
  
     callback = DefaultCallback(uncs, [],outcomes, constraints,
                                nr_experiments=nr_experiments,
                                reporting_interval=1)
     model_outcomes = {outcomes[0].name: 1}
     model_constraints = {}
     callback(experiment, model_outcomes, model_constraints)
      
     experiments, _ = callback.get_results()
     design = case
     design['policy'] = policy.name
     design['model'] = model.name
     design['scenario_id'] = scenario.name
     
     names = rf.get_names(experiments.dtype)
     for name in names:
         entry_a = experiments[name][0]
         entry_b = design[name]
         
         self.assertEqual(entry_a, entry_b, "failed for "+name)
          
     # with levers
     nr_experiments = 3
     uncs = [RealParameter("a", 0, 1),
             RealParameter("b", 0, 1)]
     levers = [RealParameter("c", 0, 1),
               RealParameter("d", 0, 1)]
     outcomes = [TimeSeriesOutcome("test")]
     case = {unc.name:random.random() for unc in uncs}
     
     model = NamedObject('test')
     policy  = Policy('policy', c=1, d=1)
     scenario = Scenario(**case)
     experiment = Case(0, model.name, policy, scenario, 0)
  
     callback = DefaultCallback(uncs, levers,outcomes,constraints, 
                                nr_experiments=nr_experiments,
                                reporting_interval=1)
     model_outcomes = {outcomes[0].name: 1}
     model_constraints = {}
     callback(experiment, model_outcomes, model_constraints)
      
     experiments, _ = callback.get_results()
     design = case
     design['c'] = 1
     design['d'] = 1
     design['policy'] = policy.name
     design['model'] = model.name
     design['scenario_id'] = scenario.name
     
     names = rf.get_names(experiments.dtype)
     
     for name in names:
         self.assertEqual(experiments[name][0], design[name])
Example #9
0
    def test_store_cases(self):
        nr_experiments = 3
        uncs = [
            RealParameter("a", 0, 1),
            RealParameter("b", 0, 1),
            CategoricalParameter('c', [0, 1, 2]),
            IntegerParameter("d", 0, 1)
        ]
        outcomes = [TimeSeriesOutcome("test")]
        case = {unc.name: random.random() for unc in uncs}
        case["c"] = int(round(case["c"] * 2))
        case["d"] = int(round(case["d"]))

        model = NamedObject('test')
        policy = Policy('policy')
        scenario = Scenario(**case)
        experiment = Case(0, model.name, policy, scenario, 0)

        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments,
                                   reporting_interval=1)
        model_outcomes = {outcomes[0].name: 1}
        callback(experiment, model_outcomes)

        experiments, _ = callback.get_results()
        design = case
        design['policy'] = policy.name
        design['model'] = model.name
        design['scenario'] = scenario.name

        names = experiments.columns.values.tolist()
        for name in names:
            entry_a = experiments[name][0]
            entry_b = design[name]

            self.assertEqual(entry_a, entry_b, "failed for " + name)

        # with levers
        nr_experiments = 3
        uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
        levers = [RealParameter("c", 0, 1), RealParameter("d", 0, 1)]
        outcomes = [TimeSeriesOutcome("test")]
        case = {unc.name: random.random() for unc in uncs}

        model = NamedObject('test')
        policy = Policy('policy', c=1, d=1)
        scenario = Scenario(**case)
        experiment = Case(0, model.name, policy, scenario, 0)

        callback = DefaultCallback(uncs,
                                   levers,
                                   outcomes,
                                   nr_experiments=nr_experiments,
                                   reporting_interval=1)
        model_outcomes = {outcomes[0].name: 1}
        callback(experiment, model_outcomes)

        experiments, _ = callback.get_results()
        design = case
        design['c'] = 1
        design['d'] = 1
        design['policy'] = policy.name
        design['model'] = model.name
        design['scenario'] = scenario.name

        names = experiments.columns.values.tolist()
        for name in names:
            self.assertEqual(experiments[name][0], design[name])
Example #10
0
    def test_store_results(self):
        nr_experiments = 3
        uncs = [RealParameter("a", 0, 1), RealParameter("b", 0, 1)]
        outcomes = [TimeSeriesOutcome("test")]
        model = NamedObject('test')

        experiment = Case(0, model, Policy('policy'), Scenario(a=1, b=0), 0)

        # case 1 scalar shape = (1)
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: 1}
        callback(experiment, model_outcomes)

        _, out = callback.get_results()

        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3, ))

        # case 2 time series shape = (1, nr_time_steps)
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: np.random.rand(10)}
        callback(experiment, model_outcomes)

        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3, 10))

        # case 3 maps etc. shape = (x,y)
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: np.random.rand(2, 2)}
        callback(experiment, model_outcomes)

        _, out = callback.get_results()
        self.assertIn(outcomes[0].name, out.keys())
        self.assertEqual(out[outcomes[0].name].shape, (3, 2, 2))

        # case 4 assert raises EMAError
        callback = DefaultCallback(uncs, [],
                                   outcomes,
                                   nr_experiments=nr_experiments)
        model_outcomes = {outcomes[0].name: np.random.rand(2, 2, 2)}
        self.assertRaises(EMAError, callback, experiment, model_outcomes)