def test_read_write_boxes(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) db = SQLiteDB() scope.store_scope(db) s1 = Box(name="Speedy", scope=scope) s1.set_upper_bound('build_travel_time', 70) s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) u = Boxes(s1, s2, scope=scope) db.write_boxes(u) scope2 = Scope(package_file('model', 'tests', 'road_test.yaml')) u2 = db.read_boxes(scope=scope2) assert u == u2 assert u["Notable"].parent_box_name == u2["Notable"].parent_box_name s1_ = db.read_box(scope.name, "Speedy") s2_ = db.read_box(scope.name, "Notable") assert s1 == s1_ assert s2 == s2_ assert s1.relevant_features == s1_.relevant_features assert s2.relevant_features == s2_.relevant_features
def test_read_db_gz(self): road_test_scope_file = emat.package_file('model', 'tests', 'road_test.yaml') with pytest.raises(FileNotFoundError): emat.Scope(emat.package_file('nope.yaml')) s = emat.Scope(road_test_scope_file) with pytest.raises(FileNotFoundError): emat.SQLiteDB(emat.package_file('nope.db.gz')) db = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.gz")) assert repr(db) == '<emat.SQLiteDB with scope "EMAT Road Test">' assert db.get_db_info()[:9] == 'SQLite @ ' assert db.get_db_info()[-11:] == 'roadtest.db' assert db.read_scope_names() == ['EMAT Road Test'] s1 = db.read_scope('EMAT Road Test') assert type(s1) == type(s) for k in ('_x_list', '_l_list', '_c_list', '_m_list', 'name', 'desc'): assert getattr(s, k) == getattr(s1, k), k assert s == s1 experiments = db.read_experiment_all('EMAT Road Test', 'lhs') assert experiments.shape == (110, 20) assert list(experiments.columns) == [ 'free_flow_time', 'initial_capacity', 'alpha', 'beta', 'input_flow', 'value_of_time', 'unit_cost_expansion', 'interest_rate', 'yield_curve', 'expand_capacity', 'amortization_period', 'debt_type', 'interest_rate_lock', 'no_build_travel_time', 'build_travel_time', 'time_savings', 'value_of_time_savings', 'net_benefits', 'cost_of_capacity_expansion', 'present_cost_expansion', ] from emat.model.core_python import Road_Capacity_Investment m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db) assert m.metamodel_id == None
def test_feature_scoring_with_nan(): road_scope = emat.Scope(emat.package_file('model','tests','road_test_bogus.yaml')) road_test = PythonCoreModel(_Road_Capacity_Investment_with_Bogus_Output, scope=road_scope) road_test_design = road_test.design_experiments(n_samples=5000, sampler='lhs') road_test_results = road_test.run_experiments(design=road_test_design) fs = feature_scores(road_scope, road_test_results, random_state=234) assert isinstance(fs, pd.io.formats.style.Styler) stable_df("./road_test_feature_scores_bogus_1.pkl.gz", fs.data)
def test_correlated_latin_hypercube_conflict(self): scope_file = emat.package_file("model", "tests", "road_test_corr_conflict.yaml") scp = Scope(scope_file) with pytest.raises(emat.exceptions.AsymmetricCorrelationError): scp.design_experiments( n_samples_per_factor=10, random_seed=1234, sampler='lhs', )
def test_correlated_latin_hypercube_bad(self): scope_file = emat.package_file("model", "tests", "road_test_corr_bad.yaml") scp = Scope(scope_file) with pytest.raises(np.linalg.LinAlgError): scp.design_experiments( n_samples_per_factor=10, random_seed=1234, sampler='lhs', )
def test_nonuniform_latin_hypercube(self): scope_file = emat.package_file("model", "tests", "road_test_nonuniform.yaml") scp = Scope(scope_file) exp_def = scp.design_experiments( n_samples_per_factor=1000, random_seed=1234, sampler='lhs', ) assert len(exp_def) == scp.n_sample_factors() * 1000 assert (exp_def['free_flow_time'] == 60).all() assert (exp_def['initial_capacity'] == 100).all() assert np.corrcoef([exp_def.alpha, exp_def.beta])[0, 1] == approx(0.75, rel=0.05) assert np.corrcoef([exp_def.alpha, exp_def.expand_capacity])[0, 1] == approx(0.0, abs=0.02) assert np.corrcoef([exp_def.input_flow, exp_def.value_of_time])[0, 1] == approx(-0.5, rel=0.05) assert np.corrcoef([exp_def.unit_cost_expansion, exp_def.value_of_time])[0, 1] == approx(0.9, rel=0.05) assert exp_def.interest_rate_lock.sum() == approx(len(exp_def) * 0.2) assert np.percentile(exp_def.alpha,np.linspace(0,100,50)) == approx( [0.10037393, 0.10722119, 0.10994485, 0.11204394, 0.11383709, 0.11544182, 0.11691345, 0.11829399, 0.11959909, 0.12084863, 0.12205279, 0.12321800, 0.12435285, 0.12546474, 0.12655958, 0.12763503, 0.12869873, 0.12975137, 0.13079620, 0.13183375, 0.13287082, 0.13390854, 0.13494651, 0.13598528, 0.13703149, 0.13808180, 0.13914411, 0.14021784, 0.14130323, 0.14240609, 0.14352608, 0.14466756, 0.14583411, 0.14702908, 0.14825720, 0.14951875, 0.15082844, 0.15218376, 0.15359963, 0.15508120, 0.15664534, 0.15831425, 0.16010073, 0.16203921, 0.16418886, 0.16662357, 0.16946999, 0.17301416, 0.17804383, 0.19662857]) assert np.percentile(exp_def.beta,np.linspace(0,100,50)) == approx( [3.51654751, 3.72503059, 3.82437701, 3.90088124, 3.96222432, 4.01360346, 4.06112277, 4.10640347, 4.14456476, 4.18084719, 4.21812584, 4.24926944, 4.28049053, 4.31181127, 4.34390502, 4.37561590, 4.40541815, 4.43276143, 4.45517485, 4.48062290, 4.50726296, 4.53334164, 4.55737738, 4.57893875, 4.60371011, 4.62590595, 4.64885523, 4.67335218, 4.69475909, 4.71546469, 4.73676622, 4.75796550, 4.77690613, 4.79738177, 4.81947505, 4.84481408, 4.86954326, 4.89379651, 4.91771359, 4.94100213, 4.97169370, 5.00298714, 5.03525103, 5.07100437, 5.11028866, 5.15061419, 5.19925284, 5.24775527, 5.32086547, 5.49345120]) assert np.percentile(exp_def.input_flow,np.linspace(0,100,50)) == approx( [ 80.06332381, 83.71770678, 85.93721426, 87.82355332, 89.52967083, 91.11148891, 92.60789787, 94.03509345, 95.41186604, 96.74598771, 98.04479355, 99.31122788, 100.55462827, 101.77808514, 102.98041937, 104.16517008, 105.33959643, 106.50165383, 107.65366600, 108.79827639, 109.93328795, 111.06707367, 112.19316408, 113.31493454, 114.43982739, 115.56182157, 116.68454749, 117.80620837, 118.93524815, 120.06752762, 121.20470208, 122.34781469, 123.50083542, 124.66086016, 125.83390291, 127.02146142, 128.22485477, 129.44681602, 130.68613841, 131.95601658, 133.25403807, 134.58951122, 135.96442305, 137.39295642, 138.89092217, 140.47204147, 142.17835057, 144.06540067, 146.28064479, 149.94588322])
def test_files_with_broken_scope(): try: import core_files_demo except: import pytest pytest.skip("core_files_demo not installed") fx = core_files_demo.RoadTestFileModel(scope_file=emat.package_file( 'model', 'tests', 'road_test_corrupt2.yaml')) design = fx.design_experiments(n_samples=2) result = fx.run_experiments(design) assert result['bogus_measure'].isna().all()
def test_multiple_connections(): import tempfile with tempfile.TemporaryDirectory() as tempdir: tempdbfile = os.path.join(tempdir, "test_db_file.db") db_test = SQLiteDB(tempdbfile, initialize=True) road_test_scope_file = emat.package_file("model", "tests", "road_test.yaml") s = emat.Scope(road_test_scope_file) db_test.store_scope(s) assert db_test.read_scope_names() == ["EMAT Road Test"] db_test2 = SQLiteDB(tempdbfile, initialize=False) with pytest.raises(KeyError): db_test2.store_scope(s) # Neither database is in a transaction assert not db_test.conn.in_transaction assert not db_test2.conn.in_transaction from emat.model.core_python import Road_Capacity_Investment m1 = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db_test) m2 = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db_test2) d1 = m1.design_experiments(n_samples=3, random_seed=1, design_name="d1") d2 = m2.design_experiments(n_samples=3, random_seed=2, design_name="d2") r1 = m1.run_experiments(design_name="d1") r2 = m2.run_experiments(design_name="d2") # Check each model can load the other's results pd.testing.assert_frame_equal( r1, m2.db.read_experiment_all(scope_name=s.name, design_name="d1", ensure_dtypes=True)[r1.columns], ) pd.testing.assert_frame_equal( r2, m1.db.read_experiment_all(scope_name=s.name, design_name="d2", ensure_dtypes=True)[r2.columns], )
def test_box_universe(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) s = Box(name="Speedy", scope=scope) s.set_upper_bound('build_travel_time', 70) s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) u = Boxes(s, s2, scope=scope) assert u.fancy_names() == [ 'Scope: EMAT Road Test', '▶ Speedy', '▷ ▶ Notable' ] assert u.plain_names() == [None, 'Speedy', 'Notable']
def test_correlated_monte_carlo(self): scope_file = emat.package_file("model", "tests", "road_test_corr.yaml") scp = Scope(scope_file) exp_def = scp.design_experiments( n_samples_per_factor=100, random_seed=1234, sampler='mc', ) assert len(exp_def) == scp.n_sample_factors() * 100 assert (exp_def['free_flow_time'] == 60).all() assert (exp_def['initial_capacity'] == 100).all() assert np.corrcoef([exp_def.alpha, exp_def.beta])[0, 1] == approx(0.75, rel=0.05) assert np.corrcoef([exp_def.alpha, exp_def.expand_capacity])[0, 1] == approx(0.0, abs=0.05) assert np.corrcoef([exp_def.input_flow, exp_def.value_of_time])[0, 1] == approx(-0.5, rel=0.05) assert np.corrcoef([exp_def.unit_cost_expansion, exp_def.value_of_time])[0, 1] == approx(0.9, rel=0.05)
def test_feature_scoring_and_prim(): road_scope = emat.Scope(emat.package_file('model','tests','road_test.yaml')) road_test = PythonCoreModel(Road_Capacity_Investment, scope=road_scope) road_test_design = road_test.design_experiments(n_samples=5000, sampler='lhs') road_test_results = road_test.run_experiments(design=road_test_design) fs = feature_scores(road_scope, road_test_results, random_state=123) assert isinstance(fs, pd.io.formats.style.Styler) stable_df("./road_test_feature_scores_1.pkl.gz", fs.data) prim1 = road_test_results.prim(target="net_benefits >= 0") pbox1 = prim1.find_box() assert pbox1._cur_box == 64 ts1 = prim1.tradeoff_selector() assert len(ts1.data) == 1 assert ts1.data[0]['x'] == approx(np.asarray([ 1., 1., 1., 1., 1., 0.99928315, 0.99856631, 0.99784946, 0.99569892, 0.99283154, 0.98924731, 0.98351254, 0.97921147, 0.97491039, 0.96702509, 0.95555556, 0.94982079, 0.94336918, 0.92903226, 0.91182796, 0.89749104, 0.87598566, 0.85304659, 0.83942652, 0.83225806, 0.82078853, 0.79713262, 0.77706093, 0.76415771, 0.75483871, 0.74480287, 0.73261649, 0.71827957, 0.70394265, 0.68100358, 0.65663082, 0.63225806, 0.61003584, 0.59569892, 0.57992832, 0.55770609, 0.54193548, 0.52759857, 0.51111111, 0.49892473, 0.48960573, 0.4781362, 0.45878136, 0.44229391, 0.42365591, 0.409319, 0.39498208, 0.38064516, 0.36487455, 0.34767025, 0.33261649, 0.31756272, 0.30322581, 0.28888889, 0.27741935, 0.26379928, 0.25089606, 0.23942652, 0.22795699, 0.2172043])) pbox1.select(40) assert pbox1._cur_box == 40 assert ts1.data[0]['marker']['symbol'] == approx(np.asarray([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])) ebox1_40 = pbox1.to_emat_box() assert ebox1_40.coverage == approx(0.5577060931899641) assert ebox1_40.density == approx(0.8356605800214822) assert ebox1_40.mass == approx(0.1862) from emat import Bounds assert ebox1_40.thresholds == {'beta': Bounds(lowerbound=3.597806324946271, upperbound=None), 'input_flow': Bounds(lowerbound=125, upperbound=None), 'value_of_time': Bounds(lowerbound=0.07705746291056698, upperbound=None), 'expand_capacity': Bounds(lowerbound=None, upperbound=95.01870815358643)} pbox1.splom() pbox1.hmm()
def test_box(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) with pytest.raises(TypeError): s = Box(scope=scope) s = Box(name="Speedy", scope=scope) s.set_upper_bound('build_travel_time', 70) with pytest.raises(ScopeError): s.set_upper_bound('not_a_thing', 70) assert len(s) == 1 assert 'build_travel_time' in s assert s.parent_box_name is None s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) assert len(s2) == 1 assert 'build_travel_time' not in s2 assert s2.parent_box_name == 'Speedy'
def test_read_write_box(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) db = SQLiteDB() scope.store_scope(db) s1 = Box(name="Speedy", scope=scope) s1.set_upper_bound('build_travel_time', 70) s1.relevant_features.add('debt_type') s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) db.write_box(s1) db.write_box(s2) s1_ = db.read_box(scope.name, "Speedy") s2_ = db.read_box(scope.name, "Notable") assert s1 == s1_ assert s2 == s2_ assert s1.thresholds == s1_.thresholds assert s2.thresholds == s2_.thresholds assert s1.relevant_features == s1_.relevant_features assert s2.relevant_features == s2_.relevant_features
class TestCoreModelMethods(unittest.TestCase): ''' tests model and meta-model methods ''' corem_scope_file = emat.package_file("model", "tests", "core_model_test.yaml") scp = Scope(corem_scope_file) corem = PythonCoreModel( function=Dummy(), configuration={ 'archive_path': config.get_subdir('test_directory', 'core_dummy_archive') }, scope=scp, ) # ============================================================================= # # Core model tests # # ============================================================================= def test_create_scenario(self): exp_def = {'Auto IVTT Sensitivity': 0.9122442817924445} self.corem.setup(exp_def) @pytest.mark.skip(reason="TODO") def test_set_invalid_exp_variable(self): exp_def = {'unsupported': 1} with self.assertRaises(KeyError): self.corem.setup(exp_def) @pytest.mark.skip(reason="TODO") def test_post_process(self): exp_def = {'Land Use - CBD Focus': 1} pm = ['Region-wide VMT'] self.corem.post_process(exp_def, pm) @pytest.mark.skip(reason="TODO") def test_archive_model(self): exp_id = 1 archive_path = self.corem.get_exp_archive_path(self.scp.scp_name, exp_id) self.corem.archive(archive_path) @pytest.mark.skip(reason="TODO") def atest_hook_presence(self): ''' confirm that hooks are present for all performance measures, exp vars''' # TODO # set experiment variables # post process # import performance measure @pytest.mark.skip(reason="TODO") def test_pm_import(self): pm = [ 'Peak Walk-to-transit Boarding', 'Total LRT Boardings', "PM Trip Time (minutes)", "Daily Transit Share", "Households within 30 min of CBD", "Number of Home-based work tours taking <= 45 minutes via transit", "Downtown to Airport Travel Time", 'OD Volume District 1 to 1', '190 Daily VHT' ] pm_vals = self.corem.import_perf_meas(pm) expected_pm = { 'Peak Walk-to-transit Boarding': 56247.88692999999, 'Total LRT Boardings': 24784.475588, "PM Trip Time (minutes)": 15.652833, "Daily Transit Share": 0.019905000000000003, "Households within 30 min of CBD": 379894, "Number of Home-based work tours taking <= 45 minutes via transit": 322069.75, "Downtown to Airport Travel Time": 14.734342999999999, 'OD Volume District 1 to 1': 55642.74609400001, '190 Daily VHT': 272612.499025 } self.assertEqual(expected_pm, pm_vals)
# %% [markdown] # The pattern has shifted, with the sine wave in *B* looking much more like the random noise, # while the linear trend in *A* is now much more important in predicting the output, and # the feature scores also shift to reflect this change. # %% [markdown] # ## Road Test Feature Scores # # We can apply the feature scoring methodology to the Road Test example # in a similar fashion. To demonstrate scoring, we'll first load and run # a sample set of experients. # %% from emat.model.core_python import Road_Capacity_Investment road_scope = emat.Scope(emat.package_file('model','tests','road_test.yaml')) road_test = PythonCoreModel(Road_Capacity_Investment, scope=road_scope) road_test_design = road_test.design_experiments(sampler='lhs') road_test_results = road_test.run_experiments(design=road_test_design) # %% [raw] raw_mimetype="text/restructuredtext" # With the experimental results in hand, we can use the same :func:`feature_scores` # function to compute the scores. # %% feature_scores(road_scope, road_test_results) # %% [raw] raw_mimetype="text/restructuredtext" # By default, the :func:`feature_scores` # function returns a stylized DataFrame, but we can also # use the `return_type` argument to get a plain 'dataframe'
class TestMetaModelMethods(unittest.TestCase): ''' tests model and meta-model methods ''' metam_scope_file = emat.package_file("model","tests","metam_test.yaml") metam_scp = Scope(metam_scope_file) # ============================================================================= # # Meta model tests # # ============================================================================= def test_derive_meta(self): from emat.examples import road_test s, db, m = road_test() db.get_db_info() m.design_experiments(n_samples=10, design_name='tiny') db.read_experiment_all(None, None) with pytest.raises(emat.PendingExperimentsError): m.create_metamodel_from_design('tiny', random_state=123) m.run_experiments(design_name='tiny') mm = emat.create_metamodel( m.scope, db.read_experiment_all(s.name, 'tiny'), random_state=123, metamodel_id=db.get_new_metamodel_id(None), ) mm.db = db # add db after creation to prevent writing it into the db assert mm.scope == m.scope tiny2 = m.design_experiments(n_samples=10, design_name='tiny2', random_seed=456) assert tiny2.iloc[0]['debt_type'] == 'GO Bond' stable_df('./test_tiny2.pkl.gz',tiny2) result2 = mm.run_experiments('tiny2') tiny2out = mm.read_experiment_measures('tiny2') stable_df('./test_tiny2out.pkl.gz', tiny2out) with pytest.raises(ValueError): # no metamodels stored mm3 = db.read_metamodel(None, None) db.write_metamodel(mm) mm2 = db.read_metamodel(None, 1) mm3 = db.read_metamodel(None, None) assert mm2 == mm == mm3 assert mm2 is not mm print(mm2.function(**(tiny2.iloc[0]))) assert mm2.function(**(tiny2.iloc[0])) == approx({ 'no_build_travel_time': 83.57502327972276, 'build_travel_time': 62.221693766038015, 'time_savings': 57.612063365257995, 'value_of_time_savings': 3749.2913256457214, 'net_benefits': 395.55020765212254, 'cost_of_capacity_expansion': 1252.6916865286616, 'present_cost_expansion': 23000.275573551233, }) mm3.metamodel_id = db.get_new_metamodel_id(None) db.write_metamodel(mm3) with pytest.raises(ValueError): # now too many to get without giving an ID mm4 = db.read_metamodel(None, None) def test_derive_meta_w_transform(self): from emat.examples import road_test s, db, m = road_test(yamlfile='road_test2.yaml') db.get_db_info() m.design_experiments(n_samples=10, design_name='tiny') db.read_experiment_all(None, None) with pytest.raises(emat.PendingExperimentsError): m.create_metamodel_from_design('tiny', random_state=123) m.run_experiments(design_name='tiny') mm = emat.create_metamodel( m.scope, db.read_experiment_all(s.name, 'tiny'), random_state=123, metamodel_id=db.get_new_metamodel_id(None), ) assert mm.scope != m.scope # now not equal as road_test2 has transforms that are stripped. mm.db = db tiny2 = m.design_experiments(n_samples=10, design_name='tiny2', random_seed=456) assert tiny2.iloc[0]['debt_type'] == 'GO Bond' assert dict(tiny2.iloc[0].drop('debt_type')) == approx({ 'alpha': 0.10428005571929212, 'amortization_period': 33, 'beta': 4.8792451185772014, 'expand_capacity': 61.4210886403998, 'input_flow': 137, 'interest_rate': 0.03099304322197216, 'interest_rate_lock': 0, 'unit_cost_expansion': 121.85520427974882, 'value_of_time': 0.002953613029133872, 'yield_curve': 0.016255990123028242, 'free_flow_time': 60, 'initial_capacity': 100}) result2 = mm.run_experiments('tiny2') # print(dict(mm.read_experiment_measures('tiny2').iloc[0])) # # print({ # 'no_build_travel_time': 81.6839454971052, # 'build_travel_time': 61.91038371206646, # 'time_savings': 44.94189289289446, # 'value_of_time_savings': 2904.081661408463, # 'net_benefits': -34.09931528157315, # 'cost_of_capacity_expansion': 1085.3565091745982, # 'present_cost_expansion': 19923.66625500023, # }) # assert dict(mm.read_experiment_measures('tiny2').iloc[0]) == approx({ 'no_build_travel_time': 81.6839454971052, 'build_travel_time': 61.91038371206646, 'log_build_travel_time': 4.120826572003798, 'time_savings': 44.94189289289446, 'value_of_time_savings': 2904.081661408463, 'net_benefits': -34.09931528157315, 'cost_of_capacity_expansion': 1085.3565091745982, 'present_cost_expansion': 19923.66625500023, }) assert m.run_experiment(tiny2.iloc[0]) == approx({ 'no_build_travel_time': 89.07004237532217, 'build_travel_time': 62.81032484779827, 'log_build_travel_time': np.log(62.81032484779827), 'time_savings': 26.259717527523904, 'value_of_time_savings': 10.62586300480175, 'present_cost_expansion': 7484.479303360477, 'cost_of_capacity_expansion': 395.69034710662226, 'net_benefits': -385.0644841018205, }) with pytest.raises(ValueError): # no metamodels stored mm3 = db.read_metamodel(None, None) db.write_metamodel(mm) mm2 = db.read_metamodel(None, 1) mm3 = db.read_metamodel(None, None) assert mm2 == mm == mm3 assert mm2 is not mm assert mm2.function(**(tiny2.iloc[0])) == approx({ 'no_build_travel_time': 81.6839454971052, 'build_travel_time': 61.91038371206646, 'log_build_travel_time': 4.120826572003798, 'time_savings': 44.94189289289446, 'value_of_time_savings': 2904.081661408463, 'net_benefits': -34.09931528157315, 'cost_of_capacity_expansion': 1085.3565091745982, 'present_cost_expansion': 19923.66625500023, }) mm3.metamodel_id = db.get_new_metamodel_id(None) db.write_metamodel(mm3) with pytest.raises(ValueError): # now too many to get without giving an ID mm4 = db.read_metamodel(None, None) def test_exogenously_stratified_k_fold(self): from emat.learn.splits import ExogenouslyStratifiedKFold X = np.arange(20) Y = np.asarray([1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1]) S = np.asarray([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0]) correct = [np.array([0, 1, 2, 14, 15]), np.array([3, 4, 5, 16]), np.array([6, 7, 8, 17]), np.array([9, 10, 11, 18]), np.array([12, 13, 19])] for j, (_, k) in zip(correct, ExogenouslyStratifiedKFold(n_splits=5, exo_data=S).split(X, Y)): assert np.array_equal(j, k)
def test_read_db_gz(): road_test_scope_file = emat.package_file("model", "tests", "road_test.yaml") with pytest.raises(FileNotFoundError): emat.Scope(emat.package_file("nope.yaml")) s = emat.Scope(road_test_scope_file) with pytest.raises(FileNotFoundError): emat.SQLiteDB(emat.package_file("nope.db.gz")) if not os.path.exists(emat.package_file("examples", "roadtest.db.gz")): db_w = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.tmp"), initialize=True) s.store_scope(db_w) s.design_experiments(n_samples=110, random_seed=1234, db=db_w, design_name="lhs") from emat.model.core_python import Road_Capacity_Investment m_w = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db_w) m_w.run_experiments(design_name="lhs", db=db_w) db_w.conn.close() import gzip import shutil with open(emat.package_file("examples", "roadtest.db.tmp"), "rb") as f_in: with gzip.open(emat.package_file("examples", "roadtest.db.gz"), "wb") as f_out: shutil.copyfileobj(f_in, f_out) db = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.gz")) assert repr(db) == '<emat.SQLiteDB with scope "EMAT Road Test">' assert db.get_db_info()[:9] == "SQLite @ " assert db.get_db_info()[-11:] == "roadtest.db" assert db.read_scope_names() == ["EMAT Road Test"] s1 = db.read_scope("EMAT Road Test") assert type(s1) == type(s) for k in ("_x_list", "_l_list", "_c_list", "_m_list", "name", "desc"): assert getattr(s, k) == getattr(s1, k), k assert s == s1 experiments = db.read_experiment_all("EMAT Road Test", "lhs") assert experiments.shape == (110, 20) assert list(experiments.columns) == [ "free_flow_time", "initial_capacity", "alpha", "beta", "input_flow", "value_of_time", "unit_cost_expansion", "interest_rate", "yield_curve", "expand_capacity", "amortization_period", "debt_type", "interest_rate_lock", "no_build_travel_time", "build_travel_time", "time_savings", "value_of_time_savings", "net_benefits", "cost_of_capacity_expansion", "present_cost_expansion", ] from emat.model.core_python import Road_Capacity_Investment m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db) assert m.metamodel_id == None
class TestMetaModelMethods(unittest.TestCase): ''' tests model and meta-model methods ''' metam_scope_file = emat.package_file("model", "tests", "metam_test.yaml") metam_scp = Scope(metam_scope_file) # ============================================================================= # # Meta model tests # # ============================================================================= def test_derive_meta(self): from emat.examples import road_test s, db, m = road_test() db.get_db_info() m.design_experiments(n_samples=10, design_name='tiny') db.read_experiment_all(None, None) with pytest.raises(emat.PendingExperimentsError): m.create_metamodel_from_design('tiny', random_state=123) m.run_experiments(design_name='tiny') mm = m.create_metamodel_from_design('tiny', random_state=123) assert mm.scope == m.scope tiny2 = m.design_experiments(n_samples=10, design_name='tiny2', random_seed=456) assert tiny2.iloc[0]['debt_type'] == 'GO Bond' assert dict(tiny2.iloc[0].drop('debt_type')) == approx({ 'alpha': 0.10428005571929212, 'amortization_period': 33, 'beta': 4.8792451185772014, 'expand_capacity': 61.4210886403998, 'input_flow': 137, 'interest_rate': 0.03099304322197216, 'interest_rate_lock': 0, 'unit_cost_expansion': 121.85520427974882, 'value_of_time': 0.002953613029133872, 'yield_curve': 0.016255990123028242, 'free_flow_time': 60, 'initial_capacity': 100 }) result2 = mm.run_experiments('tiny2') assert dict(mm.read_experiment_measures('tiny2').iloc[0]) == approx({ 'no_build_travel_time': 81.6839454971052, 'build_travel_time': 61.91038371206646, 'time_savings': 44.94189289289446, 'value_of_time_savings': 2904.081661408463, 'net_benefits': -34.09931528157315, 'cost_of_capacity_expansion': 1085.3565091745982, 'present_cost_expansion': 19923.66625500023 }) assert m.function(**(tiny2.iloc[0])) == approx({ 'no_build_travel_time': 89.07004237532217, 'build_travel_time': 62.81032484779827, 'time_savings': 26.259717527523904, 'value_of_time_savings': 10.62586300480175, 'present_cost_expansion': 7484.479303360477, 'cost_of_capacity_expansion': 395.69034710662226, 'net_benefits': -385.0644841018205 }) with pytest.raises(ValueError): # no metamodels stored mm3 = db.read_metamodel(None, None) db.write_metamodel(mm) mm2 = db.read_metamodel(None, 1) mm3 = db.read_metamodel(None, None) assert mm2 == mm == mm3 assert mm2 is not mm assert mm2.function(**(tiny2.iloc[0])) == approx({ 'no_build_travel_time': 81.6839454971052, 'build_travel_time': 61.91038371206646, 'time_savings': 44.94189289289446, 'value_of_time_savings': 2904.081661408463, 'net_benefits': -34.09931528157315, 'cost_of_capacity_expansion': 1085.3565091745982, 'present_cost_expansion': 19923.66625500023 }) mm3.metamodel_id = db.get_new_metamodel_id(None) db.write_metamodel(mm3) with pytest.raises(ValueError): # now too many to get without giving an ID mm4 = db.read_metamodel(None, None) def test_derive_meta_w_transform(self): from emat.examples import road_test s, db, m = road_test(yamlfile='road_test2.yaml') db.get_db_info() m.design_experiments(n_samples=10, design_name='tiny') db.read_experiment_all(None, None) with pytest.raises(emat.PendingExperimentsError): m.create_metamodel_from_design('tiny', random_state=123) m.run_experiments(design_name='tiny') mm = m.create_metamodel_from_design('tiny', random_state=123) assert mm.scope != m.scope # now not equal as road_test2 has transforms that are stripped. tiny2 = m.design_experiments(n_samples=10, design_name='tiny2', random_seed=456) assert tiny2.iloc[0]['debt_type'] == 'GO Bond' assert dict(tiny2.iloc[0].drop('debt_type')) == approx({ 'alpha': 0.10428005571929212, 'amortization_period': 33, 'beta': 4.8792451185772014, 'expand_capacity': 61.4210886403998, 'input_flow': 137, 'interest_rate': 0.03099304322197216, 'interest_rate_lock': 0, 'unit_cost_expansion': 121.85520427974882, 'value_of_time': 0.002953613029133872, 'yield_curve': 0.016255990123028242, 'free_flow_time': 60, 'initial_capacity': 100 }) result2 = mm.run_experiments('tiny2') # print(dict(mm.read_experiment_measures('tiny2').iloc[0])) # # print({ # 'no_build_travel_time': 81.6839454971052, # 'build_travel_time': 61.91038371206646, # 'time_savings': 44.94189289289446, # 'value_of_time_savings': 2904.081661408463, # 'net_benefits': -34.09931528157315, # 'cost_of_capacity_expansion': 1085.3565091745982, # 'present_cost_expansion': 19923.66625500023, # }) # assert dict(mm.read_experiment_measures('tiny2').iloc[0]) == approx({ 'no_build_travel_time': 81.6839454971052, 'build_travel_time': 61.91038371206646, 'log_build_travel_time': 4.120826572003798, 'time_savings': 44.94189289289446, 'value_of_time_savings': 2904.081661408463, 'net_benefits': -34.09931528157315, 'cost_of_capacity_expansion': 1085.3565091745982, 'present_cost_expansion': 19923.66625500023, }) assert m.run_experiment(tiny2.iloc[0]) == approx({ 'no_build_travel_time': 89.07004237532217, 'build_travel_time': 62.81032484779827, 'log_build_travel_time': np.log(62.81032484779827), 'time_savings': 26.259717527523904, 'value_of_time_savings': 10.62586300480175, 'present_cost_expansion': 7484.479303360477, 'cost_of_capacity_expansion': 395.69034710662226, 'net_benefits': -385.0644841018205, }) with pytest.raises(ValueError): # no metamodels stored mm3 = db.read_metamodel(None, None) db.write_metamodel(mm) mm2 = db.read_metamodel(None, 1) mm3 = db.read_metamodel(None, None) assert mm2 == mm == mm3 assert mm2 is not mm assert mm2.function(**(tiny2.iloc[0])) == approx({ 'no_build_travel_time': 81.6839454971052, 'build_travel_time': 61.91038371206646, 'log_build_travel_time': 4.120826572003798, 'time_savings': 44.94189289289446, 'value_of_time_savings': 2904.081661408463, 'net_benefits': -34.09931528157315, 'cost_of_capacity_expansion': 1085.3565091745982, 'present_cost_expansion': 19923.66625500023, }) mm3.metamodel_id = db.get_new_metamodel_id(None) db.write_metamodel(mm3) with pytest.raises(ValueError): # now too many to get without giving an ID mm4 = db.read_metamodel(None, None)
def test_read_db_gz(self): road_test_scope_file = emat.package_file('model', 'tests', 'road_test.yaml') with pytest.raises(FileNotFoundError): emat.Scope(emat.package_file('nope.yaml')) s = emat.Scope(road_test_scope_file) with pytest.raises(FileNotFoundError): emat.SQLiteDB(emat.package_file('nope.db.gz')) if not os.path.exists(emat.package_file("examples", "roadtest.db.gz")): db_w = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.tmp"), initialize=True) s.store_scope(db_w) s.design_experiments(n_samples=110, random_seed=1234, db=db_w, design_name='lhs') from emat.model.core_python import Road_Capacity_Investment m_w = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db_w) m_w.run_experiments(design_name='lhs', db=db_w) db_w.conn.close() import gzip import shutil with open(emat.package_file("examples", "roadtest.db.tmp"), 'rb') as f_in: with gzip.open(emat.package_file("examples", "roadtest.db.gz"), 'wb') as f_out: shutil.copyfileobj(f_in, f_out) db = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.gz")) assert repr(db) == '<emat.SQLiteDB with scope "EMAT Road Test">' assert db.get_db_info()[:9] == 'SQLite @ ' assert db.get_db_info()[-11:] == 'roadtest.db' assert db.read_scope_names() == ['EMAT Road Test'] s1 = db.read_scope('EMAT Road Test') assert type(s1) == type(s) for k in ('_x_list', '_l_list', '_c_list', '_m_list', 'name', 'desc'): assert getattr(s, k) == getattr(s1, k), k assert s == s1 experiments = db.read_experiment_all('EMAT Road Test', 'lhs') assert experiments.shape == (110, 20) assert list(experiments.columns) == [ 'free_flow_time', 'initial_capacity', 'alpha', 'beta', 'input_flow', 'value_of_time', 'unit_cost_expansion', 'interest_rate', 'yield_curve', 'expand_capacity', 'amortization_period', 'debt_type', 'interest_rate_lock', 'no_build_travel_time', 'build_travel_time', 'time_savings', 'value_of_time_savings', 'net_benefits', 'cost_of_capacity_expansion', 'present_cost_expansion', ] from emat.model.core_python import Road_Capacity_Investment m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db) assert m.metamodel_id == None
'initial_capacity', 'alpha', 'beta', 'input_flow', 'value_of_time', 'unit_cost_expansion', 'interest_rate', 'yield_curve', 'expand_capacity', 'amortization_period', 'debt_type', 'interest_rate_lock', 'no_build_travel_time', 'build_travel_time', 'time_savings', 'value_of_time_savings', 'net_benefits', 'cost_of_capacity_expansion', 'present_cost_expansion', ] from emat.model.core_python import Road_Capacity_Investment m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db) assert m.metamodel_id == None emat.package_file('model', 'tests', 'road_test.yaml') if __name__ == '__main__': unittest.main()
class TestExperimentMethods(unittest.TestCase): ''' tests generating experiments ''' # # one time test setup # scope_file = emat.package_file("scope", "tests", "scope_test.yaml") scp = Scope(scope_file) db_test = SQLiteDB(":memory:", initialize=True) scp.store_scope(db_test) def test_latin_hypercube(self): exp_def = self.scp.design_experiments( n_samples_per_factor=10, random_seed=1234, sampler='lhs', db=self.db_test, ) assert len(exp_def) == self.scp.n_sample_factors() * 10 assert (exp_def['TestRiskVar'] == 1.0).all() assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.0326, abs=1e-3) assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=1e-3) exp_def2 = self.db_test.read_experiment_parameters( self.scp.name, 'lhs') assert (exp_def[exp_def2.columns] == exp_def2).all().all() def test_latin_hypercube_not_joint(self): exp_def = self.scp.design_experiments( n_samples_per_factor=5, random_seed=1234, sampler='lhs', db=self.db_test, jointly=False, design_name='lhs_not_joint', ) # assert len(exp_def) == len(self.scp.get_uncertainties())*5 * len(self.scp.get_levers())*5 assert len( exp_def ) == 80 # there are only 4 unique policies, times 2 scenarios assert (exp_def['TestRiskVar'] == 1.0).all() assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.033, abs=1e-2) assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=1e-2) exp_def2 = self.db_test.read_experiment_parameters( self.scp.name, 'lhs_not_joint') assert (exp_def[exp_def2.columns] == exp_def2).all().all() def test_monte_carlo(self): exp_def = self.scp.design_experiments( n_samples_per_factor=10, random_seed=1234, sampler='mc', db=self.db_test, ) assert len(exp_def) == self.scp.n_sample_factors() * 10 assert (exp_def['TestRiskVar'] == 1.0).all() assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.0326, abs=0.01) assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=0.01) exp_def2 = self.db_test.read_experiment_parameters(self.scp.name, 'mc') assert (exp_def[exp_def2.columns] == exp_def2).all().all() def test_sensitivity_tests(self): exp_def = self.scp.design_experiments( sampler='uni', db=self.db_test, ) cols = [ 'TestRiskVar', 'Land Use - CBD Focus', 'Freeway Capacity', 'Auto IVTT Sensitivity', 'Shared Mobility', 'Kensington Decommissioning', 'LRT Extension' ] correct = '{"TestRiskVar":{"0":1.0,"1":1.0,"2":1.0,"3":1.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \ '"Land Use - CBD Focus":{"0":1.0,"1":0.82,"2":1.37,"3":1.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \ '"Freeway Capacity":{"0":1.0,"1":1.0,"2":1.0,"3":2.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \ '"Auto IVTT Sensitivity":{"0":1.0,"1":1.0,"2":1.0,"3":1.0,"4":0.75,"5":1.0,"6":1.0,"7":1.0},' \ '"Shared Mobility":{"0":0.0,"1":0.0,"2":0.0,"3":0.0,"4":0.0,"5":1.0,"6":0.0,"7":0.0},' \ '"Kensington Decommissioning":{"0":false,"1":false,"2":false,"3":false,"4":false,' \ '"5":false,"6":true,"7":false},"LRT Extension":{"0":false,"1":false,"2":false,"3":false,' \ '"4":false,"5":false,"6":false,"7":true}}' correct = pd.read_json(correct) for k in cols: assert (exp_def[k].values == approx(correct[k].values)) exp_def2 = self.db_test.read_experiment_parameters( self.scp.name, 'uni') for k in cols: assert (exp_def2[k].values == approx(correct[k].values)) def test_nonuniform_latin_hypercube(self): scope_file = emat.package_file("model", "tests", "road_test_nonuniform.yaml") scp = Scope(scope_file) exp_def = scp.design_experiments( n_samples_per_factor=1000, random_seed=1234, sampler='lhs', ) assert len(exp_def) == scp.n_sample_factors() * 1000 assert (exp_def['free_flow_time'] == 60).all() assert (exp_def['initial_capacity'] == 100).all() assert np.corrcoef([exp_def.alpha, exp_def.beta])[0, 1] == approx(0.75, rel=0.05) assert np.corrcoef([exp_def.alpha, exp_def.expand_capacity])[0, 1] == approx(0.0, abs=0.02) assert np.corrcoef([exp_def.input_flow, exp_def.value_of_time])[0, 1] == approx(-0.5, rel=0.05) assert np.corrcoef( [exp_def.unit_cost_expansion, exp_def.value_of_time])[0, 1] == approx(0.9, rel=0.05) assert exp_def.interest_rate_lock.sum() == approx(len(exp_def) * 0.2) assert np.percentile(exp_def.alpha, np.linspace(0, 100, 50)) == approx( [ 0.10037393, 0.10722119, 0.10994485, 0.11204394, 0.11383709, 0.11544182, 0.11691345, 0.11829399, 0.11959909, 0.12084863, 0.12205279, 0.12321800, 0.12435285, 0.12546474, 0.12655958, 0.12763503, 0.12869873, 0.12975137, 0.13079620, 0.13183375, 0.13287082, 0.13390854, 0.13494651, 0.13598528, 0.13703149, 0.13808180, 0.13914411, 0.14021784, 0.14130323, 0.14240609, 0.14352608, 0.14466756, 0.14583411, 0.14702908, 0.14825720, 0.14951875, 0.15082844, 0.15218376, 0.15359963, 0.15508120, 0.15664534, 0.15831425, 0.16010073, 0.16203921, 0.16418886, 0.16662357, 0.16946999, 0.17301416, 0.17804383, 0.19662857 ]) assert np.percentile(exp_def.beta, np.linspace(0, 100, 50)) == approx([ 3.51654751, 3.72503059, 3.82437701, 3.90088124, 3.96222432, 4.01360346, 4.06112277, 4.10640347, 4.14456476, 4.18084719, 4.21812584, 4.24926944, 4.28049053, 4.31181127, 4.34390502, 4.37561590, 4.40541815, 4.43276143, 4.45517485, 4.48062290, 4.50726296, 4.53334164, 4.55737738, 4.57893875, 4.60371011, 4.62590595, 4.64885523, 4.67335218, 4.69475909, 4.71546469, 4.73676622, 4.75796550, 4.77690613, 4.79738177, 4.81947505, 4.84481408, 4.86954326, 4.89379651, 4.91771359, 4.94100213, 4.97169370, 5.00298714, 5.03525103, 5.07100437, 5.11028866, 5.15061419, 5.19925284, 5.24775527, 5.32086547, 5.49345120 ]) assert np.percentile(exp_def.input_flow, np.linspace( 0, 100, 50)) == approx([ 80.06332381, 83.71770678, 85.93721426, 87.82355332, 89.52967083, 91.11148891, 92.60789787, 94.03509345, 95.41186604, 96.74598771, 98.04479355, 99.31122788, 100.55462827, 101.77808514, 102.98041937, 104.16517008, 105.33959643, 106.50165383, 107.65366600, 108.79827639, 109.93328795, 111.06707367, 112.19316408, 113.31493454, 114.43982739, 115.56182157, 116.68454749, 117.80620837, 118.93524815, 120.06752762, 121.20470208, 122.34781469, 123.50083542, 124.66086016, 125.83390291, 127.02146142, 128.22485477, 129.44681602, 130.68613841, 131.95601658, 133.25403807, 134.58951122, 135.96442305, 137.39295642, 138.89092217, 140.47204147, 142.17835057, 144.06540067, 146.28064479, 149.94588322 ])
def test_road_test(self): import os test_dir = os.path.dirname(__file__) os.chdir(test_dir) road_test_scope_file = emat.package_file('model', 'tests', 'road_test.yaml') road_scope = emat.Scope(road_test_scope_file) # <emat.Scope with 2 constants, 7 uncertainties, 4 levers, 7 measures> assert len(road_scope.get_measures()) == 7 assert len(road_scope.get_levers()) == 4 assert len(road_scope.get_uncertainties()) == 7 assert len(road_scope.get_constants()) == 2 emat_db = emat.SQLiteDB() road_scope.store_scope(emat_db) with pytest.raises(KeyError): road_scope.store_scope(emat_db) assert emat_db.read_scope_names() == ['EMAT Road Test'] design = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs') design.head() large_design = design_experiments(road_scope, db=emat_db, n_samples=5000, sampler='lhs', design_name='lhs_large') large_design.head() assert list(large_design.columns) == [ 'alpha', 'amortization_period', 'beta', 'debt_type', 'expand_capacity', 'input_flow', 'interest_rate', 'interest_rate_lock', 'unit_cost_expansion', 'value_of_time', 'yield_curve', 'free_flow_time', 'initial_capacity', ] assert list(large_design.head().index) == [111, 112, 113, 114, 115] assert emat_db.read_design_names('EMAT Road Test') == ['lhs', 'lhs_large'] m = PythonCoreModel(Road_Capacity_Investment, scope=road_scope, db=emat_db) with SequentialEvaluator(m) as eval_seq: lhs_results = m.run_experiments(design_name='lhs', evaluator=eval_seq) lhs_results.head() assert lhs_results.head()['present_cost_expansion'].values == approx( [2154.41598475, 12369.38053473, 4468.50683924, 6526.32517089, 2460.91070514]) assert lhs_results.head()['net_benefits'].values == approx( [ -22.29090499, -16.84301382, -113.98841188, 11.53956058, 78.03661612]) assert lhs_results.tail()['present_cost_expansion'].values == approx( [2720.51645703, 4000.91232689, 6887.83193063, 3739.47839941, 1582.52899124]) assert lhs_results.tail()['net_benefits'].values == approx( [841.46278175, -146.71279267, -112.5681036, 25.48055303, 127.31154155]) with SequentialEvaluator(m) as eval_seq: lhs_large_results = m.run_experiments(design_name='lhs_large', evaluator=eval_seq) lhs_large_results.head() assert lhs_large_results.head()['net_benefits'].values == approx( [-522.45283083, -355.1599307 , -178.6623215 , 23.46263498, -301.17700968]) lhs_outcomes = m.read_experiment_measures(design_name='lhs') assert lhs_outcomes.head()['time_savings'].values == approx( [13.4519273, 26.34172999, 12.48385198, 15.10165981, 15.48056139]) scores = m.get_feature_scores('lhs', random_state=123) stable_df("./road_test_feature_scores.pkl.gz", scores.data) from emat.workbench.analysis import prim x = m.read_experiment_parameters(design_name='lhs_large') prim_alg = prim.Prim( m.read_experiment_parameters(design_name='lhs_large'), m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0, threshold=0.4, ) box1 = prim_alg.find_box() stable_df("./road_test_box1_peeling_trajectory.pkl.gz", box1.peeling_trajectory) from emat.util.xmle import Show from emat.util.xmle.elem import Elem assert isinstance(Show(box1.show_tradeoff()), Elem) from emat.workbench.analysis import cart cart_alg = cart.CART( m.read_experiment_parameters(design_name='lhs_large'), m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0, ) cart_alg.build_tree() stable_df("./road_test_cart_box0.pkl.gz", cart_alg.boxes[0]) cart_dict = dict(cart_alg.boxes[0].iloc[0]) assert cart_dict['debt_type'] == {'GO Bond', 'Paygo', 'Rev Bond'} #assert cart_dict['interest_rate_lock'] == {False, True} assert isinstance(Show(cart_alg.show_tree(format='svg')), Elem) from emat import Measure MAXIMIZE = Measure.MAXIMIZE MINIMIZE = Measure.MINIMIZE robustness_functions = [ Measure( 'Expected Net Benefit', kind=Measure.INFO, variable_name='net_benefits', function=numpy.mean, # min=-150, # max=50, ), Measure( 'Probability of Net Loss', kind=MINIMIZE, variable_name='net_benefits', function=lambda x: numpy.mean(x < 0), min=0, max=1, ), Measure( '95%ile Travel Time', kind=MINIMIZE, variable_name='build_travel_time', function=functools.partial(numpy.percentile, q=95), min=60, max=150, ), Measure( '99%ile Present Cost', kind=Measure.INFO, variable_name='present_cost_expansion', function=functools.partial(numpy.percentile, q=99), # min=0, # max=10, ), Measure( 'Expected Present Cost', kind=Measure.INFO, variable_name='present_cost_expansion', function=numpy.mean, # min=0, # max=10, ), ] from emat import Constraint constraint_1 = Constraint( "Maximum Log Expected Present Cost", outcome_names="Expected Present Cost", function=Constraint.must_be_less_than(4000), ) constraint_2 = Constraint( "Minimum Capacity Expansion", parameter_names="expand_capacity", function=Constraint.must_be_greater_than(10), ) constraint_3 = Constraint( "Maximum Paygo", parameter_names='debt_type', outcome_names='99%ile Present Cost', function=lambda i, j: max(0, j - 1500) if i == 'Paygo' else 0, ) from emat.optimization import HyperVolume, EpsilonProgress, SolutionViewer, ConvergenceMetrics convergence_metrics = ConvergenceMetrics( HyperVolume.from_outcomes(robustness_functions), EpsilonProgress(), SolutionViewer.from_model_and_outcomes(m, robustness_functions), ) with SequentialEvaluator(m) as eval_seq: robust = m.robust_optimize( robustness_functions, scenarios=20, nfe=5, constraints=[ constraint_1, constraint_2, constraint_3, ], epsilons=[0.05, ] * len(robustness_functions), convergence=convergence_metrics, evaluator=eval_seq, ) robust_results, convergence = robust.result, robust.convergence assert isinstance(robust_results, pandas.DataFrame) mm = m.create_metamodel_from_design('lhs') design2 = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs', random_seed=2) design2_results = mm.run_experiments(design2)
def test_database_merging(): import emat road_test_scope_file = emat.package_file("model", "tests", "road_test.yaml") road_scope = emat.Scope(road_test_scope_file) emat_db = emat.SQLiteDB() road_scope.store_scope(emat_db) assert emat_db.read_scope_names() == ["EMAT Road Test"] from emat.experiment.experimental_design import design_experiments design = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler="lhs") large_design = design_experiments(road_scope, db=emat_db, n_samples=500, sampler="lhs", design_name="lhs_large") assert emat_db.read_design_names("EMAT Road Test") == ["lhs", "lhs_large"] from emat.model.core_python import PythonCoreModel, Road_Capacity_Investment m = PythonCoreModel(Road_Capacity_Investment, scope=road_scope, db=emat_db) lhs_results = m.run_experiments(design_name="lhs") lhs_large_results = m.run_experiments(design_name="lhs_large") reload_results = m.read_experiments(design_name="lhs") pd.testing.assert_frame_equal( reload_results, lhs_results, check_like=True, ) lhs_params = m.read_experiment_parameters(design_name="lhs") assert len(lhs_params) == 110 assert len(lhs_params.columns) == 13 lhs_outcomes = m.read_experiment_measures(design_name="lhs") assert len(lhs_outcomes) == 110 assert len(lhs_outcomes.columns) == 7 mm = m.create_metamodel_from_design("lhs") assert mm.metamodel_id == 1 assert isinstance(mm.function, emat.MetaModel) design2 = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler="lhs", random_seed=2) design2_results = mm.run_experiments(design2) assert len(design2_results) == 110 assert len(design2_results.columns) == 20 assert emat_db.read_design_names(None) == ["lhs", "lhs_2", "lhs_large"] check = emat_db.read_experiment_measures(None, "lhs_2") assert len(check) == 110 assert len(check.columns) == 7 assert emat_db.read_experiment_measure_sources(None, "lhs_2") == [1] m.allow_short_circuit = False design2_results0 = m.run_experiments(design2.iloc[:5]) assert len(design2_results0) == 5 assert len(design2_results0.columns) == 20 with pytest.raises(ValueError): # now there are two sources of some measures emat_db.read_experiment_measures(None, "lhs_2") assert set(emat_db.read_experiment_measure_sources(None, "lhs_2")) == {0, 1} check = emat_db.read_experiment_measures(None, "lhs_2", source=0) assert len(check) == 5 check = emat_db.read_experiment_measures(None, "lhs_2", source=1) assert len(check) == 110 import emat.examples s2, db2, m2 = emat.examples.road_test() # write the design for lhs_2 into a different database. # it ends up giving different experient id's to these, which is fine. db2.write_experiment_parameters( None, "lhs_2", emat_db.read_experiment_parameters(None, "lhs_2")) check = db2.read_experiment_parameters( None, "lhs_2", ) assert len(check) == 110 assert len(check.columns) == 13 pd.testing.assert_frame_equal( design2.reset_index(drop=True), check.reset_index(drop=True), check_like=True, ) design2_results2 = m2.run_experiments("lhs_2") check = emat_db.read_experiment_measures(None, "lhs_2", source=0) assert len(check) == 5 assert len(check.columns) == 7 check = emat_db.read_experiment_measures(None, "lhs_2", runs="valid") assert len(check) == 115 emat_db.merge_database(db2) check = emat_db.read_experiment_measures(None, "lhs_2", source=0) assert len(check) == 110 assert len(check.columns) == 7 check = emat_db.read_experiment_measures(None, "lhs_2", runs="valid") assert len(check) == 225
def test_road_test(self): road_test_scope_file = emat.package_file('model', 'tests', 'road_test.yaml') road_scope = emat.Scope(road_test_scope_file) # <emat.Scope with 2 constants, 7 uncertainties, 4 levers, 7 measures> assert len(road_scope.get_measures()) == 7 assert len(road_scope.get_levers()) == 4 assert len(road_scope.get_uncertainties()) == 7 assert len(road_scope.get_constants()) == 2 emat_db = emat.SQLiteDB() road_scope.store_scope(emat_db) with pytest.raises(KeyError): road_scope.store_scope(emat_db) assert emat_db.read_scope_names() == ['EMAT Road Test'] design = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs') design.head() large_design = design_experiments(road_scope, db=emat_db, n_samples=5000, sampler='lhs', design_name='lhs_large') large_design.head() assert list(large_design.columns) == [ 'alpha', 'amortization_period', 'beta', 'debt_type', 'expand_capacity', 'input_flow', 'interest_rate', 'interest_rate_lock', 'unit_cost_expansion', 'value_of_time', 'yield_curve', 'free_flow_time', 'initial_capacity', ] assert list(large_design.head().index) == [111, 112, 113, 114, 115] assert emat_db.read_design_names('EMAT Road Test') == ['lhs', 'lhs_large'] m = PythonCoreModel(Road_Capacity_Investment, scope=road_scope, db=emat_db) with SequentialEvaluator(m) as eval_seq: lhs_results = m.run_experiments(design_name='lhs', evaluator=eval_seq) lhs_results.head() assert lhs_results.head()['present_cost_expansion'].values == approx( [2154.41598475, 12369.38053473, 4468.50683924, 6526.32517089, 2460.91070514]) assert lhs_results.head()['net_benefits'].values == approx( [-79.51551505, -205.32148044, -151.94431822, -167.62487134, -3.97293985]) with SequentialEvaluator(m) as eval_seq: lhs_large_results = m.run_experiments(design_name='lhs_large', evaluator=eval_seq) lhs_large_results.head() assert lhs_large_results.head()['net_benefits'].values == approx( [-584.36098322, -541.5458395, -185.16661464, -135.85689709, -357.36106457]) lhs_outcomes = m.read_experiment_measures(design_name='lhs') assert lhs_outcomes.head()['time_savings'].values == approx( [13.4519273, 26.34172999, 12.48385198, 15.10165981, 15.48056139]) correct_scores = numpy.array( [[0.06603461, 0.04858595, 0.06458574, 0.03298163, 0.05018515, 0., 0., 0.53156587, 0.05060416, 0.02558088, 0.04676956, 0.04131266, 0.04179378], [0.06003223, 0.04836434, 0.06059554, 0.03593644, 0.27734396, 0., 0., 0.28235419, 0.05303979, 0.03985181, 0.04303371, 0.05004349, 0.04940448], [0.08760605, 0.04630414, 0.0795043, 0.03892201, 0.10182534, 0., 0., 0.42508457, 0.04634321, 0.03216387, 0.0497183, 0.04953772, 0.0429905], [0.08365598, 0.04118732, 0.06716887, 0.03789444, 0.06509519, 0., 0., 0.31494171, 0.06517462, 0.02895742, 0.04731707, 0.17515158, 0.07345581], [0.06789382, 0.07852257, 0.05066944, 0.04807088, 0.32054735, 0., 0., 0.15953055, 0.05320201, 0.02890069, 0.07033928, 0.06372418, 0.05859923], [0.05105435, 0.09460353, 0.04614178, 0.04296901, 0.45179611, 0., 0., 0.04909801, 0.05478798, 0.023099, 0.08160785, 0.05642169, 0.04842069], [0.04685703, 0.03490931, 0.03214081, 0.03191602, 0.56130318, 0., 0., 0.04011044, 0.04812986, 0.02228924, 0.09753361, 0.04273004, 0.04208045], ]) scores = m.get_feature_scores('lhs', random_state=123) for _i in range(scores.metadata.values.shape[0]): for _j in range(scores.metadata.values.shape[1]): assert scores.metadata.values[_i,_j] == approx(correct_scores[_i,_j], rel=.1) from ema_workbench.analysis import prim x = m.read_experiment_parameters(design_name='lhs_large') prim_alg = prim.Prim( m.read_experiment_parameters(design_name='lhs_large'), m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0, threshold=0.4, ) box1 = prim_alg.find_box() assert dict(box1.peeling_trajectory.iloc[45]) == approx({ 'coverage': 0.8014705882352942, 'density': 0.582109479305741, 'id': 45, 'mass': 0.1498, 'mean': 0.582109479305741, 'res_dim': 4, }) from emat.util.xmle import Show from emat.util.xmle.elem import Elem assert isinstance(Show(box1.show_tradeoff()), Elem) from ema_workbench.analysis import cart cart_alg = cart.CART( m.read_experiment_parameters(design_name='lhs_large'), m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0, ) cart_alg.build_tree() cart_dict = dict(cart_alg.boxes[0].iloc[0]) assert cart_dict['debt_type'] == {'GO Bond', 'Paygo', 'Rev Bond'} assert cart_dict['interest_rate_lock'] == {False, True} del cart_dict['debt_type'] del cart_dict['interest_rate_lock'] assert cart_dict == approx({ 'free_flow_time': 60, 'initial_capacity': 100, 'alpha': 0.10001988547129116, 'beta': 3.500215589924521, 'input_flow': 80.0, 'value_of_time': 0.00100690634109406, 'unit_cost_expansion': 95.00570832093116, 'interest_rate': 0.0250022738169142, 'yield_curve': -0.0024960505548531774, 'expand_capacity': 0.0006718732232418368, 'amortization_period': 15, }) assert isinstance(Show(cart_alg.show_tree(format='svg')), Elem) from emat import Measure MAXIMIZE = Measure.MAXIMIZE MINIMIZE = Measure.MINIMIZE robustness_functions = [ Measure( 'Expected Net Benefit', kind=Measure.INFO, variable_name='net_benefits', function=numpy.mean, # min=-150, # max=50, ), Measure( 'Probability of Net Loss', kind=MINIMIZE, variable_name='net_benefits', function=lambda x: numpy.mean(x < 0), min=0, max=1, ), Measure( '95%ile Travel Time', kind=MINIMIZE, variable_name='build_travel_time', function=functools.partial(numpy.percentile, q=95), min=60, max=150, ), Measure( '99%ile Present Cost', kind=Measure.INFO, variable_name='present_cost_expansion', function=functools.partial(numpy.percentile, q=99), # min=0, # max=10, ), Measure( 'Expected Present Cost', kind=Measure.INFO, variable_name='present_cost_expansion', function=numpy.mean, # min=0, # max=10, ), ] from emat import Constraint constraint_1 = Constraint( "Maximum Log Expected Present Cost", outcome_names="Expected Present Cost", function=Constraint.must_be_less_than(4000), ) constraint_2 = Constraint( "Minimum Capacity Expansion", parameter_names="expand_capacity", function=Constraint.must_be_greater_than(10), ) constraint_3 = Constraint( "Maximum Paygo", parameter_names='debt_type', outcome_names='99%ile Present Cost', function=lambda i, j: max(0, j - 1500) if i == 'Paygo' else 0, ) from emat.optimization import HyperVolume, EpsilonProgress, SolutionViewer, ConvergenceMetrics convergence_metrics = ConvergenceMetrics( HyperVolume.from_outcomes(robustness_functions), EpsilonProgress(), SolutionViewer.from_model_and_outcomes(m, robustness_functions), ) with SequentialEvaluator(m) as eval_seq: robust_results, convergence = m.robust_optimize( robustness_functions, scenarios=20, nfe=5, constraints=[ constraint_1, constraint_2, constraint_3, ], epsilons=[0.05, ] * len(robustness_functions), convergence=convergence_metrics, evaluator=eval_seq, ) assert isinstance(robust_results, pandas.DataFrame) mm = m.create_metamodel_from_design('lhs') design2 = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs', random_seed=2) design2_results = mm.run_experiments(design2)
class TestScopeMethods(unittest.TestCase): ''' tests parsing scope file ''' # # one time test setup # scope_file = emat.package_file("model", "tests", "model_test.yaml") db_test = SQLiteDB( config.get("test_db_filename", ":memory:"), initialize=True, ) # # Tests # def test_dump_scope(self): scp = Scope(self.scope_file) dumped = scp.dump() # print("="*40) # print(dumped) # print("="*40) loaded = Scope(scope_def=dumped, scope_file="fake/filename.yaml") assert loaded == scp # filename is intentionally different but let it go # but everything else is the same assert loaded.name == scp.name assert loaded.get_measures() == scp.get_measures() assert loaded.get_parameters() == scp.get_parameters() assert loaded.scope_file != scp.scope_file assert loaded.scope_file == "fake/filename.yaml" # fix name, still get equality loaded.scope_file = scp.scope_file assert loaded == scp def test_save_scope(self): scp = Scope(self.scope_file) scp.store_scope(self.db_test) def test_null_scope(self): scp = Scope(None) assert repr(scp) == "<emat.Scope with no content>" assert len(scp.get_measures()) == 0 assert len(scp.get_parameters()) == 0 def test_box(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) with pytest.raises(TypeError): s = Box(scope=scope) s = Box(name="Speedy", scope=scope) s.set_upper_bound('build_travel_time', 70) with pytest.raises(ScopeError): s.set_upper_bound('not_a_thing', 70) assert len(s) == 1 assert 'build_travel_time' in s assert s.parent_box_name is None s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) assert len(s2) == 1 assert 'build_travel_time' not in s2 assert s2.parent_box_name == 'Speedy' def test_box_universe(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) s = Box(name="Speedy", scope=scope) s.set_upper_bound('build_travel_time', 70) s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) u = Boxes(s, s2, scope=scope) assert u.fancy_names() == [ 'Scope: EMAT Road Test', '▶ Speedy', '▷ ▶ Notable' ] assert u.plain_names() == [None, 'Speedy', 'Notable'] def test_read_write_box(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) db = SQLiteDB() scope.store_scope(db) s1 = Box(name="Speedy", scope=scope) s1.set_upper_bound('build_travel_time', 70) s1.relevant_features.add('debt_type') s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) db.write_box(s1) db.write_box(s2) s1_ = db.read_box(scope.name, "Speedy") s2_ = db.read_box(scope.name, "Notable") assert s1 == s1_ assert s2 == s2_ assert s1.thresholds == s1_.thresholds assert s2.thresholds == s2_.thresholds assert s1.relevant_features == s1_.relevant_features assert s2.relevant_features == s2_.relevant_features def test_read_write_boxes(self): scope = Scope(package_file('model', 'tests', 'road_test.yaml')) db = SQLiteDB() scope.store_scope(db) s1 = Box(name="Speedy", scope=scope) s1.set_upper_bound('build_travel_time', 70) s2 = Box(name="Notable", scope=scope, parent="Speedy") s2.set_lower_bound('expand_capacity', 20) u = Boxes(s1, s2, scope=scope) db.write_boxes(u) scope2 = Scope(package_file('model', 'tests', 'road_test.yaml')) u2 = db.read_boxes(scope=scope2) assert u == u2 assert u["Notable"].parent_box_name == u2["Notable"].parent_box_name s1_ = db.read_box(scope.name, "Speedy") s2_ = db.read_box(scope.name, "Notable") assert s1 == s1_ assert s2 == s2_ assert s1.relevant_features == s1_.relevant_features assert s2.relevant_features == s2_.relevant_features