def test_read_write_boxes(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
u = Boxes(s1, s2, scope=scope)
db.write_boxes(u)
scope2 = Scope(package_file('model', 'tests', 'road_test.yaml'))
u2 = db.read_boxes(scope=scope2)
assert u == u2
assert u["Notable"].parent_box_name == u2["Notable"].parent_box_name
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features
def test_read_db_gz(self):
road_test_scope_file = emat.package_file('model', 'tests',
'road_test.yaml')
with pytest.raises(FileNotFoundError):
emat.Scope(emat.package_file('nope.yaml'))
s = emat.Scope(road_test_scope_file)
with pytest.raises(FileNotFoundError):
emat.SQLiteDB(emat.package_file('nope.db.gz'))
db = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.gz"))
assert repr(db) == '<emat.SQLiteDB with scope "EMAT Road Test">'
assert db.get_db_info()[:9] == 'SQLite @ '
assert db.get_db_info()[-11:] == 'roadtest.db'
assert db.read_scope_names() == ['EMAT Road Test']
s1 = db.read_scope('EMAT Road Test')
assert type(s1) == type(s)
for k in ('_x_list', '_l_list', '_c_list', '_m_list', 'name', 'desc'):
assert getattr(s, k) == getattr(s1, k), k
assert s == s1
experiments = db.read_experiment_all('EMAT Road Test', 'lhs')
assert experiments.shape == (110, 20)
assert list(experiments.columns) == [
'free_flow_time',
'initial_capacity',
'alpha',
'beta',
'input_flow',
'value_of_time',
'unit_cost_expansion',
'interest_rate',
'yield_curve',
'expand_capacity',
'amortization_period',
'debt_type',
'interest_rate_lock',
'no_build_travel_time',
'build_travel_time',
'time_savings',
'value_of_time_savings',
'net_benefits',
'cost_of_capacity_expansion',
'present_cost_expansion',
]
from emat.model.core_python import Road_Capacity_Investment
m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db)
assert m.metamodel_id == None
def test_feature_scoring_with_nan():
road_scope = emat.Scope(emat.package_file('model','tests','road_test_bogus.yaml'))
road_test = PythonCoreModel(_Road_Capacity_Investment_with_Bogus_Output, scope=road_scope)
road_test_design = road_test.design_experiments(n_samples=5000, sampler='lhs')
road_test_results = road_test.run_experiments(design=road_test_design)
fs = feature_scores(road_scope, road_test_results, random_state=234)
assert isinstance(fs, pd.io.formats.style.Styler)
stable_df("./road_test_feature_scores_bogus_1.pkl.gz", fs.data)
def test_correlated_latin_hypercube_conflict(self):
scope_file = emat.package_file("model", "tests", "road_test_corr_conflict.yaml")
scp = Scope(scope_file)
with pytest.raises(emat.exceptions.AsymmetricCorrelationError):
scp.design_experiments(
n_samples_per_factor=10,
random_seed=1234,
sampler='lhs',
)
def test_correlated_latin_hypercube_bad(self):
scope_file = emat.package_file("model", "tests", "road_test_corr_bad.yaml")
scp = Scope(scope_file)
with pytest.raises(np.linalg.LinAlgError):
scp.design_experiments(
n_samples_per_factor=10,
random_seed=1234,
sampler='lhs',
)
def test_nonuniform_latin_hypercube(self):
scope_file = emat.package_file("model", "tests", "road_test_nonuniform.yaml")
scp = Scope(scope_file)
exp_def = scp.design_experiments(
n_samples_per_factor=1000,
random_seed=1234,
sampler='lhs',
)
assert len(exp_def) == scp.n_sample_factors() * 1000
assert (exp_def['free_flow_time'] == 60).all()
assert (exp_def['initial_capacity'] == 100).all()
assert np.corrcoef([exp_def.alpha, exp_def.beta])[0, 1] == approx(0.75, rel=0.05)
assert np.corrcoef([exp_def.alpha, exp_def.expand_capacity])[0, 1] == approx(0.0, abs=0.02)
assert np.corrcoef([exp_def.input_flow, exp_def.value_of_time])[0, 1] == approx(-0.5, rel=0.05)
assert np.corrcoef([exp_def.unit_cost_expansion, exp_def.value_of_time])[0, 1] == approx(0.9, rel=0.05)
assert exp_def.interest_rate_lock.sum() == approx(len(exp_def) * 0.2)
assert np.percentile(exp_def.alpha,np.linspace(0,100,50)) == approx(
[0.10037393, 0.10722119, 0.10994485, 0.11204394, 0.11383709,
0.11544182, 0.11691345, 0.11829399, 0.11959909, 0.12084863,
0.12205279, 0.12321800, 0.12435285, 0.12546474, 0.12655958,
0.12763503, 0.12869873, 0.12975137, 0.13079620, 0.13183375,
0.13287082, 0.13390854, 0.13494651, 0.13598528, 0.13703149,
0.13808180, 0.13914411, 0.14021784, 0.14130323, 0.14240609,
0.14352608, 0.14466756, 0.14583411, 0.14702908, 0.14825720,
0.14951875, 0.15082844, 0.15218376, 0.15359963, 0.15508120,
0.15664534, 0.15831425, 0.16010073, 0.16203921, 0.16418886,
0.16662357, 0.16946999, 0.17301416, 0.17804383, 0.19662857])
assert np.percentile(exp_def.beta,np.linspace(0,100,50)) == approx(
[3.51654751, 3.72503059, 3.82437701, 3.90088124, 3.96222432,
4.01360346, 4.06112277, 4.10640347, 4.14456476, 4.18084719,
4.21812584, 4.24926944, 4.28049053, 4.31181127, 4.34390502,
4.37561590, 4.40541815, 4.43276143, 4.45517485, 4.48062290,
4.50726296, 4.53334164, 4.55737738, 4.57893875, 4.60371011,
4.62590595, 4.64885523, 4.67335218, 4.69475909, 4.71546469,
4.73676622, 4.75796550, 4.77690613, 4.79738177, 4.81947505,
4.84481408, 4.86954326, 4.89379651, 4.91771359, 4.94100213,
4.97169370, 5.00298714, 5.03525103, 5.07100437, 5.11028866,
5.15061419, 5.19925284, 5.24775527, 5.32086547, 5.49345120])
assert np.percentile(exp_def.input_flow,np.linspace(0,100,50)) == approx(
[ 80.06332381, 83.71770678, 85.93721426, 87.82355332,
89.52967083, 91.11148891, 92.60789787, 94.03509345,
95.41186604, 96.74598771, 98.04479355, 99.31122788,
100.55462827, 101.77808514, 102.98041937, 104.16517008,
105.33959643, 106.50165383, 107.65366600, 108.79827639,
109.93328795, 111.06707367, 112.19316408, 113.31493454,
114.43982739, 115.56182157, 116.68454749, 117.80620837,
118.93524815, 120.06752762, 121.20470208, 122.34781469,
123.50083542, 124.66086016, 125.83390291, 127.02146142,
128.22485477, 129.44681602, 130.68613841, 131.95601658,
133.25403807, 134.58951122, 135.96442305, 137.39295642,
138.89092217, 140.47204147, 142.17835057, 144.06540067,
146.28064479, 149.94588322])
def test_files_with_broken_scope():
try:
import core_files_demo
except:
import pytest
pytest.skip("core_files_demo not installed")
fx = core_files_demo.RoadTestFileModel(scope_file=emat.package_file(
'model', 'tests', 'road_test_corrupt2.yaml'))
design = fx.design_experiments(n_samples=2)
result = fx.run_experiments(design)
assert result['bogus_measure'].isna().all()
def test_multiple_connections():
import tempfile
with tempfile.TemporaryDirectory() as tempdir:
tempdbfile = os.path.join(tempdir, "test_db_file.db")
db_test = SQLiteDB(tempdbfile, initialize=True)
road_test_scope_file = emat.package_file("model", "tests",
"road_test.yaml")
s = emat.Scope(road_test_scope_file)
db_test.store_scope(s)
assert db_test.read_scope_names() == ["EMAT Road Test"]
db_test2 = SQLiteDB(tempdbfile, initialize=False)
with pytest.raises(KeyError):
db_test2.store_scope(s)
# Neither database is in a transaction
assert not db_test.conn.in_transaction
assert not db_test2.conn.in_transaction
from emat.model.core_python import Road_Capacity_Investment
m1 = emat.PythonCoreModel(Road_Capacity_Investment,
scope=s,
db=db_test)
m2 = emat.PythonCoreModel(Road_Capacity_Investment,
scope=s,
db=db_test2)
d1 = m1.design_experiments(n_samples=3,
random_seed=1,
design_name="d1")
d2 = m2.design_experiments(n_samples=3,
random_seed=2,
design_name="d2")
r1 = m1.run_experiments(design_name="d1")
r2 = m2.run_experiments(design_name="d2")
# Check each model can load the other's results
pd.testing.assert_frame_equal(
r1,
m2.db.read_experiment_all(scope_name=s.name,
design_name="d1",
ensure_dtypes=True)[r1.columns],
)
pd.testing.assert_frame_equal(
r2,
m1.db.read_experiment_all(scope_name=s.name,
design_name="d2",
ensure_dtypes=True)[r2.columns],
)
def test_box_universe(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
s = Box(name="Speedy", scope=scope)
s.set_upper_bound('build_travel_time', 70)
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
u = Boxes(s, s2, scope=scope)
assert u.fancy_names() == [
'Scope: EMAT Road Test', '▶ Speedy', '▷ ▶ Notable'
]
assert u.plain_names() == [None, 'Speedy', 'Notable']
def test_correlated_monte_carlo(self):
scope_file = emat.package_file("model", "tests", "road_test_corr.yaml")
scp = Scope(scope_file)
exp_def = scp.design_experiments(
n_samples_per_factor=100,
random_seed=1234,
sampler='mc',
)
assert len(exp_def) == scp.n_sample_factors() * 100
assert (exp_def['free_flow_time'] == 60).all()
assert (exp_def['initial_capacity'] == 100).all()
assert np.corrcoef([exp_def.alpha, exp_def.beta])[0, 1] == approx(0.75, rel=0.05)
assert np.corrcoef([exp_def.alpha, exp_def.expand_capacity])[0, 1] == approx(0.0, abs=0.05)
assert np.corrcoef([exp_def.input_flow, exp_def.value_of_time])[0, 1] == approx(-0.5, rel=0.05)
assert np.corrcoef([exp_def.unit_cost_expansion, exp_def.value_of_time])[0, 1] == approx(0.9, rel=0.05)
def test_feature_scoring_and_prim():
road_scope = emat.Scope(emat.package_file('model','tests','road_test.yaml'))
road_test = PythonCoreModel(Road_Capacity_Investment, scope=road_scope)
road_test_design = road_test.design_experiments(n_samples=5000, sampler='lhs')
road_test_results = road_test.run_experiments(design=road_test_design)
fs = feature_scores(road_scope, road_test_results, random_state=123)
assert isinstance(fs, pd.io.formats.style.Styler)
stable_df("./road_test_feature_scores_1.pkl.gz", fs.data)
prim1 = road_test_results.prim(target="net_benefits >= 0")
pbox1 = prim1.find_box()
assert pbox1._cur_box == 64
ts1 = prim1.tradeoff_selector()
assert len(ts1.data) == 1
assert ts1.data[0]['x'] == approx(np.asarray([
1., 1., 1., 1., 1.,
0.99928315, 0.99856631, 0.99784946, 0.99569892, 0.99283154,
0.98924731, 0.98351254, 0.97921147, 0.97491039, 0.96702509,
0.95555556, 0.94982079, 0.94336918, 0.92903226, 0.91182796,
0.89749104, 0.87598566, 0.85304659, 0.83942652, 0.83225806,
0.82078853, 0.79713262, 0.77706093, 0.76415771, 0.75483871,
0.74480287, 0.73261649, 0.71827957, 0.70394265, 0.68100358,
0.65663082, 0.63225806, 0.61003584, 0.59569892, 0.57992832,
0.55770609, 0.54193548, 0.52759857, 0.51111111, 0.49892473,
0.48960573, 0.4781362, 0.45878136, 0.44229391, 0.42365591,
0.409319, 0.39498208, 0.38064516, 0.36487455, 0.34767025,
0.33261649, 0.31756272, 0.30322581, 0.28888889, 0.27741935,
0.26379928, 0.25089606, 0.23942652, 0.22795699, 0.2172043]))
pbox1.select(40)
assert pbox1._cur_box == 40
assert ts1.data[0]['marker']['symbol'] == approx(np.asarray([
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]))
ebox1_40 = pbox1.to_emat_box()
assert ebox1_40.coverage == approx(0.5577060931899641)
assert ebox1_40.density == approx(0.8356605800214822)
assert ebox1_40.mass == approx(0.1862)
from emat import Bounds
assert ebox1_40.thresholds == {'beta': Bounds(lowerbound=3.597806324946271, upperbound=None),
'input_flow': Bounds(lowerbound=125, upperbound=None),
'value_of_time': Bounds(lowerbound=0.07705746291056698, upperbound=None),
'expand_capacity': Bounds(lowerbound=None, upperbound=95.01870815358643)}
pbox1.splom()
pbox1.hmm()
def test_box(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
with pytest.raises(TypeError):
s = Box(scope=scope)
s = Box(name="Speedy", scope=scope)
s.set_upper_bound('build_travel_time', 70)
with pytest.raises(ScopeError):
s.set_upper_bound('not_a_thing', 70)
assert len(s) == 1
assert 'build_travel_time' in s
assert s.parent_box_name is None
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
assert len(s2) == 1
assert 'build_travel_time' not in s2
assert s2.parent_box_name == 'Speedy'
def test_read_write_box(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s1.relevant_features.add('debt_type')
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
db.write_box(s1)
db.write_box(s2)
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.thresholds == s1_.thresholds
assert s2.thresholds == s2_.thresholds
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features
class TestCoreModelMethods(unittest.TestCase):
'''
tests model and meta-model methods
'''
corem_scope_file = emat.package_file("model", "tests",
"core_model_test.yaml")
scp = Scope(corem_scope_file)
corem = PythonCoreModel(
function=Dummy(),
configuration={
'archive_path':
config.get_subdir('test_directory', 'core_dummy_archive')
},
scope=scp,
)
# =============================================================================
#
# Core model tests
#
# =============================================================================
def test_create_scenario(self):
exp_def = {'Auto IVTT Sensitivity': 0.9122442817924445}
self.corem.setup(exp_def)
@pytest.mark.skip(reason="TODO")
def test_set_invalid_exp_variable(self):
exp_def = {'unsupported': 1}
with self.assertRaises(KeyError):
self.corem.setup(exp_def)
@pytest.mark.skip(reason="TODO")
def test_post_process(self):
exp_def = {'Land Use - CBD Focus': 1}
pm = ['Region-wide VMT']
self.corem.post_process(exp_def, pm)
@pytest.mark.skip(reason="TODO")
def test_archive_model(self):
exp_id = 1
archive_path = self.corem.get_exp_archive_path(self.scp.scp_name,
exp_id)
self.corem.archive(archive_path)
@pytest.mark.skip(reason="TODO")
def atest_hook_presence(self):
''' confirm that hooks are present for all performance measures, exp vars'''
# TODO
# set experiment variables
# post process
# import performance measure
@pytest.mark.skip(reason="TODO")
def test_pm_import(self):
pm = [
'Peak Walk-to-transit Boarding', 'Total LRT Boardings',
"PM Trip Time (minutes)", "Daily Transit Share",
"Households within 30 min of CBD",
"Number of Home-based work tours taking <= 45 minutes via transit",
"Downtown to Airport Travel Time", 'OD Volume District 1 to 1',
'190 Daily VHT'
]
pm_vals = self.corem.import_perf_meas(pm)
expected_pm = {
'Peak Walk-to-transit Boarding': 56247.88692999999,
'Total LRT Boardings': 24784.475588,
"PM Trip Time (minutes)": 15.652833,
"Daily Transit Share": 0.019905000000000003,
"Households within 30 min of CBD": 379894,
"Number of Home-based work tours taking <= 45 minutes via transit":
322069.75,
"Downtown to Airport Travel Time": 14.734342999999999,
'OD Volume District 1 to 1': 55642.74609400001,
'190 Daily VHT': 272612.499025
}
self.assertEqual(expected_pm, pm_vals)
# %% [markdown]
# The pattern has shifted, with the sine wave in *B* looking much more like the random noise,
# while the linear trend in *A* is now much more important in predicting the output, and
# the feature scores also shift to reflect this change.
# %% [markdown]
# ## Road Test Feature Scores
#
# We can apply the feature scoring methodology to the Road Test example
# in a similar fashion. To demonstrate scoring, we'll first load and run
# a sample set of experients.
# %%
from emat.model.core_python import Road_Capacity_Investment
road_scope = emat.Scope(emat.package_file('model','tests','road_test.yaml'))
road_test = PythonCoreModel(Road_Capacity_Investment, scope=road_scope)
road_test_design = road_test.design_experiments(sampler='lhs')
road_test_results = road_test.run_experiments(design=road_test_design)
# %% [raw] raw_mimetype="text/restructuredtext"
# With the experimental results in hand, we can use the same :func:`feature_scores`
# function to compute the scores.
# %%
feature_scores(road_scope, road_test_results)
# %% [raw] raw_mimetype="text/restructuredtext"
# By default, the :func:`feature_scores`
# function returns a stylized DataFrame, but we can also
# use the `return_type` argument to get a plain 'dataframe'
class TestMetaModelMethods(unittest.TestCase):
'''
tests model and meta-model methods
'''
metam_scope_file = emat.package_file("model","tests","metam_test.yaml")
metam_scp = Scope(metam_scope_file)
# =============================================================================
#
# Meta model tests
#
# =============================================================================
def test_derive_meta(self):
from emat.examples import road_test
s, db, m = road_test()
db.get_db_info()
m.design_experiments(n_samples=10, design_name='tiny')
db.read_experiment_all(None, None)
with pytest.raises(emat.PendingExperimentsError):
m.create_metamodel_from_design('tiny', random_state=123)
m.run_experiments(design_name='tiny')
mm = emat.create_metamodel(
m.scope,
db.read_experiment_all(s.name, 'tiny'),
random_state=123,
metamodel_id=db.get_new_metamodel_id(None),
)
mm.db = db # add db after creation to prevent writing it into the db
assert mm.scope == m.scope
tiny2 = m.design_experiments(n_samples=10, design_name='tiny2', random_seed=456)
assert tiny2.iloc[0]['debt_type'] == 'GO Bond'
stable_df('./test_tiny2.pkl.gz',tiny2)
result2 = mm.run_experiments('tiny2')
tiny2out = mm.read_experiment_measures('tiny2')
stable_df('./test_tiny2out.pkl.gz', tiny2out)
with pytest.raises(ValueError):
# no metamodels stored
mm3 = db.read_metamodel(None, None)
db.write_metamodel(mm)
mm2 = db.read_metamodel(None, 1)
mm3 = db.read_metamodel(None, None)
assert mm2 == mm == mm3
assert mm2 is not mm
print(mm2.function(**(tiny2.iloc[0])))
assert mm2.function(**(tiny2.iloc[0])) == approx({
'no_build_travel_time': 83.57502327972276,
'build_travel_time': 62.221693766038015,
'time_savings': 57.612063365257995,
'value_of_time_savings': 3749.2913256457214,
'net_benefits': 395.55020765212254,
'cost_of_capacity_expansion': 1252.6916865286616,
'present_cost_expansion': 23000.275573551233,
})
mm3.metamodel_id = db.get_new_metamodel_id(None)
db.write_metamodel(mm3)
with pytest.raises(ValueError):
# now too many to get without giving an ID
mm4 = db.read_metamodel(None, None)
def test_derive_meta_w_transform(self):
from emat.examples import road_test
s, db, m = road_test(yamlfile='road_test2.yaml')
db.get_db_info()
m.design_experiments(n_samples=10, design_name='tiny')
db.read_experiment_all(None, None)
with pytest.raises(emat.PendingExperimentsError):
m.create_metamodel_from_design('tiny', random_state=123)
m.run_experiments(design_name='tiny')
mm = emat.create_metamodel(
m.scope,
db.read_experiment_all(s.name, 'tiny'),
random_state=123,
metamodel_id=db.get_new_metamodel_id(None),
)
assert mm.scope != m.scope # now not equal as road_test2 has transforms that are stripped.
mm.db = db
tiny2 = m.design_experiments(n_samples=10, design_name='tiny2', random_seed=456)
assert tiny2.iloc[0]['debt_type'] == 'GO Bond'
assert dict(tiny2.iloc[0].drop('debt_type')) == approx({
'alpha': 0.10428005571929212,
'amortization_period': 33,
'beta': 4.8792451185772014,
'expand_capacity': 61.4210886403998,
'input_flow': 137,
'interest_rate': 0.03099304322197216,
'interest_rate_lock': 0,
'unit_cost_expansion': 121.85520427974882,
'value_of_time': 0.002953613029133872,
'yield_curve': 0.016255990123028242,
'free_flow_time': 60,
'initial_capacity': 100})
result2 = mm.run_experiments('tiny2')
# print(dict(mm.read_experiment_measures('tiny2').iloc[0]))
#
# print({
# 'no_build_travel_time': 81.6839454971052,
# 'build_travel_time': 61.91038371206646,
# 'time_savings': 44.94189289289446,
# 'value_of_time_savings': 2904.081661408463,
# 'net_benefits': -34.09931528157315,
# 'cost_of_capacity_expansion': 1085.3565091745982,
# 'present_cost_expansion': 19923.66625500023,
# })
#
assert dict(mm.read_experiment_measures('tiny2').iloc[0]) == approx({
'no_build_travel_time': 81.6839454971052,
'build_travel_time': 61.91038371206646,
'log_build_travel_time': 4.120826572003798,
'time_savings': 44.94189289289446,
'value_of_time_savings': 2904.081661408463,
'net_benefits': -34.09931528157315,
'cost_of_capacity_expansion': 1085.3565091745982,
'present_cost_expansion': 19923.66625500023,
})
assert m.run_experiment(tiny2.iloc[0]) == approx({
'no_build_travel_time': 89.07004237532217,
'build_travel_time': 62.81032484779827,
'log_build_travel_time': np.log(62.81032484779827),
'time_savings': 26.259717527523904,
'value_of_time_savings': 10.62586300480175,
'present_cost_expansion': 7484.479303360477,
'cost_of_capacity_expansion': 395.69034710662226,
'net_benefits': -385.0644841018205,
})
with pytest.raises(ValueError):
# no metamodels stored
mm3 = db.read_metamodel(None, None)
db.write_metamodel(mm)
mm2 = db.read_metamodel(None, 1)
mm3 = db.read_metamodel(None, None)
assert mm2 == mm == mm3
assert mm2 is not mm
assert mm2.function(**(tiny2.iloc[0])) == approx({
'no_build_travel_time': 81.6839454971052,
'build_travel_time': 61.91038371206646,
'log_build_travel_time': 4.120826572003798,
'time_savings': 44.94189289289446,
'value_of_time_savings': 2904.081661408463,
'net_benefits': -34.09931528157315,
'cost_of_capacity_expansion': 1085.3565091745982,
'present_cost_expansion': 19923.66625500023,
})
mm3.metamodel_id = db.get_new_metamodel_id(None)
db.write_metamodel(mm3)
with pytest.raises(ValueError):
# now too many to get without giving an ID
mm4 = db.read_metamodel(None, None)
def test_exogenously_stratified_k_fold(self):
from emat.learn.splits import ExogenouslyStratifiedKFold
X = np.arange(20)
Y = np.asarray([1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1])
S = np.asarray([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0])
correct = [np.array([0, 1, 2, 14, 15]),
np.array([3, 4, 5, 16]),
np.array([6, 7, 8, 17]),
np.array([9, 10, 11, 18]),
np.array([12, 13, 19])]
for j, (_, k) in zip(correct, ExogenouslyStratifiedKFold(n_splits=5, exo_data=S).split(X, Y)):
assert np.array_equal(j, k)
def test_read_db_gz():
road_test_scope_file = emat.package_file("model", "tests",
"road_test.yaml")
with pytest.raises(FileNotFoundError):
emat.Scope(emat.package_file("nope.yaml"))
s = emat.Scope(road_test_scope_file)
with pytest.raises(FileNotFoundError):
emat.SQLiteDB(emat.package_file("nope.db.gz"))
if not os.path.exists(emat.package_file("examples", "roadtest.db.gz")):
db_w = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.tmp"),
initialize=True)
s.store_scope(db_w)
s.design_experiments(n_samples=110,
random_seed=1234,
db=db_w,
design_name="lhs")
from emat.model.core_python import Road_Capacity_Investment
m_w = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db_w)
m_w.run_experiments(design_name="lhs", db=db_w)
db_w.conn.close()
import gzip
import shutil
with open(emat.package_file("examples", "roadtest.db.tmp"),
"rb") as f_in:
with gzip.open(emat.package_file("examples", "roadtest.db.gz"),
"wb") as f_out:
shutil.copyfileobj(f_in, f_out)
db = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.gz"))
assert repr(db) == '<emat.SQLiteDB with scope "EMAT Road Test">'
assert db.get_db_info()[:9] == "SQLite @ "
assert db.get_db_info()[-11:] == "roadtest.db"
assert db.read_scope_names() == ["EMAT Road Test"]
s1 = db.read_scope("EMAT Road Test")
assert type(s1) == type(s)
for k in ("_x_list", "_l_list", "_c_list", "_m_list", "name", "desc"):
assert getattr(s, k) == getattr(s1, k), k
assert s == s1
experiments = db.read_experiment_all("EMAT Road Test", "lhs")
assert experiments.shape == (110, 20)
assert list(experiments.columns) == [
"free_flow_time",
"initial_capacity",
"alpha",
"beta",
"input_flow",
"value_of_time",
"unit_cost_expansion",
"interest_rate",
"yield_curve",
"expand_capacity",
"amortization_period",
"debt_type",
"interest_rate_lock",
"no_build_travel_time",
"build_travel_time",
"time_savings",
"value_of_time_savings",
"net_benefits",
"cost_of_capacity_expansion",
"present_cost_expansion",
]
from emat.model.core_python import Road_Capacity_Investment
m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db)
assert m.metamodel_id == None
class TestMetaModelMethods(unittest.TestCase):
'''
tests model and meta-model methods
'''
metam_scope_file = emat.package_file("model", "tests", "metam_test.yaml")
metam_scp = Scope(metam_scope_file)
# =============================================================================
#
# Meta model tests
#
# =============================================================================
def test_derive_meta(self):
from emat.examples import road_test
s, db, m = road_test()
db.get_db_info()
m.design_experiments(n_samples=10, design_name='tiny')
db.read_experiment_all(None, None)
with pytest.raises(emat.PendingExperimentsError):
m.create_metamodel_from_design('tiny', random_state=123)
m.run_experiments(design_name='tiny')
mm = m.create_metamodel_from_design('tiny', random_state=123)
assert mm.scope == m.scope
tiny2 = m.design_experiments(n_samples=10,
design_name='tiny2',
random_seed=456)
assert tiny2.iloc[0]['debt_type'] == 'GO Bond'
assert dict(tiny2.iloc[0].drop('debt_type')) == approx({
'alpha':
0.10428005571929212,
'amortization_period':
33,
'beta':
4.8792451185772014,
'expand_capacity':
61.4210886403998,
'input_flow':
137,
'interest_rate':
0.03099304322197216,
'interest_rate_lock':
0,
'unit_cost_expansion':
121.85520427974882,
'value_of_time':
0.002953613029133872,
'yield_curve':
0.016255990123028242,
'free_flow_time':
60,
'initial_capacity':
100
})
result2 = mm.run_experiments('tiny2')
assert dict(mm.read_experiment_measures('tiny2').iloc[0]) == approx({
'no_build_travel_time':
81.6839454971052,
'build_travel_time':
61.91038371206646,
'time_savings':
44.94189289289446,
'value_of_time_savings':
2904.081661408463,
'net_benefits':
-34.09931528157315,
'cost_of_capacity_expansion':
1085.3565091745982,
'present_cost_expansion':
19923.66625500023
})
assert m.function(**(tiny2.iloc[0])) == approx({
'no_build_travel_time':
89.07004237532217,
'build_travel_time':
62.81032484779827,
'time_savings':
26.259717527523904,
'value_of_time_savings':
10.62586300480175,
'present_cost_expansion':
7484.479303360477,
'cost_of_capacity_expansion':
395.69034710662226,
'net_benefits':
-385.0644841018205
})
with pytest.raises(ValueError):
# no metamodels stored
mm3 = db.read_metamodel(None, None)
db.write_metamodel(mm)
mm2 = db.read_metamodel(None, 1)
mm3 = db.read_metamodel(None, None)
assert mm2 == mm == mm3
assert mm2 is not mm
assert mm2.function(**(tiny2.iloc[0])) == approx({
'no_build_travel_time':
81.6839454971052,
'build_travel_time':
61.91038371206646,
'time_savings':
44.94189289289446,
'value_of_time_savings':
2904.081661408463,
'net_benefits':
-34.09931528157315,
'cost_of_capacity_expansion':
1085.3565091745982,
'present_cost_expansion':
19923.66625500023
})
mm3.metamodel_id = db.get_new_metamodel_id(None)
db.write_metamodel(mm3)
with pytest.raises(ValueError):
# now too many to get without giving an ID
mm4 = db.read_metamodel(None, None)
def test_derive_meta_w_transform(self):
from emat.examples import road_test
s, db, m = road_test(yamlfile='road_test2.yaml')
db.get_db_info()
m.design_experiments(n_samples=10, design_name='tiny')
db.read_experiment_all(None, None)
with pytest.raises(emat.PendingExperimentsError):
m.create_metamodel_from_design('tiny', random_state=123)
m.run_experiments(design_name='tiny')
mm = m.create_metamodel_from_design('tiny', random_state=123)
assert mm.scope != m.scope # now not equal as road_test2 has transforms that are stripped.
tiny2 = m.design_experiments(n_samples=10,
design_name='tiny2',
random_seed=456)
assert tiny2.iloc[0]['debt_type'] == 'GO Bond'
assert dict(tiny2.iloc[0].drop('debt_type')) == approx({
'alpha':
0.10428005571929212,
'amortization_period':
33,
'beta':
4.8792451185772014,
'expand_capacity':
61.4210886403998,
'input_flow':
137,
'interest_rate':
0.03099304322197216,
'interest_rate_lock':
0,
'unit_cost_expansion':
121.85520427974882,
'value_of_time':
0.002953613029133872,
'yield_curve':
0.016255990123028242,
'free_flow_time':
60,
'initial_capacity':
100
})
result2 = mm.run_experiments('tiny2')
# print(dict(mm.read_experiment_measures('tiny2').iloc[0]))
#
# print({
# 'no_build_travel_time': 81.6839454971052,
# 'build_travel_time': 61.91038371206646,
# 'time_savings': 44.94189289289446,
# 'value_of_time_savings': 2904.081661408463,
# 'net_benefits': -34.09931528157315,
# 'cost_of_capacity_expansion': 1085.3565091745982,
# 'present_cost_expansion': 19923.66625500023,
# })
#
assert dict(mm.read_experiment_measures('tiny2').iloc[0]) == approx({
'no_build_travel_time':
81.6839454971052,
'build_travel_time':
61.91038371206646,
'log_build_travel_time':
4.120826572003798,
'time_savings':
44.94189289289446,
'value_of_time_savings':
2904.081661408463,
'net_benefits':
-34.09931528157315,
'cost_of_capacity_expansion':
1085.3565091745982,
'present_cost_expansion':
19923.66625500023,
})
assert m.run_experiment(tiny2.iloc[0]) == approx({
'no_build_travel_time':
89.07004237532217,
'build_travel_time':
62.81032484779827,
'log_build_travel_time':
np.log(62.81032484779827),
'time_savings':
26.259717527523904,
'value_of_time_savings':
10.62586300480175,
'present_cost_expansion':
7484.479303360477,
'cost_of_capacity_expansion':
395.69034710662226,
'net_benefits':
-385.0644841018205,
})
with pytest.raises(ValueError):
# no metamodels stored
mm3 = db.read_metamodel(None, None)
db.write_metamodel(mm)
mm2 = db.read_metamodel(None, 1)
mm3 = db.read_metamodel(None, None)
assert mm2 == mm == mm3
assert mm2 is not mm
assert mm2.function(**(tiny2.iloc[0])) == approx({
'no_build_travel_time':
81.6839454971052,
'build_travel_time':
61.91038371206646,
'log_build_travel_time':
4.120826572003798,
'time_savings':
44.94189289289446,
'value_of_time_savings':
2904.081661408463,
'net_benefits':
-34.09931528157315,
'cost_of_capacity_expansion':
1085.3565091745982,
'present_cost_expansion':
19923.66625500023,
})
mm3.metamodel_id = db.get_new_metamodel_id(None)
db.write_metamodel(mm3)
with pytest.raises(ValueError):
# now too many to get without giving an ID
mm4 = db.read_metamodel(None, None)
def test_read_db_gz(self):
road_test_scope_file = emat.package_file('model', 'tests',
'road_test.yaml')
with pytest.raises(FileNotFoundError):
emat.Scope(emat.package_file('nope.yaml'))
s = emat.Scope(road_test_scope_file)
with pytest.raises(FileNotFoundError):
emat.SQLiteDB(emat.package_file('nope.db.gz'))
if not os.path.exists(emat.package_file("examples", "roadtest.db.gz")):
db_w = emat.SQLiteDB(emat.package_file("examples",
"roadtest.db.tmp"),
initialize=True)
s.store_scope(db_w)
s.design_experiments(n_samples=110,
random_seed=1234,
db=db_w,
design_name='lhs')
from emat.model.core_python import Road_Capacity_Investment
m_w = emat.PythonCoreModel(Road_Capacity_Investment,
scope=s,
db=db_w)
m_w.run_experiments(design_name='lhs', db=db_w)
db_w.conn.close()
import gzip
import shutil
with open(emat.package_file("examples", "roadtest.db.tmp"),
'rb') as f_in:
with gzip.open(emat.package_file("examples", "roadtest.db.gz"),
'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
db = emat.SQLiteDB(emat.package_file("examples", "roadtest.db.gz"))
assert repr(db) == '<emat.SQLiteDB with scope "EMAT Road Test">'
assert db.get_db_info()[:9] == 'SQLite @ '
assert db.get_db_info()[-11:] == 'roadtest.db'
assert db.read_scope_names() == ['EMAT Road Test']
s1 = db.read_scope('EMAT Road Test')
assert type(s1) == type(s)
for k in ('_x_list', '_l_list', '_c_list', '_m_list', 'name', 'desc'):
assert getattr(s, k) == getattr(s1, k), k
assert s == s1
experiments = db.read_experiment_all('EMAT Road Test', 'lhs')
assert experiments.shape == (110, 20)
assert list(experiments.columns) == [
'free_flow_time',
'initial_capacity',
'alpha',
'beta',
'input_flow',
'value_of_time',
'unit_cost_expansion',
'interest_rate',
'yield_curve',
'expand_capacity',
'amortization_period',
'debt_type',
'interest_rate_lock',
'no_build_travel_time',
'build_travel_time',
'time_savings',
'value_of_time_savings',
'net_benefits',
'cost_of_capacity_expansion',
'present_cost_expansion',
]
from emat.model.core_python import Road_Capacity_Investment
m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db)
assert m.metamodel_id == None
'initial_capacity',
'alpha',
'beta',
'input_flow',
'value_of_time',
'unit_cost_expansion',
'interest_rate',
'yield_curve',
'expand_capacity',
'amortization_period',
'debt_type',
'interest_rate_lock',
'no_build_travel_time',
'build_travel_time',
'time_savings',
'value_of_time_savings',
'net_benefits',
'cost_of_capacity_expansion',
'present_cost_expansion',
]
from emat.model.core_python import Road_Capacity_Investment
m = emat.PythonCoreModel(Road_Capacity_Investment, scope=s, db=db)
assert m.metamodel_id == None
emat.package_file('model', 'tests', 'road_test.yaml')
if __name__ == '__main__':
unittest.main()
class TestExperimentMethods(unittest.TestCase):
'''
tests generating experiments
'''
#
# one time test setup
#
scope_file = emat.package_file("scope", "tests", "scope_test.yaml")
scp = Scope(scope_file)
db_test = SQLiteDB(":memory:", initialize=True)
scp.store_scope(db_test)
def test_latin_hypercube(self):
exp_def = self.scp.design_experiments(
n_samples_per_factor=10,
random_seed=1234,
sampler='lhs',
db=self.db_test,
)
assert len(exp_def) == self.scp.n_sample_factors() * 10
assert (exp_def['TestRiskVar'] == 1.0).all()
assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.0326,
abs=1e-3)
assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=1e-3)
exp_def2 = self.db_test.read_experiment_parameters(
self.scp.name, 'lhs')
assert (exp_def[exp_def2.columns] == exp_def2).all().all()
def test_latin_hypercube_not_joint(self):
exp_def = self.scp.design_experiments(
n_samples_per_factor=5,
random_seed=1234,
sampler='lhs',
db=self.db_test,
jointly=False,
design_name='lhs_not_joint',
)
# assert len(exp_def) == len(self.scp.get_uncertainties())*5 * len(self.scp.get_levers())*5
assert len(
exp_def
) == 80 # there are only 4 unique policies, times 2 scenarios
assert (exp_def['TestRiskVar'] == 1.0).all()
assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.033,
abs=1e-2)
assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=1e-2)
exp_def2 = self.db_test.read_experiment_parameters(
self.scp.name, 'lhs_not_joint')
assert (exp_def[exp_def2.columns] == exp_def2).all().all()
def test_monte_carlo(self):
exp_def = self.scp.design_experiments(
n_samples_per_factor=10,
random_seed=1234,
sampler='mc',
db=self.db_test,
)
assert len(exp_def) == self.scp.n_sample_factors() * 10
assert (exp_def['TestRiskVar'] == 1.0).all()
assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.0326,
abs=0.01)
assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=0.01)
exp_def2 = self.db_test.read_experiment_parameters(self.scp.name, 'mc')
assert (exp_def[exp_def2.columns] == exp_def2).all().all()
def test_sensitivity_tests(self):
exp_def = self.scp.design_experiments(
sampler='uni',
db=self.db_test,
)
cols = [
'TestRiskVar', 'Land Use - CBD Focus', 'Freeway Capacity',
'Auto IVTT Sensitivity', 'Shared Mobility',
'Kensington Decommissioning', 'LRT Extension'
]
correct = '{"TestRiskVar":{"0":1.0,"1":1.0,"2":1.0,"3":1.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \
'"Land Use - CBD Focus":{"0":1.0,"1":0.82,"2":1.37,"3":1.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \
'"Freeway Capacity":{"0":1.0,"1":1.0,"2":1.0,"3":2.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \
'"Auto IVTT Sensitivity":{"0":1.0,"1":1.0,"2":1.0,"3":1.0,"4":0.75,"5":1.0,"6":1.0,"7":1.0},' \
'"Shared Mobility":{"0":0.0,"1":0.0,"2":0.0,"3":0.0,"4":0.0,"5":1.0,"6":0.0,"7":0.0},' \
'"Kensington Decommissioning":{"0":false,"1":false,"2":false,"3":false,"4":false,' \
'"5":false,"6":true,"7":false},"LRT Extension":{"0":false,"1":false,"2":false,"3":false,' \
'"4":false,"5":false,"6":false,"7":true}}'
correct = pd.read_json(correct)
for k in cols:
assert (exp_def[k].values == approx(correct[k].values))
exp_def2 = self.db_test.read_experiment_parameters(
self.scp.name, 'uni')
for k in cols:
assert (exp_def2[k].values == approx(correct[k].values))
def test_nonuniform_latin_hypercube(self):
scope_file = emat.package_file("model", "tests",
"road_test_nonuniform.yaml")
scp = Scope(scope_file)
exp_def = scp.design_experiments(
n_samples_per_factor=1000,
random_seed=1234,
sampler='lhs',
)
assert len(exp_def) == scp.n_sample_factors() * 1000
assert (exp_def['free_flow_time'] == 60).all()
assert (exp_def['initial_capacity'] == 100).all()
assert np.corrcoef([exp_def.alpha,
exp_def.beta])[0, 1] == approx(0.75, rel=0.05)
assert np.corrcoef([exp_def.alpha,
exp_def.expand_capacity])[0, 1] == approx(0.0,
abs=0.02)
assert np.corrcoef([exp_def.input_flow,
exp_def.value_of_time])[0, 1] == approx(-0.5,
rel=0.05)
assert np.corrcoef(
[exp_def.unit_cost_expansion,
exp_def.value_of_time])[0, 1] == approx(0.9, rel=0.05)
assert exp_def.interest_rate_lock.sum() == approx(len(exp_def) * 0.2)
assert np.percentile(exp_def.alpha, np.linspace(0, 100, 50)) == approx(
[
0.10037393, 0.10722119, 0.10994485, 0.11204394, 0.11383709,
0.11544182, 0.11691345, 0.11829399, 0.11959909, 0.12084863,
0.12205279, 0.12321800, 0.12435285, 0.12546474, 0.12655958,
0.12763503, 0.12869873, 0.12975137, 0.13079620, 0.13183375,
0.13287082, 0.13390854, 0.13494651, 0.13598528, 0.13703149,
0.13808180, 0.13914411, 0.14021784, 0.14130323, 0.14240609,
0.14352608, 0.14466756, 0.14583411, 0.14702908, 0.14825720,
0.14951875, 0.15082844, 0.15218376, 0.15359963, 0.15508120,
0.15664534, 0.15831425, 0.16010073, 0.16203921, 0.16418886,
0.16662357, 0.16946999, 0.17301416, 0.17804383, 0.19662857
])
assert np.percentile(exp_def.beta, np.linspace(0, 100, 50)) == approx([
3.51654751, 3.72503059, 3.82437701, 3.90088124, 3.96222432,
4.01360346, 4.06112277, 4.10640347, 4.14456476, 4.18084719,
4.21812584, 4.24926944, 4.28049053, 4.31181127, 4.34390502,
4.37561590, 4.40541815, 4.43276143, 4.45517485, 4.48062290,
4.50726296, 4.53334164, 4.55737738, 4.57893875, 4.60371011,
4.62590595, 4.64885523, 4.67335218, 4.69475909, 4.71546469,
4.73676622, 4.75796550, 4.77690613, 4.79738177, 4.81947505,
4.84481408, 4.86954326, 4.89379651, 4.91771359, 4.94100213,
4.97169370, 5.00298714, 5.03525103, 5.07100437, 5.11028866,
5.15061419, 5.19925284, 5.24775527, 5.32086547, 5.49345120
])
assert np.percentile(exp_def.input_flow, np.linspace(
0, 100, 50)) == approx([
80.06332381, 83.71770678, 85.93721426, 87.82355332,
89.52967083, 91.11148891, 92.60789787, 94.03509345,
95.41186604, 96.74598771, 98.04479355, 99.31122788,
100.55462827, 101.77808514, 102.98041937, 104.16517008,
105.33959643, 106.50165383, 107.65366600, 108.79827639,
109.93328795, 111.06707367, 112.19316408, 113.31493454,
114.43982739, 115.56182157, 116.68454749, 117.80620837,
118.93524815, 120.06752762, 121.20470208, 122.34781469,
123.50083542, 124.66086016, 125.83390291, 127.02146142,
128.22485477, 129.44681602, 130.68613841, 131.95601658,
133.25403807, 134.58951122, 135.96442305, 137.39295642,
138.89092217, 140.47204147, 142.17835057, 144.06540067,
146.28064479, 149.94588322
])
def test_road_test(self):
import os
test_dir = os.path.dirname(__file__)
os.chdir(test_dir)
road_test_scope_file = emat.package_file('model', 'tests', 'road_test.yaml')
road_scope = emat.Scope(road_test_scope_file)
# <emat.Scope with 2 constants, 7 uncertainties, 4 levers, 7 measures>
assert len(road_scope.get_measures()) == 7
assert len(road_scope.get_levers()) == 4
assert len(road_scope.get_uncertainties()) == 7
assert len(road_scope.get_constants()) == 2
emat_db = emat.SQLiteDB()
road_scope.store_scope(emat_db)
with pytest.raises(KeyError):
road_scope.store_scope(emat_db)
assert emat_db.read_scope_names() == ['EMAT Road Test']
design = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs')
design.head()
large_design = design_experiments(road_scope, db=emat_db, n_samples=5000, sampler='lhs',
design_name='lhs_large')
large_design.head()
assert list(large_design.columns) == [
'alpha',
'amortization_period',
'beta',
'debt_type',
'expand_capacity',
'input_flow',
'interest_rate',
'interest_rate_lock',
'unit_cost_expansion',
'value_of_time',
'yield_curve',
'free_flow_time',
'initial_capacity',
]
assert list(large_design.head().index) == [111, 112, 113, 114, 115]
assert emat_db.read_design_names('EMAT Road Test') == ['lhs', 'lhs_large']
m = PythonCoreModel(Road_Capacity_Investment, scope=road_scope, db=emat_db)
with SequentialEvaluator(m) as eval_seq:
lhs_results = m.run_experiments(design_name='lhs', evaluator=eval_seq)
lhs_results.head()
assert lhs_results.head()['present_cost_expansion'].values == approx(
[2154.41598475, 12369.38053473, 4468.50683924, 6526.32517089, 2460.91070514])
assert lhs_results.head()['net_benefits'].values == approx(
[ -22.29090499, -16.84301382, -113.98841188, 11.53956058, 78.03661612])
assert lhs_results.tail()['present_cost_expansion'].values == approx(
[2720.51645703, 4000.91232689, 6887.83193063, 3739.47839941, 1582.52899124])
assert lhs_results.tail()['net_benefits'].values == approx(
[841.46278175, -146.71279267, -112.5681036, 25.48055303, 127.31154155])
with SequentialEvaluator(m) as eval_seq:
lhs_large_results = m.run_experiments(design_name='lhs_large', evaluator=eval_seq)
lhs_large_results.head()
assert lhs_large_results.head()['net_benefits'].values == approx(
[-522.45283083, -355.1599307 , -178.6623215 , 23.46263498, -301.17700968])
lhs_outcomes = m.read_experiment_measures(design_name='lhs')
assert lhs_outcomes.head()['time_savings'].values == approx(
[13.4519273, 26.34172999, 12.48385198, 15.10165981, 15.48056139])
scores = m.get_feature_scores('lhs', random_state=123)
stable_df("./road_test_feature_scores.pkl.gz", scores.data)
from emat.workbench.analysis import prim
x = m.read_experiment_parameters(design_name='lhs_large')
prim_alg = prim.Prim(
m.read_experiment_parameters(design_name='lhs_large'),
m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0,
threshold=0.4,
)
box1 = prim_alg.find_box()
stable_df("./road_test_box1_peeling_trajectory.pkl.gz", box1.peeling_trajectory)
from emat.util.xmle import Show
from emat.util.xmle.elem import Elem
assert isinstance(Show(box1.show_tradeoff()), Elem)
from emat.workbench.analysis import cart
cart_alg = cart.CART(
m.read_experiment_parameters(design_name='lhs_large'),
m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0,
)
cart_alg.build_tree()
stable_df("./road_test_cart_box0.pkl.gz", cart_alg.boxes[0])
cart_dict = dict(cart_alg.boxes[0].iloc[0])
assert cart_dict['debt_type'] == {'GO Bond', 'Paygo', 'Rev Bond'}
#assert cart_dict['interest_rate_lock'] == {False, True}
assert isinstance(Show(cart_alg.show_tree(format='svg')), Elem)
from emat import Measure
MAXIMIZE = Measure.MAXIMIZE
MINIMIZE = Measure.MINIMIZE
robustness_functions = [
Measure(
'Expected Net Benefit',
kind=Measure.INFO,
variable_name='net_benefits',
function=numpy.mean,
# min=-150,
# max=50,
),
Measure(
'Probability of Net Loss',
kind=MINIMIZE,
variable_name='net_benefits',
function=lambda x: numpy.mean(x < 0),
min=0,
max=1,
),
Measure(
'95%ile Travel Time',
kind=MINIMIZE,
variable_name='build_travel_time',
function=functools.partial(numpy.percentile, q=95),
min=60,
max=150,
),
Measure(
'99%ile Present Cost',
kind=Measure.INFO,
variable_name='present_cost_expansion',
function=functools.partial(numpy.percentile, q=99),
# min=0,
# max=10,
),
Measure(
'Expected Present Cost',
kind=Measure.INFO,
variable_name='present_cost_expansion',
function=numpy.mean,
# min=0,
# max=10,
),
]
from emat import Constraint
constraint_1 = Constraint(
"Maximum Log Expected Present Cost",
outcome_names="Expected Present Cost",
function=Constraint.must_be_less_than(4000),
)
constraint_2 = Constraint(
"Minimum Capacity Expansion",
parameter_names="expand_capacity",
function=Constraint.must_be_greater_than(10),
)
constraint_3 = Constraint(
"Maximum Paygo",
parameter_names='debt_type',
outcome_names='99%ile Present Cost',
function=lambda i, j: max(0, j - 1500) if i == 'Paygo' else 0,
)
from emat.optimization import HyperVolume, EpsilonProgress, SolutionViewer, ConvergenceMetrics
convergence_metrics = ConvergenceMetrics(
HyperVolume.from_outcomes(robustness_functions),
EpsilonProgress(),
SolutionViewer.from_model_and_outcomes(m, robustness_functions),
)
with SequentialEvaluator(m) as eval_seq:
robust = m.robust_optimize(
robustness_functions,
scenarios=20,
nfe=5,
constraints=[
constraint_1,
constraint_2,
constraint_3,
],
epsilons=[0.05, ] * len(robustness_functions),
convergence=convergence_metrics,
evaluator=eval_seq,
)
robust_results, convergence = robust.result, robust.convergence
assert isinstance(robust_results, pandas.DataFrame)
mm = m.create_metamodel_from_design('lhs')
design2 = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs', random_seed=2)
design2_results = mm.run_experiments(design2)
def test_database_merging():
import emat
road_test_scope_file = emat.package_file("model", "tests",
"road_test.yaml")
road_scope = emat.Scope(road_test_scope_file)
emat_db = emat.SQLiteDB()
road_scope.store_scope(emat_db)
assert emat_db.read_scope_names() == ["EMAT Road Test"]
from emat.experiment.experimental_design import design_experiments
design = design_experiments(road_scope,
db=emat_db,
n_samples_per_factor=10,
sampler="lhs")
large_design = design_experiments(road_scope,
db=emat_db,
n_samples=500,
sampler="lhs",
design_name="lhs_large")
assert emat_db.read_design_names("EMAT Road Test") == ["lhs", "lhs_large"]
from emat.model.core_python import PythonCoreModel, Road_Capacity_Investment
m = PythonCoreModel(Road_Capacity_Investment, scope=road_scope, db=emat_db)
lhs_results = m.run_experiments(design_name="lhs")
lhs_large_results = m.run_experiments(design_name="lhs_large")
reload_results = m.read_experiments(design_name="lhs")
pd.testing.assert_frame_equal(
reload_results,
lhs_results,
check_like=True,
)
lhs_params = m.read_experiment_parameters(design_name="lhs")
assert len(lhs_params) == 110
assert len(lhs_params.columns) == 13
lhs_outcomes = m.read_experiment_measures(design_name="lhs")
assert len(lhs_outcomes) == 110
assert len(lhs_outcomes.columns) == 7
mm = m.create_metamodel_from_design("lhs")
assert mm.metamodel_id == 1
assert isinstance(mm.function, emat.MetaModel)
design2 = design_experiments(road_scope,
db=emat_db,
n_samples_per_factor=10,
sampler="lhs",
random_seed=2)
design2_results = mm.run_experiments(design2)
assert len(design2_results) == 110
assert len(design2_results.columns) == 20
assert emat_db.read_design_names(None) == ["lhs", "lhs_2", "lhs_large"]
check = emat_db.read_experiment_measures(None, "lhs_2")
assert len(check) == 110
assert len(check.columns) == 7
assert emat_db.read_experiment_measure_sources(None, "lhs_2") == [1]
m.allow_short_circuit = False
design2_results0 = m.run_experiments(design2.iloc[:5])
assert len(design2_results0) == 5
assert len(design2_results0.columns) == 20
with pytest.raises(ValueError):
# now there are two sources of some measures
emat_db.read_experiment_measures(None, "lhs_2")
assert set(emat_db.read_experiment_measure_sources(None,
"lhs_2")) == {0, 1}
check = emat_db.read_experiment_measures(None, "lhs_2", source=0)
assert len(check) == 5
check = emat_db.read_experiment_measures(None, "lhs_2", source=1)
assert len(check) == 110
import emat.examples
s2, db2, m2 = emat.examples.road_test()
# write the design for lhs_2 into a different database.
# it ends up giving different experient id's to these, which is fine.
db2.write_experiment_parameters(
None, "lhs_2", emat_db.read_experiment_parameters(None, "lhs_2"))
check = db2.read_experiment_parameters(
None,
"lhs_2",
)
assert len(check) == 110
assert len(check.columns) == 13
pd.testing.assert_frame_equal(
design2.reset_index(drop=True),
check.reset_index(drop=True),
check_like=True,
)
design2_results2 = m2.run_experiments("lhs_2")
check = emat_db.read_experiment_measures(None, "lhs_2", source=0)
assert len(check) == 5
assert len(check.columns) == 7
check = emat_db.read_experiment_measures(None, "lhs_2", runs="valid")
assert len(check) == 115
emat_db.merge_database(db2)
check = emat_db.read_experiment_measures(None, "lhs_2", source=0)
assert len(check) == 110
assert len(check.columns) == 7
check = emat_db.read_experiment_measures(None, "lhs_2", runs="valid")
assert len(check) == 225
def test_road_test(self):
road_test_scope_file = emat.package_file('model', 'tests', 'road_test.yaml')
road_scope = emat.Scope(road_test_scope_file)
# <emat.Scope with 2 constants, 7 uncertainties, 4 levers, 7 measures>
assert len(road_scope.get_measures()) == 7
assert len(road_scope.get_levers()) == 4
assert len(road_scope.get_uncertainties()) == 7
assert len(road_scope.get_constants()) == 2
emat_db = emat.SQLiteDB()
road_scope.store_scope(emat_db)
with pytest.raises(KeyError):
road_scope.store_scope(emat_db)
assert emat_db.read_scope_names() == ['EMAT Road Test']
design = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs')
design.head()
large_design = design_experiments(road_scope, db=emat_db, n_samples=5000, sampler='lhs',
design_name='lhs_large')
large_design.head()
assert list(large_design.columns) == [
'alpha',
'amortization_period',
'beta',
'debt_type',
'expand_capacity',
'input_flow',
'interest_rate',
'interest_rate_lock',
'unit_cost_expansion',
'value_of_time',
'yield_curve',
'free_flow_time',
'initial_capacity',
]
assert list(large_design.head().index) == [111, 112, 113, 114, 115]
assert emat_db.read_design_names('EMAT Road Test') == ['lhs', 'lhs_large']
m = PythonCoreModel(Road_Capacity_Investment, scope=road_scope, db=emat_db)
with SequentialEvaluator(m) as eval_seq:
lhs_results = m.run_experiments(design_name='lhs', evaluator=eval_seq)
lhs_results.head()
assert lhs_results.head()['present_cost_expansion'].values == approx(
[2154.41598475, 12369.38053473, 4468.50683924, 6526.32517089, 2460.91070514])
assert lhs_results.head()['net_benefits'].values == approx(
[-79.51551505, -205.32148044, -151.94431822, -167.62487134, -3.97293985])
with SequentialEvaluator(m) as eval_seq:
lhs_large_results = m.run_experiments(design_name='lhs_large', evaluator=eval_seq)
lhs_large_results.head()
assert lhs_large_results.head()['net_benefits'].values == approx(
[-584.36098322, -541.5458395, -185.16661464, -135.85689709, -357.36106457])
lhs_outcomes = m.read_experiment_measures(design_name='lhs')
assert lhs_outcomes.head()['time_savings'].values == approx(
[13.4519273, 26.34172999, 12.48385198, 15.10165981, 15.48056139])
correct_scores = numpy.array(
[[0.06603461, 0.04858595, 0.06458574, 0.03298163, 0.05018515, 0., 0., 0.53156587, 0.05060416, 0.02558088,
0.04676956, 0.04131266, 0.04179378],
[0.06003223, 0.04836434, 0.06059554, 0.03593644, 0.27734396, 0., 0., 0.28235419, 0.05303979, 0.03985181,
0.04303371, 0.05004349, 0.04940448],
[0.08760605, 0.04630414, 0.0795043, 0.03892201, 0.10182534, 0., 0., 0.42508457, 0.04634321, 0.03216387,
0.0497183, 0.04953772, 0.0429905],
[0.08365598, 0.04118732, 0.06716887, 0.03789444, 0.06509519, 0., 0., 0.31494171, 0.06517462, 0.02895742,
0.04731707, 0.17515158, 0.07345581],
[0.06789382, 0.07852257, 0.05066944, 0.04807088, 0.32054735, 0., 0., 0.15953055, 0.05320201, 0.02890069,
0.07033928, 0.06372418, 0.05859923],
[0.05105435, 0.09460353, 0.04614178, 0.04296901, 0.45179611, 0., 0., 0.04909801, 0.05478798, 0.023099,
0.08160785, 0.05642169, 0.04842069],
[0.04685703, 0.03490931, 0.03214081, 0.03191602, 0.56130318, 0., 0., 0.04011044, 0.04812986, 0.02228924,
0.09753361, 0.04273004, 0.04208045], ])
scores = m.get_feature_scores('lhs', random_state=123)
for _i in range(scores.metadata.values.shape[0]):
for _j in range(scores.metadata.values.shape[1]):
assert scores.metadata.values[_i,_j] == approx(correct_scores[_i,_j], rel=.1)
from ema_workbench.analysis import prim
x = m.read_experiment_parameters(design_name='lhs_large')
prim_alg = prim.Prim(
m.read_experiment_parameters(design_name='lhs_large'),
m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0,
threshold=0.4,
)
box1 = prim_alg.find_box()
assert dict(box1.peeling_trajectory.iloc[45]) == approx({
'coverage': 0.8014705882352942,
'density': 0.582109479305741,
'id': 45,
'mass': 0.1498,
'mean': 0.582109479305741,
'res_dim': 4,
})
from emat.util.xmle import Show
from emat.util.xmle.elem import Elem
assert isinstance(Show(box1.show_tradeoff()), Elem)
from ema_workbench.analysis import cart
cart_alg = cart.CART(
m.read_experiment_parameters(design_name='lhs_large'),
m.read_experiment_measures(design_name='lhs_large')['net_benefits'] > 0,
)
cart_alg.build_tree()
cart_dict = dict(cart_alg.boxes[0].iloc[0])
assert cart_dict['debt_type'] == {'GO Bond', 'Paygo', 'Rev Bond'}
assert cart_dict['interest_rate_lock'] == {False, True}
del cart_dict['debt_type']
del cart_dict['interest_rate_lock']
assert cart_dict == approx({
'free_flow_time': 60,
'initial_capacity': 100,
'alpha': 0.10001988547129116,
'beta': 3.500215589924521,
'input_flow': 80.0,
'value_of_time': 0.00100690634109406,
'unit_cost_expansion': 95.00570832093116,
'interest_rate': 0.0250022738169142,
'yield_curve': -0.0024960505548531774,
'expand_capacity': 0.0006718732232418368,
'amortization_period': 15,
})
assert isinstance(Show(cart_alg.show_tree(format='svg')), Elem)
from emat import Measure
MAXIMIZE = Measure.MAXIMIZE
MINIMIZE = Measure.MINIMIZE
robustness_functions = [
Measure(
'Expected Net Benefit',
kind=Measure.INFO,
variable_name='net_benefits',
function=numpy.mean,
# min=-150,
# max=50,
),
Measure(
'Probability of Net Loss',
kind=MINIMIZE,
variable_name='net_benefits',
function=lambda x: numpy.mean(x < 0),
min=0,
max=1,
),
Measure(
'95%ile Travel Time',
kind=MINIMIZE,
variable_name='build_travel_time',
function=functools.partial(numpy.percentile, q=95),
min=60,
max=150,
),
Measure(
'99%ile Present Cost',
kind=Measure.INFO,
variable_name='present_cost_expansion',
function=functools.partial(numpy.percentile, q=99),
# min=0,
# max=10,
),
Measure(
'Expected Present Cost',
kind=Measure.INFO,
variable_name='present_cost_expansion',
function=numpy.mean,
# min=0,
# max=10,
),
]
from emat import Constraint
constraint_1 = Constraint(
"Maximum Log Expected Present Cost",
outcome_names="Expected Present Cost",
function=Constraint.must_be_less_than(4000),
)
constraint_2 = Constraint(
"Minimum Capacity Expansion",
parameter_names="expand_capacity",
function=Constraint.must_be_greater_than(10),
)
constraint_3 = Constraint(
"Maximum Paygo",
parameter_names='debt_type',
outcome_names='99%ile Present Cost',
function=lambda i, j: max(0, j - 1500) if i == 'Paygo' else 0,
)
from emat.optimization import HyperVolume, EpsilonProgress, SolutionViewer, ConvergenceMetrics
convergence_metrics = ConvergenceMetrics(
HyperVolume.from_outcomes(robustness_functions),
EpsilonProgress(),
SolutionViewer.from_model_and_outcomes(m, robustness_functions),
)
with SequentialEvaluator(m) as eval_seq:
robust_results, convergence = m.robust_optimize(
robustness_functions,
scenarios=20,
nfe=5,
constraints=[
constraint_1,
constraint_2,
constraint_3,
],
epsilons=[0.05, ] * len(robustness_functions),
convergence=convergence_metrics,
evaluator=eval_seq,
)
assert isinstance(robust_results, pandas.DataFrame)
mm = m.create_metamodel_from_design('lhs')
design2 = design_experiments(road_scope, db=emat_db, n_samples_per_factor=10, sampler='lhs', random_seed=2)
design2_results = mm.run_experiments(design2)
class TestScopeMethods(unittest.TestCase):
'''
tests parsing scope file
'''
#
# one time test setup
#
scope_file = emat.package_file("model", "tests", "model_test.yaml")
db_test = SQLiteDB(
config.get("test_db_filename", ":memory:"),
initialize=True,
)
#
# Tests
#
def test_dump_scope(self):
scp = Scope(self.scope_file)
dumped = scp.dump()
# print("="*40)
# print(dumped)
# print("="*40)
loaded = Scope(scope_def=dumped, scope_file="fake/filename.yaml")
assert loaded == scp # filename is intentionally different but let it go
# but everything else is the same
assert loaded.name == scp.name
assert loaded.get_measures() == scp.get_measures()
assert loaded.get_parameters() == scp.get_parameters()
assert loaded.scope_file != scp.scope_file
assert loaded.scope_file == "fake/filename.yaml"
# fix name, still get equality
loaded.scope_file = scp.scope_file
assert loaded == scp
def test_save_scope(self):
scp = Scope(self.scope_file)
scp.store_scope(self.db_test)
def test_null_scope(self):
scp = Scope(None)
assert repr(scp) == "<emat.Scope with no content>"
assert len(scp.get_measures()) == 0
assert len(scp.get_parameters()) == 0
def test_box(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
with pytest.raises(TypeError):
s = Box(scope=scope)
s = Box(name="Speedy", scope=scope)
s.set_upper_bound('build_travel_time', 70)
with pytest.raises(ScopeError):
s.set_upper_bound('not_a_thing', 70)
assert len(s) == 1
assert 'build_travel_time' in s
assert s.parent_box_name is None
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
assert len(s2) == 1
assert 'build_travel_time' not in s2
assert s2.parent_box_name == 'Speedy'
def test_box_universe(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
s = Box(name="Speedy", scope=scope)
s.set_upper_bound('build_travel_time', 70)
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
u = Boxes(s, s2, scope=scope)
assert u.fancy_names() == [
'Scope: EMAT Road Test', '▶ Speedy', '▷ ▶ Notable'
]
assert u.plain_names() == [None, 'Speedy', 'Notable']
def test_read_write_box(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s1.relevant_features.add('debt_type')
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
db.write_box(s1)
db.write_box(s2)
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.thresholds == s1_.thresholds
assert s2.thresholds == s2_.thresholds
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features
def test_read_write_boxes(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
u = Boxes(s1, s2, scope=scope)
db.write_boxes(u)
scope2 = Scope(package_file('model', 'tests', 'road_test.yaml'))
u2 = db.read_boxes(scope=scope2)
assert u == u2
assert u["Notable"].parent_box_name == u2["Notable"].parent_box_name
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features
