def test_multiple_connections():
import tempfile
with tempfile.TemporaryDirectory() as tempdir:
tempdbfile = os.path.join(tempdir, "test_db_file.db")
db_test = SQLiteDB(tempdbfile, initialize=True)
road_test_scope_file = emat.package_file("model", "tests",
"road_test.yaml")
s = emat.Scope(road_test_scope_file)
db_test.store_scope(s)
assert db_test.read_scope_names() == ["EMAT Road Test"]
db_test2 = SQLiteDB(tempdbfile, initialize=False)
with pytest.raises(KeyError):
db_test2.store_scope(s)
# Neither database is in a transaction
assert not db_test.conn.in_transaction
assert not db_test2.conn.in_transaction
from emat.model.core_python import Road_Capacity_Investment
m1 = emat.PythonCoreModel(Road_Capacity_Investment,
scope=s,
db=db_test)
m2 = emat.PythonCoreModel(Road_Capacity_Investment,
scope=s,
db=db_test2)
d1 = m1.design_experiments(n_samples=3,
random_seed=1,
design_name="d1")
d2 = m2.design_experiments(n_samples=3,
random_seed=2,
design_name="d2")
r1 = m1.run_experiments(design_name="d1")
r2 = m2.run_experiments(design_name="d2")
# Check each model can load the other's results
pd.testing.assert_frame_equal(
r1,
m2.db.read_experiment_all(scope_name=s.name,
design_name="d1",
ensure_dtypes=True)[r1.columns],
)
pd.testing.assert_frame_equal(
r2,
m1.db.read_experiment_all(scope_name=s.name,
design_name="d2",
ensure_dtypes=True)[r2.columns],
)
def test_read_write_boxes(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
u = Boxes(s1, s2, scope=scope)
db.write_boxes(u)
scope2 = Scope(package_file('model', 'tests', 'road_test.yaml'))
u2 = db.read_boxes(scope=scope2)
assert u == u2
assert u["Notable"].parent_box_name == u2["Notable"].parent_box_name
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features
def db_setup():
db_test = SQLiteDB(config.get("test_db_filename", ":memory:"),
initialize=True)
# load experiment variables and performance measures
scp_xl = [("constant", "constant"), ("exp_var1", "risk"),
("exp_var2", "strategy")]
scp_m = [("pm_1", "none"), ("pm_2", "ln")]
db_test.init_xlm(scp_xl, scp_m)
# create emat scope
scope_name = "test"
sheet = "emat_scope1.yaml"
ex_xl = ["constant", "exp_var1", "exp_var2"]
ex_m = ["pm_1", "pm_2"]
db_test.delete_scope(scope_name)
scope = emat.Scope(sheet, scope_yaml)
db_test._write_scope(
scope_name,
sheet,
ex_xl,
ex_m,
scope,
)
yield Bunch(
db_test=db_test,
scope_name=scope_name,
sheet=sheet,
scp_xl=scp_xl,
scp_m=scp_m,
ex_m=ex_m,
ex_xl=ex_xl,
)
db_test.delete_scope(scope_name)
def test_read_write_box(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s1.relevant_features.add('debt_type')
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
db.write_box(s1)
db.write_box(s2)
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.thresholds == s1_.thresholds
assert s2.thresholds == s2_.thresholds
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features
class TestDatabaseMethods(unittest.TestCase):
'''
tests writing and reading experiments to database
'''
#
# one time test setup
#
db_test = SQLiteDB(config.get("test_db_filename", ":memory:"),
initialize=True)
# load experiment variables and performance measures
scp_xl = [('constant', 'constant'), ('exp_var1', 'risk'),
('exp_var2', 'strategy')]
scp_m = [('pm_1', 'none'), ('pm_2', 'ln')]
db_test.init_xlm(scp_xl, scp_m)
# create emat scope
scope_name = 'test'
sheet = 'emat_scope1.yaml'
ex_xl = ['constant', 'exp_var1', 'exp_var2']
ex_m = ['pm_1', 'pm_2']
db_test.delete_scope(scope_name)
def setUp(self):
# create emat scope
self.db_test.write_scope(self.scope_name, self.sheet, self.ex_xl,
self.ex_m)
def tearDown(self):
self.db_test.delete_scope(self.scope_name)
#
# Tests
#
def test_delete_experiment(self):
# write experiment definition
xl_df = pd.DataFrame({
'constant': [1, 1],
'exp_var1': [1.1, 1.2],
'exp_var2': [2.1, 2.2]
})
design = 'lhs'
self.db_test.write_experiment_parameters(self.scope_name, design,
xl_df)
self.db_test.delete_experiments(self.scope_name, design)
xl_readback = self.db_test.read_experiment_parameters(
self.scope_name, design)
#note - indexes may not match
self.assertTrue(xl_readback.empty)
def test_create_experiment(self):
# write experiment definition
xl_df = pd.DataFrame({
'constant': [1, 1],
'exp_var1': [1.1, 1.2],
'exp_var2': [2.1, 2.2]
})
design = 'lhs'
self.db_test.write_experiment_parameters(self.scope_name, design,
xl_df)
xl_readback = self.db_test.read_experiment_parameters(
self.scope_name, design)
#note - indexes may not match
self.assertTrue(np.array_equal(xl_readback.values, xl_df.values))
def test_write_pm(self):
# write experiment definition
xl_df = pd.DataFrame({
'constant': [1, 1],
'exp_var1': [1.1, 1.2],
'exp_var2': [2.1, 2.2]
})
design = 'lhs'
self.db_test.write_experiment_parameters(self.scope_name, design,
xl_df)
# get experiment ids
exp_with_ids = self.db_test.read_experiment_parameters(
self.scope_name, design)
exp_with_ids['pm_1'] = [4.0, 5.0]
exp_with_ids['pm_2'] = [6.0, 7.0]
# write performance measures
self.db_test.write_experiment_measures(self.scope_name,
SOURCE_IS_CORE_MODEL,
exp_with_ids)
xlm_readback = self.db_test.read_experiment_all(
self.scope_name, design)
self.assertTrue(exp_with_ids.equals(xlm_readback))
def test_write_partial_pm(self):
# write experiment definition
xl_df = pd.DataFrame({
'constant': [1, 1],
'exp_var1': [1.1, 1.2],
'exp_var2': [2.1, 2.2]
})
design = 'lhs'
self.db_test.write_experiment_parameters(self.scope_name, design,
xl_df)
# get experiment ids
exp_with_ids = self.db_test.read_experiment_parameters(
self.scope_name, design)
exp_with_ids['pm_1'] = [4.0, 5.0]
# write performance measures
self.db_test.write_experiment_measures(self.scope_name,
SOURCE_IS_CORE_MODEL,
exp_with_ids)
xlm_readback = self.db_test.read_experiment_all(
self.scope_name, design)
self.assertTrue(exp_with_ids.equals(xlm_readback))
def test_write_experiment(self):
# write experiment definition
xlm_df = pd.DataFrame({
'constant': [1, 1],
'exp_var1': [1.1, 1.2],
'exp_var2': [2.1, 2.2],
'pm_1': [4.0, 5.0],
'pm_2': [6.0, 7.0]
})
design = 'lhs'
core_model = True
self.db_test.write_experiment_all(self.scope_name, design,
SOURCE_IS_CORE_MODEL, xlm_df)
xlm_readback = self.db_test.read_experiment_all(
self.scope_name, design)
# index may not match
self.assertTrue(np.array_equal(xlm_readback.values, xlm_df.values))
# set experiment without all variables defined
def test_incomplete_experiment(self):
xl_df = pd.DataFrame({'exp_var1': [1]})
design = 'lhs'
with self.assertRaises(KeyError):
self.db_test.write_experiment_parameters(self.scope_name, design,
xl_df)
# try to overwrite existing scope
def test_scope_overwrite(self):
with self.assertRaises(KeyError):
self.db_test.write_scope(self.scope_name, self.sheet, self.scp_xl,
self.scp_m)
# scope with invalid risk variables
def test_scope_invalid_risk(self):
with self.assertRaises(KeyError):
self.db_test.write_scope('test2', self.sheet, ['exp_var3'],
self.ex_m)
self.db_test.delete_scope('test2')
# scope with invalid performance measures
def test_scope_invalid_pm(self):
with self.assertRaises(KeyError):
self.db_test.write_scope('test2', self.sheet, self.ex_xl, ['pm_3'])
self.db_test.delete_scope('test2')
# scope with invalid performance measures
def test_scope_add_pm(self):
# add new pm
self.db_test.init_xlm([], [('pm_4', 'none')])
self.db_test.add_scope_meas(self.scope_name, self.ex_m + ['pm_4'])
class TestExperimentMethods(unittest.TestCase):
'''
tests generating experiments
'''
#
# one time test setup
#
scope_file = emat.package_file("scope", "tests", "scope_test.yaml")
scp = Scope(scope_file)
db_test = SQLiteDB(":memory:", initialize=True)
scp.store_scope(db_test)
def test_latin_hypercube(self):
exp_def = self.scp.design_experiments(
n_samples_per_factor=10,
random_seed=1234,
sampler='lhs',
db=self.db_test,
)
assert len(exp_def) == self.scp.n_sample_factors() * 10
assert (exp_def['TestRiskVar'] == 1.0).all()
assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.0326,
abs=1e-3)
assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=1e-3)
exp_def2 = self.db_test.read_experiment_parameters(
self.scp.name, 'lhs')
assert (exp_def[exp_def2.columns] == exp_def2).all().all()
def test_latin_hypercube_not_joint(self):
exp_def = self.scp.design_experiments(
n_samples_per_factor=5,
random_seed=1234,
sampler='lhs',
db=self.db_test,
jointly=False,
design_name='lhs_not_joint',
)
# assert len(exp_def) == len(self.scp.get_uncertainties())*5 * len(self.scp.get_levers())*5
assert len(
exp_def
) == 80 # there are only 4 unique policies, times 2 scenarios
assert (exp_def['TestRiskVar'] == 1.0).all()
assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.033,
abs=1e-2)
assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=1e-2)
exp_def2 = self.db_test.read_experiment_parameters(
self.scp.name, 'lhs_not_joint')
assert (exp_def[exp_def2.columns] == exp_def2).all().all()
def test_monte_carlo(self):
exp_def = self.scp.design_experiments(
n_samples_per_factor=10,
random_seed=1234,
sampler='mc',
db=self.db_test,
)
assert len(exp_def) == self.scp.n_sample_factors() * 10
assert (exp_def['TestRiskVar'] == 1.0).all()
assert (exp_def['Land Use - CBD Focus']).mean() == approx(1.0326,
abs=0.01)
assert (exp_def['Freeway Capacity']).mean() == approx(1.5, abs=0.01)
exp_def2 = self.db_test.read_experiment_parameters(self.scp.name, 'mc')
assert (exp_def[exp_def2.columns] == exp_def2).all().all()
def test_sensitivity_tests(self):
exp_def = self.scp.design_experiments(
sampler='uni',
db=self.db_test,
)
cols = [
'TestRiskVar', 'Land Use - CBD Focus', 'Freeway Capacity',
'Auto IVTT Sensitivity', 'Shared Mobility',
'Kensington Decommissioning', 'LRT Extension'
]
correct = '{"TestRiskVar":{"0":1.0,"1":1.0,"2":1.0,"3":1.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \
'"Land Use - CBD Focus":{"0":1.0,"1":0.82,"2":1.37,"3":1.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \
'"Freeway Capacity":{"0":1.0,"1":1.0,"2":1.0,"3":2.0,"4":1.0,"5":1.0,"6":1.0,"7":1.0},' \
'"Auto IVTT Sensitivity":{"0":1.0,"1":1.0,"2":1.0,"3":1.0,"4":0.75,"5":1.0,"6":1.0,"7":1.0},' \
'"Shared Mobility":{"0":0.0,"1":0.0,"2":0.0,"3":0.0,"4":0.0,"5":1.0,"6":0.0,"7":0.0},' \
'"Kensington Decommissioning":{"0":false,"1":false,"2":false,"3":false,"4":false,' \
'"5":false,"6":true,"7":false},"LRT Extension":{"0":false,"1":false,"2":false,"3":false,' \
'"4":false,"5":false,"6":false,"7":true}}'
correct = pd.read_json(correct)
for k in cols:
assert (exp_def[k].values == approx(correct[k].values))
exp_def2 = self.db_test.read_experiment_parameters(
self.scp.name, 'uni')
for k in cols:
assert (exp_def2[k].values == approx(correct[k].values))
def test_nonuniform_latin_hypercube(self):
scope_file = emat.package_file("model", "tests",
"road_test_nonuniform.yaml")
scp = Scope(scope_file)
exp_def = scp.design_experiments(
n_samples_per_factor=1000,
random_seed=1234,
sampler='lhs',
)
assert len(exp_def) == scp.n_sample_factors() * 1000
assert (exp_def['free_flow_time'] == 60).all()
assert (exp_def['initial_capacity'] == 100).all()
assert np.corrcoef([exp_def.alpha,
exp_def.beta])[0, 1] == approx(0.75, rel=0.05)
assert np.corrcoef([exp_def.alpha,
exp_def.expand_capacity])[0, 1] == approx(0.0,
abs=0.02)
assert np.corrcoef([exp_def.input_flow,
exp_def.value_of_time])[0, 1] == approx(-0.5,
rel=0.05)
assert np.corrcoef(
[exp_def.unit_cost_expansion,
exp_def.value_of_time])[0, 1] == approx(0.9, rel=0.05)
assert exp_def.interest_rate_lock.sum() == approx(len(exp_def) * 0.2)
assert np.percentile(exp_def.alpha, np.linspace(0, 100, 50)) == approx(
[
0.10037393, 0.10722119, 0.10994485, 0.11204394, 0.11383709,
0.11544182, 0.11691345, 0.11829399, 0.11959909, 0.12084863,
0.12205279, 0.12321800, 0.12435285, 0.12546474, 0.12655958,
0.12763503, 0.12869873, 0.12975137, 0.13079620, 0.13183375,
0.13287082, 0.13390854, 0.13494651, 0.13598528, 0.13703149,
0.13808180, 0.13914411, 0.14021784, 0.14130323, 0.14240609,
0.14352608, 0.14466756, 0.14583411, 0.14702908, 0.14825720,
0.14951875, 0.15082844, 0.15218376, 0.15359963, 0.15508120,
0.15664534, 0.15831425, 0.16010073, 0.16203921, 0.16418886,
0.16662357, 0.16946999, 0.17301416, 0.17804383, 0.19662857
])
assert np.percentile(exp_def.beta, np.linspace(0, 100, 50)) == approx([
3.51654751, 3.72503059, 3.82437701, 3.90088124, 3.96222432,
4.01360346, 4.06112277, 4.10640347, 4.14456476, 4.18084719,
4.21812584, 4.24926944, 4.28049053, 4.31181127, 4.34390502,
4.37561590, 4.40541815, 4.43276143, 4.45517485, 4.48062290,
4.50726296, 4.53334164, 4.55737738, 4.57893875, 4.60371011,
4.62590595, 4.64885523, 4.67335218, 4.69475909, 4.71546469,
4.73676622, 4.75796550, 4.77690613, 4.79738177, 4.81947505,
4.84481408, 4.86954326, 4.89379651, 4.91771359, 4.94100213,
4.97169370, 5.00298714, 5.03525103, 5.07100437, 5.11028866,
5.15061419, 5.19925284, 5.24775527, 5.32086547, 5.49345120
])
assert np.percentile(exp_def.input_flow, np.linspace(
0, 100, 50)) == approx([
80.06332381, 83.71770678, 85.93721426, 87.82355332,
89.52967083, 91.11148891, 92.60789787, 94.03509345,
95.41186604, 96.74598771, 98.04479355, 99.31122788,
100.55462827, 101.77808514, 102.98041937, 104.16517008,
105.33959643, 106.50165383, 107.65366600, 108.79827639,
109.93328795, 111.06707367, 112.19316408, 113.31493454,
114.43982739, 115.56182157, 116.68454749, 117.80620837,
118.93524815, 120.06752762, 121.20470208, 122.34781469,
123.50083542, 124.66086016, 125.83390291, 127.02146142,
128.22485477, 129.44681602, 130.68613841, 131.95601658,
133.25403807, 134.58951122, 135.96442305, 137.39295642,
138.89092217, 140.47204147, 142.17835057, 144.06540067,
146.28064479, 149.94588322
])
class TestScopeMethods(unittest.TestCase):
'''
tests parsing scope file
'''
#
# one time test setup
#
scope_file = emat.package_file("model", "tests", "model_test.yaml")
db_test = SQLiteDB(
config.get("test_db_filename", ":memory:"),
initialize=True,
)
#
# Tests
#
def test_dump_scope(self):
scp = Scope(self.scope_file)
dumped = scp.dump()
# print("="*40)
# print(dumped)
# print("="*40)
loaded = Scope(scope_def=dumped, scope_file="fake/filename.yaml")
assert loaded == scp # filename is intentionally different but let it go
# but everything else is the same
assert loaded.name == scp.name
assert loaded.get_measures() == scp.get_measures()
assert loaded.get_parameters() == scp.get_parameters()
assert loaded.scope_file != scp.scope_file
assert loaded.scope_file == "fake/filename.yaml"
# fix name, still get equality
loaded.scope_file = scp.scope_file
assert loaded == scp
def test_save_scope(self):
scp = Scope(self.scope_file)
scp.store_scope(self.db_test)
def test_null_scope(self):
scp = Scope(None)
assert repr(scp) == "<emat.Scope with no content>"
assert len(scp.get_measures()) == 0
assert len(scp.get_parameters()) == 0
def test_box(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
with pytest.raises(TypeError):
s = Box(scope=scope)
s = Box(name="Speedy", scope=scope)
s.set_upper_bound('build_travel_time', 70)
with pytest.raises(ScopeError):
s.set_upper_bound('not_a_thing', 70)
assert len(s) == 1
assert 'build_travel_time' in s
assert s.parent_box_name is None
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
assert len(s2) == 1
assert 'build_travel_time' not in s2
assert s2.parent_box_name == 'Speedy'
def test_box_universe(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
s = Box(name="Speedy", scope=scope)
s.set_upper_bound('build_travel_time', 70)
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
u = Boxes(s, s2, scope=scope)
assert u.fancy_names() == [
'Scope: EMAT Road Test', '▶ Speedy', '▷ ▶ Notable'
]
assert u.plain_names() == [None, 'Speedy', 'Notable']
def test_read_write_box(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s1.relevant_features.add('debt_type')
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
db.write_box(s1)
db.write_box(s2)
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.thresholds == s1_.thresholds
assert s2.thresholds == s2_.thresholds
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features
def test_read_write_boxes(self):
scope = Scope(package_file('model', 'tests', 'road_test.yaml'))
db = SQLiteDB()
scope.store_scope(db)
s1 = Box(name="Speedy", scope=scope)
s1.set_upper_bound('build_travel_time', 70)
s2 = Box(name="Notable", scope=scope, parent="Speedy")
s2.set_lower_bound('expand_capacity', 20)
u = Boxes(s1, s2, scope=scope)
db.write_boxes(u)
scope2 = Scope(package_file('model', 'tests', 'road_test.yaml'))
u2 = db.read_boxes(scope=scope2)
assert u == u2
assert u["Notable"].parent_box_name == u2["Notable"].parent_box_name
s1_ = db.read_box(scope.name, "Speedy")
s2_ = db.read_box(scope.name, "Notable")
assert s1 == s1_
assert s2 == s2_
assert s1.relevant_features == s1_.relevant_features
assert s2.relevant_features == s2_.relevant_features