def test_versioning_with_disaggregate(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id':array([1,2,3,4]),
'id2':array([1,2,1,2])
}
)
storage.write_table(table_name='faz',
table_data={
'my_variable':array([4,8]),
'id2':array([1,2])
}
)
ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone")
ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds)
dataset_pool._add_dataset('myfaz', ds2)
var = "my_var = myzone.disaggregate(10.0*myfaz.my_variable)"
ds.modify_attribute("id2", array([2,1,2,1])) # should have version 1
ds.compute_variables([var], dataset_pool=dataset_pool)
self.assert_(ds.get_version("my_var")==0)
ds.compute_variables([var], dataset_pool=dataset_pool)
self.assert_(ds.get_version("my_var")==0) # version should stay the same, i.e. it should not recompute
ds.touch_attribute("id2") # has version 2
ds.compute_variables([var], dataset_pool=dataset_pool)
self.assert_(ds.get_version("my_var")==1) # version should be 1, i.e. it should recompute when id changes
def test_versioning_with_aggregate(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data={
'my_variable': array([4, 8, 2, 1, 40, 23, 78]),
'id0': arange(7) + 1,
'id1': array([1, 3, 1, 2, 3, 2, 1])
})
storage.write_table(table_name='fazes',
table_data={
'id1': array([1, 2, 3]),
'id2': array([1, 2, 1])
})
storage.write_table(table_name='fazdistr',
table_data={'id2': array([1, 2])})
ds0 = Dataset(in_storage=storage,
in_table_name='households',
id_name="id0",
dataset_name="myhousehold")
ds1 = Dataset(in_storage=storage,
in_table_name='fazes',
id_name="id1",
dataset_name="myfaz")
ds2 = Dataset(in_storage=storage,
in_table_name='fazdistr',
id_name="id2",
dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myhousehold', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
ds0.modify_attribute("id1", array([1, 3, 1, 2, 3, 2,
1])) # has version 1
variable = 'my_var = myfazdistr.aggregate(10.0*myhousehold.my_variable, intermediates=[myfaz])'
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(ds2.get_version("my_var") == 0)
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(
ds2.get_version("my_var") ==
0) # version should stay the same, i.e. it should not recompute
ds0.touch_attribute("id1") # has version 2
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(
ds2.get_version("my_var") ==
1) # version should be 1, i.e. it should recompute when id changes
ds1.touch_attribute("id2") # has version 1
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(
ds2.get_version("my_var") ==
2) # version should be 2, i.e. it should recompute when id changes
def test_number_of_agents(self):
expr = "mygridcell.number_of_agents(myjob)"
storage = StorageFactory().get_storage('dict_storage')
gridcell_grid_id = array([1, 2, 3])
job_grid_id = array([2, 1, 3, 1]) #specify an array of 4 jobs, 1st job's grid_id = 2 (it's in gridcell 2), etc.
storage.write_table(table_name='gridcells', table_data={'gid':gridcell_grid_id})
storage.write_table(table_name='jobs', table_data={'jid':arange(4)+1, 'gid':job_grid_id})
gs = Dataset(in_storage=storage, in_table_name='gridcells', id_name="gid", dataset_name="mygridcell")
jobs = Dataset(in_storage=storage, in_table_name='jobs', id_name="jid", dataset_name="myjob")
values = gs.compute_variables([expr], resources=Resources({"myjob":jobs, "mygridcell":gs}))
should_be = array([2, 1, 1])
self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg = "Error in " + expr)
# change gids of jobs (to test if computing dependencies is working)
jobs.modify_attribute(name="gid", data=array([1,1,1,1]))
values2 = gs.compute_variables([expr], resources=Resources({"myjob":jobs, "mygridcell":gs}))
should_be2 = array([4, 0, 0])
self.assert_(ma.allclose(values2, should_be2, rtol=1e-7), msg = "Error in " + expr)
def test_alias_attribute_with_modification(self):
# this tests an expression consisting of an alias for a primary attribute that is modified
expr = "p = persons"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"persons":array([1,5,10]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
# modify the primary attribute 'persons'
new_values = array([3, 0, 100])
dataset.modify_attribute('persons', new_values)
# result should have the new values
result = dataset.compute_variables([expr])
self.assertEqual(ma.allclose(result, new_values, rtol=1e-7), True, msg="error in test_alias_attribute_with_modification")
def test_alias_attribute_with_modification(self):
# this tests an expression consisting of an alias for a primary attribute that is modified
expr = "p = persons"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"persons": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
# modify the primary attribute 'persons'
new_values = array([3, 0, 100])
dataset.modify_attribute('persons', new_values)
# result should have the new values
result = dataset.compute_variables([expr])
self.assertEqual(ma.allclose(result, new_values, rtol=1e-7),
True,
msg="error in test_alias_attribute_with_modification")
def test_versioning_with_aggregate(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data={
'my_variable':array([4,8,2,1,40,23,78]),
'id0':arange(7)+1,
'id1':array([1,3,1,2,3,2,1])
}
)
storage.write_table(table_name='fazes',
table_data={
'id1':array([1,2,3]),
'id2':array([1,2,1])
}
)
storage.write_table(table_name='fazdistr',
table_data={
'id2':array([1,2])
}
)
ds0 = Dataset(in_storage=storage, in_table_name='households', id_name="id0", dataset_name="myhousehold")
ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz")
ds2 = Dataset(in_storage=storage, in_table_name='fazdistr', id_name="id2", dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myhousehold', ds0)
dataset_pool._add_dataset('myfaz',ds1)
dataset_pool._add_dataset('myfazdistr',ds2)
ds0.modify_attribute("id1", array([1,3,1,2,3,2,1])) # has version 1
variable = 'my_var = myfazdistr.aggregate(10.0*myhousehold.my_variable, intermediates=[myfaz])'
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(ds2.get_version("my_var")==0)
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(ds2.get_version("my_var")==0) # version should stay the same, i.e. it should not recompute
ds0.touch_attribute("id1") # has version 2
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(ds2.get_version("my_var")==1) # version should be 1, i.e. it should recompute when id changes
ds1.touch_attribute("id2") # has version 1
ds2.compute_variables([variable], dataset_pool=dataset_pool)
self.assert_(ds2.get_version("my_var")==2) # version should be 2, i.e. it should recompute when id changes
def test_variable_dependencies_tree_with_versioning(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
'id':array([2,4]),
'a_dependent_variable':array([4,7]),
'a_dependent_variable2':array([10,1])
}
)
ds = Dataset(in_storage=storage, in_table_name='tests', id_name='id', dataset_name='tests')
ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"])
self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==0) #initially version=0
self.assert_(ds.get_version("a_dependent_variable")==0)
self.assert_(ds.get_version("a_dependent_variable2")==0)
ds.modify_attribute("a_dependent_variable", array([0,0]))
self.assert_(ds.get_version("a_dependent_variable")==1) # version=1
ds.modify_attribute("a_dependent_variable", array([1,1]))
self.assert_(ds.get_version("a_dependent_variable")==2) # version=2
ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"])
self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==1)
ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"])
self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==1) # version does not change
autogen_variable = "my_var = 3 * opus_core.tests.a_dependent_variable"
ds.compute_variables([autogen_variable])
self.assert_(ds.get_version("my_var")==0)
ds.compute_variables([autogen_variable])
self.assert_(ds.get_version("my_var")==0)
def test_versioning_with_disaggregate(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id': array([1, 2, 3, 4]),
'id2': array([1, 2, 1, 2])
})
storage.write_table(table_name='faz',
table_data={
'my_variable': array([4, 8]),
'id2': array([1, 2])
})
ds = Dataset(in_storage=storage,
in_table_name='zones',
id_name="id",
dataset_name="myzone")
ds2 = Dataset(in_storage=storage,
in_table_name='faz',
id_name="id2",
dataset_name="myfaz")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds)
dataset_pool._add_dataset('myfaz', ds2)
var = "my_var = myzone.disaggregate(10.0*myfaz.my_variable)"
ds.modify_attribute("id2", array([2, 1, 2,
1])) # should have version 1
ds.compute_variables([var], dataset_pool=dataset_pool)
self.assert_(ds.get_version("my_var") == 0)
ds.compute_variables([var], dataset_pool=dataset_pool)
self.assert_(
ds.get_version("my_var") ==
0) # version should stay the same, i.e. it should not recompute
ds.touch_attribute("id2") # has version 2
ds.compute_variables([var], dataset_pool=dataset_pool)
self.assert_(
ds.get_version("my_var") ==
1) # version should be 1, i.e. it should recompute when id changes
households.get_attribute_names()
households.get_id_attribute()
households.size()
households.get_attribute("income")
households.get_attribute_names()
households.load_dataset()
households.get_attribute_names()
#households.plot_histogram("income", bins = 10)
#households.r_histogram("income")
#households.r_scatter("persons", "income")
households.correlation_coefficient("persons", "income")
households.correlation_matrix(["persons", "income"])
households.summary()
households.add_primary_attribute(data=[4,6,9,2,4,8,2,1,3,2], name="location")
households.get_attribute_names()
households.modify_attribute(name="location", data=[0,0], index=[0,1])
households.get_attribute("location")
households.get_data_element_by_id(5).location
#households.write_dataset(out_storage=storage, out_table_name="households_output")
households.get_dataset_name()
# Working with models
from opus_core.choice_model import ChoiceModel
choicemodel = ChoiceModel(choice_set=[1,2,3],
utilities = "opus_core.linear_utilities",
probabilities = "opus_core.mnl_probabilities",
choices = "opus_core.random_choices")
from numpy import array
households.get_attribute_names()
households.get_id_attribute()
households.size()
households.get_attribute("income")
households.get_attribute_names()
households.load_dataset()
households.get_attribute_names()
#households.plot_histogram("income", bins = 10)
#households.r_histogram("income")
#households.r_scatter("persons", "income")
households.correlation_coefficient("persons", "income")
households.correlation_matrix(["persons", "income"])
households.summary()
households.add_primary_attribute(data=[4,6,9,2,4,8,2,1,3,2], name="location")
households.get_attribute_names()
households.modify_attribute(name="location", data=[0,0], index=[0,1])
households.get_attribute("location")
households.get_data_element_by_id(5).location
#households.write_dataset(out_storage=storage, out_table_name="households_output")
households.get_dataset_name()
# Working with models
from opus_core.choice_model import ChoiceModel
choicemodel = ChoiceModel(choice_set=[1,2,3],
utilities = "opus_core.linear_utilities",
probabilities = "opus_core.mnl_probabilities",
choices = "opus_core.random_choices")
from numpy import array
