def prepare_for_estimate(self, specification_dict = None,
specification_storage=None,
specification_table=None,
events_for_estimation_storage=None,
events_for_estimation_table=None,
agents_filter='',
compute_variables=[],
data_objects={}):
specification = get_specification_for_estimation(specification_dict,
specification_storage,
specification_table)
projects = None
# create agents for estimation
if events_for_estimation_storage is not None:
projects = Dataset(in_storage = events_for_estimation_storage,
in_table_name= events_for_estimation_table,
id_name=[], dataset_name='development_project'
)
if compute_variables:
projects.compute_variables(compute_variables, resources=Resources(data_objects))
# needs to be a primary attribute because of the join method below
#projects.add_primary_attribute(estimation_set.get_attribute(location_id_variable),
# VariableName(location_id_variable).get_alias())
if agents_filter:
values = projects.compute_variables(agents_filter, resources=Resources(data_objects))
index = where(values > 0)[0]
projects.subset_by_index(index, flush_attributes_if_not_loaded=False)
return (specification, projects)
def test_fully_qualified_variable(self):
# this tests an expression consisting of a fully-qualified variable
expr = "opus_core.test_agent.income_times_2"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="test_agents", table_data={"income": array([1, 5, 10]), "id": array([1, 3, 4])})
dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array([2, 10, 20])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_variable")
# check that expr is in the cache of known expressions
# (normally we shouldn't be accessing this private field, but just this once ...)
cache = VariableName._cache
self.assert_(expr in cache, msg="did not find expr in cache")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
self.assertEqual(name.get_package_name(), "opus_core", msg="bad value for package")
self.assertEqual(name.get_dataset_name(), "test_agent", msg="bad value for dataset")
self.assertEqual(name.get_short_name(), "income_times_2", msg="bad value for shortname")
self.assertEqual(name.get_alias(), "income_times_2", msg="bad value for alias")
self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
# test that the variable can now also be accessed using its short name in an expression
result2 = dataset.compute_variables(["income_times_2"])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable")
# check that the cache uses the variable name with whitespace removed
oldsize = len(cache)
expr_with_spaces = "opus_core . test_agent. income_times_2 "
name2 = VariableName(expr_with_spaces)
newsize = len(cache)
self.assertEqual(oldsize, newsize, msg="caching error")
self.assert_(expr_with_spaces not in cache, msg="caching error")
self.assertEqual(expr_with_spaces, name2.get_expression(), msg="caching error")
self.assertEqual(name2.get_short_name(), "income_times_2", msg="bad value for shortname")
def test_simple_lag_variable(self):
test_data = {
1000:{
'tests':{
'id':array([1,2,3]),
'attr1':array([10,20,30]),
},
},
1001:{
'tests':{
'id':array([1,2,3]),
'attr1':array([111,222,333]),
},
},
}
cache_creator = CreateTestAttributeCache()
cache_creator.create_attribute_cache_with_data(self._temp_dir, test_data)
SimulationState().set_current_time(1001)
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True,
package_order=['opus_core'],
in_storage=attribute_cache)
ds = Dataset(in_storage = attribute_cache,
in_table_name = 'tests',
id_name = ['id'],
dataset_name = 'tests')
ds.compute_variables(['opus_core.tests.attr1'])
self.assert_(ma.allequal(ds.get_attribute('attr1'), array([111,222,333])))
ds.compute_variables(['opus_core.tests.attr1_lag1'])
self.assert_(ma.allequal(ds.get_attribute('attr1_lag1'), array([10,20,30])))
def test_fully_qualified_DDD_SSS_variable(self):
# this should use the test variable a_test_SSS_variable_DDD_SSS
expr = "opus_core.tests.a_test_squid_variable_42_clam"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"a_dependent_variable":array([1,5,10]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([10,50,100])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_DDD_SSS_variable")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
self.assertEqual(name.get_package_name(), 'opus_core', msg="bad value for package")
self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset")
self.assertEqual(name.get_short_name(), 'a_test_squid_variable_42_clam', msg="bad value for shortname")
self.assertEqual(name.get_alias(), 'a_test_squid_variable_42_clam', msg="bad value for alias")
self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
# test that the variable can now also be accessed using its short name in an expression
result2 = dataset.compute_variables(['a_test_squid_variable_42_clam'])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_squid_variable_42_clam")
def _compute_variable_for_one_run(self, run_index, variable, dataset_name, year, dataset_arguments={}):
dataset_pool = self._setup_environment(self.cache_set[run_index], year)
try:
ds = dataset_pool.get_dataset(dataset_name, dataset_arguments=dataset_arguments)
except:
logger.log_warning('Dataset %s could not be loaded from dataset pool using package order %s' % (dataset_name, self.package_order))
logger.log_warning('Generic dataset will be created.')
ds = Dataset(in_table_name=dataset_name, in_storage=dataset_pool.get_storage(), dataset_name=dataset_name, id_name=[])
dataset_pool._add_dataset(dataset_name, ds)
ds.compute_variables(variable, dataset_pool=dataset_pool)
return ds
def prepare_for_estimate(specification_dict = None,
specification_storage=None,
specification_table=None,
agent_set=None,
household_set=None,
agents_for_estimation_storage=None,
agents_for_estimation_table=None,
households_for_estimation_table=None,
join_datasets=False,
filter=None,
data_objects=None):
specification = get_specification_for_estimation(specification_dict,
specification_storage,
specification_table)
if agents_for_estimation_storage is not None:
estimation_set = Dataset(in_storage = agents_for_estimation_storage,
in_table_name=agents_for_estimation_table,
id_name=agent_set.get_id_name(),
dataset_name=agent_set.get_dataset_name())
hh_estimation_set = None
if households_for_estimation_table is not None:
hh_estimation_set = Dataset(in_storage = agents_for_estimation_storage,
in_table_name=households_for_estimation_table,
id_name=household_set.get_id_name(),
dataset_name=household_set.get_dataset_name())
filter_index = arange(estimation_set.size())
if filter:
estimation_set.compute_variables(filter, resources=Resources(data_objects))
filter_index = where(estimation_set.get_attribute(filter) > 0)[0]
#estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False)
if join_datasets:
if hh_estimation_set is not None:
household_set.join_by_rows(hh_estimation_set, require_all_attributes=False,
change_ids_if_not_unique=True)
agent_set.join_by_rows(estimation_set, require_all_attributes=False,
change_ids_if_not_unique=True)
index = arange(agent_set.size() - estimation_set.size(), agent_set.size())[filter_index]
else:
index = agent_set.get_id_index(estimation_set.get_id_attribute()[filter_index])
else:
if agent_set is not None:
index = arange(agent_set.size())
else:
index = None
return (specification, index)
def test_two_expressions(self):
# test having two different expressions (to make sure having two autogen'd classes at once is working)
expr1 = "2*sqrt(my_variable+10)"
expr2 = "3*sqrt(my_variable+10)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="dataset", table_data={"my_variable": array([4, -8, 0.5, 1]), "id": array([1, 2, 3, 4])}
)
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result1 = dataset.compute_variables([expr1])
should_be1 = array([7.48331477, 2.82842712, 6.4807407, 6.63324958])
self.assert_(ma.allclose(result1, should_be1, rtol=1e-6), "Error in test_two_expressions")
result2 = dataset.compute_variables([expr2])
should_be2 = array([11.22497216, 4.24264068, 9.72111105, 9.94987437])
self.assert_(ma.allclose(result2, should_be2, rtol=1e-6), "Error in test_two_expressions")
def test_aggregate_all(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={'my_variable': array([4,8,0.5,1]), 'id': array([1,2,3,4])})
storage.write_table(table_name='regions',
table_data={'id': array([1])})
ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone")
ds2 = Dataset(in_storage=storage, in_table_name='regions', id_name="id", dataset_name="myregion")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds)
dataset_pool._add_dataset('myregion', ds2)
ds2.compute_variables(["myvar = myregion.aggregate_all(myzone.my_variable)"], dataset_pool=dataset_pool)
values = ds2.get_attribute("myvar")
should_be = array([13.5])
self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_all")
def define_submarket(choice_set, submarket_id_expression, compute_variables=[], filter=None):
submarket_ids = choice_set.compute_variables("submarket_id=" + submarket_id_expression)
unique_submarket_ids = unique(submarket_ids)
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="submarkets", table_data={"submarket_id": unique_submarket_ids})
submarkets = Dataset(
in_storage=storage, in_table_name="submarkets", id_name="submarket_id", dataset_name="submarket"
)
if len(compute_variables):
submarkets.compute_variables(compute_variables)
if filter is not None:
from numpy import logical_not
submarkets.remove_elements(index=where(logical_not(submarkets.compute_variables(filter)))[0])
# submarkets = DatasetSubset(submarkets, index=where(submarkets.compute_variables(filter))[0])
return submarkets
def test_constants(self):
# test an expression involving two dataset names, one of which is *_constant
expr = "test_agent.age<=opus_constant.young_age"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='test_agents',
table_data={
"age":array([30,20,60,80]),
"id":array([1,3,4,10])
}
)
storage.write_table(
table_name='opus_constants',
table_data={
"young_age":array([35]),
"opus_constant_id":array([1])
}
)
dataset_pool = DatasetPool(storage=storage)
# Test that the dataset name is correct for expr. It should be test_agent -- opus_constant just holds constants,
# and is ignored as far as finding the dataset name for the expression.
name = VariableName(expr)
autogen = name.get_autogen_class()
self.assertEqual(name.get_package_name(), None)
self.assertEqual(name.get_dataset_name(), 'test_agent')
# make an instance of the class and check the dependencies (it shouldn't depend on urbansim_constant)
self.assertEqual(autogen().dependencies(), ['test_agent.age'])
dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr], dataset_pool=dataset_pool)
should_be = array( [True,True,False,False] )
self.assertEqual( ma.allequal( result, should_be), True)
def test_disaggregate_one_level(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id0':arange(7)+1,
'id1':array([1,3,1,2,3,2,1])
}
)
storage.write_table(table_name='fazes',
table_data={
'id1':array([1,2,3]),
'id2':array([1,2,1])
}
)
storage.write_table(table_name='fazdistr',
table_data={
'my_variable':array([40,50]),
'id2':array([1,2])
}
)
ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone")
ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz")
ds2 = Dataset(in_storage=storage, in_table_name='fazdistr', id_name="id2", dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
values = ds0.compute_variables(["myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])"], dataset_pool=dataset_pool)
should_be = array([40, 40, 40, 50, 40,50, 40])
self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_one_level")
def test_disaggregate_and_multiply(self):
# Perform two different disaggregations and multiply the results. This tests using a dataset name in both the
# list of intermediates and as the dataset being disaggregated (myfaz in this case).
expr = "myzone.disaggregate(myfaz.fazsqft) * myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id0':arange(7)+1,
'id1':array([1,3,1,2,3,2,1])
}
)
storage.write_table(table_name='fazes',
table_data={
'id1':array([1,2,3]),
'id2':array([1,2,1]),
'fazsqft':array([10,50,100])
}
)
storage.write_table(table_name='fazdistrs',
table_data={
'my_variable':array([40,50]),
'id2':array([1,2])
}
)
ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone")
ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz")
ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
values = ds0.compute_variables([expr], dataset_pool=dataset_pool)
should_be = array([400, 4000, 400, 2500, 4000, 2500, 400])
self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_and_multiply")
def test_unary_functions_fully_qualified_name(self):
# this tests expressions with unary functions applied to a fully qualified name
expr = "sqrt(opus_core.tests.a_test_variable)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="tests", table_data={"a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4])}
)
dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([3.16227766, 7.0710678, 10])
self.assertEqual(
ma.allclose(result, should_be, rtol=1e-3), True, msg="error in test_unary_functions_fully_qualified_name"
)
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
autogen = name.get_autogen_class()
self.assert_(issubclass(autogen, Variable), msg="autogen'd class isn't a Variable")
self.assertEqual(name.get_package_name(), None, msg="bad value for package")
self.assertEqual(name.get_dataset_name(), "tests", msg="bad value for dataset")
self.assertEqual(name.get_short_name(), autogen.__name__, msg="bad value for shortname")
self.assertEqual(name.get_alias(), autogen.__name__, msg="bad value for alias")
# make an instance of the class and check the dependencies (since the dependent variables
# all have fully-qualifed names we don't need to associate a dataset with the variable
# for this test)
self.assertEqual(
autogen().dependencies(), ["opus_core.tests.a_test_variable"], msg="dependencies are incorrect"
)
def test_number_of_agents(self):
expr = "mygridcell.number_of_agents(myjob)"
storage = StorageFactory().get_storage('dict_storage')
gridcell_grid_id = array([1, 2, 3])
job_grid_id = array([2, 1, 3, 1]) #specify an array of 4 jobs, 1st job's grid_id = 2 (it's in gridcell 2), etc.
storage.write_table(table_name='gridcells', table_data={'gid':gridcell_grid_id})
storage.write_table(table_name='jobs', table_data={'jid':arange(4)+1, 'gid':job_grid_id})
gs = Dataset(in_storage=storage, in_table_name='gridcells', id_name="gid", dataset_name="mygridcell")
jobs = Dataset(in_storage=storage, in_table_name='jobs', id_name="jid", dataset_name="myjob")
values = gs.compute_variables([expr], resources=Resources({"myjob":jobs, "mygridcell":gs}))
should_be = array([2, 1, 1])
self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg = "Error in " + expr)
# change gids of jobs (to test if computing dependencies is working)
jobs.modify_attribute(name="gid", data=array([1,1,1,1]))
values2 = gs.compute_variables([expr], resources=Resources({"myjob":jobs, "mygridcell":gs}))
should_be2 = array([4, 0, 0])
self.assert_(ma.allclose(values2, should_be2, rtol=1e-7), msg = "Error in " + expr)
def test_rand(self):
# test numpy.random.rand (this exercises 0-argument functions)
expr = "numpy.random.rand()"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
self.assert_(result >= 0 and result < 1, "Error in test_rand")
def test_numpy_arange_constant(self):
# test another constant - syntactically this looks like a method call, so it exercises that part of the code
expr = "numpy.arange(5)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([0, 1, 2, 3, 4])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant")
def test_numpy_arange_constant2(self):
# same as test_numpy_arange_constant, except provide 2 arguments
expr = "numpy.arange(2,5)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([2, 3, 4])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant2")
def test_alias_complex_expression(self):
# aliasing a complex expression
expr = "x = 2*sqrt(var1+var2)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={"var1": array([4,-8,0.5,1]), "var2": array([3,3,7,7]), "id": array([1,2,3,4])}
)
dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([ 5.29150262, 0.0, 5.47722558, 5.65685425])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_alias_complex_expression")
# check that the new var has x as an alias
v = VariableName(expr)
self.assertEqual(v.get_alias(), 'x', msg="bad value for alias")
# check that the alias gives the correct value
result2 = dataset.compute_variables(['x'])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable")
def test_ones_like(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={"attr1": array([5, 10, 0, 3]), "attr2": array([5.0, 0.0, 8.0, 1.1]), "id": arange(4)}
)
dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset")
expr1 = 'ones_like(attr1)'
result = dataset.compute_variables([expr1])
should_be = array([1, 1, 1, 1])
self.assert_(numpy.allclose(result, array(should_be), rtol=1e-6), "Error in ones_like")
self.assert_(result.dtype==should_be.dtype, "Error in ones_like")
expr2 = 'ones_like(attr1)'
result = dataset.compute_variables([expr2])
should_be = array([1, 1, 1, 1])
self.assert_(numpy.allclose(result, array(should_be), rtol=1e-6), "Error in ones_like")
self.assertEqual(result.dtype, should_be.dtype, "Error in ones_like")
def function_tester(self, func, values, should_be, optional_args=None):
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"my_variable": array(values), "id": arange(len(values))})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
if optional_args is None:
expr = "%s(my_variable)" % func
else:
expr = "%s(my_variable, %s)" % (func, optional_args)
result = dataset.compute_variables([expr])
self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in " + func)
def skip_test_dataset_qualified_name(self):
# this tests expressions with a dataset-qualified name
expr = "sqrt(tests.a_test_variable)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="tests", table_data={"a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4])}
)
dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([3.16227766, 7.0710678, 10])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-5), True)
def test_condition(self):
# test using a condition to return an array of True and False values
expr = "opus_core.test_agent.income>4"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="test_agents", table_data={"income": array([1, 5, 10, 3]), "id": array([1, 3, 4, 10])}
)
dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array([False, True, True, False])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_condition")
def test_true_false(self):
# make sure True and False can be used in an expression
expr = "array([True, False, False])"
# we're not actually using this dataset in the expression, but expressions are computed
# with respect to a dataset ...
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="test_agents", table_data={"income": array([10]), "id": array([1])})
dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array([True, False, False])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_true_false")
def test_simple_lag_variable2(self):
test_data = {
1000: {"tests": {"id": array([1, 2, 3, 4]), "attr1": array([10, 20, 30, 40])}},
1001: {"tests": {"id": array([1, 2, 3, 5]), "attr1": array([111, 222, 333, 555])}},
}
cache_creator = CreateTestAttributeCache()
cache_creator.create_attribute_cache_with_data(self._temp_dir, test_data)
SimulationState().set_current_time(1001)
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True, package_order=["opus_core"], in_storage=attribute_cache)
ds = Dataset(in_storage=attribute_cache, in_table_name="tests", id_name=["id"], dataset_name="tests")
ds.compute_variables(["opus_core.tests.attr1"])
self.assert_(ma.allequal(ds.get_attribute("attr1"), array([111, 222, 333, 555])))
ds.compute_variables(["opus_core.tests.attr1_lag1"])
self.assert_(ma.allequal(ds.get_attribute("attr1_lag1"), array([10, 20, 30, 555])))
def test_safe_array_divide_with_return_value(self):
expr = 'safe_array_divide(numerator, denominator, return_value_if_denominator_is_zero=100)'
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={"numerator": array([5, 10, 0, 3]), "denominator": array([5, 0, 8, 1]), "id": arange(4)}
)
dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([1, 100, 0, 3])
self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in safe_array_divide")
def test_safe_array_divide(self):
expr = "safe_array_divide(numerator, denominator)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="dataset",
table_data={"numerator": array([5, 10, 0, 3]), "denominator": array([5, 0, 8, 1]), "id": arange(4)},
)
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([1, 0, 0, 3])
self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in safe_array_divide")
def test_where(self):
# test using the numpy where function
expr = "where(opus_core.test_agent.income>4, 100, 200)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="test_agents", table_data={"income": array([1, 5, 10, 3]), "id": array([1, 3, 4, 10])}
)
dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array([200, 100, 100, 200])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_where")
def prepare_for_estimate(self, specification_dict = None, specification_storage=None,
specification_table=None, agent_set=None,
agents_for_estimation_storage=None,
agents_for_estimation_table=None, join_datasets=False,
index_to_unplace=None, portion_to_unplace=1.0,
agent_filter=None,
data_objects={}):
from opus_core.models.model import get_specification_for_estimation
specification = get_specification_for_estimation(specification_dict,
specification_storage,
specification_table)
if (agent_set is not None) and (index_to_unplace is not None):
if self.location_id_string is not None:
agent_set.compute_variables(self.location_id_string, resources=Resources(data_objects))
if portion_to_unplace < 1:
unplace_size = int(portion_to_unplace*index_to_unplace.size)
end_index_to_unplace = sample_noreplace(index_to_unplace, unplace_size)
else:
end_index_to_unplace = index_to_unplace
logger.log_status("Unplace " + str(end_index_to_unplace.size) + " agents.")
agent_set.modify_attribute(self.choice_set.get_id_name()[0],
-1*ones(end_index_to_unplace.size), end_index_to_unplace)
# create agents for estimation
if agents_for_estimation_storage is not None:
estimation_set = Dataset(in_storage = agents_for_estimation_storage,
in_table_name=agents_for_estimation_table,
id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name())
if agent_filter is not None:
estimation_set.compute_variables(agent_filter, resources=Resources(data_objects))
index = where(estimation_set.get_attribute(agent_filter) > 0)[0]
estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False)
if join_datasets:
agent_set.join_by_rows(estimation_set, require_all_attributes=False,
change_ids_if_not_unique=True)
index = arange(agent_set.size()-estimation_set.size(),agent_set.size())
else:
index = agent_set.get_id_index(estimation_set.get_id_attribute())
else:
index = arange(agent_set.size())
return (specification, index)
def test_casting(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"a_dependent_variable":array([1,5,1000]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
logger.enable_hidden_error_and_warning_words()
# Next line should cause a 'WARNING' to be logged.
dataset.compute_variables("opus_core.tests.a_test_variable",
resources=Resources({"check_variables":"*"}))
logger.disable_hidden_error_and_warning_words()
values = dataset.get_attribute("a_test_variable")
self.assertEqual(values.dtype.type, int8)
def test_compute_unloads_from_memory(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
'a_dependent_variable':array([1,5,10]),
'id':array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name='id', dataset_name='tests')
SessionConfiguration(in_storage=storage)["flush_variables"] = True
dataset.get_attribute("a_dependent_variable")
self.assert_("a_dependent_variable" in dataset.get_attributes_in_memory())
dataset.compute_variables("opus_core.tests.a_test_variable")
self.assert_("a_dependent_variable" not in dataset.get_attributes_in_memory())
self.assert_("a_test_variable" in dataset.get_attributes_in_memory())
SimulationState().remove_singleton(delete_cache=True)
def test_aggregate_sum_two_levels(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'my_variable':
array([4, 8, 2, 1, 40, 23, 78, 20, 25]),
'id0':
arange(9) + 1,
'id1':
array([1, 3, 1, 2, 3, 2, 1, 4, 4])
})
storage.write_table(table_name='fazes',
table_data={
'id1': array([1, 2, 3, 4]),
'id2': array([1, 2, 1, 3])
})
storage.write_table(table_name='fazdistrs',
table_data={
'id2': array([1, 2, 3]),
'id3': array([1, 2, 1])
})
storage.write_table(table_name='neighborhoods',
table_data={"id3": array([1, 2])})
ds0 = Dataset(in_storage=storage,
in_table_name='zones',
id_name="id0",
dataset_name="myzone")
ds1 = Dataset(in_storage=storage,
in_table_name='fazes',
id_name="id1",
dataset_name="myfaz")
ds2 = Dataset(in_storage=storage,
in_table_name='fazdistrs',
id_name="id2",
dataset_name="myfazdistr")
ds3 = Dataset(in_storage=storage,
in_table_name='neighborhoods',
id_name="id3",
dataset_name="myneighborhood")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
dataset_pool._add_dataset('myneighborhood', ds3)
values = ds3.compute_variables([
'myneighborhood.aggregate(10.0*myzone.my_variable, intermediates=[myfaz,myfazdistr], function=sum)'
],
dataset_pool=dataset_pool)
should_be = array([1770, 240])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in aggregate_sum_two_levels")
def test_fully_qualified_DDD_SSS_variable(self):
# this should use the test variable a_test_SSS_variable_DDD_SSS
expr = "opus_core.tests.a_test_squid_variable_42_clam"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"a_dependent_variable": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([10, 50, 100])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_fully_qualified_DDD_SSS_variable")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
self.assertEqual(name.get_package_name(),
'opus_core',
msg="bad value for package")
self.assertEqual(name.get_dataset_name(),
'tests',
msg="bad value for dataset")
self.assertEqual(name.get_short_name(),
'a_test_squid_variable_42_clam',
msg="bad value for shortname")
self.assertEqual(name.get_alias(),
'a_test_squid_variable_42_clam',
msg="bad value for alias")
self.assertEqual(name.get_autogen_class(),
None,
msg="bad value for autogen_class")
# test that the variable can now also be accessed using its short name in an expression
result2 = dataset.compute_variables(['a_test_squid_variable_42_clam'])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6),
"Error in accessing a_test_squid_variable_42_clam")
def test_simple_lag_variable(self):
test_data = {
1000: {
'tests': {
'id': array([1, 2, 3]),
'attr1': array([10, 20, 30]),
},
},
1001: {
'tests': {
'id': array([1, 2, 3]),
'attr1': array([111, 222, 333]),
},
},
}
cache_creator = CreateTestAttributeCache()
cache_creator.create_attribute_cache_with_data(self._temp_dir,
test_data)
SimulationState().set_current_time(1001)
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True,
package_order=['opus_core'],
in_storage=attribute_cache)
ds = Dataset(in_storage=attribute_cache,
in_table_name='tests',
id_name=['id'],
dataset_name='tests')
ds.compute_variables(['opus_core.tests.attr1'])
self.assert_(
ma.allequal(ds.get_attribute('attr1'), array([111, 222, 333])))
ds.compute_variables(['opus_core.tests.attr1_lag1'])
self.assert_(
ma.allequal(ds.get_attribute('attr1_lag1'), array([10, 20, 30])))
def test_casting(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"a_dependent_variable": array([1, 5, 1000]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
logger.enable_hidden_error_and_warning_words()
# Next line should cause a 'WARNING' to be logged.
dataset.compute_variables("opus_core.tests.a_test_variable",
resources=Resources({"check_variables":
"*"}))
logger.disable_hidden_error_and_warning_words()
values = dataset.get_attribute("a_test_variable")
self.assertEqual(values.dtype.type, int8)
def test_casts_attribute(self):
expr1 = "persons.astype(float64)"
expr2 = "persons.astype(float64)**2"
expr3 = "(2*persons).astype(float64)"
error_msg = "Error in test_casts_attribute"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"persons":array([1,5,10]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
result1 = dataset.compute_variables([expr1])
self.assertEqual( type(result1[0]), float64, error_msg)
self.assert_(ma.allclose(result1, array([1, 5, 10]), rtol=1e-6), error_msg)
result2 = dataset.compute_variables([expr2])
self.assertEqual( type(result2[0]), float64, error_msg)
self.assert_(ma.allclose(result2, array([1, 25, 100]), rtol=1e-6), error_msg)
result3 = dataset.compute_variables([expr3])
self.assertEqual( type(result3[0]), float64, error_msg)
self.assert_(ma.allclose(result3, array([2, 10, 20]), rtol=1e-6), error_msg)
def test_numpy_arange_constant(self):
# test another constant - syntactically this looks like a method call, so it exercises that part of the code
expr = "numpy.arange(5)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([0, 1, 2, 3, 4])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_numpy_arange_constant")
def test_numpy_arange_constant2(self):
# same as test_numpy_arange_constant, except provide 2 arguments
expr = "numpy.arange(2,5)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([2, 3, 4])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_numpy_arange_constant2")
def test_casts_fully_qualified_variable(self):
expr1 = "opus_core.test_agent.income_times_10.astype(int32)"
expr2 = "opus_core.test_agent.income_times_10.astype(int32)**2"
expr3 = "(2*opus_core.test_agent.income_times_10).astype(int32)"
error_msg = "Error in test_casts_fully_qualified_variable"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='test_agents',
table_data={
"income":array([1,5,10]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent")
result1 = dataset.compute_variables([expr1])
self.assertEqual( type(result1[0]), int32, error_msg)
self.assert_(ma.allclose(result1, array([10, 50, 100]), rtol=1e-6), error_msg)
result2 = dataset.compute_variables([expr2])
self.assertEqual( type(result2[0]), int32, error_msg)
self.assert_(ma.allclose(result2, array([100, 2500, 10000]), rtol=1e-6), error_msg)
result3 = dataset.compute_variables([expr3])
self.assertEqual( type(result3[0]), int32, error_msg)
self.assert_(ma.allclose(result3, array([20, 100, 200]), rtol=1e-6), error_msg)
def test_disaggregate_two_levels(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='gridcells',
table_data={
'id': arange(9) + 1,
'id0': array([7, 6, 1, 3, 4, 4, 5, 2, 5])
})
storage.write_table(table_name='zones',
table_data={
'id0': arange(7) + 1,
'id1': array([1, 3, 1, 2, 3, 2, 1])
})
storage.write_table(table_name='fazes',
table_data={
'id1': array([1, 2, 3]),
'id2': array([1, 2, 1])
})
storage.write_table(table_name='fazdistrs',
table_data={
'my_variable': array([40, 50]),
'id2': array([1, 2])
})
ds = Dataset(in_storage=storage,
in_table_name='gridcells',
id_name="id0",
dataset_name="mygridcell")
ds0 = Dataset(in_storage=storage,
in_table_name='zones',
id_name="id0",
dataset_name="myzone")
ds1 = Dataset(in_storage=storage,
in_table_name='fazes',
id_name="id1",
dataset_name="myfaz")
ds2 = Dataset(in_storage=storage,
in_table_name='fazdistrs',
id_name="id2",
dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('mygridcell', ds)
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
values = ds.compute_variables([
"mygridcell.disaggregate(myfazdistr.my_variable, intermediates=[myfaz,myzone])"
],
dataset_pool=dataset_pool)
should_be = array([40, 50, 40, 40, 50, 50, 40, 40, 40])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in disaggregate_two_levels")
def test_variable_dependencies_tree_with_versioning(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
'id':array([2,4]),
'a_dependent_variable':array([4,7]),
'a_dependent_variable2':array([10,1])
}
)
ds = Dataset(in_storage=storage, in_table_name='tests', id_name='id', dataset_name='tests')
ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"])
self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==0) #initially version=0
self.assert_(ds.get_version("a_dependent_variable")==0)
self.assert_(ds.get_version("a_dependent_variable2")==0)
ds.modify_attribute("a_dependent_variable", array([0,0]))
self.assert_(ds.get_version("a_dependent_variable")==1) # version=1
ds.modify_attribute("a_dependent_variable", array([1,1]))
self.assert_(ds.get_version("a_dependent_variable")==2) # version=2
ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"])
self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==1)
ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"])
self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==1) # version does not change
autogen_variable = "my_var = 3 * opus_core.tests.a_dependent_variable"
ds.compute_variables([autogen_variable])
self.assert_(ds.get_version("my_var")==0)
ds.compute_variables([autogen_variable])
self.assert_(ds.get_version("my_var")==0)
def skip_test_dataset_qualified_name(self):
# this tests expressions with a dataset-qualified name
expr = "sqrt(tests.a_test_variable)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"a_dependent_variable":array([1,5,10]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([3.16227766, 7.0710678, 10])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-5), True)
def test_compute_unloads_from_memory(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
'a_dependent_variable': array([1, 5, 10]),
'id': array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name='id',
dataset_name='tests')
SessionConfiguration(in_storage=storage)["flush_variables"] = True
dataset.get_attribute("a_dependent_variable")
self.assert_(
"a_dependent_variable" in dataset.get_attributes_in_memory())
dataset.compute_variables("opus_core.tests.a_test_variable")
self.assert_(
"a_dependent_variable" not in dataset.get_attributes_in_memory())
self.assert_("a_test_variable" in dataset.get_attributes_in_memory())
SimulationState().remove_singleton(delete_cache=True)
def test_condition(self):
# test using a condition to return an array of True and False values
expr = "opus_core.test_agent.income>4"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='test_agents',
table_data={
"income":array([1,5,10,3]),
"id":array([1,3,4,10])
}
)
dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array( [False,True,True,False] )
self.assertEqual( ma.allclose( result, should_be, rtol=1e-7), True, msg = "Error in test_condition")
def test_where(self):
# test using the numpy where function
expr = "where(opus_core.test_agent.income>4, 100, 200)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='test_agents',
table_data={
"income":array([1,5,10,3]),
"id":array([1,3,4,10])
}
)
dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array( [200, 100, 100, 200] )
self.assertEqual( ma.allclose( result, should_be, rtol=1e-7), True, msg = "Error in test_where")
def prepare_for_estimate(specification_dict=None,
specification_storage=None,
specification_table=None,
agent_set=None,
agents_for_estimation_storage=None,
agents_for_estimation_table=None,
join_datasets=False,
filter=None,
agents_filter=None,
data_objects=None):
"""
filter - alias to agents_filter for backforward compatibility, which is more specific
"""
if agents_filter is None and filter is not None:
agents_filter = filter
specification = get_specification_for_estimation(specification_dict,
specification_storage,
specification_table)
if agents_for_estimation_storage is not None:
estimation_set = Dataset(in_storage=agents_for_estimation_storage,
in_table_name=agents_for_estimation_table,
id_name=agent_set.get_id_name(),
dataset_name=agent_set.get_dataset_name())
filter_index = arange(estimation_set.size())
if agents_filter:
filter_condition = estimation_set.compute_variables(
agents_filter, resources=Resources(data_objects))
filter_index = where(filter_condition)[0]
if join_datasets:
agent_set.join_by_rows(estimation_set,
require_all_attributes=False,
change_ids_if_not_unique=True)
index = arange(agent_set.size() - estimation_set.size(),
agent_set.size())[filter_index]
else:
index = agent_set.get_id_index(
estimation_set.get_id_attribute()[filter_index])
else:
if agent_set is not None:
index = arange(agent_set.size())
else:
index = None
return (specification, index)
def test_sqrt_constant(self):
# test an expression that is constant -- should have no dependencies
expr = "sqrt(25)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={"id": array([1,2])}
)
# we don't actually use anything in the dataset
dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
self.assert_(4.99<result and result<5.01, "Error in test_sqrt_constant")
# check the dependencies
v = VariableName(expr)
var = VariableFactory().get_variable(v, dataset)
self.assertEqual(var.dependencies(), [], msg="dependencies are incorrect")
def test_safe_array_divide_with_return_value(self):
expr = 'safe_array_divide(numerator, denominator, return_value_if_denominator_is_zero=100)'
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"numerator": array([5, 10, 0, 3]),
"denominator": array([5, 0, 8, 1]),
"id": arange(4)
})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([1, 100, 0, 3])
self.assert_(ma.allclose(result, array(should_be), rtol=1e-6),
"Error in safe_array_divide")
def test_expression_1var_2times(self):
# test an expression with two occurences of the same variable
# (the var should just occur once in dependencies)
expr = "var1+sqrt(var1)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={"var1": array([4,25,0,1]), "id": array([1,2,3,4])}
)
dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([ 6, 30, 0, 2])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_expression_2vars")
# check the dependencies
v = VariableName(expr)
var = VariableFactory().get_variable(v, dataset)
self.assertEqual(var.dependencies(), ['mydataset.var1'], msg="dependencies are incorrect")
def test_true_false(self):
# make sure True and False can be used in an expression
expr = "array([True, False, False])"
# we're not actually using this dataset in the expression, but expressions are computed
# with respect to a dataset ...
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='test_agents',
table_data={
"income":array([10]),
"id":array([1])
}
)
dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array( [True, False, False] )
self.assertEqual( ma.allclose( result, should_be, rtol=1e-7), True, msg = "Error in test_true_false")
def test_expression(self):
dataset_pool = DatasetPool(package_order=['urbansim'],
storage=AttributeCache())
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={
"id": array([1, 2, 3, 4]),
"year": array([1968, 1989, 1750,
0]) # absolute_min_year = 1800
})
ds = Dataset(in_storage=storage, in_table_name='dataset', id_name="id")
result = ds.compute_variables([
'is_correct_year = dataset.year >= urbansim_constant.absolute_min_year'
],
dataset_pool=dataset_pool)
self.assertEqual(ma.allequal(result, array([1, 1, 0, 0])), True)
def function_tester(self, func, values, should_be, optional_args=None):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"my_variable": array(values),
"id": arange(len(values))
})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
if optional_args is None:
expr = "%s(my_variable)" % func
else:
expr = "%s(my_variable, %s)" % (func, optional_args)
result = dataset.compute_variables([expr])
self.assert_(ma.allclose(result, array(should_be), rtol=1e-6),
"Error in " + func)
def test_expression_2vars(self):
# test an expression with 2 variables
expr = "2*sqrt(var1+var2)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={"var1": array([4,-8,0.5,1]), "var2": array([3,3,7,7]), "id": array([1,2,3,4])}
)
dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([ 5.29150262, 0.0, 5.47722558, 5.65685425])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_expression_2vars")
# check the dependencies (will depend on two different other variables)
v = VariableName(expr)
var = VariableFactory().get_variable(v, dataset)
# use sets for the equality test, since we don't know in what order the dependencies will be returned
self.assertEqual(set(var.dependencies()), set(['mydataset.var1', 'mydataset.var2']),
msg="dependencies are incorrect")
def test_attr_power(self):
# Attributes and fully-qualified names to a power require separate parse tree patterns,
# which are tested in the following two tests.
# test attribute to a power
expr = "var1**3"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={"var1": array([4,-8,0.5,1]), "id": array([1,2,3,4])}
)
dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([64, -512, 0.125, 1])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_attr_power")
# check the dependencies (trickier for ** because we need a separate attribute tree pattern)
v = VariableName(expr)
var = VariableFactory().get_variable(v, dataset)
self.assertEqual(var.dependencies(), ['mydataset.var1'], msg="dependencies are incorrect")
def test_aggregate_sum(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'my_variable':array([4,8,0.5,1]),
'id':array([1,2,3,4]),
'id2':array([1,2,1,2]),
}
)
storage.write_table(table_name='faz', table_data={"id2":array([1,2])})
ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone")
ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds)
dataset_pool._add_dataset('myfaz', ds2)
values = ds2.compute_variables(['myfaz.aggregate(myzone.my_variable, function=sum)'], dataset_pool=dataset_pool)
should_be = array([4.5, 9])
self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_sum")
def test_fully_qualified_name_power(self):
# test fully qualified name to a power
expr = "opus_core.tests.a_test_variable**2"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"a_dependent_variable":array([1,0]),
"id":array([1,3])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([100,0])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-5), True, msg="error in test_fully_qualified_name_power")
# check the dependencies
v = VariableName(expr)
var = VariableFactory().get_variable(v, dataset)
self.assertEqual(var.dependencies(), ['opus_core.tests.a_test_variable'], msg="dependencies are incorrect")
def test_unary_functions_fully_qualified_name(self):
# this tests expressions with unary functions applied to a fully qualified name
expr = "sqrt(opus_core.tests.a_test_variable)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"a_dependent_variable": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([3.16227766, 7.0710678, 10])
self.assertEqual(
ma.allclose(result, should_be, rtol=1e-5),
True,
msg="error in test_unary_functions_fully_qualified_name")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
autogen = name.get_autogen_class()
self.assert_(issubclass(autogen, Variable),
msg="autogen'd class isn't a Variable")
self.assertEqual(name.get_package_name(),
None,
msg="bad value for package")
self.assertEqual(name.get_dataset_name(),
'tests',
msg="bad value for dataset")
self.assertEqual(name.get_short_name(),
autogen.__name__,
msg="bad value for shortname")
self.assertEqual(name.get_alias(),
autogen.__name__,
msg="bad value for alias")
# make an instance of the class and check the dependencies (since the dependent variables
# all have fully-qualifed names we don't need to associate a dataset with the variable
# for this test)
self.assertEqual(autogen().dependencies(),
['opus_core.tests.a_test_variable'],
msg="dependencies are incorrect")
def test_disaggregate_and_multiply(self):
# Perform two different disaggregations and multiply the results. This tests using a dataset name in both the
# list of intermediates and as the dataset being disaggregated (myfaz in this case).
expr = "myzone.disaggregate(myfaz.fazsqft) * myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id0': arange(7) + 1,
'id1': array([1, 3, 1, 2, 3, 2, 1])
})
storage.write_table(table_name='fazes',
table_data={
'id1': array([1, 2, 3]),
'id2': array([1, 2, 1]),
'fazsqft': array([10, 50, 100])
})
storage.write_table(table_name='fazdistrs',
table_data={
'my_variable': array([40, 50]),
'id2': array([1, 2])
})
ds0 = Dataset(in_storage=storage,
in_table_name='zones',
id_name="id0",
dataset_name="myzone")
ds1 = Dataset(in_storage=storage,
in_table_name='fazes',
id_name="id1",
dataset_name="myfaz")
ds2 = Dataset(in_storage=storage,
in_table_name='fazdistrs',
id_name="id2",
dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
values = ds0.compute_variables([expr], dataset_pool=dataset_pool)
should_be = array([400, 4000, 400, 2500, 4000, 2500, 400])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in disaggregate_and_multiply")
def test_alias_attribute_with_modification(self):
# this tests an expression consisting of an alias for a primary attribute that is modified
expr = "p = persons"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"persons": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
# modify the primary attribute 'persons'
new_values = array([3, 0, 100])
dataset.modify_attribute('persons', new_values)
# result should have the new values
result = dataset.compute_variables([expr])
self.assertEqual(ma.allclose(result, new_values, rtol=1e-7),
True,
msg="error in test_alias_attribute_with_modification")
def test_aggregate_sum_one_level(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'my_variable': array([4, 8, 2, 1, 40, 23, 78]),
'id0': arange(7) + 1,
'id1': array([1, 3, 1, 2, 3, 2, 1])
})
storage.write_table(table_name='fazes',
table_data={
'id1': array([1, 2, 3]),
'id2': array([1, 2, 1])
})
storage.write_table(table_name='fazdistr',
table_data={'id2': array([1, 2])})
ds0 = Dataset(in_storage=storage,
in_table_name='zones',
id_name="id0",
dataset_name="myzone")
ds1 = Dataset(in_storage=storage,
in_table_name='fazes',
id_name="id1",
dataset_name="myfaz")
ds2 = Dataset(in_storage=storage,
in_table_name='fazdistr',
id_name="id2",
dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
values = ds2.compute_variables([
'myfazdistr.aggregate(myzone.my_variable, intermediates=[myfaz])'
],
dataset_pool=dataset_pool)
should_be = array([132, 24])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in aggregate_sum_one_level")
def test_aggregate_squared(self):
# more exercising the SUBPATTERN_NUMBER_OF_AGENTS_WITH_CAST tree pattern
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'zone_id':array([1,2]),
}
)
storage.write_table(table_name='gridcells',
table_data={
'my_variable':array([4,8,0.5,1]),
'grid_id':array([1,2,3,4]),
'zone_id':array([1,2,1,2]),
}
)
zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name='zone')
gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell')
dataset_pool = DatasetPool()
dataset_pool._add_dataset('gridcell', gridcell_dataset)
dataset_pool._add_dataset('zone', zone_dataset)
values = zone_dataset.compute_variables(['zone.aggregate(gridcell.my_variable)**2'], dataset_pool=dataset_pool)
should_be = array([4.5*4.5, 9*9])
self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate")
